diff --git a/packages/data_designer_nemo/src/data_designer_nemo/fileset_filesystem_provider.py b/packages/data_designer_nemo/src/data_designer_nemo/fileset_filesystem_provider.py index 3c418321b5..6ae662aba3 100644 --- a/packages/data_designer_nemo/src/data_designer_nemo/fileset_filesystem_provider.py +++ b/packages/data_designer_nemo/src/data_designer_nemo/fileset_filesystem_provider.py @@ -13,9 +13,9 @@ SeedReaderFileSystemContext, ) from data_designer_nemo.filesystem import make_filesystem +from filesets import FilesetFileSystem, FilesetPathError, build_fileset_ref, parse_fileset_ref from fsspec.implementations.dirfs import DirFileSystem from nemo_platform import AsyncNeMoPlatform, NeMoPlatform -from nemo_platform.filesets import FilesetFileSystem, FilesetPathError, build_fileset_ref, parse_fileset_ref from nemo_platform_plugin.files.client import AsyncFilesClient, FilesClient diff --git a/packages/data_designer_nemo/src/data_designer_nemo/filesystem.py b/packages/data_designer_nemo/src/data_designer_nemo/filesystem.py index 689e5c6d62..7968b59503 100644 --- a/packages/data_designer_nemo/src/data_designer_nemo/filesystem.py +++ b/packages/data_designer_nemo/src/data_designer_nemo/filesystem.py @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 from data_designer_nemo.sdk_translation import async_to_sync_sdk +from filesets import FilesetFileSystem from nemo_platform import AsyncNeMoPlatform, NeMoPlatform -from nemo_platform.filesets import FilesetFileSystem from nemo_platform_plugin.client.adapter import client_from_platform from nemo_platform_plugin.files.client import FilesClient diff --git a/packages/data_designer_nemo/src/data_designer_nemo/seed.py b/packages/data_designer_nemo/src/data_designer_nemo/seed.py index 8bd3a492c8..82fb956797 100644 --- a/packages/data_designer_nemo/src/data_designer_nemo/seed.py +++ b/packages/data_designer_nemo/src/data_designer_nemo/seed.py @@ -10,8 +10,8 @@ from data_designer_nemo.fileset_file_seed_source import FilesetFileSeedSource from data_designer_nemo.fileset_filesystem_provider import is_local_directory from data_designer_nemo.secret_resolver import validate_secret +from filesets import FilesetPathError, build_fileset_ref, parse_fileset_ref from nemo_platform import AsyncNeMoPlatform -from nemo_platform.filesets import FilesetPathError, build_fileset_ref, parse_fileset_ref from nemo_platform_plugin.client.adapter import client_from_platform from nemo_platform_plugin.client.errors import NotFoundError, PermissionDeniedError from nemo_platform_plugin.files.client import AsyncFilesClient diff --git a/packages/filesets/pyproject.toml b/packages/filesets/pyproject.toml index 9274914a4e..3ee6047145 100644 --- a/packages/filesets/pyproject.toml +++ b/packages/filesets/pyproject.toml @@ -31,6 +31,7 @@ packages = ["src/filesets"] [tool.vendor-package] package = "filesets" package_root = "packages/filesets" +sdk_include_mode = "source-package" target_sdk_module = "filesets" included_paths = [ "**/*.py", diff --git a/packages/filesets/src/filesets/resources.py b/packages/filesets/src/filesets/resources.py index de4b204ba0..f9032b09d9 100644 --- a/packages/filesets/src/filesets/resources.py +++ b/packages/filesets/src/filesets/resources.py @@ -12,10 +12,16 @@ from dataclasses import dataclass from functools import cached_property from pathlib import PurePath -from typing import Protocol, runtime_checkable +from typing import Any, Protocol, runtime_checkable from fsspec.callbacks import DEFAULT_CALLBACK, Callback from fsspec.core import has_magic +from nemo_platform.resources.files.files import ( + AsyncFilesResource as GeneratedAsyncFilesResource, +) +from nemo_platform.resources.files.files import ( + FilesResource as GeneratedFilesResource, +) from nemo_platform.resources.files.filesets import AsyncFilesetsResource, FilesetsResource from nemo_platform.resources.files.otlp.otlp import AsyncOtlpResource, OtlpResource from nemo_platform_plugin.files.client import AsyncFilesClient, FilesClient @@ -150,6 +156,7 @@ def __init__( # Retain the platform client so the generated fileset/otlp sub-resources # (which speak to the platform client, not the FilesClient) can be exposed. self._platform_client = client + self._generated_files = GeneratedFilesResource(client) self._async_client = async_files_client if files_client is not None: self._client = files_client @@ -163,6 +170,9 @@ def client(self) -> FilesClient: """Access the underlying FilesClient for direct API calls.""" return self._client + def __getattr__(self, name: str) -> Any: + return getattr(self._generated_files, name) + @cached_property def filesets(self) -> FilesetsResource: """Fileset entity CRUD (create/list/get/update/delete) via the generated SDK resource.""" @@ -255,7 +265,7 @@ def download( ... ) # With progress callback - >>> from nemo_platform.filesets import RichProgressCallback + >>> from filesets import RichProgressCallback >>> with RichProgressCallback(description="Downloading") as cb: ... sdk.files.download( ... remote_path="my-fileset#", @@ -693,6 +703,7 @@ def __init__(self, client, *, files_client: AsyncFilesClient | None = None) -> N # Retain the platform client so the generated fileset/otlp sub-resources # (which speak to the platform client, not the FilesClient) can be exposed. self._platform_client = client + self._generated_files = GeneratedAsyncFilesResource(client) if files_client is not None: self._client = files_client else: @@ -705,6 +716,9 @@ def client(self) -> AsyncFilesClient: """Access the underlying AsyncFilesClient for direct API calls.""" return self._client + def __getattr__(self, name: str) -> Any: + return getattr(self._generated_files, name) + @cached_property def filesets(self) -> AsyncFilesetsResource: """Fileset entity CRUD (create/list/get/update/delete) via the generated SDK resource.""" diff --git a/packages/models/pyproject.toml b/packages/models/pyproject.toml index 9f7c49c9d5..1f66650b7e 100644 --- a/packages/models/pyproject.toml +++ b/packages/models/pyproject.toml @@ -27,6 +27,7 @@ dev = [] [tool.vendor-package] package = "models" package_root = "packages/models" +sdk_include_mode = "source-package" target_sdk_module = "models" included_paths = [ "**/*.py", diff --git a/packages/models/tests/test_client.py b/packages/models/tests/test_client.py index f5d3ee849c..42fa9a71f4 100644 --- a/packages/models/tests/test_client.py +++ b/packages/models/tests/test_client.py @@ -449,8 +449,8 @@ def test_wait_for_openai_model_bounds_sleep_to_remaining_timeout(sdk): "get", side_effect=_not_found_error(), ), - patch("nemo_platform.models.resources.time.time", side_effect=clock.time), - patch("nemo_platform.models.resources.time.sleep", side_effect=clock.sleep), + patch("models.resources.time.time", side_effect=clock.time), + patch("models.resources.time.sleep", side_effect=clock.sleep), ): with pytest.raises(TimeoutError, match="OpenAI model ws/model-a not available"): sdk.models.wait_for_openai_model("model-a", workspace="ws", timeout=1.25, poll_interval=5) @@ -631,8 +631,8 @@ async def test_async_wait_for_openai_model_bounds_sleep_to_remaining_timeout(asy with ( patch.object(async_sdk.inference.gateway.openai.v1.models, "get", mock_get), - patch("nemo_platform.models.resources.time.time", side_effect=clock.time), - patch("nemo_platform.models.resources.asyncio.sleep", side_effect=clock.async_sleep), + patch("models.resources.time.time", side_effect=clock.time), + patch("models.resources.asyncio.sleep", side_effect=clock.async_sleep), ): with pytest.raises(TimeoutError, match="OpenAI model ws/model-a not available"): await async_sdk.models.wait_for_openai_model("model-a", workspace="ws", timeout=1.25, poll_interval=5) diff --git a/packages/nemo_evaluator_sdk/pyproject.toml b/packages/nemo_evaluator_sdk/pyproject.toml index 4eaac9d8bf..8685fd3e1a 100644 --- a/packages/nemo_evaluator_sdk/pyproject.toml +++ b/packages/nemo_evaluator_sdk/pyproject.toml @@ -108,6 +108,7 @@ addopts = ["--import-mode=importlib"] [tool.vendor-package] package = "nemo_evaluator_sdk" package_root = "packages/nemo_evaluator_sdk" +sdk_include_mode = "source-package" source_module = "nemo_evaluator_sdk" target_sdk_module = "beta.evaluator" sdk_optional_dependencies_name = "nemo-evaluator-sdk" diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_evidence.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_evidence.py index ce42b78215..f6265d1078 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_evidence.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_evidence.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import asyncio +import importlib import json import os import time @@ -372,9 +373,10 @@ async def test_trace_handle_exposes_typed_tool_evidence_and_retains_modeled_fiel def test_observation_models_are_exported_from_source_and_vendored_values_packages() -> None: from nemo_evaluator_sdk.values import Observation as SourceObservation from nemo_evaluator_sdk.values import ObservationResult as SourceObservationResult - from nemo_platform.beta.evaluator.values import Observation as VendoredObservation - from nemo_platform.beta.evaluator.values import ObservationResult as VendoredObservationResult + vendored_values = importlib.import_module("nemo_platform.beta.evaluator.values") + VendoredObservation = vendored_values.Observation + VendoredObservationResult = vendored_values.ObservationResult assert SourceObservation.__name__ == VendoredObservation.__name__ == "Observation" assert SourceObservationResult.__name__ == VendoredObservationResult.__name__ == "ObservationResult" diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_sandbox_compose_vendored_import.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_sandbox_compose_vendored_import.py index bf0bb5e905..a77a56925c 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_sandbox_compose_vendored_import.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_sandbox_compose_vendored_import.py @@ -8,20 +8,16 @@ import importlib from pathlib import Path -from nemo_platform.beta.evaluator.agent_eval.runtimes.sandbox.providers.compose import ( - ComposeCleanupError, - ComposeCommandResult, - ComposeServiceTopology, - ComposeTeardownContext, - DockerComposeSandboxProvider, - ProgressCallback, - PullPolicy, - TeardownHook, -) - def test_vendored_compose_public_imports_are_constructible_without_docker(tmp_path: Path) -> None: """The vendored public Compose façade remains importable without Docker.""" + compose = importlib.import_module("nemo_platform.beta.evaluator.agent_eval.runtimes.sandbox.providers.compose") + ComposeCleanupError = compose.ComposeCleanupError + ComposeCommandResult = compose.ComposeCommandResult + ComposeServiceTopology = compose.ComposeServiceTopology + ComposeTeardownContext = compose.ComposeTeardownContext + DockerComposeSandboxProvider = compose.DockerComposeSandboxProvider + for cls in ( ComposeCleanupError, ComposeCommandResult, @@ -31,9 +27,9 @@ def test_vendored_compose_public_imports_are_constructible_without_docker(tmp_pa ): assert getattr(importlib.import_module(cls.__module__), cls.__name__) is cls - assert ProgressCallback is not None - assert PullPolicy is not None - assert TeardownHook is not None + assert compose.ProgressCallback is not None + assert compose.PullPolicy is not None + assert compose.TeardownHook is not None topology = ComposeServiceTopology("agent", frozenset({"agent"})) command_result = ComposeCommandResult(("docker", "compose", "ps"), 0, "", "") diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_values.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_values.py index f738d7a6d7..c1438469ea 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_values.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_values.py @@ -6,7 +6,9 @@ import json import math from collections.abc import Iterator +from importlib import import_module from pathlib import Path +from typing import Any import pytest from nemo_evaluator_sdk.agent_eval.results import ( @@ -30,6 +32,10 @@ from pydantic import RootModel, ValidationError +def _vendored_module(name: str) -> Any: + return import_module(f"nemo_platform.beta.evaluator.agent_eval.{name}") + + class _TokenCount(RootModel[int]): """A free-model output: numeric, but a measurement rather than a per-trial score.""" @@ -543,7 +549,7 @@ def test_summary_without_task_metric_values_loads_as_empty() -> None: def test_vendored_summary_accepts_task_metric_values() -> None: - from nemo_platform.beta.evaluator.agent_eval.results import AgentEvalSummary as VendoredAgentEvalSummary + VendoredAgentEvalSummary = _vendored_module("results").AgentEvalSummary payload = { "task_metric_values": { @@ -559,18 +565,11 @@ def test_vendored_summary_accepts_task_metric_values() -> None: def test_vendored_module_exposes_the_public_value_api() -> None: # The byte-copy test below proves file parity, not that the names are usable through the shipped # package. These are the surface a consumer of nemo-platform actually imports. - from nemo_platform.beta.evaluator.agent_eval.results import ( - AgentEvalSummary as VendoredSummary, - ) - from nemo_platform.beta.evaluator.agent_eval.results import ( - TrialMetricValue as VendoredValue, - ) - from nemo_platform.beta.evaluator.agent_eval.results import ( - TrialMetricValueType as VendoredType, - ) - from nemo_platform.beta.evaluator.agent_eval.results import ( - numeric_metric_values as vendored_numeric, - ) + vendored_results = _vendored_module("results") + VendoredSummary = vendored_results.AgentEvalSummary + VendoredValue = vendored_results.TrialMetricValue + VendoredType = vendored_results.TrialMetricValueType + vendored_numeric = vendored_results.numeric_metric_values records = [VendoredValue(trial_id="t0", value=1.0), VendoredValue(trial_id="t1", value="good")] assert vendored_numeric(records) == [1.0] # the label is dropped, as in the source module @@ -581,22 +580,14 @@ def test_vendored_module_exposes_the_public_value_api() -> None: assert outcomes.task_id == "task-a" and outcomes.outcomes[0].metric_name == "reward.score" -def test_vendored_results_module_is_a_verbatim_copy_of_this_one() -> None: - # `make vendor` mirrors this module into the SDK, rewriting only the package root. Validating the - # field shape (above) would still pass against a stale copy carrying older filtering or docs, so - # pin the whole file: any edit here that is not mirrored is drift between two live code paths. +def test_legacy_results_import_resolves_to_the_source_module() -> None: + # The SDK exposes this legacy path through a runtime alias, not a rewritten copy, so import + # compatibility should point at the canonical source file. import nemo_evaluator_sdk.agent_eval.results as source - import nemo_platform.beta.evaluator.agent_eval.results as vendored - expected = ( - Path(source.__file__) - .read_text(encoding="utf-8") - .replace("from nemo_evaluator_sdk.", "from nemo_platform.beta.evaluator.") - ) + legacy = _vendored_module("results") - assert Path(vendored.__file__).read_text(encoding="utf-8") == expected, ( - "sdk/python/.../beta/evaluator/agent_eval/results.py is out of sync; re-run `make vendor`" - ) + assert Path(legacy.__file__).resolve() == Path(source.__file__).resolve() def test_gym_example_rejects_a_bundle_written_before_task_metric_values(tmp_path: Path) -> None: diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_trial_error_rollup.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_trial_error_rollup.py index 0a2811b9b9..2054c5b715 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_trial_error_rollup.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_trial_error_rollup.py @@ -5,6 +5,9 @@ from __future__ import annotations +from importlib import import_module +from typing import Any + import pytest from nemo_evaluator_sdk.agent_eval.results import AgentEvalSummary, _error_trial_ids from nemo_evaluator_sdk.agent_eval.scores import AgentEvalScoreStatus, AgentEvalTaskScore @@ -18,6 +21,10 @@ from pydantic import ValidationError +def _vendored_module(name: str) -> Any: + return import_module(f"nemo_platform.beta.evaluator.agent_eval.{name}") + + def _trial( trial_id: str, *, @@ -133,17 +140,13 @@ def test_summary_round_trips_the_rollup_through_json() -> None: def test_vendored_module_exposes_the_error_rollup_surface() -> None: # The byte-copy pin proves file parity, not that these names are importable through the shipped - # package — which is the path a nemo-platform consumer actually uses. - from nemo_platform.beta.evaluator.agent_eval.results import AgentEvalSummary as VendoredSummary - from nemo_platform.beta.evaluator.agent_eval.trials import ( - AgentEvalTrial as VendoredTrial, - ) - from nemo_platform.beta.evaluator.agent_eval.trials import ( - AgentEvalTrialStatus as VendoredStatus, - ) - from nemo_platform.beta.evaluator.agent_eval.trials import ( - TrialError as VendoredError, - ) + # package -- which is the path a nemo-platform consumer actually uses. + vendored_results = _vendored_module("results") + vendored_trials = _vendored_module("trials") + VendoredSummary = vendored_results.AgentEvalSummary + VendoredTrial = vendored_trials.AgentEvalTrial + VendoredStatus = vendored_trials.AgentEvalTrialStatus + VendoredError = vendored_trials.TrialError trial = VendoredTrial( id="t0", diff --git a/packages/nemo_platform/pyproject.toml b/packages/nemo_platform/pyproject.toml index bfbc14c546..1ee4d8cc56 100644 --- a/packages/nemo_platform/pyproject.toml +++ b/packages/nemo_platform/pyproject.toml @@ -126,6 +126,7 @@ files-service = [ "ngcsdk>=4.9.10", "duckdb>=1.1.3", "pandas>=1.5.3", + "filesets", "opentelemetry-proto>=1.28.2", "aioboto3>=15.5.0", "types-aioboto3[s3]>=15.5.0", @@ -390,24 +391,24 @@ nemo-platform-plugin = [ # Generated from [tool.bundle-package]; do not edit by hand. nemo-platform-sdk = [ - "httpx>=0.23.0, <1", - "pydantic>=2.0.0,<3", - "typing-extensions>=4.14, <5", - "anyio>=4.0.0,<5", - "distro>=1.7.0, <2", - "sniffio", "nemo-platform-plugin", "typer>=0.20.0", "rich>=13.7.1", "prompt_toolkit>=3.0.0", "requests>=2.31.0", + "pydantic>=2.0.0,<3", "pyyaml>=6.0.0", "docker>=7.0.0", "ngcsdk>=4.8.2", "nvidia-ml-py>=13.0.0", "psutil>=5.9.0", + "httpx>=0.23.0,<1", "openai", + "anyio>=4.0.0,<5", "fsspec>=2023.1.0", + "typing-extensions>=4.14, <5", + "distro>=1.7.0, <2", + "sniffio", ] # Generated from [tool.bundle-package]; do not edit by hand. @@ -504,9 +505,9 @@ services = [ "nmp-common", "pyleak>=0.1.0", "rich>=14.1.0", - "nemo-platform[platform-seed-service]", "nemo-platform[core-service]", "nemo-platform[studio-service]", + "nemo-platform[platform-seed-service]", "nemo-platform[intake-service]", "nemo-platform[hello-world-service]", "nemo-platform[guardrails-service]", @@ -671,6 +672,8 @@ deployments = "nemo_deployments_plugin.skills:skills_dir" nemo-platform-sdk = { workspace = true } nemo-platform-ext = { workspace = true } nemo-evaluator-sdk = { workspace = true } +models = { workspace = true } +filesets = { workspace = true } nmp-build-tools = { workspace = true } [tool.hatch.version] @@ -707,6 +710,9 @@ only-include = ["_empty"] # Library packages nmp-common = { source = "../../packages/nmp_common/src/nmp/common", module = "nmp/common", inherit = { "entry-points" = ["nemo.*"] } } nemo-platform-plugin = { source = "../../packages/nemo_platform_plugin/src/nemo_platform_plugin", module = "nemo_platform_plugin" } +nemo-platform-ext = { source = "../../packages/nemo_platform_ext/src/nemo_platform_ext", module = "nemo_platform_ext", deps_group = "nemo-platform-sdk", force_include = { "../../../../docs" = "nemo_platform_ext/cli/docs" } } +models = { source = "../../packages/models/src/models", module = "models", deps_group = "nemo-platform-sdk" } +filesets = { source = "../../packages/filesets/src/filesets", module = "filesets", deps_group = "nemo-platform-sdk" } nemo-evaluator-sdk = { source = "../../packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk", module = "nemo_evaluator_sdk" } data-designer-nemo = { source = "../../packages/data_designer_nemo/src/data_designer_nemo", module = "data_designer_nemo" } nmp-platform-runner = { source = "../../packages/nmp_platform_runner/src/nmp/platform_runner", module = "nmp/platform_runner", deps_group = "services" } @@ -748,7 +754,6 @@ module = "nemo_platform" inherit."optional-dependencies" = true inherit.scripts = true inherit."entry-points" = ["nemo.*"] -force_include."../../../../../docs" = "nemo_platform/cli/docs" [tool.bundle-package.nemo-agents-example-calculator] source = "../../plugins/nemo-agents/examples/calculator-agent/src/calculator_agent" diff --git a/packages/nemo_platform_ext/pyproject.toml b/packages/nemo_platform_ext/pyproject.toml index 7136be3899..64c5bf5972 100644 --- a/packages/nemo_platform_ext/pyproject.toml +++ b/packages/nemo_platform_ext/pyproject.toml @@ -88,6 +88,7 @@ packages = ["src/nemo_platform_ext"] [tool.vendor-package] package = "nemo_platform_ext" package_root = "packages/nemo_platform_ext" +sdk_include_mode = "source-package" # Globs are evaluated per top-level module (e.g. `skills/`, `cli/`, `quickstart/`). # `**/*.md` recursively includes SKILL.md plus any companion markdown files # skills ship under `resources/` (e.g. notes, sub-docs, prompts). Add further diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/files/__init__.py b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/files/__init__.py index f5adedbda5..c4d8e3200f 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/files/__init__.py +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/files/__init__.py @@ -69,7 +69,7 @@ def upload_files( if workspace is None: workspace = client._get_workspace_path_param() - from nemo_platform.filesets import RichProgressCallback + from filesets import RichProgressCallback with RichProgressCallback(description="Uploading") as callback: if fileset is not None: @@ -136,7 +136,7 @@ def download_files( if workspace is None: workspace = client._get_workspace_path_param() - from nemo_platform.filesets import RichProgressCallback + from filesets import RichProgressCallback with RichProgressCallback(description="Downloading") as callback: client.files.download( diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/client/enhanced.py b/packages/nemo_platform_ext/src/nemo_platform_ext/client/enhanced.py index af6451b3ed..f8980ae6fe 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/client/enhanced.py +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/client/enhanced.py @@ -7,7 +7,7 @@ import os from pathlib import Path -from typing import Any, Mapping +from typing import Any, Mapping, Self import httpx from httpx import Timeout @@ -49,6 +49,20 @@ def _should_bootstrap_config( ) +def _copy_requires_bootstrap( + *, + config_path: Path | None, + context_name: str | None, + access_token: str | None, +) -> bool: + return ( + config_path is not None + or context_name is not None + or access_token is not None + or bool(os.environ.get(WORKLOAD_IDENTITY_TOKEN_FILE_ENVVAR)) + ) + + class NeMoPlatform(SyncAPIClient): def __init__( self, @@ -133,6 +147,9 @@ def __init__( http_client: Custom ``httpx.Client`` instance. When provided, the auth bootstrap is skipped entirely regardless of other parameters. """ + env_base_url = os.environ.get("NEMO_PLATFORM_BASE_URL") + bootstrap_base_url = base_url if base_url is not None else env_base_url + should_bootstrap = _should_bootstrap_config( http_client=http_client, base_url=base_url, @@ -146,7 +163,7 @@ def __init__( client_init_kwargs = build_client_init_kwargs( config_path=config_path, - base_url=base_url, + base_url=bootstrap_base_url, context_name=context_name, access_token=access_token, extra_headers=default_headers, @@ -155,9 +172,19 @@ def __init__( if workspace is None: workspace = client_init_kwargs.workspace default_headers = client_init_kwargs.default_headers + if client_init_kwargs.http_client is not None and not isinstance( + client_init_kwargs.http_client, httpx.Client + ): + raise TypeError("Expected httpx.Client from sync client factory") http_client = client_init_kwargs.http_client except Exception as e: - raise RuntimeError(f"NeMoPlatform client initialization failed: {e}") + raise RuntimeError(f"NeMoPlatform client initialization failed: {e}") from e + + if base_url is None: + base_url = bootstrap_base_url + + if base_url is None: + raise RuntimeError("NeMoPlatform client initialization failed: base_url is required") client_verify = client_verify_from_env() if http_client is None and client_verify is not True: @@ -194,6 +221,66 @@ def __getattr__(self, name: str) -> Any: self.__dict__[name] = instance return instance + def copy( + self, + *, + workspace: str | None = None, + base_url: str | httpx.URL | None = None, + inference_base_url: str | httpx.URL | None = None, + config_path: Path | None = None, + context_name: str | None = None, + access_token: str | None = None, + timeout: float | Timeout | None | NotGiven = not_given, + http_client: httpx.Client | None = None, + max_retries: int | NotGiven = not_given, + default_headers: Mapping[str, str] | None = None, + set_default_headers: Mapping[str, str] | None = None, + default_query: Mapping[str, object] | None = None, + set_default_query: Mapping[str, object] | None = None, + _extra_kwargs: Mapping[str, Any] = {}, + ) -> Self: + """ + Create a new client instance re-using the same options given to the current client with optional overriding. + """ + if default_headers is not None and set_default_headers is not None: + raise ValueError("The `default_headers` and `set_default_headers` arguments are mutually exclusive") + + if default_query is not None and set_default_query is not None: + raise ValueError("The `default_query` and `set_default_query` arguments are mutually exclusive") + + headers = self._custom_headers + if default_headers is not None: + headers = {**headers, **default_headers} + elif set_default_headers is not None: + headers = set_default_headers + + params = self._custom_query + if default_query is not None: + params = {**params, **default_query} + elif set_default_query is not None: + params = set_default_query + + if http_client is None and not _copy_requires_bootstrap( + config_path=config_path, + context_name=context_name, + access_token=access_token, + ): + http_client = self._client + return self.__class__( + workspace=workspace or self.workspace, + base_url=base_url or self.base_url, + inference_base_url=inference_base_url or self.inference_base_url, + config_path=config_path, + context_name=context_name, + access_token=access_token, + timeout=self.timeout if isinstance(timeout, NotGiven) else timeout, + http_client=http_client, + max_retries=self.max_retries if isinstance(max_retries, NotGiven) else max_retries, + default_headers=headers, + default_query=params, + **_extra_kwargs, + ) + class AsyncNeMoPlatform(AsyncAPIClient): # client options @@ -295,6 +382,9 @@ async def main() -> None: http_client: Custom ``httpx.AsyncClient`` instance. When provided, the auth bootstrap is skipped entirely regardless of other parameters. """ + env_base_url = os.environ.get("NEMO_PLATFORM_BASE_URL") + bootstrap_base_url = base_url if base_url is not None else env_base_url + should_bootstrap = _should_bootstrap_config( http_client=http_client, base_url=base_url, @@ -308,7 +398,7 @@ async def main() -> None: client_init_kwargs = build_async_client_init_kwargs( config_path=config_path, - base_url=base_url, + base_url=bootstrap_base_url, context_name=context_name, access_token=access_token, extra_headers=default_headers, @@ -317,9 +407,19 @@ async def main() -> None: if workspace is None: workspace = client_init_kwargs.workspace default_headers = client_init_kwargs.default_headers + if client_init_kwargs.http_client is not None and not isinstance( + client_init_kwargs.http_client, httpx.AsyncClient + ): + raise TypeError("Expected httpx.AsyncClient from async client factory") http_client = client_init_kwargs.http_client except Exception as e: - raise RuntimeError(f"NeMoPlatform client initialization failed: {e}") + raise RuntimeError(f"NeMoPlatform client initialization failed: {e}") from e + + if base_url is None: + base_url = bootstrap_base_url + + if base_url is None: + raise RuntimeError("NeMoPlatform client initialization failed: base_url is required") client_verify = client_verify_from_env() if http_client is None and client_verify is not True: @@ -358,3 +458,63 @@ def __getattr__(self, name: str) -> Any: instance = resource_cls(self) self.__dict__[name] = instance return instance + + def copy( + self, + *, + workspace: str | None = None, + base_url: str | httpx.URL | None = None, + inference_base_url: str | httpx.URL | None = None, + config_path: Path | None = None, + context_name: str | None = None, + access_token: str | None = None, + timeout: float | Timeout | None | NotGiven = not_given, + http_client: httpx.AsyncClient | None = None, + max_retries: int | NotGiven = not_given, + default_headers: Mapping[str, str] | None = None, + set_default_headers: Mapping[str, str] | None = None, + default_query: Mapping[str, object] | None = None, + set_default_query: Mapping[str, object] | None = None, + _extra_kwargs: Mapping[str, Any] = {}, + ) -> Self: + """ + Create a new client instance re-using the same options given to the current client with optional overriding. + """ + if default_headers is not None and set_default_headers is not None: + raise ValueError("The `default_headers` and `set_default_headers` arguments are mutually exclusive") + + if default_query is not None and set_default_query is not None: + raise ValueError("The `default_query` and `set_default_query` arguments are mutually exclusive") + + headers = self._custom_headers + if default_headers is not None: + headers = {**headers, **default_headers} + elif set_default_headers is not None: + headers = set_default_headers + + params = self._custom_query + if default_query is not None: + params = {**params, **default_query} + elif set_default_query is not None: + params = set_default_query + + if http_client is None and not _copy_requires_bootstrap( + config_path=config_path, + context_name=context_name, + access_token=access_token, + ): + http_client = self._client + return self.__class__( + workspace=workspace or self.workspace, + base_url=base_url or self.base_url, + inference_base_url=inference_base_url or self.inference_base_url, + config_path=config_path, + context_name=context_name, + access_token=access_token, + timeout=self.timeout if isinstance(timeout, NotGiven) else timeout, + http_client=http_client, + max_retries=self.max_retries if isinstance(max_retries, NotGiven) else max_retries, + default_headers=headers, + default_query=params, + **_extra_kwargs, + ) diff --git a/packages/nemo_platform_ext/tests/client/test_client.py b/packages/nemo_platform_ext/tests/client/test_client.py index 785b72c160..e4b79332f9 100644 --- a/packages/nemo_platform_ext/tests/client/test_client.py +++ b/packages/nemo_platform_ext/tests/client/test_client.py @@ -299,8 +299,8 @@ def test_exchanges_workload_identity_token_file(self, mock_exchange, _mock_disco assert mock_exchange.call_args.kwargs["scope"] == "openid email groups" @pytest.mark.asyncio - @patch("nemo_platform.client.factory.discover_nmp_config", return_value=_MOCK_WORKLOAD_NMP_CONFIG) - @patch("nemo_platform.auth.workload_exchange.token_exchange_grant") + @patch("nemo_platform_ext.client.factory.discover_nmp_config", return_value=_MOCK_WORKLOAD_NMP_CONFIG) + @patch("nemo_platform_ext.auth.workload_exchange.token_exchange_grant") async def test_async_exchanges_workload_identity_token_file_at_request_time( self, mock_exchange, _mock_discover, tmp_path, monkeypatch ): @@ -540,7 +540,7 @@ def test_refresh_grant_failure_surfaces_clear_error(self, mock_post, _mock_disco class TestClientConstructorBootstrapBypass: - @patch("nemo_platform.client.factory.build_client_init_kwargs") + @patch("nemo_platform_ext.client.factory.build_client_init_kwargs") def test_sync_constructor_with_base_url_skips_config_bootstrap(self, mock_build_client_kwargs): mock_build_client_kwargs.side_effect = AssertionError("bootstrap should not be called") @@ -553,8 +553,29 @@ def test_sync_constructor_with_base_url_skips_config_bootstrap(self, mock_build_ mock_build_client_kwargs.assert_not_called() + @patch("nemo_platform_ext.client.factory.build_client_init_kwargs") + def test_sync_constructor_env_base_url_still_bootstraps_when_base_url_omitted( + self, mock_build_client_kwargs, monkeypatch + ): + monkeypatch.setenv("NEMO_PLATFORM_BASE_URL", "http://env-host:8081") + mock_build_client_kwargs.return_value = MagicMock( + base_url="http://env-host:8081", + workspace="test-workspace", + default_headers=None, + http_client=None, + ) + + client = NeMoPlatform() + try: + assert str(client.base_url).rstrip("/") == "http://env-host:8081" + assert client.workspace == "test-workspace" + finally: + client.close() + + assert mock_build_client_kwargs.call_args.kwargs["base_url"] == "http://env-host:8081" + @patch("nemo_platform._client.DefaultHttpxClient") - @patch("nemo_platform.client.factory.build_client_init_kwargs") + @patch("nemo_platform_ext.client.factory.build_client_init_kwargs") def test_sync_constructor_direct_mode_uses_nemo_scoped_ca_bundle( self, mock_build_client_kwargs, mock_default_httpx_client, monkeypatch ): @@ -571,7 +592,27 @@ def test_sync_constructor_direct_mode_uses_nemo_scoped_ca_bundle( mock_build_client_kwargs.assert_not_called() mock_default_httpx_client.assert_called_once_with(verify="/tmp/nemo-ca.pem") - @patch("nemo_platform.client.factory.build_client_init_kwargs") + @patch("nemo_platform_ext.client.factory.build_client_init_kwargs") + def test_sync_copy_with_access_token_bootstraps_instead_of_reusing_http_client(self, mock_build_client_kwargs): + mock_build_client_kwargs.return_value = MagicMock( + base_url="http://override-host:8081", + workspace="test-workspace", + default_headers={"Authorization": "Bearer replacement-token"}, + http_client=None, + ) + + client = NeMoPlatform(base_url="http://original-host:8081", workspace="original-workspace") + original_http_client = client._client + copied = client.copy(access_token="replacement-token") + try: + assert copied._client is not original_http_client + finally: + copied.close() + client.close() + + assert mock_build_client_kwargs.call_args.kwargs["access_token"] == "replacement-token" + + @patch("nemo_platform_ext.client.factory.build_client_init_kwargs") def test_sync_constructor_with_workload_file_and_base_url_bootstraps(self, mock_build_client_kwargs, monkeypatch): monkeypatch.setenv(WORKLOAD_IDENTITY_TOKEN_FILE_ENVVAR, "/var/run/secrets/nemo-platform/workload/token") mock_build_client_kwargs.return_value = MagicMock( @@ -589,7 +630,7 @@ def test_sync_constructor_with_workload_file_and_base_url_bootstraps(self, mock_ assert mock_build_client_kwargs.call_args.kwargs["base_url"] == "http://override-host:8081" - @patch("nemo_platform.client.factory.build_client_init_kwargs") + @patch("nemo_platform_ext.client.factory.build_client_init_kwargs") def test_sync_constructor_passes_context_name_to_bootstrap(self, mock_build_client_kwargs): mock_build_client_kwargs.return_value = MagicMock( base_url="http://override-host:8081", @@ -609,9 +650,9 @@ def test_sync_constructor_passes_context_name_to_bootstrap(self, mock_build_clie def test_sync_constructor_rejects_legacy_context_argument(self): with pytest.raises(TypeError, match="unexpected keyword argument 'context'"): - NeMoPlatform(context="ctx-b") + NeMoPlatform(context="ctx-b") # ty: ignore[unknown-argument] - @patch("nemo_platform.client.factory.build_client_init_kwargs") + @patch("nemo_platform_ext.client.factory.build_client_init_kwargs") def test_sync_constructor_with_http_client_skips_config_bootstrap(self, mock_build_client_kwargs): mock_build_client_kwargs.side_effect = AssertionError("bootstrap should not be called") @@ -630,7 +671,7 @@ def test_sync_constructor_with_http_client_skips_config_bootstrap(self, mock_bui mock_build_client_kwargs.assert_not_called() @pytest.mark.asyncio - @patch("nemo_platform.client.factory.build_async_client_init_kwargs") + @patch("nemo_platform_ext.client.factory.build_async_client_init_kwargs") async def test_async_constructor_with_base_url_skips_config_bootstrap(self, mock_build_client_kwargs): mock_build_client_kwargs.side_effect = AssertionError("bootstrap should not be called") @@ -643,9 +684,31 @@ async def test_async_constructor_with_base_url_skips_config_bootstrap(self, mock mock_build_client_kwargs.assert_not_called() + @pytest.mark.asyncio + @patch("nemo_platform_ext.client.factory.build_async_client_init_kwargs") + async def test_async_constructor_env_base_url_still_bootstraps_when_base_url_omitted( + self, mock_build_client_kwargs, monkeypatch + ): + monkeypatch.setenv("NEMO_PLATFORM_BASE_URL", "http://env-host:8081") + mock_build_client_kwargs.return_value = MagicMock( + base_url="http://env-host:8081", + workspace="test-workspace", + default_headers=None, + http_client=None, + ) + + client = AsyncNeMoPlatform() + try: + assert str(client.base_url).rstrip("/") == "http://env-host:8081" + assert client.workspace == "test-workspace" + finally: + await client.close() + + assert mock_build_client_kwargs.call_args.kwargs["base_url"] == "http://env-host:8081" + @pytest.mark.asyncio @patch("nemo_platform._client.DefaultAsyncHttpxClient") - @patch("nemo_platform.client.factory.build_async_client_init_kwargs") + @patch("nemo_platform_ext.client.factory.build_async_client_init_kwargs") async def test_async_constructor_direct_mode_uses_nemo_scoped_ca_bundle( self, mock_build_client_kwargs, mock_default_httpx_client, monkeypatch ): @@ -663,7 +726,30 @@ async def test_async_constructor_direct_mode_uses_nemo_scoped_ca_bundle( mock_default_httpx_client.assert_called_once_with(verify="/tmp/nemo-ca.pem") @pytest.mark.asyncio - @patch("nemo_platform.client.factory.build_async_client_init_kwargs") + @patch("nemo_platform_ext.client.factory.build_async_client_init_kwargs") + async def test_async_copy_with_access_token_bootstraps_instead_of_reusing_http_client( + self, mock_build_client_kwargs + ): + mock_build_client_kwargs.return_value = MagicMock( + base_url="http://override-host:8081", + workspace="test-workspace", + default_headers={"Authorization": "Bearer replacement-token"}, + http_client=None, + ) + + client = AsyncNeMoPlatform(base_url="http://original-host:8081", workspace="original-workspace") + original_http_client = client._client + copied = client.copy(access_token="replacement-token") + try: + assert copied._client is not original_http_client + finally: + await copied.close() + await client.close() + + assert mock_build_client_kwargs.call_args.kwargs["access_token"] == "replacement-token" + + @pytest.mark.asyncio + @patch("nemo_platform_ext.client.factory.build_async_client_init_kwargs") async def test_async_constructor_with_workload_file_and_base_url_bootstraps( self, mock_build_client_kwargs, monkeypatch ): @@ -684,7 +770,7 @@ async def test_async_constructor_with_workload_file_and_base_url_bootstraps( assert mock_build_client_kwargs.call_args.kwargs["base_url"] == "http://override-host:8081" @pytest.mark.asyncio - @patch("nemo_platform.client.factory.build_async_client_init_kwargs") + @patch("nemo_platform_ext.client.factory.build_async_client_init_kwargs") async def test_async_constructor_with_http_client_skips_config_bootstrap(self, mock_build_client_kwargs): mock_build_client_kwargs.side_effect = AssertionError("bootstrap should not be called") @@ -703,7 +789,7 @@ async def test_async_constructor_with_http_client_skips_config_bootstrap(self, m mock_build_client_kwargs.assert_not_called() @pytest.mark.asyncio - @patch("nemo_platform.client.factory.build_async_client_init_kwargs") + @patch("nemo_platform_ext.client.factory.build_async_client_init_kwargs") async def test_async_constructor_passes_context_name_to_bootstrap(self, mock_build_client_kwargs): mock_build_client_kwargs.return_value = MagicMock( base_url="http://override-host:8081", diff --git a/packages/nemo_platform_plugin/pyproject.toml b/packages/nemo_platform_plugin/pyproject.toml index 57408c9e29..11da24291b 100644 --- a/packages/nemo_platform_plugin/pyproject.toml +++ b/packages/nemo_platform_plugin/pyproject.toml @@ -42,6 +42,25 @@ Source = "https://github.com/NVIDIA-NeMo/nemo-platform/tree/main/packages/nemo_p Documentation = "https://github.com/NVIDIA-NeMo/nemo-platform/tree/main/packages/nemo_platform_plugin/src/nemo_platform_plugin/docs" [project.optional-dependencies] +# Generated from [tool.bundle-package]; do not edit by hand. +nemo-evaluator-sdk = [ + "pydantic>=2.10.6", + "jinja2>=3.1.6", + "jsonschema>=4.23.0", + "jsonpath-ng>=1.7.0", + "pyarrow>=19.0.1", + "pandas>=1.5.3", + "openai>=1.61.0", + "httpx>=0.27.0,<1", + "sacrebleu>=2.5.1", + "rouge_score==0.1.2", + "ragas==0.4.3", + "langchain-openai>=1.3.5", + "langchain-nvidia-ai-endpoints>=1.4.3,<2.0.0", + "nemo-relay>=0.7.2,<0.8", + "nemo-fabric>=0.2.0,<0.3.0", +] + # Generated from [tool.bundle-package]; do not edit by hand. nemo-platform-sdk = [ "httpx>=0.23.0, <1", @@ -83,3 +102,7 @@ packages = ["src/nemo_platform_plugin"] [tool.bundle-package] nemo-platform-sdk = { source = "../../sdk/python/nemo-platform/src/nemo_platform", module = "nemo_platform" } +nemo-platform-ext = { source = "../nemo_platform_ext/src/nemo_platform_ext", module = "nemo_platform_ext", deps_group = "nemo-platform-sdk", force_include = { "../../../../docs" = "nemo_platform_ext/cli/docs" } } +models = { source = "../models/src/models", module = "models", deps_group = "nemo-platform-sdk" } +filesets = { source = "../filesets/src/filesets", module = "filesets", deps_group = "nemo-platform-sdk" } +nemo-evaluator-sdk = { source = "../nemo_evaluator_sdk/src/nemo_evaluator_sdk", module = "nemo_evaluator_sdk" } diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/commands.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/commands.py index add676eb86..e3fafb0f59 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/commands.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/commands.py @@ -1319,7 +1319,7 @@ def _post_function_submit( def _resolve_cluster_name_to_base_url(cluster_name: str) -> str: """Resolve a configured cluster name to its base URL.""" - from nemo_platform.config.config import Config + from nemo_platform_ext.config.config import Config config = Config.load() for cluster in config.get_config_file().clusters: diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/jobs/file_manager.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/jobs/file_manager.py index effcf552e9..cd613d19c0 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/jobs/file_manager.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/jobs/file_manager.py @@ -11,8 +11,8 @@ import anyio import fsspec.asyn +from filesets import FilesetFileSystem, build_fileset_ref, parse_fileset_ref from nemo_platform import AsyncNeMoPlatform, NeMoPlatform -from nemo_platform.filesets import FilesetFileSystem, build_fileset_ref, parse_fileset_ref from nemo_platform_plugin.files.types import CreateFilesetRequest from nemo_platform_plugin.jobs.schemas import FileStorageType diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/jobs/result_manager.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/jobs/result_manager.py index 3ab12e2fc6..cac46051d9 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/jobs/result_manager.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/jobs/result_manager.py @@ -9,8 +9,8 @@ from pathlib import Path from typing import Generic, Literal, Type, TypeVar, overload +from filesets import parse_fileset_ref from nemo_platform import AsyncNeMoPlatform, NeMoPlatform -from nemo_platform.filesets import parse_fileset_ref from nemo_platform_plugin.client.adapter import client_from_platform from nemo_platform_plugin.client.errors import ConflictError as ClientConflictError from nemo_platform_plugin.client.errors import NemoClientError diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/nooa_model_client.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/nooa_model_client.py index db0cd22a77..2b677a3cad 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/nooa_model_client.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/nooa_model_client.py @@ -15,9 +15,9 @@ from dataclasses import dataclass from nemo_platform import AsyncNeMoPlatform -from nemo_platform.config import get_context from nemo_platform.types.inference import ModelProvider from nemo_platform.types.models import ModelEntity +from nemo_platform_ext.config import get_context from nooa.unifiedllm import CompletionClient, UnifiedLLM _PLACEHOLDER_API_KEY = "not-needed" diff --git a/packages/nemo_platform_plugin/tests/test_commands.py b/packages/nemo_platform_plugin/tests/test_commands.py index 78795143a4..ab9c08e050 100644 --- a/packages/nemo_platform_plugin/tests/test_commands.py +++ b/packages/nemo_platform_plugin/tests/test_commands.py @@ -319,7 +319,7 @@ def get_config_file(self) -> SimpleNamespace: else: monkeypatch.setenv("NMP_BASE_URL", env_base_url) monkeypatch.setattr("nemo_platform_plugin.scheduler.NemoJobScheduler.submit_remote", _capture) - monkeypatch.setattr("nemo_platform.config.config.Config.load", lambda: _FakeConfig()) + monkeypatch.setattr("nemo_platform_ext.config.config.Config.load", lambda: _FakeConfig()) app = _app_with_jobs(_GreetJob) state = _State(context_base_url) diff --git a/packages/nmp_common/src/nmp/common/auth/testing.py b/packages/nmp_common/src/nmp/common/auth/testing.py index 696b18e7d3..55c748f47f 100644 --- a/packages/nmp_common/src/nmp/common/auth/testing.py +++ b/packages/nmp_common/src/nmp/common/auth/testing.py @@ -44,7 +44,7 @@ from typing import Any, Dict, Optional import httpx -from nemo_platform.auth.helpers import generate_unsigned_jwt as generate_unsigned_jwt_helper +from nemo_platform_ext.auth.helpers import generate_unsigned_jwt as generate_unsigned_jwt_helper # Some packages do not have respx as a dependency try: diff --git a/packages/nmp_common/tests/sdk_factory/test_sdk.py b/packages/nmp_common/tests/sdk_factory/test_sdk.py index e3df60674d..e981cb5f53 100644 --- a/packages/nmp_common/tests/sdk_factory/test_sdk.py +++ b/packages/nmp_common/tests/sdk_factory/test_sdk.py @@ -7,7 +7,7 @@ import httpx import pytest -from nemo_platform.auth.helpers import NMPOIDCConfig +from nemo_platform_ext.auth.helpers import NMPOIDCConfig from nemo_platform_plugin.client.constants import WORKLOAD_IDENTITY_TOKEN_FILE_ENVVAR from nmp.common.config import Configuration, PlatformConfig from nmp.common.http_clients import shared_async_http_client, shared_sync_http_client @@ -189,8 +189,10 @@ def token_exchange_grant(**kwargs): monkeypatch.setenv(WORKLOAD_IDENTITY_TOKEN_FILE_ENVVAR, str(subject_token_file)) monkeypatch.setenv("NMP_PRINCIPAL", json.dumps({"id": "creator@example.com", "email": "creator@example.com"})) monkeypatch.delenv("NMP_ACCESS_TOKEN", raising=False) - monkeypatch.setattr("nemo_platform.client.factory.discover_nmp_config", lambda _base_url: _workload_oidc_config()) - monkeypatch.setattr("nemo_platform.auth.workload_exchange.token_exchange_grant", token_exchange_grant) + monkeypatch.setattr( + "nemo_platform_ext.client.factory.discover_nmp_config", lambda _base_url: _workload_oidc_config() + ) + monkeypatch.setattr("nemo_platform_ext.auth.workload_exchange.token_exchange_grant", token_exchange_grant) sdk = get_platform_sdk() try: @@ -388,8 +390,10 @@ def token_exchange_grant(**kwargs): monkeypatch.setenv(WORKLOAD_IDENTITY_TOKEN_FILE_ENVVAR, str(subject_token_file)) monkeypatch.setenv("NMP_PRINCIPAL", json.dumps({"id": "creator@example.com", "email": "creator@example.com"})) monkeypatch.delenv("NMP_ACCESS_TOKEN", raising=False) - monkeypatch.setattr("nemo_platform.client.factory.discover_nmp_config", lambda _base_url: _workload_oidc_config()) - monkeypatch.setattr("nemo_platform.auth.workload_exchange.token_exchange_grant", token_exchange_grant) + monkeypatch.setattr( + "nemo_platform_ext.client.factory.discover_nmp_config", lambda _base_url: _workload_oidc_config() + ) + monkeypatch.setattr("nemo_platform_ext.auth.workload_exchange.token_exchange_grant", token_exchange_grant) sdk = get_task_sdk(as_service="customizer") try: diff --git a/packages/nmp_testing/src/nmp/testing/client.py b/packages/nmp_testing/src/nmp/testing/client.py index 8d7433a846..5113ba6e25 100644 --- a/packages/nmp_testing/src/nmp/testing/client.py +++ b/packages/nmp_testing/src/nmp/testing/client.py @@ -176,7 +176,7 @@ def _create_svc( def _install_asgi_files_resource(sdk: NeMoPlatform, async_http_client: httpx.AsyncClient) -> None: """Route sync SDK file uploads through the in-process test app.""" - from nemo_platform.filesets.resources import FilesResource + from filesets.resources import FilesResource from nemo_platform_plugin.client.adapter import client_from_platform from nemo_platform_plugin.files.client import AsyncFilesClient, FilesClient diff --git a/plugins/nemo-agents/src/nemo_agents_plugin/cli.py b/plugins/nemo-agents/src/nemo_agents_plugin/cli.py index 7fbb32a182..e57b299889 100644 --- a/plugins/nemo-agents/src/nemo_agents_plugin/cli.py +++ b/plugins/nemo-agents/src/nemo_agents_plugin/cli.py @@ -74,7 +74,7 @@ ) from nemo_agents_plugin.leaderboard.cli import register_leaderboard_commands from nemo_agents_plugin.usage.cli import register_usage_commands -from nemo_platform.cli.core.formatters import Column, format_output +from nemo_platform_ext.cli.core.formatters import Column, format_output from nemo_platform_plugin.cli import NemoCLI from nemo_platform_plugin.cli_errors import print_http_request_error, print_http_status_error from nemo_platform_plugin.cli_progress import request_progress diff --git a/plugins/nemo-agents/src/nemo_agents_plugin/leaderboard/cli.py b/plugins/nemo-agents/src/nemo_agents_plugin/leaderboard/cli.py index b8b7bd0d5f..e16104d0b2 100644 --- a/plugins/nemo-agents/src/nemo_agents_plugin/leaderboard/cli.py +++ b/plugins/nemo-agents/src/nemo_agents_plugin/leaderboard/cli.py @@ -24,7 +24,7 @@ from nemo_agents_plugin.leaderboard.rank import rank_entries from nemo_agents_plugin.leaderboard.render import render_entries from nemo_agents_plugin.leaderboard.types import AgentLeaderboardEntry -from nemo_platform.cli.core.help_formatter import create_typer_app +from nemo_platform_ext.cli.core.help_formatter import create_typer_app def register_leaderboard_commands(app: typer.Typer) -> None: diff --git a/plugins/nemo-agents/src/nemo_agents_plugin/leaderboard/render.py b/plugins/nemo-agents/src/nemo_agents_plugin/leaderboard/render.py index d507a3cf77..08bee0a931 100644 --- a/plugins/nemo-agents/src/nemo_agents_plugin/leaderboard/render.py +++ b/plugins/nemo-agents/src/nemo_agents_plugin/leaderboard/render.py @@ -13,7 +13,7 @@ from io import StringIO from nemo_agents_plugin.leaderboard.types import AgentLeaderboard, AgentLeaderboardEntry -from nemo_platform.cli.core.help_formatter import _get_terminal_width +from nemo_platform_ext.cli.core.help_formatter import _get_terminal_width from rich.console import Console from rich.table import Table diff --git a/plugins/nemo-agents/src/nemo_agents_plugin/tasks/execute/workdir.py b/plugins/nemo-agents/src/nemo_agents_plugin/tasks/execute/workdir.py index f22c258005..e2b07bb837 100644 --- a/plugins/nemo-agents/src/nemo_agents_plugin/tasks/execute/workdir.py +++ b/plugins/nemo-agents/src/nemo_agents_plugin/tasks/execute/workdir.py @@ -9,7 +9,7 @@ from pathlib import Path, PurePosixPath from typing import Any, Protocol -from nemo_platform.filesets import FilesetPathError, build_fileset_ref, parse_fileset_ref +from filesets import FilesetPathError, build_fileset_ref, parse_fileset_ref from pydantic import BaseModel, Field, field_validator, model_validator diff --git a/plugins/nemo-agents/src/nemo_agents_plugin/utils.py b/plugins/nemo-agents/src/nemo_agents_plugin/utils.py index bc2c8e2ffb..eae83701c7 100644 --- a/plugins/nemo-agents/src/nemo_agents_plugin/utils.py +++ b/plugins/nemo-agents/src/nemo_agents_plugin/utils.py @@ -195,7 +195,7 @@ def get_internal_base_url() -> str | None: def get_default_model() -> str | None: """Return the default model for the platform from the SDK context.""" - from nemo_platform.config import get_context + from nemo_platform_ext.config import get_context return get_context().default_model diff --git a/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/app/input.py b/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/app/input.py index 4d58885d2b..b9d48168ae 100644 --- a/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/app/input.py +++ b/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/app/input.py @@ -14,9 +14,9 @@ import anyio from anonymizer.config.anonymizer_config import AnonymizerInput +from filesets import FilesetPathError, build_fileset_ref, parse_fileset_ref from nemo_anonymizer_plugin.app.errors import AnonymizerInvalidConfigError from nemo_platform import AsyncNeMoPlatform, NeMoPlatform -from nemo_platform.filesets import FilesetPathError, build_fileset_ref, parse_fileset_ref from nemo_platform_plugin.jobs.file_manager import AsyncFilesetFileManager, FilesetFileManager, TmpDirPath from pydantic import BaseModel, Field, ValidationError diff --git a/plugins/nemo-data-designer/src/nemo_data_designer_plugin/sdk/resources.py b/plugins/nemo-data-designer/src/nemo_data_designer_plugin/sdk/resources.py index 4d4a61ed19..34c8207ec4 100644 --- a/plugins/nemo-data-designer/src/nemo_data_designer_plugin/sdk/resources.py +++ b/plugins/nemo-data-designer/src/nemo_data_designer_plugin/sdk/resources.py @@ -16,6 +16,7 @@ from data_designer.config.preview_results import PreviewResults from data_designer.config.utils.info import InterfaceInfo from data_designer.logging import RandomEmoji +from models.resources import AsyncModelsResource, ModelsResource from nemo_data_designer_plugin.functions._types import ( AnalysisFrame, DatasetFrame, @@ -42,7 +43,6 @@ validate_config_sync, ) from nemo_platform import AsyncNeMoPlatform, NeMoPlatform -from nemo_platform.models.resources import AsyncModelsResource, ModelsResource from nemo_platform.types.inference import ModelProvider as NMPModelProvider from nemo_platform_plugin.functions.frames import Done, Error, Heartbeat from nemo_platform_plugin.sdk import NemoPluginSDKResources diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/filesets.py b/plugins/nemo-evaluator/src/nemo_evaluator/filesets.py index 44ffe32cff..d7925b23dd 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/filesets.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/filesets.py @@ -9,8 +9,8 @@ from pathlib import Path import fsspec.asyn +from filesets import FilesetFileSystem from nemo_platform import AsyncNeMoPlatform, NeMoPlatform -from nemo_platform.filesets import FilesetFileSystem from nemo_platform_plugin.client.adapter import client_from_platform from nemo_platform_plugin.files.client import AsyncFilesClient, FilesClient from pydantic import Field, RootModel diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/client.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/client.py index cbaa1b8e07..31deb18ee4 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/client.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/client.py @@ -20,8 +20,8 @@ from urllib.parse import urlparse from nemo_platform import AsyncNeMoPlatform -from nemo_platform.auth.helpers import discover_nmp_config -from nemo_platform.config.config import Config +from nemo_platform_ext.auth.helpers import discover_nmp_config +from nemo_platform_ext.config.config import Config # Loopback hosts are served by an unauthenticated local platform; attaching # (and refreshing) OAuth tokens there is both unnecessary and a failure mode diff --git a/plugins/nemo-experimentalist/tests/test_client.py b/plugins/nemo-experimentalist/tests/test_client.py index 7c43e2bc58..d5362b2cd0 100644 --- a/plugins/nemo-experimentalist/tests/test_client.py +++ b/plugins/nemo-experimentalist/tests/test_client.py @@ -5,7 +5,7 @@ import pytest from nemo_experimentalist_plugin.client import make_client -from nemo_platform.auth.helpers import NMPOIDCConfig +from nemo_platform_ext.auth.helpers import NMPOIDCConfig REMOTE_URL = "https://nemo-platform.example.com" diff --git a/plugins/nemo-insights/evaluation/export.py b/plugins/nemo-insights/evaluation/export.py index 278b5b11a5..cc893adfbe 100644 --- a/plugins/nemo-insights/evaluation/export.py +++ b/plugins/nemo-insights/evaluation/export.py @@ -23,7 +23,7 @@ from urllib.parse import urlparse from nemo_platform import AsyncNeMoPlatform -from nemo_platform.config.config import Config +from nemo_platform_ext.config.config import Config EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc) PAGE_SIZE = 1000 diff --git a/plugins/nemo-insights/evaluation/ingest.py b/plugins/nemo-insights/evaluation/ingest.py index def4ba6697..5c02afee1f 100644 --- a/plugins/nemo-insights/evaluation/ingest.py +++ b/plugins/nemo-insights/evaluation/ingest.py @@ -11,7 +11,7 @@ import httpx from nemo_platform import NeMoPlatform -from nemo_platform.config.config import Config +from nemo_platform_ext.config.config import Config _LOOPBACK_HOSTS = frozenset({"localhost", "127.0.0.1", "::1", "0.0.0.0"}) diff --git a/plugins/nemo-insights/src/nemo_insights_plugin/analyst/observability.py b/plugins/nemo-insights/src/nemo_insights_plugin/analyst/observability.py index 4da978ee2a..c156666661 100644 --- a/plugins/nemo-insights/src/nemo_insights_plugin/analyst/observability.py +++ b/plugins/nemo-insights/src/nemo_insights_plugin/analyst/observability.py @@ -8,7 +8,7 @@ from uuid import uuid4 from nemo_insights_plugin.client import LOOPBACK_HOSTS -from nemo_platform.config.config import Config +from nemo_platform_ext.config.config import Config from nooa.tracing import enable_tracing, exporters, flush_traces, set_session ANALYST_OBSERVABILITY_ENV = "NEMO_INSIGHTS_ANALYST_OBSERVABILITY" diff --git a/plugins/nemo-insights/src/nemo_insights_plugin/client.py b/plugins/nemo-insights/src/nemo_insights_plugin/client.py index 777ad04b06..1f4c32e8dd 100644 --- a/plugins/nemo-insights/src/nemo_insights_plugin/client.py +++ b/plugins/nemo-insights/src/nemo_insights_plugin/client.py @@ -20,8 +20,8 @@ from urllib.parse import urlparse from nemo_platform import AsyncNeMoPlatform -from nemo_platform.auth.helpers import discover_nmp_config -from nemo_platform.config.config import Config +from nemo_platform_ext.auth.helpers import discover_nmp_config +from nemo_platform_ext.config.config import Config # Loopback hosts are served by an unauthenticated local platform; attaching # (and refreshing) OAuth tokens there is both unnecessary and a failure mode diff --git a/plugins/nemo-insights/tests/test_client.py b/plugins/nemo-insights/tests/test_client.py index 4be5bb9762..dd42937c4e 100644 --- a/plugins/nemo-insights/tests/test_client.py +++ b/plugins/nemo-insights/tests/test_client.py @@ -4,7 +4,7 @@ from unittest.mock import MagicMock, patch from nemo_insights_plugin.client import make_client -from nemo_platform.auth.helpers import NMPOIDCConfig +from nemo_platform_ext.auth.helpers import NMPOIDCConfig REMOTE_URL = "https://nemo-platform.example.com" diff --git a/plugins/nemo-iron-swarm/src/nemo_iron_swarm_plugin/filesets.py b/plugins/nemo-iron-swarm/src/nemo_iron_swarm_plugin/filesets.py index b454c16a5f..adc9a450d4 100644 --- a/plugins/nemo-iron-swarm/src/nemo_iron_swarm_plugin/filesets.py +++ b/plugins/nemo-iron-swarm/src/nemo_iron_swarm_plugin/filesets.py @@ -21,9 +21,9 @@ from pathlib import Path import fsspec.asyn +from filesets import FilesetFileSystem from nemo_agents_plugin.container.template import DOCKERIGNORE_TEMPLATE from nemo_platform import NeMoPlatform -from nemo_platform.filesets import FilesetFileSystem from nemo_platform_plugin.client.adapter import client_from_platform from nemo_platform_plugin.files.client import FilesClient diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/api/v2/jobs/endpoints.py b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/api/v2/jobs/endpoints.py index 78b4fc3a45..48fd82f8e1 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/api/v2/jobs/endpoints.py +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/api/v2/jobs/endpoints.py @@ -11,8 +11,8 @@ from typing import Any from urllib.parse import urlparse +from filesets import FilesetPathError, parse_fileset_ref from nemo_platform import AsyncNeMoPlatform, NotFoundError, PermissionDeniedError -from nemo_platform.filesets import FilesetPathError, parse_fileset_ref from nemo_platform_plugin.authz import AuthzScope from nemo_platform_plugin.client.adapter import client_from_platform from nemo_platform_plugin.client.errors import NotFoundError as ClientNotFoundError diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/tasks/safe_synthesizer/__main__.py b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/tasks/safe_synthesizer/__main__.py index d3e03b51e1..0d263fcfc6 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/tasks/safe_synthesizer/__main__.py +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/tasks/safe_synthesizer/__main__.py @@ -25,8 +25,8 @@ import pandas as pd from datasets import Dataset, DatasetDict, load_dataset +from filesets import parse_fileset_ref from nemo_platform import NeMoPlatform -from nemo_platform.filesets import parse_fileset_ref from nemo_platform_plugin.client.adapter import client_from_platform from nemo_platform_plugin.config import get_platform_config from nemo_platform_plugin.jobs.client import JobsClient diff --git a/sdk/python/nemo-platform/hatch_build.py b/sdk/python/nemo-platform/hatch_build.py new file mode 100644 index 0000000000..52ff76325b --- /dev/null +++ b/sdk/python/nemo-platform/hatch_build.py @@ -0,0 +1,221 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# ruff: noqa: I001 + +from __future__ import annotations + +import collections.abc +import shutil +import tempfile +from pathlib import Path + +from hatchling.builders.hooks.plugin.interface import BuildHookInterface + +GENERATED_INIT_FILE = """# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +""" + + +class SourcePackage: + def __init__(self, *, source: str, target: str, include: tuple[str, ...]) -> None: + self.source = source + self.target = target + self.include = include + + +class CustomBuildHook(BuildHookInterface): + """Stage source SDK extensions into SDK build artifacts.""" + + def initialize(self, version: str, build_data: dict[str, object]) -> None: + if version == "editable": + # Editable installs run against the repo/workspace sources, where + # these packages are already importable as workspace packages. + return + + packages = _source_packages(self.config) + project_root = Path(self.root).resolve() + source_base = _find_source_base(project_root, packages) + self._stage_tmp = tempfile.TemporaryDirectory(prefix="nmp-sdk-stage-") + stage_root = Path(self._stage_tmp.name) + + force_include = _force_include(build_data) + if self.target_name == "sdist": + _stage_sdist_sources(source_base, stage_root, force_include, packages) + patched_pyproject = _write_sdist_pyproject(project_root, stage_root, self.metadata.version) + _replace_force_include_target(force_include, source=patched_pyproject, target="pyproject.toml") + else: + _stage_wheel_sources(source_base, stage_root, force_include, packages) + + build_data["force_include"] = force_include + + def finalize(self, _version: str, _build_data: dict[str, object], _artifact_path: str) -> None: + stage_tmp = getattr(self, "_stage_tmp", None) + if stage_tmp is not None: + stage_tmp.cleanup() + + +def _source_packages(config: collections.abc.Mapping[str, object]) -> tuple[SourcePackage, ...]: + packages = [] + for entry in _config_entries(config, "source-packages"): + packages.append( + SourcePackage( + source=_required_string(entry, "source", "source-packages"), + target=_required_string(entry, "target", "source-packages"), + include=_include_patterns(entry), + ) + ) + return tuple(packages) + + +def _force_include(build_data: collections.abc.Mapping[str, object]) -> dict[str, str]: + existing_force_include = build_data.get("force_include") + if not isinstance(existing_force_include, dict): + return {} + return {str(source): str(target) for source, target in existing_force_include.items()} + + +def _find_source_base(project_root: Path, packages: tuple[SourcePackage, ...]) -> Path: + """Find the root containing the configured source package paths. + + Monorepo builds run from ``sdk/python/nemo-platform`` while wheels built + from an sdist run from the extracted sdist root. Walking upward supports + both layouts without hard-coding a fixed number of parent directories. + """ + for candidate in (project_root, *project_root.parents): + if all((candidate / package.source).is_dir() for package in packages): + return candidate + + missing = ", ".join(package.source for package in packages) + raise FileNotFoundError(f"Could not find SDK source package roots from {project_root}: {missing}") + + +def _stage_wheel_sources( + source_base: Path, + stage_root: Path, + force_include: dict[str, str], + packages: tuple[SourcePackage, ...], +) -> None: + for package in packages: + source_root = source_base / package.source + package_stage = stage_root / package.target + _copy_included_paths(source_root, package_stage, package.include) + _ensure_init_files(package_stage) + force_include[str(package_stage)] = package.target + + +def _stage_sdist_sources( + source_base: Path, + stage_root: Path, + force_include: dict[str, str], + packages: tuple[SourcePackage, ...], +) -> None: + for package in packages: + source_root = source_base / package.source + package_stage = stage_root / package.source + _copy_included_paths(source_root, package_stage, package.include) + force_include[str(package_stage)] = package.source + + +def _write_sdist_pyproject(project_root: Path, stage_root: Path, version: str) -> Path: + """Write a self-contained sdist pyproject. + + The monorepo SDK pyproject uses ``nmp-build-tools`` from the uv workspace + for dynamic versioning. An sdist is outside that workspace, so wheel builds + from the sdist need static version metadata and no workspace-only build + dependency. + """ + source = project_root / "pyproject.toml" + destination = stage_root / "pyproject.toml" + destination.write_text(_sdist_pyproject(source.read_text(encoding="utf-8"), version), encoding="utf-8") + return destination + + +def _sdist_pyproject(content: str, version: str) -> str: + content = content.replace('dynamic = ["readme", "version"]', f'dynamic = ["readme"]\nversion = "{version}"') + content = content.replace('"hatch-fancy-pypi-readme", "nmp-build-tools"', '"hatch-fancy-pypi-readme"') + content = "\n".join(line for line in content.splitlines() if not _is_nmp_build_tools_workspace_source(line)) + content = _remove_toml_section(content, "[tool.hatch.version]") + return f"{content.rstrip()}\n" + + +def _is_nmp_build_tools_workspace_source(line: str) -> bool: + return line.strip().replace(" ", "") == "nmp-build-tools={workspace=true}" + + +def _remove_toml_section(content: str, section_header: str) -> str: + lines = content.splitlines() + output = [] + skipping = False + + for line in lines: + stripped = line.strip() + if stripped == section_header: + skipping = True + continue + if skipping and stripped.startswith("[") and stripped.endswith("]"): + skipping = False + if not skipping: + output.append(line) + + return "\n".join(output) + + +def _replace_force_include_target(force_include: dict[str, str], *, source: Path, target: str) -> None: + for existing_source, existing_target in tuple(force_include.items()): + if existing_target == target: + del force_include[existing_source] + force_include[str(source)] = target + + +def _config_entries( + config: collections.abc.Mapping[str, object], key: str +) -> tuple[collections.abc.Mapping[str, object], ...]: + raw_entries = config.get(key, []) + if not isinstance(raw_entries, list): + raise TypeError(f"`{key}` must be an array") + + entries = [] + for entry in raw_entries: + if not isinstance(entry, dict): + raise TypeError(f"`{key}` entries must be tables") + entries.append(entry) + return tuple(entries) + + +def _required_string(entry: collections.abc.Mapping[str, object], key: str, section: str) -> str: + value = entry.get(key) + if not isinstance(value, str) or not value: + raise TypeError(f"`{section}` entries must define a non-empty `{key}` string") + return value + + +def _include_patterns(entry: collections.abc.Mapping[str, object]) -> tuple[str, ...]: + raw_patterns = entry.get("include", ["**/*.py"]) + if not isinstance(raw_patterns, list) or any( + not isinstance(pattern, str) or not pattern for pattern in raw_patterns + ): + raise TypeError("`source-packages` entries must define `include` as an array of non-empty strings") + return tuple(raw_patterns) + + +def _copy_included_paths(source_root: Path, target_root: Path, patterns: tuple[str, ...]) -> None: + if not source_root.is_dir(): + raise FileNotFoundError(f"Source package path does not exist: {source_root}") + + seen: set[Path] = set() + for pattern in patterns: + for source_file in source_root.glob(pattern): + if not source_file.is_file() or source_file in seen: + continue + seen.add(source_file) + relative_path = source_file.relative_to(source_root) + target_file = target_root / relative_path + target_file.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source_file, target_file) + + +def _ensure_init_files(package_root: Path) -> None: + for directory in (package_root, *(path for path in package_root.rglob("*") if path.is_dir())): + init_file = directory / "__init__.py" + if not init_file.exists(): + init_file.write_text(GENERATED_INIT_FILE, encoding="utf-8") diff --git a/sdk/python/nemo-platform/pyproject.toml b/sdk/python/nemo-platform/pyproject.toml index 27d96b9f5e..558e50bcf3 100644 --- a/sdk/python/nemo-platform/pyproject.toml +++ b/sdk/python/nemo-platform/pyproject.toml @@ -118,8 +118,26 @@ include = [ [tool.hatch.build.targets.wheel] packages = ["src/nemo_platform"] -[tool.hatch.build.targets.wheel.force-include] -"../../../docs" = "nemo_platform/cli/docs" +[tool.hatch.build.targets.wheel.hooks.custom] +path = "hatch_build.py" + +[[tool.hatch.build.targets.wheel.hooks.custom.source-packages]] +source = "packages/nemo_platform_ext/src/nemo_platform_ext" +target = "nemo_platform_ext" +include = ["**/*.py", "skills/**/*.md", "skills/**/*.yaml", "skills/**/*.yml", "skills/**/*.json"] + +[[tool.hatch.build.targets.wheel.hooks.custom.source-packages]] +source = "packages/models/src/models" +target = "models" + +[[tool.hatch.build.targets.wheel.hooks.custom.source-packages]] +source = "packages/filesets/src/filesets" +target = "filesets" + +[[tool.hatch.build.targets.wheel.hooks.custom.source-packages]] +source = "packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk" +target = "nemo_evaluator_sdk" +include = ["**/*.py", "agent_eval/runtimes/fabric/sandbox.Dockerfile"] [tool.hatch.build.targets.sdist] # Basically everything except hidden files/directories (such as .github, .devcontainers, .python-version, etc) include = [ @@ -135,6 +153,26 @@ include = [ "tests/*", ] +[tool.hatch.build.targets.sdist.hooks.custom] +path = "hatch_build.py" + +[[tool.hatch.build.targets.sdist.hooks.custom.source-packages]] +source = "packages/nemo_platform_ext/src/nemo_platform_ext" +target = "nemo_platform_ext" +include = ["**/*.py", "skills/**/*.md", "skills/**/*.yaml", "skills/**/*.yml", "skills/**/*.json"] + +[[tool.hatch.build.targets.sdist.hooks.custom.source-packages]] +source = "packages/models/src/models" +target = "models" + +[[tool.hatch.build.targets.sdist.hooks.custom.source-packages]] +source = "packages/filesets/src/filesets" +target = "filesets" + +[[tool.hatch.build.targets.sdist.hooks.custom.source-packages]] +source = "packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk" +target = "nemo_evaluator_sdk" +include = ["**/*.py", "agent_eval/runtimes/fabric/sandbox.Dockerfile"] [tool.hatch.metadata.hooks.fancy-pypi-readme] content-type = "text/markdown" diff --git a/sdk/python/nemo-platform/src/nemo_platform/_alias.py b/sdk/python/nemo-platform/src/nemo_platform/_alias.py new file mode 100644 index 0000000000..cbae279921 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/_alias.py @@ -0,0 +1,220 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# ruff: noqa: I001 - the generated SDK and workspace use different import-order settings. + +from __future__ import annotations + +import sys +from importlib import import_module, util +from importlib.abc import Loader, MetaPathFinder +from importlib.machinery import ModuleSpec +from types import ModuleType +from typing import Any + + +class _AliasLoader(Loader): + def __init__(self, alias_name: str, target_name: str) -> None: + self._alias_name = alias_name + self._target_name = target_name + + def create_module(self, spec: ModuleSpec) -> ModuleType: + del spec + target = import_module(self._target_name) + module = ModuleType(self._alias_name, target.__doc__) + _populate_alias_namespace(module.__dict__, self._alias_name, self._target_name, target) + return module + + def exec_module(self, module: ModuleType) -> None: + del module + return None + + def get_resource_reader(self, fullname: str) -> Any: + if fullname != self._alias_name: + return None + + target = import_module(self._target_name) + target_loader = getattr(target, "__loader__", None) + get_resource_reader = getattr(target_loader, "get_resource_reader", None) + if get_resource_reader is None: + return None + return get_resource_reader(self._target_name) + + +class _AliasFinder(MetaPathFinder): + def __init__(self) -> None: + self._aliases: dict[str, str] = {} + + def add_alias(self, alias_name: str, target_name: str) -> None: + self._aliases[alias_name] = target_name + + def find_spec( + self, + fullname: str, + _path: object | None = None, + _target: ModuleType | None = None, + ) -> ModuleSpec | None: + target_name = self._target_for(fullname) + if target_name is None: + return None + + target_spec = util.find_spec(target_name) + if target_spec is None: + return None + + is_package = target_spec.submodule_search_locations is not None + spec = ModuleSpec( + fullname, + _AliasLoader(fullname, target_name), + origin=target_spec.origin, + is_package=is_package, + ) + spec.cached = target_spec.cached + spec.has_location = target_spec.has_location + if is_package: + spec.submodule_search_locations = target_spec.submodule_search_locations + return spec + + def _target_for(self, fullname: str) -> str | None: + for alias_name, target_name in sorted(self._aliases.items(), key=lambda item: len(item[0]), reverse=True): + if fullname == alias_name: + return target_name + prefix = f"{alias_name}." + if fullname.startswith(prefix): + suffix = fullname[len(alias_name) :] + return f"{target_name}{suffix}" + return None + + +_FINDER: _AliasFinder | None = None + +_MODULE_METADATA_NAMES = frozenset( + { + "__builtins__", + "__cached__", + "__dir__", + "__doc__", + "__file__", + "__getattr__", + "__loader__", + "__name__", + "__package__", + "__path__", + "__spec__", + } +) + + +def _module_alias_name(value: ModuleType, alias_name: str, target_name: str) -> str | None: + module_name = value.__name__ + if module_name == target_name: + return alias_name + + target_prefix = f"{target_name}." + if module_name.startswith(target_prefix): + return f"{alias_name}{module_name[len(target_name) :]}" + + return None + + +def _alias_value(value: Any, alias_name: str, target_name: str) -> Any: + if not isinstance(value, ModuleType): + return value + + alias_module_name = _module_alias_name(value, alias_name, target_name) + if alias_module_name is None: + return value + + if alias_module_name == alias_name: + return sys.modules.get(alias_name, value) + + return import_module(alias_module_name) + + +def _alias_spec(alias_name: str, target: ModuleType) -> ModuleSpec | None: + target_spec = target.__spec__ + if target_spec is None: + return None + + is_package = target_spec.submodule_search_locations is not None + spec = ModuleSpec( + alias_name, + _AliasLoader(alias_name, target.__name__), + origin=target_spec.origin, + is_package=is_package, + ) + spec.cached = target_spec.cached + spec.has_location = target_spec.has_location + if is_package: + spec.submodule_search_locations = target_spec.submodule_search_locations + return spec + + +def _populate_alias_namespace( + namespace: dict[str, Any], + alias_name: str, + target_name: str, + target: ModuleType, +) -> None: + namespace["__doc__"] = target.__doc__ + namespace["__package__"] = alias_name if hasattr(target, "__path__") else alias_name.rpartition(".")[0] + + alias_spec = _alias_spec(alias_name, target) + if alias_spec is not None: + namespace["__spec__"] = alias_spec + namespace["__loader__"] = alias_spec.loader + + target_file = getattr(target, "__file__", None) + if target_file is not None: + namespace["__file__"] = target_file + + target_cached = getattr(target, "__cached__", None) + if target_cached is not None: + namespace["__cached__"] = target_cached + + target_path = getattr(target, "__path__", None) + if target_path is not None: + namespace["__path__"] = list(target_path) + + for name, value in target.__dict__.items(): + if name in _MODULE_METADATA_NAMES: + continue + if isinstance(value, ModuleType) and _module_alias_name(value, alias_name, target_name) is not None: + continue + namespace.setdefault(name, value) + + def __getattr__(name: str) -> Any: + value = _alias_value(getattr(target, name), alias_name, target_name) + namespace[name] = value + return value + + def __dir__() -> list[str]: + return sorted({*namespace, *dir(target)}) + + namespace["__getattr__"] = __getattr__ + namespace["__dir__"] = __dir__ + + +def alias_package(target_name: str, namespace: dict[str, Any]) -> ModuleType: + """Expose a native source package through a ``nemo_platform`` package path. + + Tiny generated ``__init__.py`` files call this from legacy SDK locations + such as ``nemo_platform.filesets``. The finder below maps submodule imports + like ``nemo_platform.filesets.resources`` to the real staged package path + (``filesets.resources``), so runtime resolution works without copying the + package tree into ``nemo_platform``. + """ + alias_name = str(namespace["__name__"]) + target = import_module(target_name) + _alias_finder().add_alias(alias_name, target_name) + _populate_alias_namespace(namespace, alias_name, target_name, target) + return target + + +def _alias_finder() -> _AliasFinder: + global _FINDER + + if _FINDER is None: + _FINDER = _AliasFinder() + sys.meta_path.insert(0, _FINDER) + return _FINDER diff --git a/sdk/python/nemo-platform/src/nemo_platform/_client.py b/sdk/python/nemo-platform/src/nemo_platform/_client.py index 9c5761f202..856213b1ee 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/_client.py +++ b/sdk/python/nemo-platform/src/nemo_platform/_client.py @@ -49,9 +49,9 @@ AsyncAPIClient, ) from nemo_platform._base_client import DefaultAsyncHttpxClient, DefaultHttpxClient -from nemo_platform.client.tls import client_verify_from_env from nemo_platform_plugin.client.constants import WORKLOAD_IDENTITY_TOKEN_FILE_ENVVAR from pathlib import Path +from nemo_platform_ext.client.tls import client_verify_from_env if TYPE_CHECKING: from .resources import ( @@ -124,6 +124,20 @@ def _should_bootstrap_config( ) +def _copy_requires_bootstrap( + *, + config_path: Path | None, + context_name: str | None, + access_token: str | None, +) -> bool: + return ( + config_path is not None + or context_name is not None + or access_token is not None + or bool(os.environ.get(WORKLOAD_IDENTITY_TOKEN_FILE_ENVVAR)) + ) + + class NeMoPlatform(SyncAPIClient): # client options workspace: str | None @@ -210,6 +224,9 @@ def __init__( http_client: Custom ``httpx.Client`` instance. When provided, the auth bootstrap is skipped entirely regardless of other parameters. """ + env_base_url = os.environ.get("NEMO_PLATFORM_BASE_URL") + bootstrap_base_url = base_url if base_url is not None else env_base_url + should_bootstrap = _should_bootstrap_config( http_client=http_client, base_url=base_url, @@ -219,11 +236,11 @@ def __init__( ) if should_bootstrap: try: - from nemo_platform.client.factory import build_client_init_kwargs + from nemo_platform_ext.client.factory import build_client_init_kwargs client_init_kwargs = build_client_init_kwargs( config_path=config_path, - base_url=base_url, + base_url=bootstrap_base_url, context_name=context_name, access_token=access_token, extra_headers=default_headers, @@ -232,9 +249,19 @@ def __init__( if workspace is None: workspace = client_init_kwargs.workspace default_headers = client_init_kwargs.default_headers + if client_init_kwargs.http_client is not None and not isinstance( + client_init_kwargs.http_client, httpx.Client + ): + raise TypeError("Expected httpx.Client from sync client factory") http_client = client_init_kwargs.http_client except Exception as e: - raise RuntimeError(f"NeMoPlatform client initialization failed: {e}") + raise RuntimeError(f"NeMoPlatform client initialization failed: {e}") from e + + if base_url is None: + base_url = bootstrap_base_url + + if base_url is None: + raise RuntimeError("NeMoPlatform client initialization failed: base_url is required") client_verify = client_verify_from_env() if http_client is None and client_verify is not True: @@ -379,6 +406,10 @@ def copy( *, workspace: str | None = None, base_url: str | httpx.URL | None = None, + inference_base_url: str | httpx.URL | None = None, + config_path: Path | None = None, + context_name: str | None = None, + access_token: str | None = None, timeout: float | Timeout | None | NotGiven = not_given, http_client: httpx.Client | None = None, max_retries: int | NotGiven = not_given, @@ -409,13 +440,22 @@ def copy( elif set_default_query is not None: params = set_default_query - http_client = http_client or self._client + if http_client is None and not _copy_requires_bootstrap( + config_path=config_path, + context_name=context_name, + access_token=access_token, + ): + http_client = self._client return self.__class__( workspace=workspace or self.workspace, base_url=base_url or self.base_url, + inference_base_url=inference_base_url or self.inference_base_url, + config_path=config_path, + context_name=context_name, + access_token=access_token, timeout=self.timeout if isinstance(timeout, NotGiven) else timeout, http_client=http_client, - max_retries=max_retries if is_given(max_retries) else self.max_retries, + max_retries=self.max_retries if isinstance(max_retries, NotGiven) else max_retries, default_headers=headers, default_query=params, **_extra_kwargs, @@ -583,6 +623,9 @@ async def main() -> None: http_client: Custom ``httpx.AsyncClient`` instance. When provided, the auth bootstrap is skipped entirely regardless of other parameters. """ + env_base_url = os.environ.get("NEMO_PLATFORM_BASE_URL") + bootstrap_base_url = base_url if base_url is not None else env_base_url + should_bootstrap = _should_bootstrap_config( http_client=http_client, base_url=base_url, @@ -592,11 +635,11 @@ async def main() -> None: ) if should_bootstrap: try: - from nemo_platform.client.factory import build_async_client_init_kwargs + from nemo_platform_ext.client.factory import build_async_client_init_kwargs client_init_kwargs = build_async_client_init_kwargs( config_path=config_path, - base_url=base_url, + base_url=bootstrap_base_url, context_name=context_name, access_token=access_token, extra_headers=default_headers, @@ -605,9 +648,19 @@ async def main() -> None: if workspace is None: workspace = client_init_kwargs.workspace default_headers = client_init_kwargs.default_headers + if client_init_kwargs.http_client is not None and not isinstance( + client_init_kwargs.http_client, httpx.AsyncClient + ): + raise TypeError("Expected httpx.AsyncClient from async client factory") http_client = client_init_kwargs.http_client except Exception as e: - raise RuntimeError(f"NeMoPlatform client initialization failed: {e}") + raise RuntimeError(f"NeMoPlatform client initialization failed: {e}") from e + + if base_url is None: + base_url = bootstrap_base_url + + if base_url is None: + raise RuntimeError("NeMoPlatform client initialization failed: base_url is required") client_verify = client_verify_from_env() if http_client is None and client_verify is not True: @@ -755,6 +808,10 @@ def copy( *, workspace: str | None = None, base_url: str | httpx.URL | None = None, + inference_base_url: str | httpx.URL | None = None, + config_path: Path | None = None, + context_name: str | None = None, + access_token: str | None = None, timeout: float | Timeout | None | NotGiven = not_given, http_client: httpx.AsyncClient | None = None, max_retries: int | NotGiven = not_given, @@ -785,13 +842,22 @@ def copy( elif set_default_query is not None: params = set_default_query - http_client = http_client or self._client + if http_client is None and not _copy_requires_bootstrap( + config_path=config_path, + context_name=context_name, + access_token=access_token, + ): + http_client = self._client return self.__class__( workspace=workspace or self.workspace, base_url=base_url or self.base_url, + inference_base_url=inference_base_url or self.inference_base_url, + config_path=config_path, + context_name=context_name, + access_token=access_token, timeout=self.timeout if isinstance(timeout, NotGiven) else timeout, http_client=http_client, - max_retries=max_retries if is_given(max_retries) else self.max_retries, + max_retries=self.max_retries if isinstance(max_retries, NotGiven) else max_retries, default_headers=headers, default_query=params, **_extra_kwargs, diff --git a/sdk/python/nemo-platform/src/nemo_platform/auth/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/auth/__init__.py index 1275d78dff..12a873240d 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/auth/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/auth/__init__.py @@ -1,15 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +from nemo_platform._alias import alias_package as _alias_package + +_alias_package("nemo_platform_ext.auth", globals()) diff --git a/sdk/python/nemo-platform/src/nemo_platform/auth/device_flow.py b/sdk/python/nemo-platform/src/nemo_platform/auth/device_flow.py deleted file mode 100644 index 06a62e1874..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/auth/device_flow.py +++ /dev/null @@ -1,304 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""OAuth 2.0 Device Authorization Flow (RFC 8628) implementation.""" - -import asyncio -import time -import webbrowser -from dataclasses import dataclass - -import httpx -from rich.console import Console -from rich.panel import Panel - -from nemo_platform.auth.token_provider import refresh_token_grant -from nemo_platform.client.tls import client_verify_from_env - -console = Console() - - -async def _async_pause(seconds: float) -> None: - await asyncio.sleep(seconds) - - -@dataclass -class DeviceCodeResponse: - """Response from device authorization endpoint.""" - - device_code: str - user_code: str - verification_uri: str - verification_uri_complete: str | None - expires_in: int - interval: int - - -@dataclass -class TokenResponse: - """Response from token endpoint.""" - - access_token: str - id_token: str | None # ID token (JWT) - refresh_token: str | None - token_type: str - expires_in: int - scope: str | None - - @property - def token_for_nmp(self) -> str: - """Return the token to use for NeMo Platform authentication.""" - return self.access_token - - -class DeviceFlowError(Exception): - """Device flow authentication error.""" - - pass - - -class DeviceFlow: - """OAuth 2.0 Device Authorization Flow client.""" - - def __init__( - self, - device_authorization_endpoint: str, - token_endpoint: str, - client_id: str, - scope: str = "openid email profile", - ): - self.device_authorization_endpoint = device_authorization_endpoint - self.token_endpoint = token_endpoint - self.client_id = client_id - self.scope = scope - - async def start_device_authorization(self) -> DeviceCodeResponse: - """Start the device authorization flow.""" - async with httpx.AsyncClient(verify=client_verify_from_env()) as client: - response = await client.post( - self.device_authorization_endpoint, - data={ - "client_id": self.client_id, - "scope": self.scope, - }, - timeout=30.0, - ) - response.raise_for_status() - data = response.json() - - return DeviceCodeResponse( - device_code=data["device_code"], - user_code=data["user_code"], - verification_uri=data["verification_uri"], - verification_uri_complete=data.get("verification_uri_complete"), - expires_in=data["expires_in"], - interval=data.get("interval", 5), - ) - - async def poll_for_token( - self, - device_code: str, - interval: int, - expires_in: int, - ) -> TokenResponse: - """Poll the token endpoint until authorization is complete.""" - start_time = time.time() - - async with httpx.AsyncClient(verify=client_verify_from_env()) as client: - while time.time() - start_time < expires_in: - await _async_pause(interval) - - response = await client.post( - self.token_endpoint, - data={ - "grant_type": "urn:ietf:params:oauth:grant-type:device_code", - "client_id": self.client_id, - "device_code": device_code, - "scope": self.scope, # Casdoor doesn't propagate scope from DeviceAuthCache - }, - timeout=30.0, - ) - - if response.status_code == 200: - data = response.json() - return TokenResponse( - access_token=data["access_token"], - id_token=data.get("id_token"), # Capture ID token if present - refresh_token=data.get("refresh_token"), - token_type=data.get("token_type", "Bearer"), - expires_in=data.get("expires_in", 3600), - scope=data.get("scope"), - ) - - error_data = response.json() - error = error_data.get("error") - - if error == "authorization_pending": - continue - elif error == "slow_down": - interval += 5 - continue - elif error == "expired_token": - raise DeviceFlowError("Authorization request expired") - elif error == "access_denied": - raise DeviceFlowError("User denied authorization") - else: - raise DeviceFlowError(f"Token request failed: {error}") - - raise DeviceFlowError("Authorization timed out") - - -async def authenticate_with_device_flow( - device_authorization_endpoint: str, - token_endpoint: str, - client_id: str, - scope: str = "openid email profile", - open_browser: bool = True, -) -> TokenResponse: - """Perform OAuth device flow authentication. - - Args: - device_authorization_endpoint: URL for device authorization - token_endpoint: URL for token exchange - client_id: OAuth client ID - scope: OAuth scopes to request - open_browser: Whether to automatically open the browser - - Returns: - TokenResponse with access and refresh tokens - """ - flow = DeviceFlow( - device_authorization_endpoint=device_authorization_endpoint, - token_endpoint=token_endpoint, - client_id=client_id, - scope=scope, - ) - - # Start device authorization - device_response = await flow.start_device_authorization() - - # Display user code and instructions - console.print() - console.print( - Panel( - f"[bold cyan]Visit:[/] {device_response.verification_uri}\n" - f"[bold cyan]Enter code:[/] [bold yellow]{device_response.user_code}[/]", - title="Authorization Required", - border_style="cyan", - ) - ) - - # Optionally open browser - if open_browser and device_response.verification_uri_complete: - console.print("\n[dim]Opening browser...[/]") - webbrowser.open(device_response.verification_uri_complete) - elif open_browser: - console.print("\n[dim]Opening browser...[/]") - webbrowser.open(device_response.verification_uri) - - console.print("\n[dim]Waiting for authorization...[/]") - - # Poll for token - token_response = await flow.poll_for_token( - device_code=device_response.device_code, - interval=device_response.interval, - expires_in=device_response.expires_in, - ) - - console.print("[green]Authorization successful![/]") - - return token_response - - -async def refresh_access_token( - token_endpoint: str, - client_id: str, - refresh_token: str, - scope: str | None = None, -) -> TokenResponse: - """ - Refresh an access token using a refresh token. - - Args: - token_endpoint: The OAuth token endpoint URL. - client_id: The OAuth client ID. - refresh_token: The refresh token from a previous authentication. - scope: OAuth scopes to request (required for some IdPs like Azure AD). - - Returns: - TokenResponse with new access_token (and possibly new refresh_token). - - Raises: - DeviceFlowError: If token refresh fails. - """ - try: - data = await asyncio.to_thread( - refresh_token_grant, - token_endpoint, - client_id, - refresh_token, - scope=scope, - ) - except RuntimeError as e: - raise DeviceFlowError(str(e)) from e - - return TokenResponse( - access_token=data["access_token"], - id_token=data.get("id_token"), - refresh_token=data.get("refresh_token"), # May be rotated - token_type=data.get("token_type", "Bearer"), - expires_in=data.get("expires_in", 3600), - scope=data.get("scope"), - ) - - -def authenticate_with_password_grant( - token_endpoint: str, - client_id: str, - username: str, - password: str, - scope: str = "openid profile email", -) -> TokenResponse: - """Obtain tokens using the Resource Owner Password Credentials grant (RFC 6749). - - Use this for non-interactive environments (e.g. CI) where no browser is available. - The IdP must have the password grant enabled for the application. - - Args: - token_endpoint: The OAuth token endpoint URL. - client_id: The OAuth client ID. - username: Resource owner username (e.g. testuser or built-in/admin). - password: Resource owner password. - scope: OAuth scopes to request. - - Returns: - TokenResponse with access_token and optional refresh_token. - - Raises: - DeviceFlowError: If the token request fails. - """ - data = { - "grant_type": "password", - "client_id": client_id, - "username": username, - "password": password, - "scope": scope, - } - with httpx.Client(verify=client_verify_from_env()) as client: - response = client.post(token_endpoint, data=data, timeout=30.0) - - if response.status_code != 200: - error_data = response.json() if response.headers.get("content-type", "").startswith("application/json") else {} - error = error_data.get("error", "unknown_error") - error_description = error_data.get("error_description", response.text) - raise DeviceFlowError(f"Token request failed: {error} - {error_description}") - - resp_data = response.json() - return TokenResponse( - access_token=resp_data["access_token"], - id_token=resp_data.get("id_token"), - refresh_token=resp_data.get("refresh_token"), - token_type=resp_data.get("token_type", "Bearer"), - expires_in=resp_data.get("expires_in", 3600), - scope=resp_data.get("scope"), - ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/auth/helpers.py b/sdk/python/nemo-platform/src/nemo_platform/auth/helpers.py deleted file mode 100644 index c3ff84c17a..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/auth/helpers.py +++ /dev/null @@ -1,224 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this code except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Auth helpers for NeMo Platform CLI (scope normalization, JWT decode, scope validation).""" - -from __future__ import annotations - -import base64 -import json -import time -from dataclasses import dataclass -from typing import Any - -import httpx - -from nemo_platform.client.tls import client_verify_from_env - -DEFAULT_OAUTH_SCOPES = "openid profile email offline_access" - - -class AuthError(Exception): - """Authentication-related error (CLI auth commands).""" - - pass - - -def normalize_scope_prefix(prefix: str | None) -> str: - """Normalize scope prefix to ensure it ends with a separator. - - Azure AD scope URIs require a '/' between the app ID and scope name. - This handles misconfigured clusters that omit the trailing slash. - - Args: - prefix: The scope prefix from cluster configuration (may be None or empty) - - Returns: - Empty string if prefix is None/empty, otherwise prefix with trailing '/' - """ - if not prefix: - return "" - return prefix if prefix.endswith("/") else f"{prefix}/" - - -def scope_short(scope: str, scope_prefix: str) -> str: - """Return scope in short form for comparison (strip prefix if present).""" - if scope_prefix and scope.startswith(scope_prefix): - return scope[len(scope_prefix) :] - return scope - - -def _decode_jwt_segment(token: str, index: int) -> dict[str, Any]: - try: - parts = token.split(".") - if len(parts) != 3: - return {} - payload = parts[index] - payload += "=" * (-len(payload) % 4) - decoded = base64.urlsafe_b64decode(payload) - data = json.loads(decoded) - return data if isinstance(data, dict) else {} - except Exception: - return {} - - -def decode_jwt_header(token: str) -> dict[str, Any]: - """Decode JWT header without verification.""" - return _decode_jwt_segment(token, 0) - - -def decode_jwt_claims(token: str) -> dict[str, Any]: - """Decode JWT claims without verification (for display purposes only).""" - return _decode_jwt_segment(token, 1) - - -def is_unsigned_jwt(token: str) -> bool: - """Return True when JWT uses ``alg=none``.""" - header = decode_jwt_header(token) - return str(header.get("alg", "")).lower() == "none" - - -def _base64url_encode_json(payload: dict[str, Any]) -> str: - encoded = json.dumps(payload, separators=(",", ":")).encode("utf-8") - return base64.urlsafe_b64encode(encoded).rstrip(b"=").decode("ascii") - - -def generate_unsigned_jwt( - principal_id: str, - *, - email: str | None = None, - groups: list[str] | None = None, - scopes: list[str] | None = None, - expires_in_seconds: int | None = 3600, - issued_at: int | None = None, - audience: str | None = None, - issuer: str | None = None, - extra_claims: dict[str, Any] | None = None, -) -> str: - """Generate an unsigned JWT (`alg=none`) for local development and testing.""" - now = issued_at if issued_at is not None else int(time.time()) - claims: dict[str, Any] = { - "sub": principal_id, - "iat": now, - } - - if email: - claims["email"] = email - if groups: - claims["groups"] = groups - if scopes: - claims["scope"] = " ".join(scopes) - if expires_in_seconds is not None: - claims["exp"] = now + expires_in_seconds - if audience: - claims["aud"] = audience - if issuer: - claims["iss"] = issuer - if extra_claims: - claims.update(extra_claims) - - header_segment = _base64url_encode_json({"alg": "none", "typ": "JWT"}) - claims_segment = _base64url_encode_json(claims) - return f"{header_segment}.{claims_segment}." - - -@dataclass(frozen=True) -class NMPOIDCConfig: - """OIDC configuration discovered from the NeMo Platform.""" - - auth_enabled: bool - issuer: str | None = None - client_id: str | None = None - token_endpoint: str | None = None - device_authorization_endpoint: str | None = None - default_scopes: str = DEFAULT_OAUTH_SCOPES - scope_prefix: str | None = None - workload_token_exchange_enabled: bool = False - workload_client_id: str | None = None - workload_token_endpoint: str | None = None - workload_audience: str | None = None - workload_scope: str | None = None - - -def discover_nmp_config(base_url: str, timeout: float = 10.0) -> NMPOIDCConfig: - """Fetch OIDC configuration from the NeMo Platform auth discovery endpoint.""" - response = httpx.get( - f"{base_url.rstrip('/')}/apis/auth/discovery", - timeout=timeout, - verify=client_verify_from_env(), - ) - response.raise_for_status() - data = response.json() - - oidc = data.get("oidc") or {} - return NMPOIDCConfig( - auth_enabled=data.get("auth_enabled", False), - issuer=oidc.get("issuer"), - client_id=oidc.get("client_id"), - token_endpoint=oidc.get("token_endpoint"), - device_authorization_endpoint=oidc.get("device_authorization_endpoint"), - default_scopes=oidc.get("default_scopes", DEFAULT_OAUTH_SCOPES), - scope_prefix=oidc.get("scope_prefix"), - workload_token_exchange_enabled=oidc.get("workload_token_exchange_enabled", False), - workload_client_id=oidc.get("workload_client_id"), - workload_token_endpoint=oidc.get("workload_token_endpoint"), - workload_audience=oidc.get("workload_audience"), - workload_scope=oidc.get("workload_scope"), - ) - - -def build_effective_scope(requested_scopes: str, scope_prefix: str | None) -> str: - """Prepend scope_prefix to custom scopes (those with ':' or ending with '.default').""" - prefix = normalize_scope_prefix(scope_prefix) - if not prefix: - return requested_scopes - expanded = [] - for s in requested_scopes.split(): - if ":" in s or s.endswith(".default"): - expanded.append(f"{prefix}{s}") - else: - expanded.append(s) - return " ".join(expanded) - - -def validate_requested_scopes_granted( - effective_scope: str, - granted_scopes: list[str], - scope_prefix: str, -) -> None: - """Validate that requested platform scopes appear in granted scopes; raise AuthError if not. - - Compares in short form so IdPs (e.g. Azure AD) that return scp as "platform:read" - match requested "api://nmp/platform:read". - """ - requested_platform = {s for s in effective_scope.split() if ":" in s} - requested_short = {scope_short(s, scope_prefix) for s in requested_platform} - granted_set = set(granted_scopes) - granted_short = {scope_short(s, scope_prefix) for s in granted_set} - missing_short = requested_short - granted_short - if not missing_short: - return - full_missing = sorted(s for s in requested_platform if scope_short(s, scope_prefix) in missing_short) - hint = "" - if scope_prefix and "api://" in scope_prefix: - hint = ( - "\nHint: For Azure AD, add the scopes in the app registration (Expose an API) and grant " - "admin consent. See tools/auth/azure/README.md." - ) - raise AuthError( - f"Token is missing requested scopes: {' '.join(full_missing)}.\n" - "The identity provider did not grant the requested scopes. " - "Check IdP configuration." + hint - ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/auth/token_provider.py b/sdk/python/nemo-platform/src/nemo_platform/auth/token_provider.py deleted file mode 100644 index 947e0a87d6..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/auth/token_provider.py +++ /dev/null @@ -1,254 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Token provider with automatic refresh for NeMo Platform SDK authentication.""" - -import asyncio -import json -import logging -import threading -import time -from collections.abc import Callable -from contextlib import AbstractContextManager, nullcontext -from dataclasses import dataclass, field - -import httpx -from typing_extensions import Self - -from nemo_platform.auth.helpers import decode_jwt_claims -from nemo_platform.client.tls import client_verify_from_env - -logger = logging.getLogger(__name__) - -# Refresh proactively when less than this many seconds remain before expiry. -DEFAULT_REFRESH_MARGIN_SECONDS = 60 - - -class TokenRefreshError(RuntimeError): - """Structured error raised for OAuth refresh_token grant failures.""" - - def __init__(self, *, error: str, error_description: str) -> None: - self.error = error - self.error_description = error_description - super().__init__(f"Token refresh failed: {error} - {error_description}") - - -def _validate_expires_in(expires_in: object) -> int | float | None: - if isinstance(expires_in, bool): - return None - return expires_in if isinstance(expires_in, int | float) else None - - -def refresh_token_grant( - token_endpoint: str, - client_id: str, - refresh_token: str, - *, - scope: str | None = None, - timeout: float = 30.0, -) -> dict: - """Execute OAuth refresh_token grant and return token response JSON.""" - data: dict[str, str] = { - "grant_type": "refresh_token", - "client_id": client_id, - "refresh_token": refresh_token, - } - if scope: - data["scope"] = scope - - response = httpx.post(token_endpoint, data=data, timeout=timeout, verify=client_verify_from_env()) - - if response.status_code != 200: - error_data: dict[str, str] = {} - if response.headers.get("content-type", "").startswith("application/json"): - try: - error_data = response.json() - except (json.JSONDecodeError, ValueError): - error_data = {} - error = error_data.get("error", "unknown_error") - error_description = error_data.get("error_description", response.text) - raise TokenRefreshError(error=error, error_description=error_description) - - return response.json() - - -@dataclass -class TokenSet: - """A pair of access + refresh tokens with expiry metadata.""" - - access_token: str - refresh_token: str | None = None - expires_at: float | None = None - - @staticmethod - def from_access_token( - access_token: str, - refresh_token: str | None = None, - expires_in: object = None, - ) -> Self: - """Create a TokenSet, extracting expiry from the JWT's `exp` claim.""" - expires_at = None - claims = decode_jwt_claims(access_token) - if claims: - expires_at = claims.get("exp") - validated_expires_in = _validate_expires_in(expires_in) - if expires_at is None and validated_expires_in is not None: - expires_at = time.time() + float(validated_expires_in) - return TokenSet( - access_token=access_token, - refresh_token=refresh_token, - expires_at=float(expires_at) if expires_at is not None else None, - ) - - def is_expired(self, margin_seconds: float = DEFAULT_REFRESH_MARGIN_SECONDS) -> bool: - """Check if the access token is expired or about to expire.""" - if self.expires_at is None: - return False - return time.time() >= (self.expires_at - margin_seconds) - - -@dataclass -class OIDCTokenProvider: - """Provides access tokens with automatic refresh via the OAuth2 refresh_token grant. - - This is the core component for SDK-level token management. It: - - Holds the current access + refresh tokens - - Proactively refreshes the access token before it expires - - Is thread-safe (uses a lock for concurrent access) - - Optionally persists refreshed tokens via a callback - - Args: - token_endpoint: The OAuth2 token endpoint URL. - client_id: The OAuth2 client ID. - tokens: The current token set. - refresh_margin_seconds: Seconds before expiry to proactively refresh. - load_tokens: Optional callback to reload tokens from a shared store (e.g. - config file) before refresh attempts. - refresh_lock: Optional context manager factory for serializing refresh - transactions across processes. - on_tokens_refreshed: Optional callback invoked with the new ``TokenSet`` - after a successful refresh. Use this to persist tokens (e.g. write - them back to ``~/.config/nmp/config.yaml``). - """ - - token_endpoint: str - client_id: str - tokens: TokenSet = field(default_factory=lambda: TokenSet(access_token="")) - refresh_margin_seconds: float = DEFAULT_REFRESH_MARGIN_SECONDS - refresh_scope: str | None = None - load_tokens: Callable[[], TokenSet | None] | None = None - refresh_lock: Callable[[], AbstractContextManager[None]] | None = None - on_tokens_refreshed: Callable[[TokenSet], None] | None = None - _lock: threading.Lock = field(default_factory=threading.Lock, repr=False) - - def get_access_token(self) -> str: - """Return a valid access token, refreshing if necessary.""" - with self._lock: - if self.tokens.is_expired(self.refresh_margin_seconds): - self._refresh() - return self.tokens.access_token - - async def get_access_token_async(self) -> str: - """Return a valid access token in async contexts. - - Runs refresh logic in a worker thread so token refresh does not block the - event loop. - """ - return await asyncio.to_thread(self.get_access_token) - - def reload_tokens(self) -> bool: - """Reload tokens from a shared store, if configured.""" - with self._lock: - return self._reload_tokens_from_source() - - def _reload_tokens_from_source(self) -> bool: - if self.load_tokens is None: - return False - - try: - loaded_tokens = self.load_tokens() - except Exception: - logger.warning("Failed to reload shared tokens", exc_info=True) - return False - - if loaded_tokens is None or loaded_tokens == self.tokens: - return False - - self.tokens = loaded_tokens - logger.debug("Reloaded shared tokens (expires_at=%s)", self.tokens.expires_at) - return True - - def _refresh(self, *, force: bool = False) -> None: - """Refresh the access token using the refresh_token grant. - - Raises: - RuntimeError: If no refresh token is available or the refresh request fails. - """ - lock_context = self.refresh_lock() if self.refresh_lock is not None else nullcontext() - with lock_context: - self._reload_tokens_from_source() - if not force and not self.tokens.is_expired(self.refresh_margin_seconds): - return - - if not self.tokens.refresh_token: - raise RuntimeError( - "Access token has expired and no refresh token is available. " - "Re-authenticate with `nemo auth login` to obtain new tokens." - ) - - logger.debug("Refreshing access token via %s", self.token_endpoint) - - token_data: dict - try: - token_data = refresh_token_grant( - token_endpoint=self.token_endpoint, - client_id=self.client_id, - refresh_token=self.tokens.refresh_token, - scope=self.refresh_scope, - ) - except TokenRefreshError as exc: - if exc.error != "invalid_grant": - raise - - if not self._reload_tokens_from_source(): - raise - - if not force and not self.tokens.is_expired(self.refresh_margin_seconds): - logger.debug("Recovered from invalid_grant with shared tokens") - return - - if not self.tokens.refresh_token: - raise RuntimeError( - "Access token has expired and no refresh token is available. " - "Re-authenticate with `nemo auth login` to obtain new tokens." - ) - - token_data = refresh_token_grant( - token_endpoint=self.token_endpoint, - client_id=self.client_id, - refresh_token=self.tokens.refresh_token, - scope=self.refresh_scope, - ) - - new_access_token = token_data["access_token"] - # The IdP may rotate the refresh token. - new_refresh_token = token_data.get("refresh_token", self.tokens.refresh_token) - - self.tokens = TokenSet.from_access_token( - new_access_token, - new_refresh_token, - expires_in=token_data.get("expires_in"), - ) - logger.debug("Access token refreshed successfully (expires_at=%s)", self.tokens.expires_at) - - if self.on_tokens_refreshed: - try: - self.on_tokens_refreshed(self.tokens) - except Exception: - logger.warning("Failed to persist refreshed tokens", exc_info=True) - - def force_refresh(self) -> str: - """Force a token refresh regardless of expiry. Returns the new access token.""" - with self._lock: - self._refresh(force=True) - return self.tokens.access_token diff --git a/sdk/python/nemo-platform/src/nemo_platform/auth/workload_exchange.py b/sdk/python/nemo-platform/src/nemo_platform/auth/workload_exchange.py deleted file mode 100644 index c54423fc01..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/auth/workload_exchange.py +++ /dev/null @@ -1,207 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Workload identity token exchange for SDK authentication.""" - -from __future__ import annotations - -import asyncio -import json -import logging -import math -import threading -from dataclasses import dataclass, field -from ipaddress import ip_address -from pathlib import Path -from urllib.parse import urlparse - -import httpx -from nemo_platform_plugin.client.constants import WORKLOAD_IDENTITY_TOKEN_FILE_ENVVAR - -from nemo_platform.auth.token_provider import DEFAULT_REFRESH_MARGIN_SECONDS, TokenSet -from nemo_platform.client.tls import client_verify_from_env - -logger = logging.getLogger(__name__) - -TOKEN_EXCHANGE_GRANT_TYPE = "urn:ietf:params:oauth:grant-type:token-exchange" -JWT_TOKEN_TYPE = "urn:ietf:params:oauth:token-type:jwt" -ACCESS_TOKEN_TYPE = "urn:ietf:params:oauth:token-type:access_token" - - -class WorkloadTokenExchangeError(RuntimeError): - """Structured error raised for RFC 8693 workload token exchange failures.""" - - def __init__(self, *, error: str, error_description: str) -> None: - self.error = error - self.error_description = error_description - super().__init__(f"Workload token exchange failed: {error} - {error_description}") - - -def read_subject_token_file(path: Path) -> str: - """Read a subject token from a workload identity token file.""" - try: - token = path.read_text(encoding="utf-8").strip() - except OSError as exc: - raise ValueError(f"Unable to read {WORKLOAD_IDENTITY_TOKEN_FILE_ENVVAR} at {path}: {exc}") from exc - if not token: - raise ValueError(f"{WORKLOAD_IDENTITY_TOKEN_FILE_ENVVAR} at {path} is empty") - return token - - -def _is_loopback_host(hostname: str | None) -> bool: - if hostname == "localhost": - return True - if hostname is None: - return False - try: - return ip_address(hostname).is_loopback - except ValueError: - return False - - -def _validate_token_endpoint(token_endpoint: str) -> None: - """Reject non-HTTPS token endpoints (except loopback for local dev).""" - parsed = urlparse(token_endpoint) - if parsed.scheme == "https": - return - if parsed.scheme == "http" and _is_loopback_host(parsed.hostname): - return - raise ValueError( - f"OIDC token endpoint must use HTTPS (got {token_endpoint!r}). " - "HTTP is only allowed for loopback addresses (localhost, 127.0.0.1, ::1)." - ) - - -def token_exchange_grant( - *, - token_endpoint: str, - client_id: str, - subject_token: str, - audience: str | None = None, - scope: str | None = None, - timeout: float = 30.0, -) -> dict[str, object]: - """Execute RFC 8693 token exchange and return token response JSON.""" - _validate_token_endpoint(token_endpoint) - data: dict[str, str] = { - "grant_type": TOKEN_EXCHANGE_GRANT_TYPE, - "client_id": client_id, - "subject_token": subject_token, - "subject_token_type": JWT_TOKEN_TYPE, - "requested_token_type": ACCESS_TOKEN_TYPE, - } - if audience: - data["audience"] = audience - if scope: - data["scope"] = scope - - response = httpx.post(token_endpoint, data=data, timeout=timeout, verify=client_verify_from_env()) - - if response.status_code != 200: - error_data: dict[str, object] = {} - if response.headers.get("content-type", "").startswith("application/json"): - error_data = _response_json_object( - response, - error_description="Token endpoint error response was not a JSON object", - ) - error = _response_string(error_data, "error", "unknown_error") - error_description = _response_string(error_data, "error_description", response.text) - raise WorkloadTokenExchangeError(error=error, error_description=error_description) - - token_data = _response_json_object( - response, - error_description="Token endpoint response was not a JSON object", - ) - _access_token_from_response(token_data) - return token_data - - -def _response_json_object(response: httpx.Response, *, error_description: str) -> dict[str, object]: - try: - payload = response.json() - except (json.JSONDecodeError, ValueError) as exc: - raise WorkloadTokenExchangeError(error="invalid_response", error_description=error_description) from exc - if not isinstance(payload, dict): - raise WorkloadTokenExchangeError(error="invalid_response", error_description=error_description) - return payload - - -def _response_string(payload: dict[str, object], key: str, default: str) -> str: - value = payload.get(key) - return value if isinstance(value, str) and value else default - - -def _access_token_from_response(token_data: dict[str, object]) -> str: - access_token = token_data.get("access_token") - if not isinstance(access_token, str) or not access_token.strip(): - raise WorkloadTokenExchangeError( - error="invalid_response", - error_description="Token endpoint response did not include a non-empty access_token", - ) - return access_token - - -def _expires_in_from_response(token_data: dict[str, object]) -> int | float | None: - expires_in = token_data.get("expires_in") - if isinstance(expires_in, bool): - return None - return expires_in if isinstance(expires_in, int | float) else None - - -@dataclass -class WorkloadTokenExchangeProvider: - """Provides access tokens by exchanging a workload identity subject token file.""" - - token_endpoint: str - client_id: str - subject_token_file: Path - audience: str | None = None - scope: str | None = None - refresh_margin_seconds: float = DEFAULT_REFRESH_MARGIN_SECONDS - tokens: TokenSet = field(default_factory=lambda: TokenSet(access_token="")) - _lock: threading.Lock = field(default_factory=threading.Lock, repr=False) - - def get_access_token(self) -> str: - """Return a valid access token, exchanging the current subject token if needed.""" - with self._lock: - if not self.tokens.access_token or self.tokens.is_expired(self.refresh_margin_seconds): - self._exchange() - return self.tokens.access_token - - async def get_access_token_async(self) -> str: - """Return a valid access token in async contexts.""" - return await asyncio.to_thread(self.get_access_token) - - def _exchange(self) -> None: - subject_token = read_subject_token_file(self.subject_token_file) - logger.debug("Exchanging workload identity token via %s", self.token_endpoint) - token_data = token_exchange_grant( - token_endpoint=self.token_endpoint, - client_id=self.client_id, - subject_token=subject_token, - audience=self.audience, - scope=self.scope, - ) - access_token = _access_token_from_response(token_data) - try: - tokens = TokenSet.from_access_token( - access_token, - refresh_token=None, - expires_in=_expires_in_from_response(token_data), - ) - except (OverflowError, TypeError, ValueError) as exc: - raise WorkloadTokenExchangeError( - error="invalid_response", - error_description="Token endpoint response did not include a usable access_token lifetime", - ) from exc - if tokens.expires_at is None or not math.isfinite(tokens.expires_at): - raise WorkloadTokenExchangeError( - error="invalid_response", - error_description="Token endpoint response did not include a usable access_token lifetime", - ) - if tokens.is_expired(0): - raise WorkloadTokenExchangeError( - error="invalid_response", - error_description="Token endpoint response returned an expired access_token", - ) - self.tokens = tokens diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/beta/__init__.py new file mode 100644 index 0000000000..8b2527cba3 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Beta SDK extensions.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/__init__.py index 114048d8af..2f271d337a 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/__init__.py @@ -1,301 +1,6 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""NeMo Evaluator SDK. +from nemo_platform._alias import alias_package as _alias_package -The public surface resolves lazily (PEP 562). Importing this package must not drag in the -execution/backend or metric stack: importing any submodule runs this module first, so eager -re-exports made ``import nemo_platform.beta.evaluator.agent_eval.runtimes.harbor_runtime`` — all the -optimizer needs — cost ~1400 modules (openai, sacrebleu, zstandard, ...) instead of ~485, and -turned every one of those transitive packages into an evaluation-time failure mode for the -SDK-backed evaluator. - -Add a new re-export to ``_LAZY_ATTRS``, the ``TYPE_CHECKING`` block and ``__all__`` — never as a -module-level import. ``tests/test_lazy_public_api.py`` locks the boundary in. -""" - -# ruff: noqa: I001 - the vendored SDK mirror uses different import-order settings. - -from importlib import import_module as _import_module -from importlib.metadata import PackageNotFoundError as _PackageNotFoundError -from importlib.metadata import version as _package_version -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - # Annotations and static analysis only; these must never execute at run time. Listing the - # names in ``__all__`` is what marks them as re-exports for ruff and the type checkers. - # - # AGENTS.md ("Python Style notes") says not to import types under TYPE_CHECKING and to use a - # regular import "when possible". A regular import is exactly what this module exists to - # remove, so the exception is deliberate: these names are re-exports, not annotations, and - # every one of them resolves for real through ``__getattr__`` below. - from nemo_platform.beta.evaluator.agent_stream_translation import ( - AgentStreamTranslation, - AgentStreamTranslationContext, - AgentStreamTranslator, - SseFrame, - ) - from nemo_platform.beta.evaluator.datasets import DatasetLoadError, load_dataset, load_dataset_as_dicts - from nemo_platform.beta.evaluator.execution.backends.local.backend import LocalBackend - from nemo_platform.beta.evaluator.execution.evaluator import Evaluator - from nemo_platform.beta.evaluator.execution.values import ( - EvaluationError, - EvaluationPhase, - ) - from nemo_platform.beta.evaluator.metrics.bleu import BLEUMetric - from nemo_platform.beta.evaluator.metrics.exact_match import ExactMatchMetric - from nemo_platform.beta.evaluator.metrics.f1 import F1Metric - from nemo_platform.beta.evaluator.metrics.llm_judge import LLMJudgeMetric - from nemo_platform.beta.evaluator.metrics.number_check import NumberCheckMetric - from nemo_platform.beta.evaluator.metrics.protocol import ( - Metric, - MetricTypeName, - validate_metric_result, - ) - from nemo_platform.beta.evaluator.metrics.remote import NemoAgentToolkitRemoteMetric, RemoteMetric - from nemo_platform.beta.evaluator.metrics.rouge import ROUGEMetric - from nemo_platform.beta.evaluator.metrics.string_check import StringCheckMetric - from nemo_platform.beta.evaluator.metrics.tool_calling import ToolCallingMetric - from nemo_platform.beta.evaluator.metrics.tunable_rag_evaluator import TunableRagEvaluatorMetric - from nemo_platform.beta.evaluator.resolver_protocols import ModelResolver, SecretResolver - from nemo_platform.beta.evaluator.resolvers import LocalModelResolver, LocalSecretResolver - from nemo_platform.beta.evaluator.structured_output import ( - InferenceFn, - InferenceStructuredOutput, - StructuredOutput, - StructuredOutputMode, - default_structured_output_mode, - detect_structured_output_mode, - ) - from nemo_platform.beta.evaluator.values import ( - Agent, - AgentBase, - BooleanValue, - CandidateOutput, - ContinuousScore, - BenchmarkEvaluationResult, - DatasetRow, - DatasetRows, - DiscreteScore, - EvaluationResult, - FieldMapping, - InferenceParams, - JSONScoreParser, - Label, - MetricDescriptor, - MetricDiagnostic, - MetricInput, - MetricOutput, - MetricOutputSpec, - MetricResult, - Model, - ModelRef, - GenericAgent, - NatAgentConfig, - NemoAgentToolkitAgent, - RangeScore, - ReasoningParams, - RemoteScore, - RubricScore, - RunConfig, - RunConfigOnline, - RunConfigOnlineModel, - SecretRef, - ) - - -def _resolve_version() -> str: - """Report the version of whichever distribution actually shipped this code. - - ``nemo-evaluator-sdk`` is not published on its own — this package is also vendored into the - ``nemo-platform`` wheel as ``nemo_platform.beta.evaluator``. There the SDK distribution does - not exist, so resolving only that name reported ``"0.0.0"`` unconditionally and any telemetry - or support log that read it got a useless constant. - """ - for distribution in ("nemo-evaluator-sdk", "nemo-platform"): - try: - return _package_version(distribution) - except _PackageNotFoundError: - continue - return "0.0.0" - - -version = _resolve_version() - -# Re-exported name -> the submodule that defines it, relative to this package. Relative on -# purpose: the vendoring tool mirrors this file into nemo_platform.beta.evaluator by rewriting -# module paths, and a relative name has nothing to rewrite, so the mirror is correct by -# construction. Mirrors the TYPE_CHECKING block above, in the same order. -_LAZY_ATTRS: dict[str, str] = { - "AgentStreamTranslation": ".agent_stream_translation", - "AgentStreamTranslationContext": ".agent_stream_translation", - "AgentStreamTranslator": ".agent_stream_translation", - "SseFrame": ".agent_stream_translation", - "DatasetLoadError": ".datasets", - "load_dataset": ".datasets", - "load_dataset_as_dicts": ".datasets", - "LocalBackend": ".execution.backends.local.backend", - "Evaluator": ".execution.evaluator", - "EvaluationError": ".execution.values", - "EvaluationPhase": ".execution.values", - "BLEUMetric": ".metrics.bleu", - "ExactMatchMetric": ".metrics.exact_match", - "F1Metric": ".metrics.f1", - "LLMJudgeMetric": ".metrics.llm_judge", - "NumberCheckMetric": ".metrics.number_check", - "Metric": ".metrics.protocol", - "MetricTypeName": ".metrics.protocol", - "validate_metric_result": ".metrics.protocol", - "NemoAgentToolkitRemoteMetric": ".metrics.remote", - "RemoteMetric": ".metrics.remote", - "ROUGEMetric": ".metrics.rouge", - "StringCheckMetric": ".metrics.string_check", - "ToolCallingMetric": ".metrics.tool_calling", - "TunableRagEvaluatorMetric": ".metrics.tunable_rag_evaluator", - "ModelResolver": ".resolver_protocols", - "SecretResolver": ".resolver_protocols", - "LocalModelResolver": ".resolvers", - "LocalSecretResolver": ".resolvers", - "InferenceFn": ".structured_output", - "InferenceStructuredOutput": ".structured_output", - "StructuredOutput": ".structured_output", - "StructuredOutputMode": ".structured_output", - "default_structured_output_mode": ".structured_output", - "detect_structured_output_mode": ".structured_output", - "Agent": ".values", - "AgentBase": ".values", - "BooleanValue": ".values", - "CandidateOutput": ".values", - "ContinuousScore": ".values", - "BenchmarkEvaluationResult": ".values", - "DatasetRow": ".values", - "DatasetRows": ".values", - "DiscreteScore": ".values", - "EvaluationResult": ".values", - "FieldMapping": ".values", - "InferenceParams": ".values", - "JSONScoreParser": ".values", - "Label": ".values", - "MetricDescriptor": ".values", - "MetricDiagnostic": ".values", - "MetricInput": ".values", - "MetricOutput": ".values", - "MetricOutputSpec": ".values", - "MetricResult": ".values", - "Model": ".values", - "ModelRef": ".values", - "GenericAgent": ".values", - "NatAgentConfig": ".values", - "NemoAgentToolkitAgent": ".values", - "RangeScore": ".values", - "ReasoningParams": ".values", - "RemoteScore": ".values", - "RubricScore": ".values", - "RunConfig": ".values", - "RunConfigOnline": ".values", - "RunConfigOnlineModel": ".values", - "SecretRef": ".values", -} - -__all__ = [ - "BLEUMetric", - "Agent", - "AgentBase", - "EvaluationError", - "EvaluationPhase", - "DatasetLoadError", - "DatasetRows", - "RunConfig", - "RunConfigOnline", - "RunConfigOnlineModel", - "BenchmarkEvaluationResult", - "EvaluationResult", - "Evaluator", - "ExactMatchMetric", - "F1Metric", - "FieldMapping", - "InferenceParams", - "InferenceFn", - "InferenceStructuredOutput", - "JSONScoreParser", - "Metric", - "MetricTypeName", - "MetricDescriptor", - "MetricDiagnostic", - "MetricInput", - "MetricOutput", - "MetricOutputSpec", - "MetricResult", - "LLMJudgeMetric", - "BooleanValue", - "CandidateOutput", - "ContinuousScore", - "DatasetRow", - "DiscreteScore", - "Label", - "LocalBackend", - "LocalModelResolver", - "LocalSecretResolver", - "Model", - "ModelRef", - "GenericAgent", - "ModelResolver", - "NatAgentConfig", - "NemoAgentToolkitAgent", - "AgentStreamTranslation", - "AgentStreamTranslationContext", - "AgentStreamTranslator", - "NemoAgentToolkitRemoteMetric", - "NumberCheckMetric", - "RangeScore", - "ReasoningParams", - "RemoteMetric", - "RemoteScore", - "ROUGEMetric", - "RubricScore", - "SecretRef", - "SecretResolver", - "SseFrame", - "StringCheckMetric", - "StructuredOutput", - "StructuredOutputMode", - "ToolCallingMetric", - "TunableRagEvaluatorMetric", - "default_structured_output_mode", - "detect_structured_output_mode", - "load_dataset", - "load_dataset_as_dicts", - "validate_metric_result", - "version", -] - - -def __getattr__(name: str) -> object: - """Import the submodule that defines ``name`` on first access (PEP 562). - - An *unknown* name raises ``AttributeError``, which is required: ``from pkg import sub`` only - falls back to importing a submodule when attribute lookup raises ``AttributeError``. - - A *known* name whose submodule fails to import propagates that ``ImportError`` unchanged, and - that is deliberate — ``ModuleNotFoundError: No module named 'sacrebleu'`` is far more useful - than an ``AttributeError`` claiming ``BLEUMetric`` does not exist. The consequence is that - ``hasattr(nemo_evaluator_sdk, name)`` raises rather than returning ``False`` when a name's - dependencies are not installed, since ``hasattr`` only swallows ``AttributeError``. To probe - for an optional part of the surface, catch ``ImportError`` around the access instead of using - ``hasattr``; to probe only for name membership, test against ``__all__``. - """ - submodule = _LAZY_ATTRS.get(name) - if submodule is None: - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") - value = getattr(_import_module(submodule, __name__), name) - globals()[name] = value # cache, so later lookups skip __getattr__ entirely - return value - - -def __dir__() -> list[str]: - # The declared surface plus any submodule the caller has already imported. Everything this - # module needs for its own machinery is imported under a leading underscore so the filter - # below keeps it out of autocomplete and inspect.getmembers without a name-by-name denylist; - # ``TYPE_CHECKING`` is the one exception, kept unaliased so type checkers still recognise it. - public = {name for name in globals() if not name.startswith("_")} - {"TYPE_CHECKING"} - return sorted(set(__all__) | public) +_alias_package("nemo_evaluator_sdk", globals()) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/dashboard.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/dashboard.py deleted file mode 100644 index 4c72c455dc..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/dashboard.py +++ /dev/null @@ -1,175 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Small HTML dashboard for standalone agent-eval result bundles.""" - -from __future__ import annotations - -import html -import json -from pathlib import Path -from typing import Any - -from nemo_platform.beta.evaluator.agent_eval.results import AgentEvalResult -from nemo_platform.beta.evaluator.agent_eval.scores import AgentEvalTaskScore -from nemo_platform.beta.evaluator.values.results import AggregateScalarScore, AggregateScore -from pydantic import BaseModel - - -def write_dashboard(result: AgentEvalResult, output_path: str | Path) -> Path: - """Write an HTML dashboard and return its path.""" - path = Path(output_path) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(render_dashboard(result), encoding="utf-8") - return path - - -def render_dashboard(result: AgentEvalResult) -> str: - """Render a compact generic report for metric outputs.""" - return f""" - - - - - Agent Eval Report - - - -
-

Agent Eval Report

-
Run {_e(result.run_id)} Β· {_e(result.summary.task_count)} tasks Β· {_e(result.summary.trial_count)} trials
-
-
-
-
Tasks{_e(result.summary.task_count)}
-
Trials{_e(result.summary.trial_count)}
-
Metric Scores{_e(result.summary.score_count)}
-
-

Metric Rollups

- {_metric_rollups(result)} -

Scores

- {_score_table(result.scores)} -
- - -""" - - -def _metric_rollups(result: AgentEvalResult) -> str: - aggregated = result.summary.scores.scores - if not aggregated: - return '

No numeric metric outputs to summarize.

' - rows: list[str] = [] - for score in sorted(aggregated, key=lambda item: item.name): - rows.append( - "" - f"{_e(score.name)}" - f"{_format_score(_headline_value(score))}" - f"{_format_score(_median(score))}" - f"{_format_score(score.sample_std_dev)}" - f"{_count(score.count)}" - f"{_e(score.nan_count)}" - "" - ) - return ( - "" - "" + "".join(rows) + "
NameValueMedianStd devCountNaN
" - ) - - -def _headline_value(score: AggregateScore) -> float | None: - """The one number to show: a scalar's ``value``, otherwise the mean of the distribution. - - A scalar score has no mean β€” rendering the column straight off ``score.mean`` would leave every - runner-imported figure blank in the table where it is the only thing worth reading. - """ - return score.value if isinstance(score, AggregateScalarScore) else score.mean - - -def _median(score: AggregateScore) -> float | None: - """The median, whether it arrived as a field or only inside a percentile distribution. - - Reading `percentiles.p50` alone would blank the column for every imported aggregate: a backend that - reports a median without a full distribution (Gym does) sets `median` and nothing else, which is the - case the field was added for. Natively computed scores populate both, identically. - """ - if score.median is not None: - return score.median - percentiles = getattr(score, "percentiles", None) - return percentiles.p50 if percentiles is not None else None - - -def _count(count: int | None) -> str: - """Sample size, or an em dash when the producer didn't report one (imported aggregates). - - Tests for None specifically: a genuine 0 means every sample was NaN, which is worth seeing. - """ - return "—" if count is None else _e(count) - - -def _score_table(scores: list[AgentEvalTaskScore]) -> str: - if not scores: - return '

No metric scores.

' - rows = [ - "" - f"{_e(score.task_id)}" - f"{_e(score.trial_id)}" - f"{_e(score.metric_type)}" - f"{_outputs(score)}" - "" - for score in scores - ] - return ( - "" - + "".join(rows) - + "
TaskTrialMetricOutputs
" - ) - - -def _outputs(score: AgentEvalTaskScore) -> str: - chunks = [] - for output in score.outputs: - chunks.append( - f'
{_e(output.name)}
{_e(_jsonish(output.value))}
' - ) - return '
' + "".join(chunks) + "
" - - -def _jsonish(value: Any) -> str: - if isinstance(value, BaseModel): - value = value.model_dump(mode="json") - try: - return json.dumps(value, indent=2, sort_keys=True) - except (TypeError, ValueError): - # ValueError covers circular references; fall back to a plain string rather than crash rendering. - return str(value) - - -def _format_score(value: float | None) -> str: - if value is None: - return "n/a" - return f"{value:.3f}" - - -def _e(value: object) -> str: - return html.escape(str(value), quote=True) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py deleted file mode 100644 index e7ca881708..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py +++ /dev/null @@ -1,825 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Standalone agent evaluation orchestration.""" - -# ruff: noqa: I001 - the vendored SDK mirror uses different import-order settings. - -from __future__ import annotations - -import asyncio -import uuid -from collections import defaultdict -from collections.abc import Awaitable, Callable, Mapping, Sequence -from datetime import UTC, datetime -from importlib.metadata import PackageNotFoundError -from importlib.metadata import version as package_version -from logging import getLogger -from pathlib import Path -from typing import Any, cast, overload -from urllib.parse import urlparse - -import httpx -import nemo_platform.beta.evaluator.inference as inference -from nemo_platform.beta.evaluator.agent_eval.results import AgentEvalResult, AgentEvalSummary, RunMetadata -from nemo_platform.beta.evaluator.agent_eval.scores import ( - AgentEvalDiagnostic, - AgentEvalDiagnosticSeverity, - AgentEvalScoreStatus, - AgentEvalTaskScore, - TRIAL_STATUS_DETAIL, -) -from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask -from nemo_platform.beta.evaluator.agent_eval.trials import ( - AgentEvalTarget, - AgentEvalTrial, - AgentEvalTrialStatus, - AgentOutput, - AgentTaskRunner, - RunAggregationsProvider, - RunnerInfo, -) -from nemo_platform.beta.evaluator.agent_inference import ( - AgentInferenceContext, - AgentInferenceFn, - AgentInferenceFnFactory, - make_agent_inference_fn, - new_agent_inference_client, -) -from nemo_platform.beta.evaluator.execution.metric_execution import ( - generate_online_sample, - resolve_target_structured_output_mode, - run_sync, -) -from nemo_platform.beta.evaluator.session import begin_evaluation_session -from nemo_platform.beta.evaluator.execution.samples import build_metric_input -from nemo_platform.beta.evaluator.inference import InferenceFn -from nemo_platform.beta.evaluator.metrics.protocol import Metric, MetricWithPreflight, validate_metric_result -from nemo_platform.beta.evaluator.metrics.utils import metric_type_name -from nemo_platform.beta.evaluator.values import ( - Agent, - AgentBase, - GenericAgent, - Model, - RunConfig, - RunConfigOnline, - RunConfigOnlineModel, -) -from nemo_platform.beta.evaluator.values.results import AggregateScore -from nemo_platform.beta.evaluator.values.evidence import ( - EVIDENCE_FORMAT_JSON, - EVIDENCE_TRACE, - CandidateEvidence, - EvidenceDescriptor, -) -from openai import AsyncOpenAI - -log = getLogger(__name__) - -_SAMPLE_KEYS_EXCLUDED_FROM_OUTPUT_METADATA = frozenset( - { - "evidence", - "invocation_metadata", - "invocation_status", - "output_text", - "response", - "trajectory", - } -) - - -class AgentEvaluator: - """Run stored-trial or live-target agent evaluations. - - The online inference seam (an optional ``inference_fn``, transport ``client``, and - ``default_headers``) is injected on the evaluator instance rather than the run config, - because these are runtime transport concerns rather than declarative run settings. A - single ``inference_fn``/``client`` pair serves both model and agent targets; leave them - unset to let the evaluator build a default client for the resolved target type. - """ - - @overload - def __init__( - self, - *, - inference_fn: InferenceFn | AgentInferenceFn | None = None, - agent_inference_fn_factory: None = None, - client: AsyncOpenAI | httpx.AsyncClient | None = None, - default_headers: dict[str, str] | None = None, - ) -> None: ... - - @overload - def __init__( - self, - *, - inference_fn: None = None, - agent_inference_fn_factory: AgentInferenceFnFactory, - client: AsyncOpenAI | httpx.AsyncClient | None = None, - default_headers: dict[str, str] | None = None, - ) -> None: ... - - def __init__( - self, - *, - inference_fn: InferenceFn | AgentInferenceFn | None = None, - agent_inference_fn_factory: AgentInferenceFnFactory | None = None, - client: AsyncOpenAI | httpx.AsyncClient | None = None, - default_headers: dict[str, str] | None = None, - ) -> None: - """Configure runtime dependencies for live target generation. - - Args: - inference_fn: Optional model or agent inference override. When omitted, the - evaluator selects the default implementation for the target type. - agent_inference_fn_factory: Optional per-task factory for agent inference. - The evaluator supplies persistence and invocation identity through an - :class:`AgentInferenceContext`. - client: Optional transport client matching the target type: ``AsyncOpenAI`` for - models or ``httpx.AsyncClient`` for agents. - default_headers: Additional HTTP headers forwarded to live inference requests. - """ - if inference_fn is not None and agent_inference_fn_factory is not None: - raise ValueError("provide either inference_fn or agent_inference_fn_factory, not both") - self.inference_fn = inference_fn - self.agent_inference_fn_factory = agent_inference_fn_factory - self.client = client - self.default_headers = default_headers - - async def run( - self, - *, - tasks: Sequence[AgentEvalTask], - trials: Sequence[AgentEvalTrial] | None = None, - target: AgentEvalTarget | None = None, - config: AgentEvalRunConfig | None = None, - ) -> AgentEvalResult: - """Evaluate imported trials or generate live trials before scoring. - - Exactly one of ``trials`` or ``target`` must be provided. - """ - resolved_config = config or AgentEvalRunConfig() - task_list = list(tasks) - if not task_list: - raise ValueError("at least one task is required") - - run_id = resolved_config.run_id or _new_run_id() - runtime_config = resolved_config.model_copy(update={"run_id": run_id}) - started_at = datetime.now(UTC) - - seam_error = "provide exactly one of trials or target" - if trials is not None and target is not None: - raise ValueError(seam_error) - - async with begin_evaluation_session(): - # Branch on which seam was supplied so the type checker narrows each of ``trials`` and - # ``target`` to a concrete type without a cast. The final arm is the "neither" case. - if trials is not None: - trial_list = list(trials) - elif target is not None: - trial_list = await self._generate_trials(tasks=task_list, target=target, config=runtime_config) - else: - raise ValueError(seam_error) - scores = await self._score_trials( - tasks=task_list, - trials=trial_list, - config=runtime_config, - run_id=run_id, - ) - runner_scores = _collect_runner_aggregate_scores(target) if target is not None else [] - finished_at = datetime.now(UTC) - metadata = RunMetadata( - labels=dict(runtime_config.labels), - target=_describe_target(target, runtime_config.params), - started_at=started_at, - finished_at=finished_at, - duration_sec=(finished_at - started_at).total_seconds(), - sdk_version=_sdk_version(), - ) - result = AgentEvalResult( - run_id=run_id, - tasks=task_list, - trials=trial_list, - scores=scores, - summary=AgentEvalSummary.from_scores( - scores, tasks=task_list, trials=trial_list, extra_scores=runner_scores - ), - metadata=metadata, - work_dir=runtime_config.work_dir, - ) - - return result - - def run_sync( - self, - *, - tasks: Sequence[AgentEvalTask], - trials: Sequence[AgentEvalTrial] | None = None, - target: AgentEvalTarget | None = None, - config: AgentEvalRunConfig | None = None, - ) -> AgentEvalResult: - """Synchronous bridge for :meth:`run`.""" - return run_sync(lambda: self.run(tasks=tasks, trials=trials, target=target, config=config)) - - async def _score_trials( - self, - *, - tasks: list[AgentEvalTask], - trials: list[AgentEvalTrial], - config: AgentEvalRunConfig, - run_id: str, - ) -> list[AgentEvalTaskScore]: - tasks_by_id = {task.id: task for task in tasks} - task_index_by_id = {task.id: index for index, task in enumerate(tasks)} - trials_by_task: dict[str, list[AgentEvalTrial]] = defaultdict(list) - for trial in trials: - if trial.task_id not in tasks_by_id: - raise ValueError(f"trial {trial.id!r} references unknown task {trial.task_id!r}") - trials_by_task[trial.task_id].append(trial) - - # Fail loudly when a task produced no trial. Imported trials or an AgentTaskRunner may omit a - # task entirely; without this an incomplete run would look successful aside from lower summary - # counts. (A richer alternative is to emit a "missing trial" failed score per metric.) - tasks_without_trials = [task.id for task in tasks if not trials_by_task.get(task.id)] - if tasks_without_trials: - raise ValueError(f"no trials produced for tasks: {sorted(tasks_without_trials)}") - - for task in tasks: - if not task.metrics: - raise ValueError(f"task {task.id!r} does not declare any metrics") - - await _preflight_task_metrics(tasks, trials_by_task) - - semaphore = asyncio.Semaphore(config.parallelism) - - async def guarded_score(task: AgentEvalTask, trial: AgentEvalTrial, metric: Metric) -> AgentEvalTaskScore: - async with semaphore: - row_index = task_index_by_id[task.id] - if trial.status == AgentEvalTrialStatus.FAILED: - return _failed_metric_score( - run_id=run_id, - task=task, - trial=trial, - metric=metric, - row_index=row_index, - diagnostic=AgentEvalDiagnostic( - severity=AgentEvalDiagnosticSeverity.ERROR, - message=f"trial {trial.id!r} is failed", - source=metric_type_name(metric), - # The key pass@k reads to tell "the agent produced nothing" (a failed - # attempt) from "the metric raised" (an unusable measurement). - details={TRIAL_STATUS_DETAIL: trial.status.value}, - ), - ) - try: - return await _score_metric( - run_id=run_id, - task=task, - trial=trial, - metric=metric, - row_index=row_index, - ) - except Exception as exc: - if config.fail_fast: - raise - log.warning( - "metric %s failed for trial %r (task %r): %s", - metric_type_name(metric), - trial.id, - task.id, - exc, - ) - return _failed_metric_score( - run_id=run_id, - task=task, - trial=trial, - metric=metric, - row_index=row_index, - diagnostic=_exception_diagnostic(exc, metric_type_name(metric)), - ) - - return await asyncio.gather( - *[ - guarded_score(task, trial, metric) - for task in tasks - for trial in trials_by_task.get(task.id, []) - for metric in task.metrics - ] - ) - - async def _generate_trials( - self, - *, - tasks: list[AgentEvalTask], - target: AgentEvalTarget, - config: AgentEvalRunConfig, - ) -> list[AgentEvalTrial]: - if isinstance(target, AgentTaskRunner): - return list(await target.run_tasks(tasks, config=config)) - if not isinstance(target, (Model, AgentBase)): - raise NotImplementedError(f"unsupported agent-eval target type: {type(target).__name__}") - - params = _resolve_live_params(config, target) - prompt_template = config.prompt_template or _default_prompt_template(target) - semaphore = asyncio.Semaphore(params.parallelism) - - # Use the injected transport client when provided; otherwise build a default for the - # resolved target type and close it when generation finishes. - client = self.client - close_client: Callable[[], Awaitable[Any]] | None = None - if client is None and self.inference_fn is None: - if isinstance(target, Model): - client = inference.new_inference_client(target) - close_client = client.close - else: - client = new_agent_inference_client() - close_client = client.aclose - - try: - # When config.params.ignore_request_failure is set, convert a failed generation request - # into a FAILED trial (which the scorer turns into failed metric scores) instead of - # aborting the whole run. This matches the existing online-evaluator contract. - async def generate_one(index: int, task: AgentEvalTask) -> AgentEvalTrial: - async with semaphore: - # Keep evaluator-owned runtime identity separate from task inputs. - # ``_generate_sample`` exposes these values to request templates under - # ``agent_eval``. For agent targets, the same values are supplied to the - # inference factory so stream translators and evidence can carry stable - # evaluation identifiers without coupling them to this evaluator. - agent_eval_context = { - "run_id": config.run_id, - "task_id": task.id, - "invocation_id": f"{config.run_id}:{task.id}:{target.name}", - } - evidence_dir = ( - _task_evidence_dir(Path(config.work_dir), index=index, task_id=task.id) - if config.work_dir is not None and isinstance(target, AgentBase) - else None - ) - resolved_inference_fn = self.inference_fn - if isinstance(target, AgentBase) and resolved_inference_fn is None: - factory = self.agent_inference_fn_factory or make_agent_inference_fn - resolved_inference_fn = factory( - AgentInferenceContext( - evidence_dir=evidence_dir, - metadata=agent_eval_context, - ) - ) - try: - sample = await _generate_sample( - target=target, - row=_task_row(task), - index=index, - prompt_template=prompt_template, - params=params, - inference_fn=resolved_inference_fn, - client=client, - default_headers=self.default_headers, - agent_eval_context=agent_eval_context, - ) - except Exception as exc: - if params.ignore_request_failure: - return _failed_generation_trial(task, target, exc) - raise - return _trial_from_sample(task, target, sample) - - return await asyncio.gather(*(generate_one(index, task) for index, task in enumerate(tasks))) - finally: - if close_client is not None: - await close_client() - - -async def _preflight_task_metrics( - tasks: Sequence[AgentEvalTask], - trials_by_task: Mapping[str, Sequence[AgentEvalTrial]], -) -> None: - """Run each distinct metric's preflight once, skipping metrics with nothing to score. - - Agent-eval scores metrics directly rather than through ``prepare_metric_for_execution``, so - nothing else runs their preflight. Failed trials short-circuit to a failed score without - invoking the metric, so a task whose trials all failed must not trigger one: preflight resolves - the judge endpoint, making it both a wasted request and a way for the run to abort. - """ - preflighted: set[int] = set() - for task in tasks: - scoreable = any(trial.status != AgentEvalTrialStatus.FAILED for trial in trials_by_task.get(task.id, ())) - if not scoreable: - continue - for metric in task.metrics: - if isinstance(metric, MetricWithPreflight) and id(metric) not in preflighted: - preflighted.add(id(metric)) - await metric.preflight() - - -async def _generate_sample( - *, - target: Model | Agent, - row: dict[str, Any], - index: int, - prompt_template: str | dict[str, Any], - params: RunConfigOnline | RunConfigOnlineModel, - inference_fn: InferenceFn | AgentInferenceFn | None, - client: AsyncOpenAI | httpx.AsyncClient | None, - default_headers: dict[str, str] | None, - agent_eval_context: dict[str, Any], -) -> dict[str, Any]: - # InferenceFn and AgentInferenceFn are callable protocols, so isinstance cannot discriminate - # the injected fn; narrow it per target type with a cast (matching execution/benchmark_execution). - # The transport client is a real class union, so isinstance narrowing is enough there. - if isinstance(target, Model): - model_params = cast(RunConfigOnlineModel, params) - preprocess_hooks, postprocess_hooks = inference.new_hooks(model_params) - model_inference_fn = ( - cast(InferenceFn, inference_fn) if inference_fn is not None else inference.make_inference_request - ) - await resolve_target_structured_output_mode( - preprocess_hooks=preprocess_hooks, - model=target, - inference_fn=model_inference_fn, - params=model_params, - ) - return await generate_online_sample( - target=target, - row=row, - index=index, - prompt_template=prompt_template, - params=model_params, - inference_fn=model_inference_fn, - client=client if isinstance(client, AsyncOpenAI) else None, - preprocess_hooks=preprocess_hooks, - postprocess_hooks=postprocess_hooks, - default_headers=default_headers, - template_context={"agent_eval": agent_eval_context}, - ) - - if inference_fn is None: - raise TypeError("expected AgentInferenceFn for Agent target") - agent_inference_fn = cast(AgentInferenceFn, inference_fn) - return await generate_online_sample( - target=target, - row=row, - index=index, - prompt_template=prompt_template, - params=params, - inference_fn=agent_inference_fn, - client=client if isinstance(client, httpx.AsyncClient) else None, - default_headers=default_headers, - template_context={"agent_eval": agent_eval_context}, - ) - - -def _trial_from_sample(task: AgentEvalTask, target: Model | Agent, sample: dict[str, Any]) -> AgentEvalTrial: - output_text = sample.get("output_text") - if not (isinstance(output_text, str) and output_text.strip()): - # Reasoning models that exhaust the token budget can return only - # `reasoning_content` with empty `content`. Fall back to that text so the - # trial stays scorable instead of being dropped as empty output. - output_text = _reasoning_content_fallback(sample.get("response")) - evidence = sample.get("evidence") - if evidence is not None and not isinstance(evidence, CandidateEvidence): - evidence = CandidateEvidence.model_validate(evidence) - - # Evidence precedence: - # - trajectory exists: merge it without replacing a typed trace. - # - no trajectory, but typed evidence exists: preserve that evidence unchanged. - # - neither exists: synthesize the fallback trace. - if "trajectory" in sample: - trace = EvidenceDescriptor(kind=EVIDENCE_TRACE, format=EVIDENCE_FORMAT_JSON, data=sample["trajectory"]) - descriptors = dict(evidence.descriptors) if evidence is not None else {} - descriptors.setdefault(EVIDENCE_TRACE, trace) - evidence = CandidateEvidence( - descriptors=descriptors, - metadata=dict(evidence.metadata) if evidence is not None else {}, - ) - elif evidence is None: - evidence = CandidateEvidence( - descriptors={ - EVIDENCE_TRACE: EvidenceDescriptor( - kind=EVIDENCE_TRACE, - format=EVIDENCE_FORMAT_JSON, - data={"task_id": task.id, "target": target.name}, - ) - } - ) - - status_value = sample.get("invocation_status", AgentEvalTrialStatus.COMPLETED.value) - status = AgentEvalTrialStatus(status_value) - invocation_metadata = sample.get("invocation_metadata") - if not isinstance(invocation_metadata, dict): - invocation_metadata = {} - - return AgentEvalTrial( - id=f"{task.id}:{target.name}", - task_id=task.id, - status=status, - output=AgentOutput( - output_text=output_text if isinstance(output_text, str) else None, - response=sample.get("response"), - metadata={ - **invocation_metadata, - **{ - key: value for key, value in sample.items() if key not in _SAMPLE_KEYS_EXCLUDED_FROM_OUTPUT_METADATA - }, - }, - ), - evidence=evidence, - metadata={ - **invocation_metadata, - "model_id": target.name, - "target_name": target.name, - "generated": True, - }, - ) - - -def _reasoning_content_fallback(response: Any) -> str | None: - if not isinstance(response, dict): - return None - choices = response.get("choices") - if not isinstance(choices, list): - return None - for choice in choices: - message = choice.get("message") if isinstance(choice, dict) else None - if not isinstance(message, dict): - continue - reasoning = message.get("reasoning_content") - if isinstance(reasoning, str) and reasoning.strip(): - return reasoning - return None - - -def _failed_generation_trial(task: AgentEvalTask, target: Model | Agent, exc: Exception) -> AgentEvalTrial: - return AgentEvalTrial( - id=f"{task.id}:{target.name}", - task_id=task.id, - status=AgentEvalTrialStatus.FAILED, - output=None, - evidence=CandidateEvidence( - descriptors={ - "error": EvidenceDescriptor( - kind="error", - data={"error_type": exc.__class__.__name__, "error": str(exc)}, - ) - } - ), - metadata={ - "model_id": target.name, - "target_name": target.name, - "generated": True, - "error_type": exc.__class__.__name__, - "error": str(exc), - }, - ) - - -async def _score_metric( - *, - run_id: str, - task: AgentEvalTask, - trial: AgentEvalTrial, - metric: Metric, - row_index: int, -) -> AgentEvalTaskScore: - output_spec = metric.output_spec() - metric_result = validate_metric_result( - await metric.compute_scores(build_metric_input(_metric_row(task, trial), _trial_sample(trial), row_index)), - output_spec, - ) - metric_type = metric_type_name(metric) - return AgentEvalTaskScore( - id=_score_id(run_id, task.id, trial.id, metric_type), - run_id=run_id, - task_id=task.id, - trial_id=trial.id, - metric_type=metric_type, - status=AgentEvalScoreStatus.COMPLETED, - outputs=metric_result.outputs, - # Persist the metric's own diagnostics (e.g. per-criterion judge verdicts) β€” the failure path - # already records diagnostics; the success path dropped them. - diagnostics=[ - AgentEvalDiagnostic( - severity=AgentEvalDiagnosticSeverity.INFO, - message=diagnostic.message, - source=metric_type, - details=diagnostic.details or {}, - ) - for diagnostic in metric_result.diagnostics - ], - metadata={ - "row_index": row_index, - "trial_metadata": trial.metadata, - }, - ) - - -def _failed_metric_score( - *, - run_id: str, - task: AgentEvalTask, - trial: AgentEvalTrial, - metric: Metric, - row_index: int, - diagnostic: AgentEvalDiagnostic, -) -> AgentEvalTaskScore: - metric_type = metric_type_name(metric) - return AgentEvalTaskScore( - id=_score_id(run_id, task.id, trial.id, metric_type), - run_id=run_id, - task_id=task.id, - trial_id=trial.id, - metric_type=metric_type, - status=AgentEvalScoreStatus.FAILED, - outputs=[], - diagnostics=[diagnostic], - metadata={ - "row_index": row_index, - "trial_metadata": trial.metadata, - }, - ) - - -def _exception_diagnostic(exc: Exception, metric_type: str) -> AgentEvalDiagnostic: - return AgentEvalDiagnostic( - severity=AgentEvalDiagnosticSeverity.ERROR, - message=str(exc) or exc.__class__.__name__, - source=metric_type, - details={"exception_type": exc.__class__.__name__}, - ) - - -def _score_id(run_id: str, task_id: str, trial_id: str, metric_type: str) -> str: - return f"{run_id}:{task_id}:{trial_id}:{metric_type}" - - -def _trial_sample(trial: AgentEvalTrial) -> dict[str, Any]: - if trial.output is None: - return {} - sample: dict[str, Any] = { - **trial.metadata, - **trial.output.metadata, - } - if trial.output.output_text is not None: - sample["output_text"] = trial.output.output_text - if trial.output.response is not None: - sample["response"] = trial.output.response - if trial.evidence is not None: - sample["evidence"] = trial.evidence - return sample - - -def _resolve_live_params( - config: AgentEvalRunConfig, - target: Model | Agent, -) -> RunConfigOnline | RunConfigOnlineModel: - params = config.params - if isinstance(target, Model): - if params is None: - return RunConfigOnlineModel(parallelism=config.parallelism) - if isinstance(params, RunConfigOnlineModel): - return params - if isinstance(params, RunConfigOnline): - return RunConfigOnlineModel(**params.model_dump(mode="python")) - if isinstance(params, RunConfig): - return RunConfigOnlineModel(**params.model_dump(mode="python")) - - if params is None: - return RunConfigOnline(parallelism=config.parallelism) - if isinstance(params, RunConfigOnlineModel): - return RunConfigOnline( - **params.model_dump( - mode="python", - exclude={"inference", "system_prompt", "reasoning", "structured_output"}, - ) - ) - if isinstance(params, RunConfigOnline): - return params - return RunConfigOnline(**params.model_dump(mode="python")) - - -def _default_prompt_template(target: Model | Agent) -> dict[str, Any] | str: - # Every default renders against the single canonical task input, ``instruction`` (see - # ``AgentEvalTask.agent_prompt``); no other input key is special. - if isinstance(target, GenericAgent): - # A generic HTTP agent defines its own request entirely through its `body` template, which - # renders against the task inputs (e.g. `{{ instruction }}`). Pass the task row through - # unchanged so `body` β€” not a chat/completions assumption β€” shapes the payload. See - # `_resolve_http_agent_invocation`, which renders `body` against this request. - return "{{ item }}" - if isinstance(target, Model) and _is_completions_endpoint(target.url): - return {"prompt": "{{item.instruction}}"} - return {"messages": [{"role": "user", "content": "{{item.instruction}}"}]} - - -def _task_row(task: AgentEvalTask) -> dict[str, Any]: - # The task inputs verbatim, plus the task id. `instruction` is the single canonical input the - # target is prompted with (see `AgentEvalTask.agent_prompt` and `_default_prompt_template`); no - # input key is synthesized or aliased here. - return {**task.inputs, "task_id": task.id} - - -def _metric_row(task: AgentEvalTask, trial: AgentEvalTrial) -> dict[str, Any]: - return { - "task": { - "id": task.id, - "intent": task.intent, - "metadata": task.metadata, - }, - "inputs": task.inputs, - # Grader-only ground truth: available to metrics here but never seeded into the agent's - # workspace (see AgentEvalTask.reference), so a metric can grade against held-out artifacts. - "reference": task.reference, - "trial": { - "id": trial.id, - "task_id": trial.task_id, - "status": trial.status.value, - # How the trial failed, for a metric that grades on it. None when the producer reported no failure. - "error": trial.error.model_dump(mode="json") if trial.error is not None else None, - "metadata": trial.metadata, - }, - } - - -def _is_completions_endpoint(url: str) -> bool: - path = urlparse(url).path.rstrip("/") - return path.endswith("/completions") and not path.endswith("/chat/completions") - - -def _sdk_version() -> str | None: - try: - return package_version("nemo-evaluator-sdk") - except PackageNotFoundError: # pragma: no cover - only when running from an uninstalled tree - return None - - -def _describe_target( - target: AgentEvalTarget | None, - params: RunConfig | RunConfigOnline | RunConfigOnlineModel | None = None, -) -> RunnerInfo: - """Identify what produced the trials, for the run's provenance. - - Runners identify themselves via the required :meth:`AgentTaskRunner.runner_info`; trials supplied - directly have no runner. - - Models and agents are described by name *and* the settings they were invoked with β€” the endpoint - ``url``, plus the whole ``params`` object (temperature, max_tokens, reasoning effort, system prompt, - retries, ...). A name alone is not an identity: the same model name served from two different URLs, - or at two different temperatures, would otherwise record identical provenance. ``params`` is dumped - whole rather than cherry-picked, because a filtered subset is what bites you later when the omitted - field turns out to be the one that mattered. It carries no credentials β€” ``Model.api_key_secret`` is - a reference on the model, and ``default_headers`` is excluded from serialization. - """ - if target is None: - return RunnerInfo(name="imported", kind="imported") - if isinstance(target, (Model, AgentBase)): - config: dict[str, Any] = {"url": getattr(target, "url", None)} - if params is not None: - config["params"] = params.model_dump(mode="json", exclude_none=True) - return RunnerInfo(name=target.name, kind="model" if isinstance(target, Model) else "agent", config=config) - return target.runner_info() - - -def _collect_runner_aggregate_scores(target: object) -> list[AggregateScore]: - """The typed subset of a runner's own aggregations, for merging into ``summary.scores``. - - A runner that maps its numbers onto aggregate scores namespaces them under ``runner..``, so - they sit alongside the SDK's own without being mistaken for them. That namespace is *enforced*, not - merely documented: ``RunAggregationsProvider`` is a public extension point, ``summary.scores`` is a - flat list, and a third-party runner returning ``gym_reward.reward`` would not overwrite the SDK's - own aggregate but sit next to it under the same name, leaving any lookup to pick one arbitrarily. - - Offending entries are dropped with a warning rather than raised on. This runs *after* ``run_tasks``, - so raising would sink a completed run β€” potentially hours of collection β€” over a naming bug, while - the numbers themselves remain in the runner's own files inside the bundle. - """ - if not isinstance(target, RunAggregationsProvider): - return [] - runner_info = getattr(target, "runner_info", None) # structurally optional: the protocol is a companion - runner_name = runner_info().name if callable(runner_info) else None - prefix = f"runner.{runner_name}." if runner_name else "runner." - collected: list[AggregateScore] = [] - for score in target.run_aggregate_scores(): - if not score.name.startswith(prefix): - log.warning( - "Dropping runner-contributed aggregate %r: RunAggregationsProvider names must be " - "namespaced %r so an imported figure is never mistaken for one the SDK computed.", - score.name, - prefix, - ) - continue - collected.append(score) - return collected - - -def _new_run_id() -> str: - timestamp = datetime.now(UTC).strftime("%Y%m%d%H%M%S") - return f"agent-eval-{timestamp}-{uuid.uuid4().hex[:8]}" - - -def _task_evidence_dir(output_dir: Path, *, index: int, task_id: str) -> Path: - safe_task_id = _safe_path_component(task_id) - task_dir = f"{index:06d}-{safe_task_id}" if safe_task_id else f"task-{index:06d}" - return output_dir / "evidence" / task_dir - - -def _safe_path_component(value: str) -> str: - sanitized = "".join(char if char.isalnum() or char in "-_." else "-" for char in value) - return sanitized.strip("-_.")[:120] diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/metrics.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/metrics.py deleted file mode 100644 index 99af728cec..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/metrics.py +++ /dev/null @@ -1,296 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Reusable agent-eval metrics and the typed view over trial measurements. - -Two complementary pieces, both keyed off ``AgentEvalTrial``: - -* Metrics (scorers) β€” ``AgentPhaseSuccessMetric`` reads the agent-phase outcome - stamped on trial metadata; ``EvidencePresenceMetric`` is a genuine - *metric-over-evidence* that scores by inspecting ``candidate.evidence`` (a - filesystem evidence handle) rather than trusting a verifier's stamped reward. -* ``TrialMeasurements`` β€” the single documented place that names the loose - metadata keys gating/reporting read, applying the fallbacks (``duration_ms`` β†’ - ``runtime_sec``, ``passed`` β†’ ``reward``). -""" - -from __future__ import annotations - -import json -import logging -import math -from collections.abc import Mapping -from typing import Any, ClassVar, Literal - -from nemo_platform.beta.evaluator.agent_eval.trials import EVIDENCE_FINAL_STATE -from nemo_platform.beta.evaluator.enums import MetricType -from nemo_platform.beta.evaluator.metrics.protocol import ( - CandidateOutput, - MetricInput, - MetricOutput, - MetricOutputSpec, - MetricResult, -) -from nemo_platform.beta.evaluator.values.atif import Trajectory -from nemo_platform.beta.evaluator.values.evidence import EVIDENCE_TRACE -from nemo_platform.beta.evaluator.values.metrics import MetricBase -from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator - -logger = logging.getLogger(__name__) - -# Token-measurement keys carried on trial metadata (and in result.json["metrics"]). -TOKEN_KEYS: tuple[str, ...] = ( - "prompt_tokens", - "completion_tokens", - "total_tokens", - "cache_creation_tokens", - "cache_read_tokens", -) - - -class AgentPhaseSuccessMetric(MetricBase): - """Emit ``True`` when the agent phase exited successfully, else ``False``. - - The output name stays ``agent_phase_success`` (which gating reads as a reward - signal β€” ``True``/``False`` coerces to ``1.0``/``0.0``). - - A built-in metric type, so it bundles inline and needs no cloudpickle opt-in to be - stored on a task. ``type`` is therefore a fixed discriminator and no longer - overridable per caller. - """ - - type: Literal[MetricType.AGENT_PHASE_SUCCESS] = MetricType.AGENT_PHASE_SUCCESS - - def output_spec(self) -> list[MetricOutputSpec]: - return [MetricOutputSpec.boolean("agent_phase_success")] - - async def compute_scores(self, input: MetricInput) -> MetricResult: - # Only an explicit boolean counts as success; a stray truthy string - # (e.g. "false") must not mark a failed trial as passed. - raw_agent_ok = input.candidate.metadata.get("agent_ok") - agent_ok = raw_agent_ok if isinstance(raw_agent_ok, bool) else False - return MetricResult(outputs=[MetricOutput(name="agent_phase_success", value=agent_ok)]) - - -class EvidencePresenceMetric(MetricBase): - """Emit ``True`` when a named filesystem evidence directory exists (and is non-empty). - - Reads ``candidate.evidence`` directly β€” the canonical metric-over-evidence - pattern β€” so the result reflects what the agent actually produced on disk, - not a reward stamped into metadata by a verifier. - """ - - type: Literal[MetricType.EVIDENCE_PRESENCE] = MetricType.EVIDENCE_PRESENCE - evidence_name: str = Field(default=EVIDENCE_FINAL_STATE, description="Evidence directory to look for.") - output_name: str = Field(default="evidence_present", description="Name of the emitted boolean score.") - require_non_empty: bool = Field( - default=True, description="Require the evidence directory to be non-empty, not merely present." - ) - - def output_spec(self) -> list[MetricOutputSpec]: - return [MetricOutputSpec.boolean(self.output_name)] - - async def compute_scores(self, input: MetricInput) -> MetricResult: - present = False - evidence = input.candidate.evidence - if evidence is not None and evidence.get(self.evidence_name) is not None: - try: - handle = await evidence.filesystem(self.evidence_name) - if await handle.exists(): - present = bool(await handle.iter_paths(recursive=True)) if self.require_non_empty else True - except (KeyError, ValueError) as exc: - logger.warning( - "EvidencePresenceMetric scored False: could not resolve evidence %r for output %r: %s", - self.evidence_name, - self.output_name, - exc, - ) - return MetricResult(outputs=[MetricOutput(name=self.output_name, value=present)]) - - -class SkillUsedMetric(MetricBase): - """Emit ``skill_present`` and ``skill_used`` so an eval can flag a failure to use an injected skill. - - * ``skill_present`` β€” ``True`` when one or more skills were injected into the trial. Reads - the ``"skills"`` metadata key a skill-aware runtime stamps β€” a list of provenance dicts - (``{"name", "hash", "mode", "adapter_id", "location", ...}``, see ``fabric.skills.SkillProvenance``). - Baseline trials carry an empty list. - * ``skill_used`` β€” best-effort ``True`` when the agent referenced *any* injected skill in its ATIF - trajectory. It matches each skill's staged ``location`` (a specific, low-false-positive path - signal β€” e.g. a read of ``.agents/skills//SKILL.md``) against tool-call names/arguments, - step messages, reasoning, and observations. A bare skill-*name* match is intentionally NOT - counted (the name commonly appears in the task prompt), so ``skill_present=True, skill_used=False`` - flags a *likely* failure to use the skill. - - Limitation: an absent trajectory reference cannot fully distinguish "not used" from "used without - leaving a filesystem trace" β€” strongest for codex-style filesystem discovery, weaker for in-context - skill loading. Authoritative usage detection via harness skill-activation events is a follow-up. - With no skill present, both outputs are ``False``. - """ - - type: Literal[MetricType.SKILL_USED] = MetricType.SKILL_USED - trace_evidence: str = Field(default=EVIDENCE_TRACE, description="Trace evidence to scan for skill usage.") - - OUTPUT_PRESENT: ClassVar[str] = "skill_present" - OUTPUT_USED: ClassVar[str] = "skill_used" - # Metadata key skill-aware runtimes stamp the provenance list under (matches the fabric runtime). - _SKILLS_KEY: ClassVar[str] = "skills" - - def output_spec(self) -> list[MetricOutputSpec]: - return [ - MetricOutputSpec.boolean(self.OUTPUT_PRESENT), - MetricOutputSpec.boolean(self.OUTPUT_USED), - ] - - async def compute_scores(self, input: MetricInput) -> MetricResult: - provenances = self._extract_provenances(input.candidate.metadata) - present = bool(provenances) - used = await self._any_skill_used(input.candidate, provenances) if present else False - return MetricResult( - outputs=[ - MetricOutput(name=self.OUTPUT_PRESENT, value=present), - MetricOutput(name=self.OUTPUT_USED, value=used), - ] - ) - - def _extract_provenances(self, metadata: Mapping[str, Any]) -> list[Mapping[str, Any]]: - skills = metadata.get(self._SKILLS_KEY) - if isinstance(skills, list): - return [p for p in skills if isinstance(p, Mapping) and p] - return [] - - async def _any_skill_used(self, candidate: CandidateOutput, provenances: list[Mapping[str, Any]]) -> bool: - locations = [loc for p in provenances if isinstance(loc := p.get("location"), str) and loc] - if not locations: - return False - evidence = candidate.evidence - if evidence is None or evidence.get(self.trace_evidence) is None: - return False - try: - trajectory = await (await evidence.trace(self.trace_evidence)).trace() - except (KeyError, ValueError, ValidationError, OSError) as exc: - # Best-effort: a missing/malformed/invalid trajectory must score skill_used=False, not raise. - # ValidationError covers Trajectory.model_validate; OSError covers the underlying file read. - logger.warning( - "SkillUsedMetric scored skill_used=False: could not read trace %r: %s", self.trace_evidence, exc - ) - return False - return any(_trajectory_references(trajectory, loc) for loc in locations) - - -class TrialMeasurements(BaseModel): - """Numeric measurements projected from trial metadata. - - Reporting/gating consume it via :meth:`from_metadata`; producers keep writing - the same keys onto ``AgentEvalTrial.metadata``. - """ - - model_config = ConfigDict(extra="forbid") - - prompt_tokens: int | None = None - completion_tokens: int | None = None - total_tokens: int | None = None - cache_creation_tokens: int | None = None - cache_read_tokens: int | None = None - runtime_sec: float | None = None - cost_usd: float | None = Field(default=None, allow_inf_nan=False) - reward: float | None = None - passed: bool | None = None - - @field_validator("cost_usd", mode="before") - @classmethod - def _reject_boolean_cost(cls, value: Any) -> Any: - """Refuse a ``bool`` cost, which coercion would otherwise hide. - - ``bool`` is an ``int`` subclass, so ``True`` would validate as a cost of 1.0. Every other - unusable value is already refused: ``allow_inf_nan=False`` covers NaN and the infinities - however they were spelled, and float coercion covers an int too large to represent. - """ - if isinstance(value, bool): - raise ValueError("cost_usd must be a number, not a bool") - return value - - @classmethod - def from_metadata(cls, metadata: Mapping[str, Any] | None) -> TrialMeasurements: - """Project loose trial metadata onto the typed contract. - - Applies the historical fallbacks so callers don't re-implement them: - ``runtime_sec`` falls back to ``duration_ms / 1000``; ``reward`` falls - back to ``1.0``/``0.0`` derived from ``passed`` when no explicit reward - is recorded. - """ - metadata = metadata or {} - - tokens = {key: _as_int(metadata.get(key)) for key in TOKEN_KEYS} - passed = metadata.get("passed") - passed = bool(passed) if isinstance(passed, bool) else None - - return cls( - **tokens, - runtime_sec=_runtime_sec(metadata), - cost_usd=_as_float(metadata.get("cost_usd")), - reward=_reward(metadata, passed), - passed=passed, - ) - - -def _trajectory_references(trajectory: Trajectory, needle: str) -> bool: - """Whether ``needle`` appears anywhere an agent action could reference the skill. - - Scans each step's message, reasoning, tool calls (name + arguments), and observation results. - """ - for step in trajectory.steps: - if needle in step.message or (step.reasoning_content is not None and needle in step.reasoning_content): - return True - for call in step.tool_calls or []: - if needle in call.function_name: - return True - if call.arguments is not None and needle in json.dumps(call.arguments, default=str): - return True - if step.observation is not None: - for result in step.observation.results: - if result.content is not None and needle in json.dumps(result.content, default=str): - return True - return False - - -def _as_int(value: Any) -> int | None: - # bool is an int subclass; never treat True/False as a token count. - if isinstance(value, bool): - return None - return value if isinstance(value, int) else None - - -def _as_float(value: Any) -> float | None: - # bool is an int subclass; never treat True/False as a measurement. NaN, the infinities, and - # integers too large to represent are rejected too: none can be serialised onto the wire, so - # recording one would fail the publish of an otherwise good trial. - if isinstance(value, bool) or not isinstance(value, int | float): - return None - try: - number = float(value) - except OverflowError: - return None - return number if math.isfinite(number) else None - - -def _runtime_sec(metadata: Mapping[str, Any]) -> float | None: - runtime_sec = metadata.get("runtime_sec") - if isinstance(runtime_sec, int | float) and not isinstance(runtime_sec, bool): - return float(runtime_sec) - duration_ms = metadata.get("duration_ms") - if isinstance(duration_ms, int | float) and not isinstance(duration_ms, bool): - return float(duration_ms) / 1000.0 - return None - - -def _reward(metadata: Mapping[str, Any], passed: bool | None) -> float | None: - reward = metadata.get("reward") - if reward is not None: - try: - return float(reward) - except (TypeError, ValueError): - return None - if passed is not None: - return 1.0 if passed else 0.0 - return None diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/persistence.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/persistence.py deleted file mode 100644 index 0f8d158c97..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/persistence.py +++ /dev/null @@ -1,172 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Persistence helpers for standalone agent-eval result bundles.""" - -from __future__ import annotations - -import json -from collections.abc import Iterator, Sequence -from pathlib import Path -from typing import Any - -from nemo_platform.beta.evaluator.agent_eval.dashboard import write_dashboard -from nemo_platform.beta.evaluator.agent_eval.results import AgentEvalResult, BundleLocation -from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial -from pydantic import BaseModel - -#: Filename of the rendered HTML dashboard inside a bundle. -DASHBOARD_FILENAME = "report.html" - - -def persist_run( - result: AgentEvalResult, - output_dir: str | Path, - *, - write_html_dashboard: bool = True, -) -> BundleLocation: - """Write a completed run to a bundle at ``output_dir`` and report where it landed. - - Explicit rather than a side effect of :meth:`AgentEvaluator.run`: computing an evaluation and - storing one are different decisions, and folding them together is what forced the result object to - carry paths it could not know at construction time. (Same reasoning as ``publish_to_intake``.) - - Set ``write_html_dashboard=False`` to skip rendering ``report.html`` β€” the dashboard is written - here so the manifest can record it in a single pass. - """ - path = Path(output_dir) - path.mkdir(parents=True, exist_ok=True) - - # Render first so the manifest below can name it; the dashboard reads only the run's own contents. - dashboard_path = write_dashboard(result, path / DASHBOARD_FILENAME) if write_html_dashboard else None - - _write_json(path / "metadata.json", result.metadata) - _write_jsonl(path / "tasks.jsonl", result.tasks) - _write_trials(path / "trials.jsonl", result.trials, base=path) - _write_jsonl(path / "scores.jsonl", result.scores) - _write_json(path / "summary.json", result.summary) - - location = BundleLocation(output_dir=path, dashboard_path=dashboard_path) - _write_json(path / "run.json", _run_manifest(result, location)) - return location - - -def _run_manifest(result: AgentEvalResult, location: BundleLocation) -> dict[str, Any]: - return { - "run_id": result.run_id, - "output_dir": str(location.output_dir), - "dashboard_path": str(location.dashboard_path) if location.dashboard_path is not None else None, - "artifacts": { - "metadata": "metadata.json", - "tasks": "tasks.jsonl", - "trials": "trials.jsonl", - "scores": "scores.jsonl", - "summary": "summary.json", - }, - } - - -def _write_json(path: Path, value: BaseModel | dict[str, Any]) -> None: - if isinstance(value, BaseModel): - payload = value.model_dump(mode="json") - else: - payload = value - path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") - - -def _write_jsonl(path: Path, rows: Sequence[BaseModel]) -> None: - # Stream row-by-row instead of joining the whole payload in memory first. - with path.open("w", encoding="utf-8") as handle: - for row in rows: - handle.write(json.dumps(row.model_dump(mode="json"), sort_keys=True)) - handle.write("\n") - - -def _write_trials(path: Path, trials: Sequence[BaseModel], *, base: Path) -> None: - """Write trials, rewriting evidence refs bundle-relative so the bundle is self-contained. - - A trial's evidence lives under the bundle (``/evidence/...``); storing the ref relative to - the bundle (rather than the launch CWD) means a moved or copied bundle re-scores without any path fixups. - Refs that point outside the bundle (rare) are left verbatim. - """ - resolved_base = base.resolve() - with path.open("w", encoding="utf-8") as handle: - row: dict[str, Any] - for trial in trials: - row = trial.model_dump(mode="json") - for descriptor in ((row.get("evidence") or {}).get("descriptors") or {}).values(): - descriptor["ref"] = _relativize_ref(descriptor.get("ref"), resolved_base) - handle.write(json.dumps(row, sort_keys=True)) - handle.write("\n") - - -def _relativize_ref(ref: str | None, base: Path) -> str | None: - """Make an evidence ref relative to the bundle dir when it lives under it; else leave it verbatim.""" - if not ref: - return ref - try: - return Path(ref).resolve().relative_to(base).as_posix() - except ValueError: - return ref # evidence written outside the bundle β€” cannot relativize - - -def read_trials(run_dir: str | Path) -> list[AgentEvalTrial]: - """Hydrate the persisted trials of a run bundle β€” the inverse of the ``trials.jsonl`` ``persist_run`` writes. - - Each row is loaded back into an ``AgentEvalTrial`` with its evidence pointing at the on-disk - deliverables, so a stored run can be **re-scored** β€” ``AgentEvaluator().run(tasks=…, trials=…)`` with - fresh metrics/judge β€” without re-running the agent. Evidence refs are resolved relative to ``run_dir`` - when the stored (launch-relative) ref no longer resolves, so a moved or copied bundle still works. - """ - directory = Path(run_dir) - trials: list[AgentEvalTrial] = [] - for row in _read_jsonl(directory / "trials.jsonl"): - for descriptor in ((row.get("evidence") or {}).get("descriptors") or {}).values(): - descriptor["ref"] = _resolve_evidence_ref(directory, descriptor.get("ref")) - trials.append(AgentEvalTrial.model_validate(row)) - return trials - - -def _resolve_evidence_ref(run_dir: Path, ref: str | None) -> str | None: - """Resolve a persisted evidence ref against the bundle dir. - - Self-contained bundles store refs relative to the bundle (see ``_write_trials``), so those resolve - directly under ``run_dir``. Falls back for still-valid absolute refs, and for moved bundles / legacy - absolute refs (rebuilt under ``run_dir`` from the ``evidence/`` tail). - """ - if not ref: - return ref - base = run_dir.resolve() - candidate = Path(ref) - if not candidate.is_absolute(): - rebuilt = run_dir / candidate - if _resolves_within(base, rebuilt): - return str(rebuilt) - elif candidate.exists(): - return ref - parts = candidate.parts - if "evidence" in parts: - rebuilt = run_dir / Path(*parts[parts.index("evidence") :]) - if _resolves_within(base, rebuilt): - return str(rebuilt) - return ref - - -def _resolves_within(base: Path, path: Path) -> bool: - """Whether ``path`` exists and stays inside ``base`` after resolving β€” no ``..``/symlink escape. - - Rebuilt refs are joined onto ``run_dir``; a bundle is designed to be moved/copied, so a ref with ``..`` - (or a symlink) must not be allowed to point the hydrated evidence outside the bundle it was loaded from. - """ - resolved = path.resolve() - if not resolved.exists(): - return False - return resolved == base or base in resolved.parents - - -def _read_jsonl(path: Path) -> Iterator[dict[str, Any]]: - with path.open(encoding="utf-8") as handle: - for line in handle: - stripped = line.strip() - if stripped: - yield json.loads(stripped) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py deleted file mode 100644 index 19430dd4e5..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py +++ /dev/null @@ -1,1414 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Aggregated summary, coverage, and the root result for a completed agent evaluation.""" - -from __future__ import annotations - -import json -import math -from collections.abc import Mapping, Sequence -from datetime import datetime -from enum import Enum -from pathlib import Path -from typing import Any - -from nemo_platform.beta.evaluator.agent_eval.scores import ( - AgentEvalDiagnosticSeverity, - AgentEvalScoreStatus, - AgentEvalTaskScore, - is_trial_failure, -) -from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalTask, SemanticReducer, ViewSignal -from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, RunnerInfo -from nemo_platform.beta.evaluator.metrics.aggregation import compute_percentiles -from nemo_platform.beta.evaluator.metrics.protocol import MetricOutput -from nemo_platform.beta.evaluator.metrics.utils import metric_type_name -from nemo_platform.beta.evaluator.values.protocol import BooleanValue, ContinuousScore, DiscreteScore, Label -from nemo_platform.beta.evaluator.values.results import ( - AggregatedMetricResult, - AggregateRangeScore, - AggregateScore, - ResultView, - flatten_dict, - format_table, - serialize_value, - summary_aggregate_record, -) -from pydantic import BaseModel, ConfigDict, Field, field_serializer, model_validator - -#: Metric-output value schemas retained in the ordered per-task value mapping. Broader than -#: :data:`_PASS_AT_K_VALUE_SCHEMAS` on purpose: a :class:`TrialMetricValue` is per-trial evidence, so a -#: count or a judge's label is worth keeping even though neither is a "did it pass?" signal. -_TASK_METRIC_VALUE_SCHEMAS = (ContinuousScore, DiscreteScore, BooleanValue, Label) - -#: Metric-output value schemas eligible for pass@k (a per-trial "did it pass?" signal). Labels, -#: discrete/count outputs, and free models (e.g. token measurements) are excluded. -_PASS_AT_K_VALUE_SCHEMAS = (ContinuousScore, BooleanValue) - -#: Score value at or above which a trial counts as a pass for pass@k. Full credit β€” pass@k answers -#: "did the agent solve the task", so partial credit is not a pass. Deliberately not configurable: -#: it's a reporting-time interpretation, and making it tunable would yield pass@k numbers that look -#: comparable across runs but aren't. -_PASS_VALUE = 1.0 - - -class AgentEvalMetricOutputCoverage(BaseModel): - """Coverage counts for one metric output across scored trials.""" - - model_config = ConfigDict(extra="forbid") - - total: int = Field(default=0, description="Total scores considered for this metric output.") - scored: int = Field(default=0, description="Scores that produced this output successfully.") - failed: int = Field(default=0, description="Scores where the metric failed to run.") - missing: int = Field(default=0, description="Scores where the output was expected but absent.") - - -#: Tokens :class:`TrialMetricValue` escapes non-finite floats as, and the floats they decode to. -#: Strict JSON has no literal for these, so they travel as strings -- which is the whole reason the -#: record carries ``value_type``: without it, a label that happens to read "NaN" is the same three -#: bytes as a real NaN. -_SPECIAL_FLOAT_TOKENS_MAP: dict[str, float] = { - "NaN": float("nan"), - "Infinity": float("inf"), - "-Infinity": float("-inf"), -} - - -def _escape_special_float(value: float) -> str: - """The token :data:`_SPECIAL_FLOAT_TOKENS_MAP` decodes back to ``value``. - - Looked up rather than spelled out a second time, so the encode and decode directions cannot - drift apart. NaN needs :func:`math.isnan` rather than equality: it is the one float that does - not equal itself, so a lookup keyed by value would miss it. - """ - for token, decoded in _SPECIAL_FLOAT_TOKENS_MAP.items(): - if decoded == value or (math.isnan(decoded) and math.isnan(value)): - return token - raise ValueError(f"{value!r} is a finite float and needs no escape") - - -class TrialMetricValueType(str, Enum): - """What kind of value one trial recorded under one metric output. - - Deliberately coarser than the declared value schemas: JSON already round-trips int, float and - bool distinctly, so a ``continuous``/``discrete``/``boolean`` split would restate what the payload - already says and give a reader two sources of truth for one fact. The only thing JSON cannot - carry is whether a string is a number's escape or a label, and that is exactly what this - discriminates. - """ - - NUMBER = "number" - LABEL = "label" - MISSING = "missing" - - -class TrialMetricValue(BaseModel): - """One trial's measured value under one metric output: which trial made it, and what it measured. - - Values keep the type the metric produced them in -- a count stays an int, a flag stays a bool, a - judge's verdict stays the string it was -- because this is one trial's measurement, not a mean or - other aggregate. Look up the matching trial by ``trial_id`` (in ``result.trials`` or - ``trials.jsonl``); do not assume list index lines up across metric outputs. Read it through - :func:`numeric_metric_values` when you intend to do arithmetic. - - Frozen because these records are handed out by reference from the summary: a consumer rescaling - values in place (Gym reports reward on 0-100 where we use 0-1) would otherwise rewrite the run's - own results, and a later persist would save the rewrite. - """ - - model_config = ConfigDict(extra="forbid", frozen=True) - - trial_id: str = Field( - description=( - "Identifier of the trial that produced this value. Joins to AgentEvalTrial.id " - "(trials.jsonl) and AgentEvalTaskScore.trial_id (scores.jsonl)." - ) - ) - # The default is never observed: `_derive_value_type` runs before validation and always supplies - # one. It exists so callers can write TrialMetricValue(trial_id=..., value=...) without - # restating what the value already says -- the type checker reads the signature, not the validator. - value_type: TrialMetricValueType = Field( - default=TrialMetricValueType.MISSING, - description=( - "Which kind of value this record holds: 'number' (float, int or bool), 'label' (a " - "categorical string), or 'missing' (the trial failed before it could be measured). " - "Always present in serialized output, because non-finite floats are escaped as strings " - "-- without it a genuine label reading 'NaN' and a real NaN are the same three bytes. " - "Derived from 'value' when omitted, so hand-built records and bundles written before " - "this field existed both load." - ), - ) - value: float | int | bool | str | None = Field( - description=( - "What the metric output measured, in the type the metric produced it in -- a number, a " - "label, or None when the trial failed before it could be measured: a trial that did " - "not pass. Required rather than defaulted: None is a load-bearing signal pass@k counts " - "as not passing, so an omitted value must not quietly become one. None never means " - "'no value of this kind'; that is what value_type is for." - ), - ) - - @model_validator(mode="before") - @classmethod - def _derive_value_type(cls, data: Any) -> Any: - """Fill in ``value_type`` when absent, and decode the escaped-float form when present. - - Runs *before* the union so the escape is undone while the discriminator is still readable: - afterwards pydantic's smart mode has already committed ``"NaN"`` to ``str``, and the record - would be a label whatever the type said. - """ - if not isinstance(data, Mapping): - return data - value = data.get("value") - declared = data.get("value_type") - - if declared is None: - # No discriminator: a hand-built record, or a bundle written before this field existed. - # The old encoding gave a string exactly one meaning -- the escape -- so honour that - # rather than reading a pre-widening NaN as the label "NaN". A label that genuinely - # reads "NaN" must therefore name its value_type explicitly. - if isinstance(value, str): - if value in _SPECIAL_FLOAT_TOKENS_MAP: - return { - **data, - "value_type": TrialMetricValueType.NUMBER, - "value": _SPECIAL_FLOAT_TOKENS_MAP[value], - } - return {**data, "value_type": TrialMetricValueType.LABEL} - return { - **data, - "value_type": TrialMetricValueType.MISSING if value is None else TrialMetricValueType.NUMBER, - } - - if TrialMetricValueType(declared) is TrialMetricValueType.NUMBER and isinstance(value, str): - decoded = _SPECIAL_FLOAT_TOKENS_MAP.get(value) - if decoded is None: - raise ValueError( - f"value_type='number' but value {value!r} is not one of the escaped-float tokens " - f"{sorted(_SPECIAL_FLOAT_TOKENS_MAP)}; a categorical value must declare value_type='label'" - ) - return {**data, "value": decoded} - return data - - @model_validator(mode="after") - def _value_matches_its_type(self) -> TrialMetricValue: - """Re-narrow the union, so a record cannot claim one kind and carry another.""" - if self.value_type is TrialMetricValueType.MISSING: - if self.value is not None: - raise ValueError("value_type='missing' requires value None (a trial that died before measurement)") - elif self.value_type is TrialMetricValueType.LABEL: - if not isinstance(self.value, str): - raise ValueError(f"value_type='label' requires a string value, got {type(self.value).__name__}") - elif not isinstance(self.value, bool | int | float): - raise ValueError(f"value_type='number' requires a numeric value, got {type(self.value).__name__}") - return self - - @field_serializer("value") - def serialize_nan(self, value: float | int | bool | str | None) -> float | int | bool | str | None: - """Escape non-finite floats as strings, so ``summary.json`` stays strict JSON. - - A metric may legitimately score a trial NaN, and this is the first summary field to carry - a raw metric value rather than a filtered aggregate. ``json.dumps`` would write a bare ``NaN`` - or ``Infinity`` token, which is valid Python but not valid JSON, so any strict reader of - ``summary.json`` would reject the whole bundle. ``value_type`` says which of these strings is - an escape and which is a label, so the round trip is lossless in both directions. - - This is deliberately *wider* than :meth:`MetricOutput.serialize_nan`, which escapes NaN only - and has no decoding validator -- an infinite value reaches ``scores.jsonl`` as ``null``. - Collapsing the SDK's several non-finite-float escapes into one pair belongs in ``values/``. - """ - if isinstance(value, float) and not math.isfinite(value): - return _escape_special_float(value) - return value - - -#: One task's recorded values, keyed ``"."``. Named because the nesting is -#: otherwise spelled out at every producer, consumer and local that touches it, and because the key -#: format is the part a reader cannot infer from ``dict[str, ...]``. -TrialValuesByMetric = dict[str, list[TrialMetricValue]] - - -class PerTaskOutcome(BaseModel): - """Every trial's value at one task under one metric output.""" - - model_config = ConfigDict(extra="forbid") - - metric_name: str = Field(description="'.', e.g. 'gym_reward.reward'.") - trials: list[TrialMetricValue] = Field( - description="Values in trial order. A value of None is a trial that died before scoring." - ) - - -class PerTaskOutcomes(BaseModel): - """One task's values across every metric output that measured it.""" - - model_config = ConfigDict(extra="forbid") - - task_id: str = Field(description="The task these outcomes belong to.") - outcomes: list[PerTaskOutcome] = Field(description="One entry per metric output, sorted by metric_name.") - - -class AgentEvalSummary(BaseModel): - """Aggregated scores, coverage, per-task metric values, and run counts for an agent-eval run.""" - - model_config = ConfigDict(extra="forbid") - - scores: AggregatedMetricResult = Field( - default_factory=lambda: AggregatedMetricResult(scores=[]), - description=( - "Aggregated statistics (mean/min/max/std_dev/nan_count) per metric output, named " - "'.', plus per-semantic-view rollups named 'view.'. " - "Failed or missing scores are surfaced as nan_count." - ), - examples=[ - # Emission order is real: metric outputs, then views, then pass@k. Note that pass@k - # counts *tasks* (4) where the metric output counts *trials* (10). - { - "scores": [ - { - "name": "harbor_reward.reward", - "score_type": "range", - "count": 10, - "nan_count": 2, - "mean": 0.6, - "min": 0.0, - "max": 1.0, - "std_dev": 0.4899, - }, - { - "name": "view.legal_quality", - "score_type": "range", - "count": 8, - "nan_count": 4, - "mean": 0.7375, - "min": 0.1, - "max": 1.0, - "std_dev": 0.3674, - }, - { - "name": "harbor_reward.reward.pass@1", - "score_type": "range", - "count": 4, - "nan_count": 1, - "mean": 0.5, - "min": 0.0, - "max": 1.0, - "std_dev": 0.3727, - }, - { - "name": "harbor_reward.reward.pass@2", - "score_type": "range", - "count": 4, - "nan_count": 1, - "mean": 0.6667, - "min": 0.0, - "max": 1.0, - "std_dev": 0.4082, - }, - ] - }, - # A separate run, because a runner's imported figures cannot co-occur with another - # runner's metrics. Scalars carry `value` and no distribution, and no `count` when the - # backend reports a figure without the sample size behind it. - { - "scores": [ - { - "name": "gym_reward.reward", - "score_type": "range", - "count": 20, - "nan_count": 0, - "mean": 0.65, - "min": 0.0, - "max": 1.0, - "std_dev": 0.477, - }, - {"name": "runner.gym.pass@1/accuracy", "score_type": "scalar", "nan_count": 0, "value": 0.68}, - ] - }, - ], - ) - metric_coverage: dict[str, dict[str, AgentEvalMetricOutputCoverage]] = Field( - default_factory=dict, - description="Per-metric, per-output coverage counts (total/scored/failed/missing).", - examples=[ - # Same 12 trials under two metrics, which is what distinguishes a low mean from low - # coverage. The two dead trials fail every metric; the judge failed once more on its own, - # and once completed without emitting its output at all (missing, not failed). - { - "harbor_reward": {"reward": {"total": 12, "scored": 10, "failed": 2, "missing": 0}}, - "rubric_judge": {"criteria_pass_rate": {"total": 12, "scored": 8, "failed": 3, "missing": 1}}, - } - ], - ) - task_metric_values: dict[str, TrialValuesByMetric] = Field( - default_factory=dict, - description=( - "Per task, the values each '.' measured, in trial order. Each " - "record names the trial that produced it, so values join across keys -- and out to " - "trials.jsonl and scores.jsonl -- by trial_id. Values keep the type the metric produced " - "them in: a count stays an int, a flag stays a bool, a judge's verdict stays a label -- " - "read them through numeric_metric_values() before doing arithmetic. A failed trial has " - "value None: a trial that did not pass. An unmeasured trial (metric failed, output " - "absent) has no entry at all, so each key's list is independent: align by trial_id, " - "never by position. An empty list means nothing was measured, including a task that " - "produced no trial." - ), - examples=[ - { - "contract-review-msa-indemnity": { - "harbor_reward.reward": [ - {"trial_id": "contract-review-msa-indemnity__k3f9wq2", "value_type": "number", "value": 1.0}, - {"trial_id": "contract-review-msa-indemnity__t7m2xb4", "value_type": "number", "value": 0.0}, - {"trial_id": "contract-review-msa-indemnity__9jr4vd1", "value_type": "number", "value": 1.0}, - ], - # A count stays an int, and t7m2xb4's judge verdict is kept as a label -- neither - # is pass@k-eligible, but both are per-trial evidence worth recording. - "steps.count": [ - {"trial_id": "contract-review-msa-indemnity__k3f9wq2", "value_type": "number", "value": 14}, - {"trial_id": "contract-review-msa-indemnity__t7m2xb4", "value_type": "number", "value": 31}, - {"trial_id": "contract-review-msa-indemnity__9jr4vd1", "value_type": "number", "value": 12}, - ], - # t7m2xb4 is absent here rather than null: its judge timed out, so that trial - # went unmeasured. Index 1 is therefore a different trial in each of these lists. - "rubric_judge.criteria_pass_rate": [ - {"trial_id": "contract-review-msa-indemnity__k3f9wq2", "value_type": "number", "value": 0.75}, - {"trial_id": "contract-review-msa-indemnity__9jr4vd1", "value_type": "number", "value": 1.0}, - ], - }, - "nda-scope-carveouts": { - # p2hn8sc died in the sandbox, so it is 'missing' in every key: a trial that - # happened and did not pass, as opposed to one that was never measured. - "harbor_reward.reward": [ - {"trial_id": "nda-scope-carveouts__p2hn8sc", "value_type": "missing", "value": None}, - {"trial_id": "nda-scope-carveouts__w5db3qy", "value_type": "number", "value": 1.0}, - {"trial_id": "nda-scope-carveouts__z8kt1nf", "value_type": "number", "value": 0.0}, - ], - "rubric_judge.verdict": [ - {"trial_id": "nda-scope-carveouts__p2hn8sc", "value_type": "missing", "value": None}, - {"trial_id": "nda-scope-carveouts__w5db3qy", "value_type": "label", "value": "compliant"}, - {"trial_id": "nda-scope-carveouts__z8kt1nf", "value_type": "label", "value": "overbroad"}, - ], - }, - # Requested, but the runner returned no trial for it: keys declared, nothing measured. - "merger-hsr-filing-threshold": { - "harbor_reward.reward": [], - "rubric_judge.verdict": [], - }, - } - ], - ) - error_trial_ids: dict[str, list[str]] = Field( - default_factory=dict, - description=( - "Trials that errored, grouped by error type -- Harbor's 'exception_stats' shape. Values " - "are trial ids, not task ids: they join to AgentEvalTrial.id (trials.jsonl), " - "AgentEvalTaskScore.trial_id (scores.jsonl), and TrialMetricValue.trial_id in " - "task_metric_values. Membership is 'the trial carries an error', with no status filter -- " - "an errored Harbor trial is PARTIAL rather than FAILED so that it is still scored, and it " - "belongs here regardless. A trial that both errored and produced a reward therefore " - "appears here AND in task_metric_values, where it may even count as a pass; that is what " - "Harbor does too. Ids are appended in trial order and never deduplicated. Key order is " - "not meaningful -- summary.json is written with sorted keys. Empty is ambiguous and " - "stays that way: it means either no trial errored or no trials were supplied to " - "from_scores(). The field always serializes (it defaults to {}), so the two cases are " - "indistinguishable in summary.json -- read trial_count, or the trials themselves, to " - "tell them apart." - ), - examples=[ - { - "RuntimeError": [ - "contract-review-msa-indemnity__k3f9wq2", - "nda-scope-carveouts__p2hn8sc", - ], - "TimeoutError": ["merger-hsr-filing-threshold__w5db3qy"], - } - ], - ) - task_count: int = Field(default=0, description="Number of tasks represented in the run.") - trial_count: int = Field(default=0, description="Number of distinct trials scored.") - score_count: int = Field(default=0, description="Total number of metric scores.") - error_count: int = Field( - default=0, - description=( - "Number of trials that errored -- Harbor's 'n_errors'. Equals the total ids across " - "error_trial_ids; stated rather than derived so a non-Python reader of summary.json need " - "not sum a nested structure, matching the other counts here." - ), - ) - - @model_validator(mode="after") - def _error_count_matches_rollup(self) -> AgentEvalSummary: - """Keep the two error fields from disagreeing when a summary is built by hand. - - ``from_scores`` derives both from one walk, but the model is public and directly - constructible -- and a count that contradicts the rollup beside it is worse than no count. - """ - total = sum(len(ids) for ids in self.error_trial_ids.values()) - if self.error_count != total: - raise ValueError(f"error_count {self.error_count} does not match {total} ids in error_trial_ids") - return self - - @property - def scores_by_name(self) -> Mapping[str, AggregateScore]: - """Aggregates keyed by name β€” see :attr:`AggregatedMetricResult.scores_by_name`.""" - return self.scores.scores_by_name - - def score(self, name: str) -> AggregateScore: - """Return the aggregate named ``name`` β€” see :meth:`AggregatedMetricResult.score`. - - Exists so callers needn't know the aggregates sit one level down, behind a field whose name - differs from the summary's own accessor by a single character. - """ - return self.scores.score(name) - - def task_outcomes(self, metric_name: str | None = None) -> list[PerTaskOutcomes]: - """:attr:`task_metric_values` as models that name their own keys, sorted by task then metric. - - A read-time *view*, not the wire format. The field itself stays a nested dict because it is - persisted per run: repeating "task_id"/"metric_name" on every row would grow ``summary.json`` - for no new information, and lookup by task and metric stays O(1). Reach for this when you - want a typed object to pass around or to hand to a template. - - ``metric_name`` narrows to one ``"."``, which is what a report over a - single metric wants:: - - summary.task_outcomes() -> every task, every metric output - summary.task_outcomes("gym_reward.reward") -> every task that metric measured - - A task the named metric never measured is **dropped**, not returned empty: it was scored by - a different metric, so reporting it as unmeasured would invent missing coverage. A task that - declared the metric but produced no usable value is different - it keeps its entry with an - empty ``trials`` list, because there the coverage really is missing. That is the same - distinction :attr:`task_metric_values` draws by having a key at all. - """ - outcomes_by_task = [ - ( - task_id, - [ - PerTaskOutcome(metric_name=key, trials=list(records)) - for key, records in sorted(by_key.items()) - if metric_name is None or key == metric_name - ], - ) - for task_id, by_key in sorted(self.task_metric_values.items()) - ] - return [ - PerTaskOutcomes(task_id=task_id, outcomes=outcomes) - for task_id, outcomes in outcomes_by_task - if metric_name is None or outcomes - ] - - @staticmethod - def from_scores( - scores: Sequence[AgentEvalTaskScore], - *, - tasks: Sequence[AgentEvalTask] | None = None, - trials: Sequence[AgentEvalTrial] | None = None, - extra_scores: Sequence[AggregateScore] = (), - ) -> AgentEvalSummary: - """Build aggregated scores, task values, coverage, and the error rollup for a set of scores. - - ``extra_scores`` are already-aggregated scores contributed by the runner (namespaced - ``runner..``), merged in so a backend's own figures are addressable the same way as ours. - - ``trials`` supplies the only thing scores cannot carry: what went wrong. Omitting it leaves - :attr:`error_trial_ids` empty rather than raising -- the same silent-skip contract ``tasks`` - already has for pass@k. It may legitimately be *wider* than ``scores`` (a caller - re-aggregating a subset), so the rollup can name trial ids absent from - :attr:`task_metric_values`. - """ - task_list = list(tasks) if tasks is not None else None - task_metric_values = _task_metric_values(scores, task_list) - error_trial_ids = _error_trial_ids(trials) - return AgentEvalSummary( - scores=_aggregate_scores( - scores, - task_list, - extra_scores, - task_metric_values=task_metric_values, - ), - metric_coverage=_metric_coverage(scores, task_list), - task_metric_values=task_metric_values, - error_trial_ids=error_trial_ids, - task_count=len(task_list) if task_list is not None else len({score.task_id for score in scores}), - trial_count=len({score.trial_id for score in scores}), - score_count=len(scores), - error_count=sum(len(ids) for ids in error_trial_ids.values()), - ) - - -class RunMetadata(BaseModel): - """Provenance for a run: what was evaluated, by what, and when. - - Answers "what produced this result?" β€” previously improvised by callers inside an untyped - ``benchmark`` dict. ``labels`` remains free-form for caller-specific tags, but the fields that - every run has are typed. - """ - - model_config = ConfigDict(extra="forbid") - - labels: dict[str, str] = Field( - default_factory=dict, - description="Caller-supplied tags for this run (e.g. benchmark, mode, backend). Free-form by design.", - ) - target: RunnerInfo | None = Field( - default=None, - description="Identity of the runner/model/agent that produced the trials; None for imported trials.", - ) - started_at: datetime | None = Field(default=None, description="UTC timestamp when the run began.") - finished_at: datetime | None = Field(default=None, description="UTC timestamp when scoring completed.") - duration_sec: float | None = Field(default=None, description="Wall-clock seconds from start to finish.") - sdk_version: str | None = Field(default=None, description="nemo-evaluator-sdk version that produced the run.") - - -class BundleLocation(BaseModel): - """Where a run was written, returned by :meth:`AgentEvalResult.persist`. - - Kept off :class:`AgentEvalResult` because it is not a property of the evaluation β€” it is the - outcome of choosing to store it. Holding one means the bundle exists, so there is no optional to - re-check; a run that was never persisted simply has no ``BundleLocation``. - """ - - model_config = ConfigDict(extra="forbid", frozen=True) - - output_dir: Path = Field(description="Directory the run bundle was written to.") - dashboard_path: Path | None = Field( - default=None, - description="Path to the rendered HTML dashboard, or None when dashboard writing was disabled.", - ) - - -class AgentEvalResult(BaseModel): - """Root result for a completed agent evaluation: tasks, trials, scores, and summary. - - Describes the evaluation and nothing else β€” storing it is a separate decision, made by calling - :meth:`persist`. Because the result carries no paths, it never holds a location that was unknown - when it was constructed, and nothing has to mutate it after the fact. - """ - - model_config = ConfigDict(extra="forbid") - - run_id: str = Field(description="Identifier of this run.") - tasks: list[AgentEvalTask] = Field(description="Immutable task definitions evaluated in this run.") - trials: list[AgentEvalTrial] = Field(description="Trials produced or imported for the run.") - scores: list[AgentEvalTaskScore] = Field(description="Metric scores computed for the trials.") - summary: AgentEvalSummary = Field(description="Derived rollups and coverage computed for the run.") - metadata: RunMetadata = Field( - default_factory=RunMetadata, - description="Run provenance: labels, target identity, timings, SDK version.", - ) - work_dir: Path | None = Field( - default=None, - description="Directory the run worked in, where its runtimes wrote trial evidence. Known " - "before the run starts (it comes from the run config), so unlike a bundle location it is " - "never attached after the fact. None for a purely in-memory run.", - ) - - def persist(self, output_dir: str | Path | None = None, *, write_dashboard: bool = True) -> BundleLocation: - """Write this run to a bundle and return where it landed. - - Deliberately a call rather than something ``AgentEvaluator.run`` does for you: computing an - evaluation and storing one are separate decisions (the same reasoning as ``publish_to_intake``). - - Defaults to :attr:`work_dir`, which is the directory the trials' evidence already lives under β€” - so the bundle is self-contained and survives being moved. Passing a different ``output_dir`` - leaves those evidence references pointing back at the original directory. That is supported (a - re-scored run may reference an earlier run's deliverables) but the resulting bundle only - resolves while the original directory is still there. - - Set ``write_dashboard=False`` to skip rendering ``report.html``. - """ - # Imported here rather than at module scope: persistence imports this module for the types it - # writes, so a top-level import would be circular. - from nemo_platform.beta.evaluator.agent_eval.persistence import persist_run - - target = output_dir if output_dir is not None else self.work_dir - if target is None: - raise ValueError( - "this run has no work_dir to persist into (it ran in memory); pass an explicit " - "output_dir, or set work_dir on the AgentEvalRunConfig so evidence and bundle share " - "a directory" - ) - return persist_run(self, target, write_html_dashboard=write_dashboard) - - def to_records(self, view: ResultView = "rows") -> list[dict[str, Any]]: - """Convert this run into flat dictionaries for export or inspection. - - ``view="rows"`` yields one record per metric score β€” the agent-eval analogue of the dataset - path's row. The fan-out is preserved rather than collapsed: ``task_id`` and ``trial_id`` are - columns, so a consumer can still group by task, which is what pass@k depends on. - - ``view="aggregate"`` matches the dataset path exactly β€” percentiles flattened, histograms - kept as JSON strings so the view stays tabular. - - Args: - view: Output projection, either ``"rows"`` or ``"aggregate"``. - - Returns: - Flat record dictionaries for downstream table/dataframe conversion. - - Raises: - ValueError: If ``view`` is unsupported. - """ - if view == "rows": - return [_score_record(score) for score in self.scores] - - if view == "aggregate": - records: list[dict[str, Any]] = [] - for score in self.summary.scores.scores: - record: dict[str, Any] = {} - for key, value in score.model_dump(mode="json").items(): - if key == "percentiles" and isinstance(value, dict): - flatten_dict("percentiles", value, record) - elif key == "histogram" and value is not None: - # Histograms stay as JSON strings so aggregate views remain tabular instead - # of expanding variable-width nested columns. - record[key] = json.dumps(value, sort_keys=True) - else: - record[key] = value - records.append(record) - return records - - raise ValueError(f"Unsupported view {view!r}. Expected 'rows' or 'aggregate'.") - - def to_table(self, view: ResultView = "rows"): - """Convert records into a ``pyarrow.Table``. - - Args: - view: Output projection, either ``"rows"`` or ``"aggregate"``. - - Columns are unioned across every record before the table is built. ``pa.Table.from_pylist`` - takes its schema from the first record alone, and in a row view ``error`` and - ``diagnostics.*`` appear only on failed scores β€” so a run whose first score succeeded would - otherwise export a table with the failure columns silently missing. ``to_pandas`` already - unions keys, and the two should not disagree about what a run contains. - - Args: - view: Output projection, either ``"rows"`` or ``"aggregate"``. - - Returns: - Table built from ``to_records(view=view)``. - """ - import pyarrow as pa - - records = self.to_records(view=view) - # dict-of-None preserves first-appearance order, matching how format_table derives columns. - columns = {key: None for record in records for key in record} - return pa.Table.from_pylist([{key: record.get(key) for key in columns} for record in records]) - - def to_pandas(self, view: ResultView = "rows"): - """Convert records into a pandas ``DataFrame``. - - Args: - view: Output projection, either ``"rows"`` or ``"aggregate"``. - - Returns: - DataFrame built from ``to_records(view=view)``. - """ - import pandas as pd - - return pd.DataFrame.from_records(self.to_records(view=view)) - - def format_summary(self, max_rows: int = 10, *, max_error_rows: int | None = None) -> str: - """Render a human-readable summary with aggregates and a score preview. - - Args: - max_rows: Maximum number of score records included in the preview. - max_error_rows: Maximum number of failed scores included in the error-details section. - Defaults to ``max_rows``. - - Returns: - Multi-line summary string suitable for terminal/notebook display. - """ - if max_error_rows is None: - max_error_rows = max_rows - aggregate_records = [summary_aggregate_record(score) for score in self.summary.scores.scores] - preview = [_score_preview_record(score) for score in self.scores[:max_rows]] - parts = [ - _agent_eval_summary_header(self), - "", - "Aggregate scores", - format_table(aggregate_records), - ] - if preview: - parts.extend( - [ - "", - f"Score preview (first {len(preview)} of {len(self.scores)})", - format_table(preview), - ] - ) - parts.extend(_format_score_errors(self.scores, max_error_rows=max_error_rows)) - return "\n".join(parts) - - def print_summary(self, max_rows: int = 10, *, max_error_rows: int | None = None) -> None: - """Print ``format_summary`` output. - - Args: - max_rows: Maximum number of score records included in the preview. - max_error_rows: Maximum number of failed scores included in the error-details section. - Defaults to ``max_rows``. - """ - print(self.format_summary(max_rows=max_rows, max_error_rows=max_error_rows)) - - def __str__(self) -> str: - """Return the default compact summary representation. - - Returns: - Summary string with up to five preview scores. - """ - return self.format_summary(max_rows=5) - - -def _score_error_text(score: AgentEvalTaskScore) -> str | None: - """Join the error-severity diagnostic messages for a score, or None when it has none.""" - messages = [ - diagnostic.message - for diagnostic in score.diagnostics - if diagnostic.severity is AgentEvalDiagnosticSeverity.ERROR - ] - return "; ".join(messages) if messages else None - - -def _score_diagnostics_columns(score: AgentEvalTaskScore) -> dict[str, str]: - """JSON-encoded diagnostic columns, keyed ``diagnostics.``. - - Encoded as compact JSON for the same reason the dataset path does it: diagnostics have a - metric-defined shape, and exports stay flat only if that shape is a string. - """ - if not score.diagnostics: - return {} - return { - f"diagnostics.{score.metric_type}": json.dumps( - [serialize_value(diagnostic) for diagnostic in score.diagnostics], sort_keys=True - ) - } - - -def _score_preview_record(score: AgentEvalTaskScore) -> dict[str, Any]: - """Identity and status columns shared by the row export and the summary preview.""" - record: dict[str, Any] = { - "task_id": score.task_id, - "trial_id": score.trial_id, - "metric_type": score.metric_type, - "status": score.status.value, - } - for output in score.outputs: - record[f"output.{output.name}"] = serialize_value(output.value) - return record - - -def _score_record(score: AgentEvalTaskScore) -> dict[str, Any]: - """Full export record for one score: identity, preview columns, error text, and diagnostics. - - Carries ``id``, ``run_id``, and ``metadata`` that the summary preview leaves out. An export is - the thing a caller joins, concatenates, and keeps: ``id`` is what a row is addressable by, - ``run_id`` keeps a frame self-describing once several runs are stacked into one, and - ``metadata`` is caller-supplied β€” dropping it silently discards data the SDK never owned. The - preview stays narrow because it is read on a terminal, the same split the dataset path makes - between ``to_records`` and ``summary_row_base_record``. - """ - record: dict[str, Any] = {"id": score.id, "run_id": score.run_id} - record.update(_score_preview_record(score)) - if error_text := _score_error_text(score): - record["error"] = error_text - record.update(_score_diagnostics_columns(score)) - # Flattened rather than JSON-encoded: metadata is free-form but usually shallow and scalar, so - # dotted columns keep it queryable. Diagnostics get the JSON treatment instead because their - # shape is metric-defined and variable-width. - flatten_dict("metadata", serialize_value(score.metadata), record) - return record - - -def _agent_eval_summary_header(result: AgentEvalResult) -> str: - """Build the header line, mirroring the shape :func:`summary_header` produces for row results. - - The counts differ because the units do β€” a run has tasks, trials, and scores where the dataset - path has rows β€” but the ``Name(field=value, ...)`` shape is the same, and a status the run never - produced is left out, matching what that header does with its zero counts. - - Statuses are counted by tallying the scores present, so an absent status simply never becomes a - key; there is no zero to filter out. - """ - status_counts: dict[str, int] = {} - for score in result.scores: - status_counts[score.status.value] = status_counts.get(score.status.value, 0) + 1 - fields = [ - f"tasks={len(result.tasks)}", - f"trials={len(result.trials)}", - f"scores={len(result.scores)}", - f"aggregate_scores={len(result.summary.scores.scores)}", - ] - fields.extend(f"{status}={count}" for status, count in sorted(status_counts.items())) - return f"AgentEvalResult({', '.join(fields)})" - - -def _format_score_errors( - scores: Sequence[AgentEvalTaskScore], - *, - max_error_rows: int | None, -) -> list[str]: - """Render the failed-score detail section, separating a failed trial from a failed metric. - - Both arrive as ``FAILED``, but they mean different things to a reader: a failed trial is one - the agent is answerable for, a failed metric is a measurement that never happened. The - dataset path has no equivalent distinction to make, so this section is agent-eval's own rather - than a reuse of :func:`format_error_details`. - """ - failed = [score for score in scores if score.status is AgentEvalScoreStatus.FAILED] - if not failed: - return [] - - # max(0, ...) guards a negative limit, which slicing would otherwise read as an offset from the - # end: failed[:-2] shows all but the last two rather than none. An over-large limit needs no - # guard, since a slice past the end is simply the whole list. Mirrors format_error_details. - shown_limit = len(failed) if max_error_rows is None else max(0, max_error_rows) - shown = failed[:shown_limit] - parts = ["", f"Error details ({len(shown)} of {len(failed)} failed scores)"] - for score in shown: - kind = "failed trial" if is_trial_failure(score) else "failed metric" - parts.extend(["", f"[{score.task_id} / {score.trial_id} / {score.metric_type}] {kind}"]) - parts.append(_score_error_text(score) or "(no error-severity diagnostic recorded)") - if len(shown) < len(failed): - parts.extend(["", f"... {len(failed) - len(shown)} more failed scores omitted"]) - return parts - - -def _aggregate_scores( - scores: Sequence[AgentEvalTaskScore], - tasks: Sequence[AgentEvalTask] | None, - extra_scores: Sequence[AggregateScore] = (), - *, - task_metric_values: dict[str, TrialValuesByMetric], -) -> AggregatedMetricResult: - """Aggregate per-metric-output, per-semantic-view, and task-level pass@k values into range scores. - - Each metric output becomes a score named ``.``, each semantic view - ``view.``, and each score-like output additionally yields ``..pass@k`` - task-level rollups. Failed and missing scores are surfaced as ``nan_count`` so coverage is visible - alongside the statistics. ``extra_scores`` (runner-contributed, ``runner.``-namespaced) are appended - as-is. - """ - aggregated: list[AggregateScore] = [] - - output_names = _metric_output_names(scores, tasks) - for metric_type, names in sorted(output_names.items()): - metric_records = [score for score in scores if score.metric_type == metric_type] - total = len(metric_records) - for output_name in names: - values: list[float] = [] - for score in metric_records: - value = None - # PARTIAL scores can still emit valid per-output values; include them so - # stats agree with coverage (which counts non-FAILED outputs as scored). - # Outputs actually missing on a PARTIAL score stay None -> counted as nan. - if score.status in (AgentEvalScoreStatus.COMPLETED, AgentEvalScoreStatus.PARTIAL): - output = _score_output(score, output_name) - value = _numeric_value(output) if output is not None else None - if value is not None: - values.append(value) - aggregated.append(_aggregate_range_score(f"{metric_type}.{output_name}", values, total)) - - for view_name, (values, total) in sorted(_semantic_view_values(scores, tasks).items()): - aggregated.append(_aggregate_range_score(f"view.{view_name}", values, total)) - - # Required rather than recomputed here: the summary needs the same mapping, and deriving it - # twice is what this rewiring exists to stop. The one caller builds it once and shares it. - aggregated.extend(_task_pass_at_k_scores(task_metric_values, tasks)) - aggregated.extend(extra_scores) - - return AggregatedMetricResult(scores=aggregated) - - -def metric_values(records: Sequence[TrialMetricValue]) -> list[float | int | bool | str | None]: - """The bare per-trial values, for consumers reading records without caring which trial made them. - - Preserves order, cardinality, type, and the None-versus-absent distinction exactly as recorded. - Reach for :func:`numeric_metric_values` before doing arithmetic: this list may hold labels, and - ``value >= 1.0`` raises on one. - """ - return [record.value for record in records] - - -def numeric_metric_values(records: Sequence[TrialMetricValue]) -> list[float | None]: - """The values that can be compared and averaged, as floats, for consumers doing arithmetic. - - A number becomes a float (a bool becomes 1.0/0.0, matching how a pass/fail flag has always been - read). A dead trial stays ``None`` -- it is a trial that definitively did not pass, and - dropping it would let a crashed rollout flatter the agent. - - A label is **dropped**, not zeroed. A categorical verdict says nothing about whether the agent - solved the task, so it is an unmeasured trial rather than a failed one -- the same reading this - module gives a metric that raised (see :func:`is_trial_failure`). Charging it as a failure would - misattribute a measurement problem to the agent, and counting it as a pass is not defined. - - A label can land under a score-like key: :func:`validate_metric_result` coerces and discards, so - a metric declaring a continuous score may still return the string ``"0.9"``, and an output one - task never declared may be score-like on another. This is where that stops being arithmetic. - """ - values: list[float | None] = [] - for record in records: - value = record.value - if value is None: - values.append(None) - elif isinstance(value, bool | int | float): - # bool first: it is a subclass of int, and False must become 0.0 rather than be dropped. - values.append(float(value)) - return values - - -def _pass_at_k(n: int, c: int, k: int) -> float: - """Unbiased pass@k estimator (Chen et al., 2021): ``1 - C(n-c, k) / C(n, k)``. - - The probability that at least one of ``k`` samples drawn without replacement from ``n`` trials - (``c`` of them passing) is a pass. Caller guarantees ``1 <= k <= n``. - """ - if n - c < k: - return 1.0 - product = 1.0 - for i in range(n - c + 1, n + 1): - product *= 1.0 - k / i - return 1.0 - product - - -def _scorelike_outputs(tasks: Sequence[AgentEvalTask] | None) -> set[tuple[str, str]]: - """``(metric_type, output_name)`` pairs whose declared value is a score (continuous or boolean). - - pass@k is only meaningful for a per-trial pass/fail signal, so labels, discrete/count outputs, - and free models (e.g. token measurements) are excluded. Needs task metric specs; with no tasks - the set is empty and pass@k is skipped. - """ - scorelike: set[tuple[str, str]] = set() - if tasks is None: - return scorelike - for task in tasks: - for metric in task.metrics: - metric_type = metric_type_name(metric) - for spec in metric.output_spec(): - if issubclass(spec.value_schema, _PASS_AT_K_VALUE_SCHEMAS): - scorelike.add((metric_type, spec.name)) - return scorelike - - -def _error_trial_ids(trials: Sequence[AgentEvalTrial] | None) -> dict[str, list[str]]: - """Trial ids grouped by error type, in trial order β€” Harbor's ``exception_stats``. - - Three trials, the middle one fine:: - - in t0 error RuntimeError - t1 (no error) - t2 error RuntimeError - t3 error TimeoutError - - out {"RuntimeError": ["t0", "t2"], "TimeoutError": ["t3"]} - - Selection is on ``trial.error``, never on ``trial.status``: an errored Harbor trial is ``PARTIAL`` - so that it still scores, and filtering by status would drop exactly the trials this exists to name. - - Ids are **appended**, never collected into a set or used as dict keys. Nothing enforces trial-id - uniqueness (Gym derives ids from a rollout index in two separate loops), and losing cardinality - here would understate the error count β€” the same rule ``task_metric_values`` follows. - """ - grouped: dict[str, list[str]] = {} - for trial in trials or (): - if trial.error is not None: - grouped.setdefault(trial.error.type, []).append(trial.id) - return grouped - - -def _task_metric_values( - scores: Sequence[AgentEvalTaskScore], - tasks: Sequence[AgentEvalTask] | None, -) -> dict[str, TrialValuesByMetric]: - """Ordered per-trial records per task, keyed ``.``. - - ``task-a`` declares ``reward.score`` (continuous), ``steps.count`` (discrete) and - ``usage.prompt_tokens`` (a free model) and runs four trials:: - - in t0 reward 1.0 steps 5 usage 1200 - t1 reward steps 9 usage 1300 # the judge died, not the agent - t2 # every metric fails as a trial failure - t3 reward 0.0 steps 7 usage 1100 - - out {"task-a": {"reward.score": [(t0, 1.0), (t2, None), (t3, 0.0)], - "steps.count": [(t0, 5), (t1, 9), (t2, None), (t3, 7)]}} - - (shown as ``(trial_id, value)`` pairs; each is an :class:`TrialMetricValue`) - - ``usage.prompt_tokens`` is absent because its declared schema is not in - :data:`_TASK_METRIC_VALUE_SCHEMAS`; t1 is missing from ``reward.score`` but present in - ``steps.count``; t2 is ``None`` in both. ``steps.count`` keeps its ints -- values are recorded in - the type the metric produced them in, not flattened to float. - - Which keys a task gets: - - - declared by its metric spec under :data:`_TASK_METRIC_VALUE_SCHEMAS` -> kept - - declared under any other schema -> dropped, even when the emitted value is numeric, so a - ``MetricOutputSpec.model("prompt_tokens", TokenCount)`` measurement never becomes a key - - undeclared, but some score emitted a recordable value for it -> kept - - ``tasks is None`` -> no specs to filter against, so every recordable output observed is kept - - What each score contributes to its key, in trial order: - - - failed trial (:func:`is_trial_failure`) -> value ``None``, a trial that did not pass - - failed metric, or the output absent -> no entry; the trial is unmeasured, not unsuccessful - - a value a metric can emit (number, bool or label) -> that value, in its own type - - anything else (a dict, a list, a literal null) -> no entry; see :func:`_native_value` - - pass@k needs that asymmetry, and it is why a list is indexed by surviving measurement rather than - by trial: above, index 1 is t2 under ``reward.score`` but t1 under ``steps.count``. Every entry - therefore names its trial, and ``trial_id`` β€” not position β€” is what joins two keys of one task, - or joins out to ``trials.jsonl`` and ``scores.jsonl``. Ids are recorded as the runner reported - them and are never deduplicated: two records sharing an id stay two records, so a runner that - reuses one costs pass@k nothing. - """ - output_keys: dict[str, set[tuple[str, str]]] = {} - # Outputs a task declared under a schema this mapping does not retain. Tracked so an emitted - # numeric value cannot add back what that task's spec filter just excluded, and carrying the task - # id because tasks in one run need not declare the same output under the same schema. - excluded: set[tuple[str, str, str]] = set() - if tasks is not None: - for task in tasks: - task_keys = output_keys.setdefault(task.id, set()) - for metric in task.metrics: - metric_type = metric_type_name(metric) - for spec in metric.output_spec(): - if issubclass(spec.value_schema, _TASK_METRIC_VALUE_SCHEMAS): - task_keys.add((metric_type, spec.name)) - else: - excluded.add((task.id, metric_type, spec.name)) - - # Materialized once: the key set has to be settled before any record can be filed (a trial - # failure reaches every key of its metric, including keys only a later score reveals), and - # `scores` is walked exactly once so a one-shot sequence still works. - ordered = list(scores) - for score in ordered: - # setdefault, not add: a task whose every score failed still earns an entry, so it reads as - # measured-and-empty rather than absent. - task_keys = output_keys.setdefault(score.task_id, set()) - if score.status in (AgentEvalScoreStatus.COMPLETED, AgentEvalScoreStatus.PARTIAL): - for output in score.outputs: - if (score.task_id, score.metric_type, output.name) in excluded: - continue - if _native_value(output) is not None: - task_keys.add((score.metric_type, output.name)) - - # Key set settled, so the records fill in score order -- which is what puts each key's list in - # trial order. - outputs_by_task_metric: dict[tuple[str, str], list[str]] = {} - by_task: dict[str, TrialValuesByMetric] = {} - for task_id, keys in output_keys.items(): - ordered_keys = sorted(keys) - by_task[task_id] = {f"{metric_type}.{name}": [] for metric_type, name in ordered_keys} - for metric_type, name in ordered_keys: - outputs_by_task_metric.setdefault((task_id, metric_type), []).append(name) - - for score in ordered: - output_names = outputs_by_task_metric.get((score.task_id, score.metric_type)) - if not output_names: - continue - task_values = by_task[score.task_id] - if is_trial_failure(score): - for name in output_names: - task_values[f"{score.metric_type}.{name}"].append( - TrialMetricValue(trial_id=score.trial_id, value_type=TrialMetricValueType.MISSING, value=None) - ) - continue - if score.status not in (AgentEvalScoreStatus.COMPLETED, AgentEvalScoreStatus.PARTIAL): - continue - # Indexed once per score rather than rescanned per output, and first-wins on a duplicate name - # to match :func:`_score_output`. - outputs: dict[str, MetricOutput] = {} - for output in score.outputs: - outputs.setdefault(output.name, output) - for name in output_names: - output = outputs.get(name) - payload = _native_value(output) if output is not None else None - if payload is not None: - value_type, value = payload - task_values[f"{score.metric_type}.{name}"].append( - TrialMetricValue(trial_id=score.trial_id, value_type=value_type, value=value) - ) - return by_task - - -def _task_pass_at_k_scores( - task_metric_values: dict[str, TrialValuesByMetric], - tasks: Sequence[AgentEvalTask] | None, -) -> list[AggregateScore]: - """Task-level pass@k over the R trials per task, aggregated across tasks (uniform for any runner). - - For each score-like metric output, group trials by task, count trials ``n`` and passes ``c`` - (value ``>= _PASS_VALUE``), then emit ``..pass@k`` for ``k`` in ``1..max(n)`` as - the across-task mean of the unbiased per-task estimator (over tasks with at least ``k`` trials). - ``pass@1`` equals the macro per-task pass rate, i.e. the task-level mean. - - **A failed trial did not pass.** It counts toward ``n`` and never toward ``c``: an agent that - solved a task once and crashed once did not go one-for-one. A failed *metric* is different β€” it - leaves the trial unmeasured rather than unsuccessful, so it stays out of ``n`` entirely rather - than being charged to the agent (see :func:`is_trial_failure`). Tasks left with no usable value at - all drop out of the estimate and are reported as ``nan_count``, uniform across ``k``, so a shrinking - denominator is never silent. (Tasks excluded from a given ``k`` merely for having fewer than ``k`` - trials are *not* counted there β€” that is the estimator working as defined, not missing data.) - - "No usable value" includes a task that was never scored at all: it declares the metric, holds an - empty value list, and lands in ``nan_count`` like any other unmeasured task. That is the same - missing coverage whether the trial died or was never produced, and excluding it would report - pass@k over a denominator quietly smaller than the task set asked for. - - Note this is reachable only through :meth:`AgentEvalSummary.from_scores` called directly with a - task list wider than the scores β€” a caller re-aggregating a subset, say. A full run cannot get - here: :meth:`AgentEvaluator._score_trials` refuses to score at all when a task produced no trial, - so a runner that drops one fails the run rather than reporting it as missing coverage. - """ - scorelike = _scorelike_outputs(tasks) - if not scorelike: - return [] - aggregated: list[AggregateScore] = [] - for metric_type, output_name in sorted(scorelike): - key = f"{metric_type}.{output_name}" - values_by_task = [ - numeric_metric_values(outputs[key]) for outputs in task_metric_values.values() if key in outputs - ] - measured = [values for values in values_by_task if values] - if not measured: - continue - # Empty value lists stay in nan_count (via total); for each k, mean the unbiased - # estimator over tasks with n >= k (None / < full credit do not count as passes). - unmeasured = sum(not values for values in values_by_task) - # (n, c) per task, counted once: neither depends on k, so counting inside the k loop would - # re-walk every task's values max_n times over. - counts = [ - (len(values), sum(value is not None and value >= _PASS_VALUE for value in values)) for values in measured - ] - max_n = max(n for n, _ in counts) - for k in range(1, max_n + 1): - per_task = [_pass_at_k(n, c, k) for n, c in counts if n >= k] - if per_task: - aggregated.append(_aggregate_range_score(f"{key}.pass@{k}", per_task, len(per_task) + unmeasured)) - return aggregated - - -def _aggregate_range_score(name: str, values: list[float], total: int) -> AggregateRangeScore: - finite = [value for value in values if math.isfinite(value)] - count = len(finite) - nan_count = max(total - count, 0) - if not finite: - return AggregateRangeScore(name=name, count=0, nan_count=nan_count) - total_sum = sum(finite) - mean = total_sum / count - # Report both conventions explicitly rather than picking one: the population figures describe the - # values actually evaluated, the sample figures estimate the process they were drawn from (which is - # what repeated trials over one task are sampling). Sample stats are undefined for a single value. - sum_sq_dev = sum((value - mean) ** 2 for value in finite) - variance = sum_sq_dev / count - sample_variance = sum_sq_dev / (count - 1) if count > 1 else None - percentiles = compute_percentiles(sorted(finite)) - return AggregateRangeScore( - name=name, - count=count, - nan_count=nan_count, - sum=total_sum, - mean=mean, - min=min(finite), - max=max(finite), - variance=variance, - std_dev=math.sqrt(variance), - sample_variance=sample_variance, - sample_std_dev=math.sqrt(sample_variance) if sample_variance is not None else None, - # Reuse the deterministic-metric percentile helper so agent-eval and metric aggregation report - # the same distribution the same way. - percentiles=percentiles, - # Surfaced alongside the other basic stats so `median` means the same thing whether a score - # was computed here or imported from a backend that reports one without a full distribution. - median=percentiles.p50, - ) - - -def _metric_coverage( - scores: Sequence[AgentEvalTaskScore], - tasks: Sequence[AgentEvalTask] | None, -) -> dict[str, dict[str, AgentEvalMetricOutputCoverage]]: - output_names = _metric_output_names(scores, tasks) - coverage: dict[str, dict[str, AgentEvalMetricOutputCoverage]] = {} - for metric_type, names in sorted(output_names.items()): - metric_records = [score for score in scores if score.metric_type == metric_type] - metric_coverage: dict[str, AgentEvalMetricOutputCoverage] = {} - for output_name in names: - total = len(metric_records) - failed = sum(1 for score in metric_records if score.status == AgentEvalScoreStatus.FAILED) - scored = sum( - 1 - for score in metric_records - if score.status != AgentEvalScoreStatus.FAILED - and any(output.name == output_name for output in score.outputs) - ) - metric_coverage[output_name] = AgentEvalMetricOutputCoverage( - total=total, - scored=scored, - failed=failed, - missing=max(total - scored - failed, 0), - ) - coverage[metric_type] = metric_coverage - return coverage - - -def _metric_output_names( - scores: Sequence[AgentEvalTaskScore], - tasks: Sequence[AgentEvalTask] | None, -) -> dict[str, list[str]]: - names: dict[str, set[str]] = {} - if tasks is not None: - for task in tasks: - for metric in task.metrics: - metric_type = metric_type_name(metric) - for output in metric.output_spec(): - names.setdefault(metric_type, set()).add(output.name) - - for score in scores: - for output in score.outputs: - names.setdefault(score.metric_type, set()).add(output.name) - return {metric_type: sorted(output_names) for metric_type, output_names in names.items()} - - -def _semantic_view_values( - scores: Sequence[AgentEvalTaskScore], - tasks: Sequence[AgentEvalTask] | None, -) -> dict[str, tuple[list[float], int]]: - """Return reduced view values and the number of attempted reductions per view. - - The integer in each tuple is the total number of trial/view reductions - attempted (the denominator for nan_count); the list holds the values that - reduced successfully. - """ - if tasks is None: - return {} - - tasks_by_id = {task.id: task for task in tasks} - # Match the stats path: PARTIAL scores may carry usable signal outputs. Missing - # signals still skip the view reduction below, so admitting PARTIAL is safe. - score_by_key = { - (score.task_id, score.trial_id, score.metric_type): score - for score in scores - if score.status in (AgentEvalScoreStatus.COMPLETED, AgentEvalScoreStatus.PARTIAL) - } - trials_by_task: dict[str, set[str]] = {} - for score in scores: - trials_by_task.setdefault(score.task_id, set()).add(score.trial_id) - - values_by_view: dict[str, list[float]] = {} - totals_by_view: dict[str, int] = {} - for task_id, trial_ids in trials_by_task.items(): - task = tasks_by_id.get(task_id) - if task is None: - continue - for trial_id in trial_ids: - for view_name, view in task.views.items(): - totals_by_view[view_name] = totals_by_view.get(view_name, 0) + 1 - signal_values: list[float] = [] - for signal in view.signals: - score = score_by_key.get((task_id, trial_id, signal.metric)) - output = _score_output(score, signal.output) if score is not None else None - value = _semantic_value(output) if output is not None else None - if value is None: - signal_values = [] - break - signal_values.append(value) - if not signal_values: - continue - reduced = _reduce_semantic_view(view.reducer, signal_values, view.signals) - if reduced is not None: - values_by_view.setdefault(view_name, []).append(reduced) - - return {view_name: (values_by_view.get(view_name, []), total) for view_name, total in totals_by_view.items()} - - -def _score_output(score: AgentEvalTaskScore | None, output_name: str) -> MetricOutput | None: - if score is None: - return None - for output in score.outputs: - if output.name == output_name: - return output - return None - - -def _reduce_semantic_view( - reducer: SemanticReducer, - values: list[float], - signals: list[ViewSignal], -) -> float | None: - if reducer == SemanticReducer.SINGLE: - return values[0] - if reducer == SemanticReducer.ALL: - return min(values) - if reducer == SemanticReducer.ANY: - return max(values) - if reducer == SemanticReducer.MEAN: - return mean_numeric(values) - weights = [signal.weight if signal.weight is not None else 1.0 for signal in signals] - denominator = sum(weights) - if denominator == 0: - return None - return sum(value * weight for value, weight in zip(values, weights, strict=True)) / denominator - - -def _numeric_value(output: MetricOutput) -> float | None: - value = output.value - if isinstance(value, bool): - return None - if isinstance(value, int | float): - return float(value) - if isinstance(value, BaseModel): - root = getattr(value, "root", None) - if isinstance(root, bool): - return None - if isinstance(root, int | float): - return float(root) - return None - - -def _native_value(output: MetricOutput) -> tuple[TrialMetricValueType, float | int | bool | str] | None: - """The payload for one metric output, in the type the metric produced it in. - - The *preserving* counterpart to :func:`_semantic_value`, which projects to a float because its - callers (aggregate stats, semantic views) do arithmetic. A :class:`TrialMetricValue` is not - arithmetic: it is the per-trial evidence a reader looks up by ``trial_id`` in ``result.trials`` - or ``trials.jsonl``, so a count stays an int, a flag stays a bool, and a judge's verdict stays - the string it was. - - Returns ``None`` -- "nothing a trial can record" -- rather than a value, so an output holding - a dict, a list, or a literal null stays *absent* from the value list. That is not the same as - the ``None`` a dead trial records, and conflating the two would charge pass@k a trial the - agent never made. - """ - value = output.value - if isinstance(value, BaseModel): - value = getattr(value, "root", None) - # bool first: it is a subclass of int, and it is a pass/fail signal rather than a measurement. - if isinstance(value, bool | int | float): - return (TrialMetricValueType.NUMBER, value) - if isinstance(value, str): - return (TrialMetricValueType.LABEL, value) - return None - - -def _semantic_value(output: MetricOutput) -> float | None: - """:func:`_native_value` projected to a float, for the callers that do arithmetic. - - The two answer different questions -- preserve versus interpret -- but they must agree on what a - metric value *is*, so the RootModel unwrap and the "what counts as numeric" rule live in - :func:`_native_value` alone. A label projects to ``None``: a view or aggregate cannot average it. - """ - payload = _native_value(output) - if payload is None or payload[0] is not TrialMetricValueType.NUMBER: - return None - return float(payload[1]) - - -def mean_numeric(values: list[float]) -> float | None: - """Return the mean of finite numeric values, ignoring missing and NaN.""" - finite = [value for value in values if math.isfinite(value)] - if not finite: - return None - return sum(finite) / len(finite) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/callable_runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/callable_runtime.py deleted file mode 100644 index d6af7e43be..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/callable_runtime.py +++ /dev/null @@ -1,122 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Minimal, dependency-light AgentTaskRunner backed by a user-supplied callable.""" - -from __future__ import annotations - -import asyncio -from collections.abc import Awaitable, Callable, Sequence -from dataclasses import dataclass, field -from typing import Any - -from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask -from nemo_platform.beta.evaluator.agent_eval.trials import ( - AgentEvalTrial, - AgentEvalTrialStatus, - AgentOutput, - RunnerInfo, - callable_identity, -) -from nemo_platform.beta.evaluator.values.evidence import CandidateEvidence - - -@dataclass(slots=True) -class TrialDraft: - """What an agent callable returns for one task: final output plus optional evidence. - - The runtime wraps this into a completed :class:`AgentEvalTrial`. Returning a - :class:`AgentOutput` or a plain string is also accepted as shorthand. - """ - - output: AgentOutput - evidence: CandidateEvidence | None = None - metadata: dict[str, Any] = field(default_factory=dict) - - -AgentTaskFn = Callable[[AgentEvalTask], Awaitable[TrialDraft | AgentOutput | str]] - - -class CallableAgentTaskRunner: - """Smallest possible :class:`AgentTaskRunner`: delegate each task to an async callable. - - The callable receives an :class:`AgentEvalTask` and returns the agent's final output as - a :class:`TrialDraft`, an :class:`AgentOutput`, or a plain string. This runtime adds only - what the ``AgentTaskRunner`` contract needs: bounded concurrency, stable trial ids, and - failure capture (an exception becomes a ``FAILED`` trial instead of aborting the batch). - It requires no Docker or external agent SDK, so it doubles as a reference for richer - runtimes and as the seam an ``AgentEvaluator`` drives via ``run(target=runner)``. - """ - - def __init__( - self, - agent_fn: AgentTaskFn, - *, - parallelism: int | None = None, - trial_id_suffix: str = "trial", - ) -> None: - self._agent_fn = agent_fn - self._parallelism = parallelism - self._trial_id_suffix = trial_id_suffix - - def runner_info(self) -> RunnerInfo: - """Identify this runner; the agent callable itself is the result-shaping detail.""" - return RunnerInfo( - name="callable", - kind="runner", - config={ - "agent_fn": callable_identity(self._agent_fn), - "parallelism": self._parallelism, - }, - ) - - async def run_tasks( - self, - tasks: Sequence[AgentEvalTask], - config: AgentEvalRunConfig | None = None, - ) -> list[AgentEvalTrial]: - """Run every task through the callable and return one trial per task, in order.""" - parallelism = self._parallelism if self._parallelism is not None else (config.parallelism if config else 4) - semaphore = asyncio.Semaphore(max(1, parallelism)) - - async def run_one(task: AgentEvalTask) -> AgentEvalTrial: - async with semaphore: - try: - result = await self._agent_fn(task) - except Exception as exc: # noqa: BLE001 - surfaced as a FAILED trial, not a crash - return self._failed_trial(task, exc) - return self._completed_trial(task, result) - - return list(await asyncio.gather(*(run_one(task) for task in tasks))) - - def _trial_id(self, task: AgentEvalTask) -> str: - return f"{task.id}:{self._trial_id_suffix}" - - def _completed_trial(self, task: AgentEvalTask, result: TrialDraft | AgentOutput | str) -> AgentEvalTrial: - draft = _as_trial_draft(result) - return AgentEvalTrial( - id=self._trial_id(task), - task_id=task.id, - status=AgentEvalTrialStatus.COMPLETED, - output=draft.output, - evidence=draft.evidence, - metadata=draft.metadata, - ) - - def _failed_trial(self, task: AgentEvalTask, exc: Exception) -> AgentEvalTrial: - return AgentEvalTrial( - id=self._trial_id(task), - task_id=task.id, - status=AgentEvalTrialStatus.FAILED, - metadata={"error": f"{type(exc).__name__}: {exc}"}, - ) - - -def _as_trial_draft(result: TrialDraft | AgentOutput | str) -> TrialDraft: - if isinstance(result, TrialDraft): - return result - if isinstance(result, AgentOutput): - return TrialDraft(output=result) - if isinstance(result, str): - return TrialDraft(output=AgentOutput(output_text=result)) - raise TypeError(f"agent callable must return TrialDraft, AgentOutput, or str; got {type(result).__name__}") diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py deleted file mode 100644 index bdf1cc979e..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py +++ /dev/null @@ -1,366 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Docker-backed sandbox runtime for agent-eval trials.""" - -from __future__ import annotations - -import asyncio -import contextlib -import inspect -import json -import re -import shutil -import tarfile -import tempfile -from collections.abc import Awaitable, Callable, Sequence -from dataclasses import dataclass -from datetime import UTC, datetime -from pathlib import Path -from typing import Any -from uuid import uuid4 - -from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask -from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput, RunnerInfo -from nemo_platform.beta.evaluator.values.evidence import CandidateEvidence, EvidenceDescriptor -from pydantic_core import to_jsonable_python - -DEFAULT_INSTRUCTIONS = ( - "Complete the task inside the sandbox workspace. Inspect the provided task files, " - "write any durable artifacts under output/, and return a concise final answer." -) -_RUNTIME_NAME = "docker_sandbox" -_SAFE_NAME_PATTERN = re.compile(r"[^A-Za-z0-9_.-]+") - - -@dataclass(frozen=True) -class SandboxSDK: - """Loaded OpenAI Agents SDK symbols used by the runtime.""" - - Runner: Any - RunConfig: Any - SandboxRunConfig: Any - Manifest: Any - SandboxAgent: Any - DockerSandboxClient: Any - DockerSandboxClientOptions: Any - File: Any - Dir: Any - LocalDir: Any - DEFAULT_PYTHON_SANDBOX_IMAGE: str - docker_from_env: Callable[[], Any] - - -def _load_agents_sdk() -> SandboxSDK: - try: - # The OpenAI Agents SDK is imported only when this Docker runtime is actually used, so it - # is absent from the default type-checking environment. - from agents.run import RunConfig # ty: ignore[unresolved-import] - from agents.sandbox import Manifest, SandboxAgent, SandboxRunConfig # ty: ignore[unresolved-import] - from agents.sandbox.config import DEFAULT_PYTHON_SANDBOX_IMAGE # ty: ignore[unresolved-import] - from agents.sandbox.entries import Dir, File, LocalDir # ty: ignore[unresolved-import] - from agents.sandbox.sandboxes.docker import ( # ty: ignore[unresolved-import] - DockerSandboxClient, - DockerSandboxClientOptions, - ) - - from agents import Runner # ty: ignore[unresolved-import] - from docker import from_env as docker_from_env - except ImportError as exc: - # Audience split is in the error text: SDK extras are not propagated into the - # vendored nemo_platform.beta.evaluator mirror. - raise RuntimeError( - "DockerSandboxAgentRuntime requires the openai-agents[docker] Python packages. " - "Standalone SDK: pip install 'nemo-evaluator-sdk[agent-runtimes]'. " - "Vendored nemo-platform.beta.evaluator (no SDK extras): " - "pip install 'openai-agents[docker]'" - ) from exc - - return SandboxSDK( - Runner=Runner, - RunConfig=RunConfig, - SandboxRunConfig=SandboxRunConfig, - Manifest=Manifest, - SandboxAgent=SandboxAgent, - DockerSandboxClient=DockerSandboxClient, - DockerSandboxClientOptions=DockerSandboxClientOptions, - File=File, - Dir=Dir, - LocalDir=LocalDir, - DEFAULT_PYTHON_SANDBOX_IMAGE=DEFAULT_PYTHON_SANDBOX_IMAGE, - docker_from_env=docker_from_env, - ) - - -class DockerSandboxAgentRuntime: - """Generate agent-eval trials by running a SandboxAgent in Docker per task.""" - - def __init__( - self, - *, - model: str | None = None, - instructions: str | None = None, - image: str | None = None, - work_root: Path | None = None, - timeout_s: float | None = None, - agent_factory: Callable[..., Any] | None = None, - sandbox_client_factory: Callable[[], Any] | None = None, - runner: Any | None = None, - ) -> None: - self._model = model - self._instructions = instructions or DEFAULT_INSTRUCTIONS - self._image = image - self._work_root = work_root - self._timeout_s = timeout_s - self._agent_factory = agent_factory - self._sandbox_client_factory = sandbox_client_factory - self._runner = runner - - def runner_info(self) -> RunnerInfo: - """Identify this runner and the sandbox settings that shape its results.""" - return RunnerInfo( - name="docker_sandbox", - kind="runner", - config={ - "model": self._model, - "image": self._image, - "timeout_s": self._timeout_s, - "instructions": self._instructions, - }, - ) - - async def run_tasks( - self, - tasks: Sequence[AgentEvalTask], - config: AgentEvalRunConfig | None = None, - ) -> Sequence[AgentEvalTrial]: - resolved_config = config or AgentEvalRunConfig() - if resolved_config.run_id is None: - resolved_config = resolved_config.model_copy(update={"run_id": _new_runtime_run_id()}) - sdk = _load_agents_sdk() - semaphore = asyncio.Semaphore(resolved_config.parallelism) - - async def run_one(index: int, task: AgentEvalTask) -> AgentEvalTrial: - async with semaphore: - return await self._run_task(index, task, resolved_config, sdk) - - return await asyncio.gather(*(run_one(index, task) for index, task in enumerate(tasks))) - - async def _run_task( - self, - index: int, - task: AgentEvalTask, - config: AgentEvalRunConfig, - sdk: SandboxSDK, - ) -> AgentEvalTrial: - evidence_dir = self._evidence_dir(index, task, config) - evidence_dir.mkdir(parents=True, exist_ok=True) - client = self._build_client(sdk) - sandbox = None - - try: - # Build the prompt inside the guarded block: an instruction-less task raises here and fails - # just this task rather than aborting the whole run. - prompt = task.agent_prompt() - manifest = self._build_manifest(task, sdk) - agent = self._build_agent(manifest, sdk) - sandbox = await client.create( - manifest=manifest, - options=sdk.DockerSandboxClientOptions(image=self._image or sdk.DEFAULT_PYTHON_SANDBOX_IMAGE), - ) - async with sandbox: - result = await self._run_agent(agent, prompt, sandbox, sdk) - return await self._completed_trial(task, result, sandbox, evidence_dir) - except Exception as exc: - return self._failed_trial(task, exc, evidence_dir) - finally: - if sandbox is not None: - with contextlib.suppress(Exception): - await client.delete(sandbox) - - def _build_manifest(self, task: AgentEvalTask, sdk: SandboxSDK) -> Any: - # Seed only the agent-facing projection of the task: the prompt (its instruction) plus any - # declared workspace files. We deliberately do NOT serialize the task object into the - # workspace β€” nothing in the runtime consumes it, and dumping the whole DTO would expose - # grader-only fields (e.g. ``reference`` held-out ground truth) to the agent. - entries: dict[str, Any] = { - "instruction.md": sdk.File(content=task.agent_prompt().encode("utf-8")), - "output": sdk.Dir(), - } - workspace_dir = task.inputs.get("workspace_dir") - if workspace_dir is not None: - entries["workspace"] = sdk.LocalDir(src=_validated_workspace_dir(workspace_dir)) - return sdk.Manifest(root="/workspace", entries=entries) - - def _build_agent(self, manifest: Any, sdk: SandboxSDK) -> Any: - agent_factory = self._agent_factory or sdk.SandboxAgent - kwargs = { - "name": "NeMo Agent Eval Docker Sandbox Runtime", - "instructions": self._instructions, - "default_manifest": manifest, - } - if self._model is not None: - kwargs["model"] = self._model - return agent_factory(**kwargs) - - def _build_client(self, sdk: SandboxSDK) -> Any: - if self._sandbox_client_factory is not None: - return self._sandbox_client_factory() - return sdk.DockerSandboxClient(sdk.docker_from_env()) - - async def _run_agent(self, agent: Any, prompt: str, sandbox: Any, sdk: SandboxSDK) -> Any: - runner = self._runner or sdk.Runner - run = runner.run( - agent, - prompt, - run_config=sdk.RunConfig(sandbox=sdk.SandboxRunConfig(session=sandbox)), - ) - if self._timeout_s is not None: - return await asyncio.wait_for(_maybe_await(run), timeout=self._timeout_s) - return await _maybe_await(run) - - async def _completed_trial( - self, - task: AgentEvalTask, - result: Any, - sandbox: Any, - evidence_dir: Path, - ) -> AgentEvalTrial: - final_output = getattr(result, "final_output", None) - final_output_text = "" if final_output is None else str(final_output) - - final_output_path = evidence_dir / "final_output.txt" - run_items_path = evidence_dir / "run_items.json" - raw_responses_path = evidence_dir / "raw_responses.json" - workspace_tar_path = evidence_dir / "workspace.tar" - final_state_dir = evidence_dir / "final_state" - - final_output_path.write_text(final_output_text, encoding="utf-8") - _write_json(run_items_path, _jsonable(getattr(result, "new_items", []))) - _write_json(raw_responses_path, _jsonable(getattr(result, "raw_responses", []))) - await _persist_workspace(sandbox, workspace_tar_path, final_state_dir) - - return AgentEvalTrial( - id=f"{task.id}:docker-sandbox", - task_id=task.id, - status=AgentEvalTrialStatus.COMPLETED, - output=AgentOutput( - output_text=final_output_text, - response={"final_output": final_output_text}, - metadata={ - "runtime": _RUNTIME_NAME, - "evidence_dir": str(evidence_dir), - }, - ), - evidence=CandidateEvidence( - descriptors={ - "final_state": EvidenceDescriptor(kind="filesystem", ref=str(final_state_dir)), - "workspace_archive": EvidenceDescriptor(kind="archive", format="tar", ref=str(workspace_tar_path)), - "run_items": EvidenceDescriptor(kind="run_items", format="json", ref=str(run_items_path)), - "raw_responses": EvidenceDescriptor( - kind="raw_responses", format="json", ref=str(raw_responses_path) - ), - "final_output": EvidenceDescriptor(kind="text", format="txt", ref=str(final_output_path)), - }, - metadata={"runtime": _RUNTIME_NAME, "sandbox_backend": "docker"}, - ), - metadata={"runtime": _RUNTIME_NAME, "generated": True}, - ) - - def _failed_trial(self, task: AgentEvalTask, exc: Exception, evidence_dir: Path) -> AgentEvalTrial: - error_path = evidence_dir / "error.json" - _write_json( - error_path, - { - "error_type": exc.__class__.__name__, - "error": str(exc), - }, - ) - return AgentEvalTrial( - id=f"{task.id}:docker-sandbox", - task_id=task.id, - status=AgentEvalTrialStatus.FAILED, - output=None, - evidence=CandidateEvidence( - descriptors={ - "error": EvidenceDescriptor(kind="error", format="json", ref=str(error_path)), - }, - metadata={"runtime": _RUNTIME_NAME, "sandbox_backend": "docker"}, - ), - metadata={ - "runtime": _RUNTIME_NAME, - "error_type": exc.__class__.__name__, - "error": str(exc), - }, - ) - - def _evidence_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> Path: - root = config.work_dir if config.work_dir is not None else self._work_root - if root is None: - root = Path(tempfile.gettempdir()) / "nemo-evaluator-agent-runtime" - run_id = config.run_id or _new_runtime_run_id() - safe_task_id = _safe_path_name(task.id) - task_name = f"{index:06d}-{safe_task_id}" if safe_task_id else f"task-{index:06d}" - return Path(root) / "agent-runtime" / run_id / task_name - - -def _validated_workspace_dir(workspace_dir: Any) -> Path: - if not isinstance(workspace_dir, (str, Path)): - raise ValueError(f"workspace_dir must be a path, got {type(workspace_dir).__name__}") - path = Path(workspace_dir).expanduser() - if not path.is_absolute(): - raise ValueError(f"workspace_dir must be an absolute path; got {workspace_dir!r}") - resolved = path.resolve() - if not resolved.is_dir(): - raise ValueError(f"workspace_dir does not exist or is not a directory: {resolved}") - return resolved - - -async def _maybe_await(value: Awaitable[Any] | Any) -> Any: - if inspect.isawaitable(value): - return await value - return value - - -async def _persist_workspace(sandbox: Any, workspace_tar_path: Path, final_state_dir: Path) -> None: - archive = await sandbox.persist_workspace() - try: - with workspace_tar_path.open("wb") as out: - shutil.copyfileobj(archive, out) - finally: - close = getattr(archive, "close", None) - if close is not None: - close() - - _extract_tar_safely(workspace_tar_path, final_state_dir) - - -def _extract_tar_safely(archive_path: Path, destination_root: Path) -> None: - if destination_root.exists(): - shutil.rmtree(destination_root) - destination_root.mkdir(parents=True, exist_ok=True) - - # The stdlib `data` filter (Python 3.12+) rejects absolute paths, parent-directory - # traversal, links, and special files, so we do not hand-roll those guards. - with tarfile.open(archive_path, "r:*") as archive: - archive.extractall(destination_root, filter="data") - - -def _write_json(path: Path, payload: Any) -> None: - path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") - - -def _jsonable(value: Any) -> Any: - # Normalize pydantic models, dataclasses, Paths, sets, etc. into JSON-safe values; - # `repr` is the last-resort fallback for anything still not serializable. - return to_jsonable_python(value, fallback=repr) - - -def _safe_path_name(value: str) -> str: - sanitized = _SAFE_NAME_PATTERN.sub("-", value).strip(".-") - return sanitized[:120] - - -def _new_runtime_run_id() -> str: - timestamp = datetime.now(UTC).strftime("%Y%m%d%H%M%S%f") - return f"agent-runtime-{timestamp}-{uuid4().hex[:8]}" diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/environment.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/environment.py deleted file mode 100644 index 94b50eb890..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/environment.py +++ /dev/null @@ -1,178 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Process/filesystem environment boundary for agent-eval runtimes. - -Sits *below* :class:`AgentTaskRunner` so a runtime needn't know whether the -agent/verifier run under Docker, locally, or another filesystem-backed sandbox. -It is a process/filesystem abstraction: :class:`EnvRunSpec`'s ``mounts``/ -``extra_args`` are filesystem hints that non-filesystem providers may ignore. -Handles route both roles through a single :meth:`AbstractEnvironmentHandle.run`. -""" - -from __future__ import annotations - -import abc -import asyncio -import os -import re -import subprocess -from collections.abc import Callable -from dataclasses import dataclass, field -from typing import Literal, Protocol, runtime_checkable - -from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask - -EnvRole = Literal["agent", "verifier"] -_SENSITIVE_MARKERS = ("KEY", "TOKEN", "SECRET", "PASSWORD") - - -def _redact_for_logging(cmd: list[str]) -> str: - """Scrub secret-looking values (``KEY=…`` tokens and ``--flag value`` pairs).""" - out: list[str] = [] - redact_next = False - for token in cmd: - if redact_next: - out.append("***REDACTED***") - redact_next = False - elif "=" in token: - left, right = token.split("=", 1) - sensitive = any(m in left.upper() for m in _SENSITIVE_MARKERS) - out.append(f"{left}=***REDACTED***" if sensitive else f"{left}={right}") - else: - normalized = token.lstrip("-").replace("-", "_").upper() - if token.startswith("-") and any(m in normalized for m in _SENSITIVE_MARKERS): - redact_next = True - out.append(token) - return " ".join(out) - - -def default_image_tag(task_id: str) -> str: - """Default task β†’ image-tag mapping (callers may inject their own). - - Sanitizes ``task_id`` to a valid Docker image name so ids with spaces or - other unsupported characters don't fail the build/run. - """ - safe = re.sub(r"[^a-z0-9_.-]+", "-", task_id.lower()).strip(".-") - return f"{safe or 'task'}:latest" - - -@dataclass(frozen=True) -class EnvCommandResult: - """Outcome of running a single command inside a prepared environment.""" - - exit_code: int - timed_out: bool = False - - @property - def ok(self) -> bool: - return self.exit_code == 0 and not self.timed_out - - -@dataclass -class EnvRunSpec: - """How to execute one command inside an environment handle. - - ``mounts``/``extra_args`` are filesystem-environment hints (e.g. Docker bind - mounts and extra CLI args). Non-filesystem providers may ignore them. - """ - - command: list[str] - env: dict[str, str] = field(default_factory=dict) - mounts: list[tuple[str, str]] = field(default_factory=list) - workdir: str | None = None - timeout: int | None = None - extra_args: list[str] = field(default_factory=list) - - -@runtime_checkable -class AgentEnvironmentHandle(Protocol): - """A prepared, single-task environment that can run agent/verifier commands.""" - - async def run_agent(self, spec: EnvRunSpec) -> EnvCommandResult: ... - - async def run_verifier(self, spec: EnvRunSpec) -> EnvCommandResult: ... - - async def close(self) -> None: ... - - -@runtime_checkable -class AgentEnvironmentProvider(Protocol): - """Creates per-task environment handles. Pluggable: Docker now, others later.""" - - async def prepare( - self, - task: AgentEvalTask, - config: AgentEvalRunConfig | None = None, - ) -> AgentEnvironmentHandle: ... - - -class AbstractEnvironmentHandle(abc.ABC): - """Base handle that routes both roles through a single :meth:`run`. - - Concrete handles implement :meth:`run`; ``run_agent``/``run_verifier`` are - role-specialized wrappers so the duplicated phase methods don't have to be - reimplemented per backend. - """ - - @abc.abstractmethod - async def run(self, spec: EnvRunSpec, role: EnvRole) -> EnvCommandResult: ... - - async def run_agent(self, spec: EnvRunSpec) -> EnvCommandResult: - return await self.run(spec, "agent") - - async def run_verifier(self, spec: EnvRunSpec) -> EnvCommandResult: - return await self.run(spec, "verifier") - - async def close(self) -> None: - return None - - -def _docker_run(image: str, spec: EnvRunSpec) -> EnvCommandResult: - """Run ``spec.command`` in a one-shot ``docker run --rm`` container. - - Shells out to the ``docker`` CLI (stdlib ``subprocess`` only), so no - ``agent-runtimes`` extra is needed β€” just a ``docker`` binary at call time. - """ - cmd = ["docker", "run", "--rm"] - if spec.workdir: - cmd += ["-w", spec.workdir] - for key, value in spec.env.items(): - cmd += ["-e", f"{key}={value}"] - for host_path, container_path in spec.mounts: - cmd += ["-v", f"{host_path}:{container_path}"] - cmd += spec.extra_args + os.environ.get("DOCKER_EXTRA_ARGS", "").split() - cmd += [image, *spec.command] - - print(f"[agent-eval-runtime] $ {_redact_for_logging(cmd)}") - try: - result = subprocess.run(cmd, check=False, text=True, timeout=spec.timeout) - except subprocess.TimeoutExpired: - return EnvCommandResult(exit_code=124, timed_out=True) - return EnvCommandResult(exit_code=result.returncode) - - -class DockerEnvironmentHandle(AbstractEnvironmentHandle): - """Docker-backed environment handle bound to one task image.""" - - def __init__(self, image: str) -> None: - self.image = image - - async def run(self, spec: EnvRunSpec, role: EnvRole = "agent") -> EnvCommandResult: - del role # Docker runs both roles identically against the same image. - return await asyncio.to_thread(_docker_run, self.image, spec) - - -class DockerEnvironmentProvider: - """Default provider that maps each task to its built Docker image.""" - - def __init__(self, *, image_tag_fn: Callable[[str], str] = default_image_tag) -> None: - self._image_tag_fn = image_tag_fn - - async def prepare( - self, - task: AgentEvalTask, - config: AgentEvalRunConfig | None = None, - ) -> DockerEnvironmentHandle: - del config - return DockerEnvironmentHandle(self._image_tag_fn(task.id)) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/_common.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/_common.py deleted file mode 100644 index 0d455fc9cf..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/_common.py +++ /dev/null @@ -1,158 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Shared helpers for the host and containerized NeMo Fabric agent-eval runtimes. - -:class:`~nemo_platform.beta.evaluator.agent_eval.runtimes.fabric.runtime.FabricAgentRuntime` (host) and -:class:`~nemo_platform.beta.evaluator.agent_eval.runtimes.fabric.container_runtime.FabricContainerRuntime` -(sandbox) map a Fabric ``RunResult`` to the *same* trial/evidence contract, so the pieces they share -live here β€” one definition, so the two runtimes cannot drift apart. - -Trajectory capture is built from ``nemo_relay``'s own typed config objects (a hard dependency), so -Relay owns its schema: a breaking Relay change fails construction here rather than silently producing -a malformed profile. -""" - -from __future__ import annotations - -import json -from collections.abc import Mapping -from pathlib import Path -from typing import Any - -from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalTask -from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus -from nemo_platform.beta.evaluator.values.evidence import CandidateEvidence, EvidenceDescriptor - -# Trajectory profile identity + the file-exporter output names we choose (Relay accepts these as -# inputs). Shared so both runtimes select/emit the trajectory under identical names. -TRAJECTORY_PROFILE_NAME = "eval_trajectory" -ATIF_FILENAME_TEMPLATE = "trajectory-{session_id}.atif.json" -ATOF_FILENAME = "events.atof.jsonl" -#: ATIF ``agent.version``. Both runtimes report the agent *framework* here so a consumer can group -#: host and container traces together; ``agent.name`` is what distinguishes them. Not a real version -#: yet β€” reporting the resolved nemo-fabric version would be the better answer. -FABRIC_AGENT_VERSION = "fabric" -# Fabric telemetry-profile selectors (Relay file exporter, no OTLP endpoint). -TELEMETRY_PROVIDER = "relay" -TELEMETRY_MODE = "sdk" - - -def safe_path_name(value: str) -> str: - """Filesystem-safe rendering of an arbitrary id (alnum/``._-`` kept, else ``-``; trimmed to 120).""" - return "".join(char if char.isalnum() or char in "._-" else "-" for char in value).strip(".-")[:120] - - -def task_subdir_name(index: int, task_id: str) -> str: - """Deterministic per-task evidence subdir name (``000000-``) shared by both runtimes.""" - safe = safe_path_name(task_id) - return f"{index:06d}-{safe}" if safe else f"task-{index:06d}" - - -def extract_output_text(output: object) -> str | None: - """Pull the user-visible message out of a Fabric output value (already unwrapped from the result). - - Harness outputs vary; adapters commonly nest the final message under ``response`` (the codex-cli - adapter does). Prefer a string ``response``/``output_text``/``text``/``message``, else stringify. - """ - if output is None: - return None - if isinstance(output, str): - return output - if isinstance(output, Mapping): - for key in ("response", "output_text", "text", "message"): - value = output.get(key) - if isinstance(value, str): - return value - return json.dumps(output, default=str) - - -def build_failed_trial( - task: AgentEvalTask, - evidence_dir: Path, - error: Exception | Mapping[str, Any], - *, - runtime_name: str, - trial_id_suffix: str, - extra_metadata: Mapping[str, Any] | None = None, -) -> AgentEvalTrial: - """Persist ``error.json`` and build a FAILED trial with the standard error evidence + metadata. - - ``error`` is either a raised exception or a Fabric error mapping (``stage``/``code``/``message``). - """ - if isinstance(error, Mapping): - error_type = str(error.get("code") or error.get("stage") or "FabricError") - error_message = str(error.get("message") or error) - else: - error_type = error.__class__.__name__ - error_message = str(error) - error_path = evidence_dir / "error.json" - error_path.write_text(json.dumps({"error_type": error_type, "error": error_message}) + "\n", encoding="utf-8") - return AgentEvalTrial( - id=f"{task.id}:{trial_id_suffix}", - task_id=task.id, - status=AgentEvalTrialStatus.FAILED, - output=None, - evidence=CandidateEvidence( - descriptors={"error": EvidenceDescriptor(kind="error", format="json", ref=str(error_path))}, - metadata={"runtime": runtime_name}, - ), - metadata={ - **(dict(extra_metadata) if extra_metadata else {}), - "runtime": runtime_name, - "error_type": error_type, - "error": error_message, - # A failed trial did not complete its agent phase; stamp it explicitly (matching the host - # Fabric/Codex runtimes) so AgentPhaseSuccessMetric scores it False rather than by omission. - "agent_ok": False, - }, - ) - - -def trajectory_telemetry(*, relay_dir: str, agent_name: str, agent_version: str) -> dict[str, Any]: - """The ``telemetry`` block of a Fabric trajectory profile: Relay's ATIF/ATOF file exporter (mode=sdk). - - Built from ``nemo_relay``'s own typed config so Relay owns its schema β€” no hand-maintained dict to - silently drift when Relay changes it. Callers wrap this in a profile with their own name + - ``runtime``/``environment`` blocks; ``relay_dir`` is where the ``trajectory-*.atif.json`` lands. - - ``nemo_relay`` is imported here rather than at module scope: it is a native extension costing - ~120ms to load, and this module is reachable from the evaluator plugin's job imports, so an - eager import would charge every consumer for trajectory capture they may never use. - """ - from nemo_relay.observability import ( - AtifConfig, - AtofConfig, - AtofFileSinkConfig, - ComponentSpec, - ObservabilityConfig, - ) - - observability = ComponentSpec( - config=ObservabilityConfig( - atif=AtifConfig( - enabled=True, - output_directory=relay_dir, - filename_template=ATIF_FILENAME_TEMPLATE, - agent_name=agent_name, - agent_version=agent_version, - ), - atof=AtofConfig( - enabled=True, - sinks=[ - AtofFileSinkConfig( - output_directory=relay_dir, - filename=ATOF_FILENAME, - mode="overwrite", - ) - ], - ), - ) - ) - return { - "enabled": True, - "provider": TELEMETRY_PROVIDER, - "mode": TELEMETRY_MODE, - "output_dir": relay_dir, - "config": {"version": 1, "components": [observability.to_dict()]}, - } diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/container_runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/container_runtime.py deleted file mode 100644 index 8668998dd3..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/container_runtime.py +++ /dev/null @@ -1,602 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Containerized NeMo Fabric agent-eval runtime. - -``FabricContainerRuntime`` is the sandboxed sibling of -:class:`~nemo_platform.beta.evaluator.agent_eval.runtimes.fabric.runtime.FabricAgentRuntime`: instead of -running the Fabric harness on the host filesystem, it runs it **inside a sandbox** (Docker now, -Kubernetes/agent-sandbox later) through the provider-neutral -:class:`~nemo_platform.beta.evaluator.agent_eval.runtimes.sandbox.api.AsyncSandbox` seam. - -Per task it: - -1. seeds ``/in`` with the composed Fabric agent config and framed input, plus the task's workspace - seed files; -2. execs Fabric's own CLI (``fabric run``), which writes a normalized ``RunResult`` to stdout and the - workspace + Relay ATIF trajectory under a fixed ``/out`` layout; -3. downloads ``/out`` across the boundary into the durable per-task evidence dir; and -4. maps it into the shared :class:`CandidateEvidence` contract the eval metrics consume β€” ``result`` - (json), ``trace`` (ATIF), plus ``workspace`` (filesystem) and ``logs`` β€” so the workspace-file, - held-out ``run_verifier``, and trajectory metrics score container trials with no metric changes. - (``FabricAgentRuntime`` surfaces ``workspace``/``logs`` only when Fabric promotes them as artifacts; - the container always captures them from the ``/out`` tree, so its evidence is a superset.) - -Relay writes ATIF **inside the image** (no host gateway), which removes the bare-``python3`` / -``tomli_w`` adapter-interpreter problem the host runtime has to work around. -""" - -from __future__ import annotations - -import asyncio -import copy -import json -import logging -import shlex -import shutil -import tempfile -from collections.abc import Mapping, Sequence -from pathlib import Path, PurePosixPath -from typing import TYPE_CHECKING, Any, cast - -from nemo_platform.beta.evaluator.agent_eval.runtimes.fabric import _common -from nemo_platform.beta.evaluator.agent_eval.runtimes.fabric.image import ensure_fabric_image -from nemo_platform.beta.evaluator.agent_eval.runtimes.fabric.skills import ( - CODEX_SKILLS_DIR, - SKILL_MODE_CODEX_SKILLS_DIR, - AgentSkill, - SkillInjectionError, - SkillMode, - SkillProvenance, - SkillSet, - resolve_skill_mode, - stage_skills_seed, -) -from nemo_platform.beta.evaluator.agent_eval.runtimes.sandbox.api import AsyncSandbox -from nemo_platform.beta.evaluator.agent_eval.runtimes.sandbox.base import SandboxExecResult, SandboxProvider, SandboxSpec -from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask -from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput, RunnerInfo -from nemo_platform.beta.evaluator.agent_eval.workspace_seeds import SEED_FILES_INPUT_KEY, seed_workspace -from nemo_platform.beta.evaluator.resolver_protocols import SecretResolver -from nemo_platform.beta.evaluator.resolvers import LocalSecretResolver -from nemo_platform.beta.evaluator.values.common import SecretRef -from nemo_platform.beta.evaluator.values.evidence import ( - EVIDENCE_FORMAT_ATIF, - EVIDENCE_LOGS, - EVIDENCE_TRACE, - CandidateEvidence, - EvidenceDescriptor, -) -from pydantic import JsonValue - -logger = logging.getLogger(__name__) - -if TYPE_CHECKING: - # nemo_fabric is an optional native dep (see FabricAgentRuntime); imported for typing only. Configs - # are consumed structurally via ``to_mapping()`` at runtime, so this module stays importable without it. - from nemo_fabric import FabricConfig # ty: ignore[unresolved-import] - -# Default per-task exec budget. Timeout is really task-specific (see AALGO-323 to move it onto -# AgentEvalTask); until then it is an internal default rather than a runtime-construction knob. -DEFAULT_FABRIC_TIMEOUT_S = 600 -_RUNTIME_NAME = "fabric_container" -_MISSING_FABRIC_MSG = ( - "FabricContainerRuntime skill injection requires the `nemo-fabric` package (native NeMo Fabric SDK) " - "on the host to resolve how a skill reaches the selected adapter; the container otherwise runs Fabric " - "only inside the sandbox." -) - -# Fixed in-container layout. The runtime seeds ``/in`` (agent config, input), execs Fabric's -# CLI, and reads the produced ``/out`` subtree back across the boundary. -_IN_DIR = "/in" -_OUT_DIR = "/out" -_WORKSPACE_DIR = f"{_OUT_DIR}/workspace" -_RELAY_DIR = f"{_OUT_DIR}/relay" -_ARTIFACTS_DIR = f"{_OUT_DIR}/artifacts" -_LOGS_DIR = f"{_OUT_DIR}/logs" -_RESULT_PATH = f"{_OUT_DIR}/fabric_result.json" -_FABRIC_STDERR = f"{_LOGS_DIR}/fabric-stderr.txt" -_AGENT_PATH = f"{_IN_DIR}/agent.yaml" -_INPUT_PATH = f"{_IN_DIR}/input.txt" -# In-sandbox root for a natively-injected skill bundle. It lives under ``/in`` (not ``/out``), so it is -# never part of the downloaded ``/out`` evidence β€” only codex-mode skills, which must sit in the workspace -# for the harness to self-discover them, need post-download cleanup. -_SKILLS_DIR = f"{_IN_DIR}/skills" -# Sentinel skill path attached only to probe Fabric's capability planner for the selected adapter's skills -# routing (mirrors the host runtime). Never staged and need not exist on disk. -_SKILL_PROBE_PATH = "nemo-eval-skill-capability-probe" - - -class FabricContainerRuntime: - """AgentTaskRunner that generates trials by running Fabric tasks inside a sandbox.""" - - def __init__( - self, - config: FabricConfig | Mapping[str, Any], - *, - provider: SandboxProvider, - secrets: Mapping[str, SecretRef] = {}, - image: str | None = None, - skills: Sequence[AgentSkill] | None = None, - ) -> None: - # The Fabric agent is fully described by its ``FabricConfig`` (harness + model + runtime); it is - # consumed structurally as a mapping to cross the sandbox boundary as JSON. - self._config = _to_mapping(config) - self._provider = provider - # ``secrets`` maps the env-var name a Fabric harness reads its credential from (declared by the - # adapter's ``requirements.env``) to a SecretRef. The runner only *declares* them; the resolver - # is owned by the orchestrator (see ``resolve_secrets``), mirroring ``MetricWithSecrets``. - self._secrets = dict(secrets) - self._resolved_env: dict[str, str] = {} - self._secrets_resolved = False - # Optional prebuilt image: the trial runs inside it, so it must contain the Fabric CLI + adapter. - # None -> stock harness-agnostic image built on first run. - self._image: str | None = image - # Optional agent skills injected per task (A/B: baseline vs. treated via ``with_skills``). How they - # reach the harness is resolved once per run (the adapter is constant across the taskset) in - # ``run_tasks``; only touched when a skill is set, so the no-skill path stays dependency-free. Names - # must be unique β€” each stages to its own ``/`` bundle, so a repeat would collide. - self._skill_set = SkillSet(tuple(skills or ())) - - def with_skills(self, skills: Sequence[AgentSkill]) -> FabricContainerRuntime: - """Return a copy of this runtime with ``skills`` *added* to its skill set; ``self`` is not modified. - - Mirrors :meth:`FabricAgentRuntime.with_skills`: additive and chainable - (``rt.with_skills([a]).with_skills([b])`` injects both), so an A/B eval derives a treated runtime - from a skill-free baseline (``baseline.with_skills(the_skills)``) and the arms differ in exactly the - injected skills. Names must be unique across the combined set (colliding ``/`` bundles), so - re-adding a present skill raises. A shallow copy suffices β€” the shared fields are immutable - config/paths/provider. (``run_tasks`` disposes the injected provider on completion, so an A/B run - over two arms should give each arm its own provider.) - """ - clone = copy.copy(self) - clone._skill_set = self._skill_set.with_skills(skills) - return clone - - def with_skill(self, skill: AgentSkill) -> FabricContainerRuntime: - """Return a copy of this runtime with ``skill`` *added*; ``self`` is not modified. - - Thin wrapper over :meth:`with_skills` for the single-skill case; equally chainable - (``rt.with_skill(a).with_skill(b)`` injects both). - """ - return self.with_skills([skill]) - - async def resolve_secrets(self, secret_resolver: SecretResolver) -> None: - """Resolve declared ``SecretRef``\\ s to values, keyed by the env var each harness reads. - - Mirrors ``MetricWithSecrets.resolve_secrets``: the resolver is owned by the orchestrator (the - AgentEvaluator / execution backend), not the runner. Call before :meth:`run_tasks`; a standalone - ``run_tasks`` falls back to local env resolution when this was not called. - """ - env: dict[str, str] = {} - for env_var, secret_ref in self._secrets.items(): - value = await secret_resolver.resolve_secret(secret_ref) - if value is None: - raise ValueError(f"could not resolve secret {secret_ref.root!r} for env var {env_var!r}") - env[env_var] = value - self._resolved_env = env - self._secrets_resolved = True - - def runner_info(self) -> RunnerInfo: - """Identify this runner and the Fabric container settings that shape its results. - - Records the provider only β€” never ``self._secrets``, which is persisted nowhere. - """ - return RunnerInfo( - name="fabric_container", - kind="runner", - config={ - "provider": self._provider.name, - "image": self._image, - "adapter_id": self._adapter_id(), - "skills": [skill.name for skill in self._skill_set.skills], - }, - ) - - async def run_tasks( - self, - tasks: Sequence[AgentEvalTask], - config: AgentEvalRunConfig | None = None, - ) -> Sequence[AgentEvalTrial]: - resolved_config = config or AgentEvalRunConfig() - semaphore = asyncio.Semaphore(resolved_config.parallelism) - - async def run_one(index: int, task: AgentEvalTask, skill_mode: SkillMode | None) -> AgentEvalTrial: - async with semaphore: - logger.info("running task", extra={"index": index + 1, "task_id": task.id}) - result = await self._run_task(index, task, resolved_config, skill_mode) - logger.info("task completed", extra={"index": index + 1, "task_id": task.id}) - return result - - try: - # Provision the harness-agnostic Fabric image once, build-if-missing (a first build compiles - # nemo-fabric β€” minutes); keep the blocking build off the shared event loop. Inside the guard - # so the provider is disposed even if provisioning or secret resolution raises. - if self._image is None: - self._image = await asyncio.to_thread(ensure_fabric_image) - if self._secrets and not self._secrets_resolved: - # No orchestrator resolved our secrets (standalone run) β€” fall back to local env resolution. - await self.resolve_secrets(LocalSecretResolver()) - # Resolve once (the adapter is constant across the taskset) how a skill reaches this harness, by - # probing Fabric's capability planner β€” the same authoritative routing the host runtime uses. - # Fail fast rather than silently run a skill-free trial mislabeled "with skill". Blocking pyo3 - # planning, so keep it off the shared event loop; only reached when a skill is set. - skill_mode = await asyncio.to_thread(self._resolve_skill_mode) if self._skill_set.skills else None - if self._skill_set.skills and skill_mode is None: - raise RuntimeError( - f"FabricContainerRuntime received one or more skills but adapter {self._adapter_id()!r} " - "has no known skill-injection strategy: Fabric does not route skills to it natively and " - "it is not a codex harness. Use a skills-native or codex harness, or drop the skills." - ) - return await asyncio.gather(*(run_one(index, task, skill_mode) for index, task in enumerate(tasks))) - finally: - # Each sandbox tears itself down; the provider is shared across the batch, so its - # process-wide resources are disposed once here, when the batch completes. - await self._provider.aclose() - - async def _run_task( - self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig, skill_mode: SkillMode | None - ) -> AgentEvalTrial: - evidence_dir = self._evidence_dir(index, task, config) - out_dir = evidence_dir / "out" - evidence_dir.mkdir(parents=True, exist_ok=True) - - # The whole per-task flow β€” framing input, seeding, exec, download, and parsing the result β€” is - # guarded so any failure (bad seed, sandbox crash, unreadable result) fails only this task's - # trial rather than aborting the gathered batch. - skill_provenances: list[SkillProvenance] = [] - try: - seed_files, skill_provenances = self._seed_files(task, skill_mode) - spec = SandboxSpec( - image=self._image, workdir=_WORKSPACE_DIR, env=dict(self._resolved_env), files=seed_files - ) - async with AsyncSandbox(self._provider, spec) as sandbox: - await sandbox.start() - await self._seed_workspace(sandbox, task) - result = await sandbox.exec(self._fabric_command(), timeout_s=DEFAULT_FABRIC_TIMEOUT_S) - await sandbox.download_dir(_OUT_DIR, out_dir) - # Codex self-injection seeds each bundle inside the workspace so the harness discovers it during - # the run; drop them from the downloaded evidence before the workspace is exposed (else the - # injected files read as agent output to workspace-reading metrics). Native staging lives under - # /in, which is never downloaded, so it never pollutes the evidence. - if skill_mode == SKILL_MODE_CODEX_SKILLS_DIR: - for provenance in skill_provenances: - await asyncio.to_thread(_remove_injected_bundle, out_dir / "workspace", provenance["location"]) - return self._to_trial(task, out_dir, evidence_dir, result, skill_provenances=skill_provenances) - except Exception as exc: # noqa: BLE001 - a task failure must not abort the whole run - # Stamp runtime + image + skills even on failures before _to_trial (startup/seeding/download). - return self._failed_trial( - task, evidence_dir, exc, extra_metadata={**self._base_metadata(), **_skill_metadata(skill_provenances)} - ) - - def _resolve_skill_mode(self) -> SkillMode | None: - """Ask Fabric how a skill would reach the selected harness, or ``None`` if it can't. - - Mirrors :meth:`FabricAgentRuntime._resolve_skill_mode`: plan a copy of the config with a sentinel - skill path attached (it need not exist on disk) and read how the adapter routes skills from the - capability plan. Querying the authoritative planner at runtime means any adapter that declares - native skills support β€” ours or an end-user's β€” is picked up without a hardcoded list. ``nemo_fabric`` - is imported lazily on the host (only when a skill is set), so the no-skill path never needs it. - """ - try: - from nemo_fabric import Fabric, FabricConfig # ty: ignore[unresolved-import] - except ImportError as exc: - raise RuntimeError(_MISSING_FABRIC_MSG) from exc - probe_config = FabricConfig.from_mapping(self._config) - probe_config.add_skill_path(_SKILL_PROBE_PATH) - plan = Fabric().plan(probe_config) - return resolve_skill_mode(capability_plan=plan.capability_plan, adapter_id=self._adapter_id()) - - def _adapter_id(self) -> str: - """The harness adapter id declared by the config mapping (for provenance + error messages).""" - harness = self._config.get("harness") - adapter_id = harness.get("adapter_id") if isinstance(harness, Mapping) else None - return str(adapter_id) if adapter_id is not None else "" - - def _fabric_command(self) -> str: - """The ``fabric run`` invocation: pre-create the /out dirs Fabric chdirs into, run, capture stdout.""" - run = f"fabric run {shlex.quote(_AGENT_PATH)} --input-file {shlex.quote(_INPUT_PATH)}" - return ( - f"mkdir -p {_WORKSPACE_DIR} {_RELAY_DIR} {_ARTIFACTS_DIR} {_LOGS_DIR} && " - f"{run} > {shlex.quote(_RESULT_PATH)} 2> {shlex.quote(_FABRIC_STDERR)}" - ) - - def _seed_files( - self, task: AgentEvalTask, skill_mode: SkillMode | None - ) -> tuple[dict[str, str], list[SkillProvenance]]: - """Return (files to seed into the sandbox, skill provenances). - - The agent config is written as JSON, which the Fabric CLI parses as YAML. Fabric 0.1.0rc2 removed - profile overlays (``--profile`` and the ``profiles`` config key are both gone), so everything β€” - the runtime's in-container settings and any natively-injected skill paths β€” is composed into the - single agent config here. When skills are injected each bundle is also rendered into the seed set - at the harness's in-sandbox discovery path (native: ``/in/skills/``; codex: - ``/.agents/skills/``). - """ - skill_paths: list[str] = [] - provenances: list[SkillProvenance] = [] - files: dict[str, str] = {_INPUT_PATH: task.agent_prompt()} - if self._skill_set.skills and skill_mode is not None: - if skill_mode == SKILL_MODE_CODEX_SKILLS_DIR: - _check_codex_skill_collision(self._skill_set.skills, task.inputs.get(SEED_FILES_INPUT_KEY) or {}) - seed = stage_skills_seed( - skills=self._skill_set.skills, - adapter_id=self._adapter_id(), - mode=skill_mode, - workspace_dir=_WORKSPACE_DIR, - skills_dir=_SKILLS_DIR, - ) - files.update(seed.files) - skill_paths = seed.skill_paths - provenances = seed.provenances - files[_AGENT_PATH] = json.dumps(self._composed_config(skill_paths)) - return files, provenances - - def _composed_config(self, skill_paths: Sequence[str] = ()) -> dict[str, Any]: - """The supplied agent config with the runtime's in-container settings merged on last. - - Mirrors the host runtime's ``_compose_config``: the workspace, artifact roots, trajectory - telemetry, and any natively-injected skill paths are evaluator-owned, so they are applied over - whatever the caller's config declared. Stays plain dicts rather than round-tripping through the - host's ``FabricConfig`` β€” the sandbox may run a different Fabric build, so the config is only - required to survive JSON transport, not to validate against the host's schema. - - Injected skill paths are APPENDED to the config's own ``skills.paths`` β€” mirroring - ``FabricConfig.add_skill_path`` β€” so skills the caller preconfigured survive injection and the - treated A/B arm differs from the baseline by exactly the injected skills. - """ - config = dict(self._config) - - # Each section is spread over the caller's, so sibling keys survive β€” pinning - # ``runtime.artifacts`` must not drop configured input/output schemas or timeouts. - config["runtime"] = {**_section(config, "runtime"), "artifacts": _ARTIFACTS_DIR} - # ``provider: local`` is required by the native planner in the container (it does not inject the - # Python default), and the workspace pins the harness cwd to the retrievable /out subtree. - config["environment"] = { - **_section(config, "environment"), - "provider": "local", - "workspace": _WORKSPACE_DIR, - "artifacts": _ARTIFACTS_DIR, - } - # Relay ATIF/ATOF file exporter (sdk mode), built from nemo_relay's typed config via the shared - # helper so it stays a single source of truth with the host runtime. Replaced wholesale. - # ``agent_name`` distinguishes this runtime from the host one; ``agent_version`` records the - # agent framework and so matches the host's value, letting an ATIF consumer group both - # runtimes' traces. (Neither is a real version yet β€” see _common.trajectory_telemetry.) - config["telemetry"] = _common.trajectory_telemetry( - relay_dir=_RELAY_DIR, agent_name=_RUNTIME_NAME, agent_version=_common.FABRIC_AGENT_VERSION - ) - - declared_paths = _section(config, "skills").get("paths") or [] - merged_paths = list(dict.fromkeys([*(str(path) for path in declared_paths), *skill_paths])) - if merged_paths: - config["skills"] = {**_section(config, "skills"), "paths": merged_paths} - return config - - async def _seed_workspace(self, sandbox: AsyncSandbox, task: AgentEvalTask) -> None: - seeds = task.inputs.get(SEED_FILES_INPUT_KEY) - if not seeds: - return - # Transient host-side staging (a tmpdir, not part of the evidence bundle): seed with the SDK - # handlers, then upload across the boundary. seed_workspace is synchronous and a handler may do - # blocking I/O (e.g. a fileset download), so run it off the event loop shared by concurrent tasks. - with tempfile.TemporaryDirectory(prefix="nemo-fabric-seed-") as staging_dir: - staging = Path(staging_dir) - await asyncio.to_thread(seed_workspace, staging, seeds) - await sandbox.upload_dir(staging, _WORKSPACE_DIR) - - def _base_metadata(self) -> dict[str, object]: - """Metadata stamped on every trial from this runtime (success or failure), incl. the resolved image.""" - return {"runtime": _RUNTIME_NAME, "image": self._image, "sandbox_provider": self._provider.name} - - def _to_trial( - self, - task: AgentEvalTask, - out_dir: Path, - evidence_dir: Path, - result: SandboxExecResult, - *, - skill_provenances: list[SkillProvenance] | None = None, - ) -> AgentEvalTrial: - # Skill provenance (name + content hash + injection mode) rides on every trial for the A/B diff: - # a ``skills`` list plus the historical lone ``skill`` field, matching the host FabricAgentRuntime. - base_metadata = {**self._base_metadata(), **_skill_metadata(skill_provenances or [])} - - # Gate on the exec outcome first: a timed-out or non-zero ``fabric run`` is untrustworthy even - # when a stale/partial fabric_result.json is left behind (the shell ``>`` redirect truncates the - # file regardless), so never grade such a run off that file. - if result.error_type or result.return_code != 0: - stderr = _read_text(out_dir / "logs" / "fabric-stderr.txt") or (result.stderr or "") - detail = stderr.strip() or result.error_type or f"exit code {result.return_code}" - return self._failed_trial( - task, evidence_dir, RuntimeError(f"fabric run failed: {detail}"), extra_metadata=base_metadata - ) - - result_path = out_dir / "fabric_result.json" - result_payload = _read_json(result_path) - # `fabric run` writes a normalized RunResult object (a failed harness run still produces one, with - # status != "succeeded"). A missing, non-object, or unreadable payload means no usable result. - if not isinstance(result_payload, Mapping): - stderr = _read_text(out_dir / "logs" / "fabric-stderr.txt") or (result.stderr or "") - return self._failed_trial( - task, - evidence_dir, - RuntimeError(f"fabric run produced no usable result: {stderr.strip()}"), - extra_metadata=base_metadata, - ) - - status = str(result_payload.get("status")) - if status != "succeeded": - return self._failed_trial(task, evidence_dir, _result_error(result_payload), extra_metadata=base_metadata) - - return AgentEvalTrial( - id=f"{task.id}:fabric_container", - task_id=task.id, - status=AgentEvalTrialStatus.COMPLETED, - output=AgentOutput( - # ``response`` is the RunResult *output* payload (matching the host FabricAgentRuntime), - # not the whole normalized envelope, so metrics reading ``sample.response`` see one shape. - output_text=_common.extract_output_text(result_payload.get("output")), - response=cast(JsonValue, result_payload.get("output")), - metadata={**base_metadata, "evidence_dir": str(evidence_dir)}, - ), - evidence=self._evidence(out_dir, result_path), - metadata={**base_metadata, "generated": True, "agent_ok": True}, - ) - - def _evidence(self, out_dir: Path, result_path: Path) -> CandidateEvidence: - descriptors: dict[str, EvidenceDescriptor] = { - "result": EvidenceDescriptor(kind="json", format="json", ref=str(result_path)), - } - workspace_dir = out_dir / "workspace" - if workspace_dir.is_dir(): - descriptors["workspace"] = EvidenceDescriptor(kind="filesystem", ref=str(workspace_dir)) - logs_dir = out_dir / "logs" - if logs_dir.is_dir(): - descriptors[EVIDENCE_LOGS] = EvidenceDescriptor(kind="logs", ref=str(logs_dir)) - atif = _find_atif(out_dir / "relay") - if atif is not None: - descriptors[EVIDENCE_TRACE] = EvidenceDescriptor( - kind=EVIDENCE_TRACE, format=EVIDENCE_FORMAT_ATIF, ref=str(atif) - ) - return CandidateEvidence( - descriptors=descriptors, - metadata={"runtime": _RUNTIME_NAME, "sandbox_provider": self._provider.name, "image": self._image}, - ) - - def _failed_trial( - self, - task: AgentEvalTask, - evidence_dir: Path, - error: Exception | Mapping[str, object], - *, - extra_metadata: Mapping[str, object] | None = None, - ) -> AgentEvalTrial: - # Bind this runtime's name + trial-id suffix to the shared FAILED-trial builder. - return _common.build_failed_trial( - task, - evidence_dir, - error, - runtime_name=_RUNTIME_NAME, - trial_id_suffix=_RUNTIME_NAME, - extra_metadata=extra_metadata, - ) - - def _evidence_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> Path: - # Evidence lands under the run's output dir (like every other runtime); the container's own - # working state lives at /out inside the sandbox and is downloaded here. - root = (config.work_dir or Path.cwd()) / "evidence" / "fabric_container" - return root / _common.task_subdir_name(index, task.id) - - -def _to_mapping(config: FabricConfig | Mapping[str, Any]) -> dict[str, Any]: - """Normalize a typed Fabric config or a plain mapping to a plain dict for JSON transport.""" - # A typed Fabric config exposes ``to_mapping()``; a plain mapping is used as-is. Both are - # str-keyed at runtime, but the getattr + optional (unresolved) ``FabricConfig`` type defeat static - # narrowing, so cast the known-good source before building the dict. - to_mapping = getattr(config, "to_mapping", None) - source = to_mapping() if callable(to_mapping) else config - return dict(cast(Mapping[str, Any], source)) - - -def _skill_metadata(provenances: list[SkillProvenance]) -> dict[str, object]: - """Trial-metadata fields describing the injected skill set (the A/B provenance). - - ``skills`` is the full list of injected-skill provenances (empty = baseline). ``skill`` keeps the - historical single-provenance field β€” the lone provenance for a one-skill run, else ``None`` β€” so - single-skill consumers (e.g. ``SkillUsedMetric``) and existing trials/tests keep working unchanged. - Mirrors ``FabricAgentRuntime._skill_metadata``. - """ - return {"skill": provenances[0] if len(provenances) == 1 else None, "skills": provenances} - - -def _check_codex_skill_collision(skills: Sequence[AgentSkill], task_files: Mapping[str, object]) -> None: - """Raise if a task seed file targets the same bundle dir as a runtime-injected codex skill. - - ``.agents/skills/`` holds skills from two independent, equally valid sources: the runtime - ``skills`` parameter (the A/B knob β€” staged into the workspace before the sandbox starts) and - the task's own ``files`` inputs (skills the task definition always ships β€” uploaded after it - starts). Tasks are free to seed their own skills there; only writing the *same* - ``.agents/skills//`` from both sources is a conflict, since the task upload lands second - and would overwrite the injected bundle, leaving the stamped provenance hash describing content - the agent never saw. Fail that case rather than emit a silently mislabeled A/B trial. - """ - for skill in skills: - injected_bundle = PurePosixPath(CODEX_SKILLS_DIR) / skill.name - for rel_path in task_files: - seed = PurePosixPath(rel_path) - if seed == injected_bundle or injected_bundle in seed.parents: - raise SkillInjectionError( - f"task seed file {str(rel_path)!r} writes into {str(injected_bundle)!r}, which is " - f"also injected as the runtime skill {skill.name!r}; the task upload would overwrite " - "the injected bundle. Inject this skill via the runtime ``skills`` parameter or ship " - "it in the task's files, not both" - ) - - -def _remove_injected_bundle(workspace_dir: Path, location: str) -> None: - """Remove the Codex-injected skill subtree from a downloaded ``workspace`` dir and prune emptied parents. - - ``location`` is workspace-relative (``.agents/skills/``). Best-effort and mirrors the host - runtime's cleanup: the skill was already captured in the run's trajectory, so SkillUsedMetric (which - reads the trace, not the workspace) is unaffected, and any filesystem error here must not fail an - otherwise-successful trial. - """ - if not workspace_dir.is_dir(): - return - workspace_root = workspace_dir.resolve() - injected = (workspace_dir / location).resolve() - # Guard against a location escaping the workspace (defensive; provenance is evaluator-authored). - if workspace_root not in injected.parents or not injected.exists(): - return - shutil.rmtree(injected, ignore_errors=True) - # Prune now-empty reserved parents (``.agents/skills``, ``.agents``) but never the workspace itself. - parent = injected.parent - while parent != workspace_root and parent.is_dir(): - try: - parent.rmdir() # only succeeds while empty - except OSError: - break - parent = parent.parent - - -def _section(config: Mapping[str, Any], name: str) -> dict[str, Any]: - """A top-level config section as a plain dict β€” ``{}`` when absent or not a mapping.""" - value = config.get(name) - return dict(value) if isinstance(value, Mapping) else {} - - -def _find_atif(relay_dir: Path) -> Path | None: - # Relay nests the trajectory under a per-run subdir (relay/runtime-/trajectory-*.atif.json), - # so search recursively rather than only relay's direct children. - if not relay_dir.is_dir(): - return None - matches = sorted(relay_dir.rglob("trajectory-*.atif.json")) - return matches[0] if matches else None - - -def _read_json(path: Path) -> JsonValue | None: - if not path.is_file(): - return None - # A truncated/binary/unreadable result (e.g. a crashed CLI that left partial or non-UTF-8 bytes) - # is treated as "no usable result" rather than propagating and aborting the batch. - try: - return json.loads(path.read_text(encoding="utf-8")) - except (json.JSONDecodeError, UnicodeDecodeError, OSError): - return None - - -def _read_text(path: Path) -> str: - try: - return path.read_text(encoding="utf-8") if path.is_file() else "" - except (UnicodeDecodeError, OSError): - return "" - - -def _result_error(payload: object) -> Mapping[str, object]: - if not isinstance(payload, Mapping): - return {"code": "FabricError", "message": "Fabric run did not produce a result"} - error = payload.get("error") - if isinstance(error, Mapping): - return {"stage": error.get("stage"), "code": error.get("code"), "message": error.get("message")} - return {"code": payload.get("status"), "message": "Fabric run did not succeed"} diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/hook_loading.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/hook_loading.py deleted file mode 100644 index 3c3d893dc1..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/hook_loading.py +++ /dev/null @@ -1,141 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Load :class:`FabricTaskRunHook` implementations from string references. - -Authors register hooks without baking agent-specific code into the platform. -YAML may point at: - -* ``ref`` β€” ``module.path:Attr`` (importable object) -* ``path`` + ``attr`` β€” Python file on disk (no package install required) -* ``entry_point`` / ``type`` β€” name under ``nemo.fabric.task_hooks`` - -Remaining mapping keys are forwarded as constructor kwargs. -""" - -import importlib -import importlib.metadata -import importlib.util -import sys -from collections.abc import Mapping -from pathlib import Path -from typing import Any - -from nemo_platform.beta.evaluator.agent_eval.runtimes.fabric.hooks import FabricTaskRunHook - -FABRIC_TASK_HOOKS_GROUP = "nemo.fabric.task_hooks" - -_RESERVED = frozenset({"ref", "path", "attr", "entry_point", "type"}) - - -class FabricTaskHookLoadError(RuntimeError): - """Raised when a Fabric task-hook reference cannot be resolved or constructed.""" - - -def load_fabric_task_hook(spec: Mapping[str, Any] | None) -> FabricTaskRunHook | None: - """Construct a task hook from a mapping, or return ``None`` when ``spec`` is unset.""" - if spec is None: - return None - if not isinstance(spec, Mapping): - raise FabricTaskHookLoadError("run_hook spec must be a mapping when set.") - - ref = _optional_str(spec.get("ref")) - path = _optional_str(spec.get("path")) - attr = _optional_str(spec.get("attr")) - entry_point = _optional_str(spec.get("entry_point")) or _optional_str(spec.get("type")) - - modes = [bool(ref), bool(path), bool(entry_point)] - if sum(modes) == 0: - raise FabricTaskHookLoadError( - "run_hook requires one of: ref (module:attr), path+attr (file), or entry_point/type (nemo.fabric.task_hooks)." - ) - if sum(modes) > 1: - raise FabricTaskHookLoadError("run_hook accepts only one of: ref, path, or entry_point/type.") - - if path and not attr: - raise FabricTaskHookLoadError("run_hook.path requires run_hook.attr (class or factory name).") - - if ref: - target = _load_from_ref(ref) - elif path: - target = _load_from_path(Path(path).expanduser(), attr=attr or "") - else: - target = _load_from_entry_point(entry_point or "") - - kwargs = {key: value for key, value in spec.items() if key not in _RESERVED} - return _construct_hook(target, kwargs) - - -def _construct_hook(target: Any, kwargs: dict[str, Any]) -> FabricTaskRunHook: - if callable(target) and not isinstance(target, type): - # Module-level factory function. - hook = target(**kwargs) if kwargs else target() - elif isinstance(target, type): - hook = target(**kwargs) if kwargs else target() - else: - if kwargs: - raise FabricTaskHookLoadError("run_hook target is already an instance; constructor kwargs are not allowed.") - hook = target - - for method in ("prepare", "after_success", "cleanup"): - if not callable(getattr(hook, method, None)): - raise FabricTaskHookLoadError(f"run_hook object missing required method {method!r}.") - return hook # type: ignore[return-value] - - -def _load_from_ref(ref: str) -> Any: - module_name, _, attr_path = ref.partition(":") - if not module_name or not attr_path: - raise FabricTaskHookLoadError(f"run_hook.ref must look like 'module.path:Attr', got {ref!r}.") - try: - module = importlib.import_module(module_name) - except ImportError as exc: - raise FabricTaskHookLoadError(f"Could not import run_hook.ref module {module_name!r}.") from exc - return _resolve_attr(module, attr_path, label=f"run_hook.ref {ref!r}") - - -def _load_from_path(path: Path, attr: str) -> Any: - resolved = path.resolve() - if not resolved.is_file(): - raise FabricTaskHookLoadError(f"run_hook.path does not exist: {resolved}") - module_name = f"_nemo_fabric_task_hook_{resolved.stem}_{abs(hash(str(resolved)))}" - spec = importlib.util.spec_from_file_location(module_name, resolved) - if spec is None or spec.loader is None: - raise FabricTaskHookLoadError(f"Could not load run_hook.path: {resolved}") - module = importlib.util.module_from_spec(spec) - sys.modules[module_name] = module - try: - spec.loader.exec_module(module) - except Exception as exc: - sys.modules.pop(module_name, None) - raise FabricTaskHookLoadError(f"Failed executing run_hook.path {resolved}: {exc}") from exc - return _resolve_attr(module, attr, label=f"run_hook.path attr {attr!r}") - - -def _load_from_entry_point(name: str) -> Any: - matches = [ep for ep in importlib.metadata.entry_points(group=FABRIC_TASK_HOOKS_GROUP) if ep.name == name] - if not matches: - raise FabricTaskHookLoadError( - f"No entry point {name!r} in group {FABRIC_TASK_HOOKS_GROUP!r}. " - "Authors register hooks via packaging entry points, or use run_hook.ref / run_hook.path." - ) - try: - return matches[0].load() - except Exception as exc: - raise FabricTaskHookLoadError(f"Failed to load entry point {name!r} from {FABRIC_TASK_HOOKS_GROUP!r}.") from exc - - -def _resolve_attr(module: Any, attr_path: str, label: str) -> Any: - current = module - for part in attr_path.split("."): - if not hasattr(current, part): - raise FabricTaskHookLoadError(f"{label} not found.") - current = getattr(current, part) - return current - - -def _optional_str(value: Any) -> str | None: - if value is None: - return None - text = str(value).strip() - return text or None diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/hooks.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/hooks.py deleted file mode 100644 index debda3c94a..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/hooks.py +++ /dev/null @@ -1,54 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Per-task lifecycle hooks for :class:`FabricAgentRuntime`. - -Fabric already accepts a complete typed config per ``Fabric.run``. These hooks -exist so callers (e.g. optimize trials) can wrap each task with agent-specific -ephemeral state β€” run-scoped MCP bindings, credential handoffs β€” without -baking that logic into the runtime or into Fabric itself. -""" - -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any, Protocol - -from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalTask - - -@dataclass -class FabricTaskRunSession: - """Mutable bag owned by a hook for one task invocation.""" - - state: dict[str, Any] = field(default_factory=dict) - - -class FabricTaskRunHook(Protocol): - """Optional prepare / after-success / cleanup around one Fabric task run.""" - - def prepare( - self, - config: Any, - task: AgentEvalTask, - evidence_dir: Path, - workspace_dir: Path, - session: FabricTaskRunSession, - ) -> Any: - """Return the config that should be passed to ``Fabric.run`` for this task. - - ``config`` is a composed ``nemo_fabric.FabricConfig`` (typed when Fabric is installed). - """ - - def after_success( - self, - task: AgentEvalTask, - result: Any, - session: FabricTaskRunSession, - ) -> dict[str, Any] | None: - """Optional extras merged into trial ``output.metadata`` / ``metadata`` on success. - - ``result`` is a Fabric ``RunResult``. Raise to fail the trial (e.g. analyzer audit failed). - """ - - def cleanup(self, session: FabricTaskRunSession) -> None: - """Always invoked in ``finally`` after the task attempt (success or failure).""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/hooks_mcp_binding.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/hooks_mcp_binding.py deleted file mode 100644 index 0c3ef853c7..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/hooks_mcp_binding.py +++ /dev/null @@ -1,440 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Platform Fabric task hook for per-task MCP bindings (path-first). - -**Static MCP (no hook):** declare ``mcp.servers`` (transport, url, exposure, env, args) in the -optimize / Fabric YAML. That is the hero path for fixed stdio MCP servers. - -**Bound MCP (this hook):** use when the agent needs run-scoped MCP state (private input -binding, audit/verify, optional credential handoff). Configure via:: - - eval: - run_hook: - type: mcp_run_binding - agent_src: ${AGENT_SRC} # path-first: checkout .../src on sys.path - bindings: - - server: my-mcp # must match mcp.servers key - binding: my_pkg.audit:RunBinding - executable: ${AGENT_MCP_BIN} # MCP process from agent's own venv - config_paths: [settings.yaml] - handoff: # optional; at most one per binding - env: NVIDIA_API_KEY - ref: my_pkg.handoff:CredentialHandoff - -``mcp.servers`` still owns transport / placeholder url / exposure / env / args. This hook -only rebinds ``url`` to ``binding.mcp_command`` after ``Binding.create``, preserving -top-level ``env`` and ``args``. - -**Agent protocol (duck-typed, in the agent checkout):** - -* ``Binding.create(prompt, parent, **kwargs) -> binding`` -* ``binding.mcp_command`` β€” path/URL for this task -* ``binding.verify()`` or ``verify_exactly_once()`` β€” fail the trial on audit breach -* ``binding.cleanup()`` -* Optional handoff: ``Handoff.start(credential, timeout_seconds=...)`` with - ``.socket_path`` / ``.token`` / ``.close()`` - -Path isolation: do **not** pip-install the agent into the platform venv. Point -``agent_src`` at the checkout and ``executable`` at the agent-owned MCP binary. -Binding/handoff modules load into the platform process β€” keep them lightly dependent; -heavy runtime stays behind the MCP stdio boundary. -""" - -from __future__ import annotations - -import importlib -import importlib.util -import inspect -import json -import logging -import os -import sys -from collections.abc import Mapping, Sequence -from pathlib import Path -from typing import Any - -logger = logging.getLogger(__name__) - - -class McpRunBindingHookError(RuntimeError): - """Raised when MCP run-binding configuration or lifecycle fails.""" - - -def _load_ref(ref: str) -> Any: - """Load ``module.path:Attr`` or ``/abs/or/rel/file.py:Attr``.""" - module_name, _, attr = ref.partition(":") - if not module_name or not attr: - raise McpRunBindingHookError(f"ref must look like 'module.path:Attr' or 'file.py:Attr', got {ref!r}") - - path = Path(module_name).expanduser() - if path.suffix == ".py" or path.is_file(): - resolved = path.resolve() - if not resolved.is_file(): - raise McpRunBindingHookError(f"ref file does not exist: {resolved}") - mod_name = f"_mcp_run_binding_{resolved.stem}_{abs(hash(str(resolved)))}" - spec = importlib.util.spec_from_file_location(mod_name, resolved) - if spec is None or spec.loader is None: - raise McpRunBindingHookError(f"could not load ref file: {resolved}") - module = importlib.util.module_from_spec(spec) - sys.modules[mod_name] = module - spec.loader.exec_module(module) - else: - module = importlib.import_module(module_name) - - current: Any = module - for part in attr.split("."): - current = getattr(current, part) - return current - - -def _resolve_target(value: Any) -> Any: - """Resolve a string ref or pass through an already-imported class/callable.""" - if isinstance(value, str): - return _load_ref(value.strip()) - if value is None: - raise McpRunBindingHookError("binding/handoff ref is required") - return value - - -def _prepend_sys_path(path: str | Path) -> None: - resolved = str(Path(path).expanduser().resolve()) - if resolved not in sys.path: - sys.path.insert(0, resolved) - - -def _as_path_list(value: Any) -> list[Path]: - if value is None: - return [] - if isinstance(value, (str, Path)): - items: Sequence[Any] = [value] - elif isinstance(value, Sequence): - items = value - else: - raise McpRunBindingHookError(f"config_paths must be a path or list of paths, got {type(value)!r}") - paths: list[Path] = [] - for item in items: - path = Path(item).expanduser() - if not path.is_file(): - raise McpRunBindingHookError(f"config path does not exist: {path}") - paths.append(path.resolve()) - return paths - - -def _filter_kwargs(fn: Any, kwargs: dict[str, Any]) -> dict[str, Any]: - try: - params = inspect.signature(fn).parameters - except (TypeError, ValueError): - return kwargs - if any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values()): - return kwargs - return {key: value for key, value in kwargs.items() if key in params} - - -def _as_str_list(value: Any) -> list[str]: - if value is None: - return [] - if isinstance(value, str): - raise McpRunBindingHookError("MCP server args must be a list of strings, not a string") - if isinstance(value, Sequence): - return [str(item) for item in value] - raise McpRunBindingHookError(f"MCP server args must be a sequence of strings, got {type(value)!r}") - - -def _as_str_map(value: Any) -> dict[str, str]: - if value is None: - return {} - if not isinstance(value, Mapping): - raise McpRunBindingHookError(f"MCP server env must be a mapping, got {type(value)!r}") - return {str(key): str(item) for key, item in value.items()} - - -def _server_snapshot(config: Any, name: str) -> dict[str, Any]: - """Return preserved ``add_mcp_server`` kwargs for an existing MCP server. - - Fabric now owns ``env`` / ``args`` as top-level MCP server fields (not - ``extra_fields``). Legacy snapshots that still stash them under - ``extra_fields`` are lifted to top-level kwargs. - """ - mcp = getattr(config, "mcp", None) - servers = getattr(mcp, "servers", None) or {} - server = servers.get(name) if isinstance(servers, Mapping) else None - if server is None: - return {"transport": "stdio", "exposure": "harness_native"} - - transport = str(getattr(server, "transport", None) or "stdio") - exposure = str(getattr(server, "exposure", None) or "harness_native") - - extra: dict[str, Any] = {} - extra_fields = getattr(server, "extra_fields", None) - if isinstance(extra_fields, Mapping): - extra = dict(extra_fields) - elif callable(extra_fields): - extra = dict(extra_fields()) - elif hasattr(server, "model_extra") and isinstance(server.model_extra, Mapping): - extra = dict(server.model_extra) - - args = _as_str_list(getattr(server, "args", None)) - if not args and "args" in extra: - args = _as_str_list(extra.pop("args")) - - env = _as_str_map(getattr(server, "env", None)) - if not env and "env" in extra: - env = _as_str_map(extra.pop("env")) - - snapshot: dict[str, Any] = {"transport": transport, "exposure": exposure} - if args: - snapshot["args"] = args - if env: - snapshot["env"] = env - if extra: - snapshot["extra_fields"] = extra - return snapshot - - -def _verify_binding(binding: Any) -> Any: - verify = getattr(binding, "verify", None) - if callable(verify): - return verify() - verify_once = getattr(binding, "verify_exactly_once", None) - if callable(verify_once): - try: - return verify_once() - except Exception as exc: - # Agents sometimes re-call the tool after a successful analysis. Prefer the - # audited analysis over failing the whole optimize sample when one exists. - fallback = _audit_from_binding_path(binding) - if fallback is not None and _result_payload(fallback) is not None: - logger.warning( - "MCP binding exactly-once verify failed (%s); using audit analysis anyway", - exc, - ) - return fallback - raise McpRunBindingHookError(str(exc)) from exc - raise McpRunBindingHookError("binding has neither verify() nor verify_exactly_once()") - - -def _audit_from_binding_path(binding: Any) -> Any | None: - """Best-effort read of ``binding.audit_path`` when strict verify fails.""" - path = getattr(binding, "audit_path", None) - if path is None: - return None - audit_path = Path(path) - if not audit_path.is_file(): - return None - try: - payload = json.loads(audit_path.read_text(encoding="utf-8")) - except (OSError, ValueError): - return None - if not isinstance(payload, Mapping): - return None - - class _AuditShim: - def __init__(self, data: Mapping[str, Any]) -> None: - self._data = dict(data) - self.analysis = data.get("analysis") - self.result = data.get("result") - - def public_mapping(self) -> dict[str, Any]: - return {key: self._data[key] for key in ("run_id", "input_sha256", "invocation_count") if key in self._data} - - return _AuditShim(payload) - - -def _audit_mapping(audit: Any) -> dict[str, Any] | None: - public = getattr(audit, "public_mapping", None) - if callable(public): - mapping = public() - return dict(mapping) if isinstance(mapping, Mapping) else {"value": mapping} - if isinstance(audit, Mapping): - return dict(audit) - return None - - -def _result_payload(audit: Any) -> Any: - for attr in ("analysis", "result"): - value = getattr(audit, attr, None) - if value is None: - continue - dump = getattr(value, "model_dump", None) - if callable(dump): - return dump(mode="json") - return value - return None - - -class McpRunBindingHook: - """Ordered per-task MCP binding lifecycle around ``Fabric.run``.""" - - def __init__( - self, - bindings: Sequence[Mapping[str, Any]] | None = None, - *, - agent_src: str | Path | None = None, - pythonpath: str | Path | None = None, - binding_parent: str | Path | None = None, - ) -> None: - src = agent_src if agent_src is not None else pythonpath - if src is not None: - _prepend_sys_path(src) - - if not bindings: - raise McpRunBindingHookError("mcp_run_binding requires a non-empty bindings list") - - self._binding_parent = Path(binding_parent).expanduser() if binding_parent else None - self._entries: list[dict[str, Any]] = [] - for index, raw in enumerate(bindings): - if not isinstance(raw, Mapping): - raise McpRunBindingHookError(f"bindings[{index}] must be a mapping") - server = str(raw.get("server") or "").strip() - if not server or raw.get("binding") is None: - raise McpRunBindingHookError(f"bindings[{index}] requires server and binding") - - handoff_raw = raw.get("handoff") - handoff_env: str | None = None - handoff_cls: Any | None = None - if handoff_raw is not None: - if not isinstance(handoff_raw, Mapping): - raise McpRunBindingHookError(f"bindings[{index}].handoff must be a mapping") - handoff_env = str(handoff_raw.get("env") or "").strip() or None - handoff_ref = handoff_raw.get("ref") - if not handoff_env or handoff_ref is None: - raise McpRunBindingHookError(f"bindings[{index}].handoff requires env and ref") - try: - handoff_cls = _resolve_target(handoff_ref) - except Exception as exc: - raise McpRunBindingHookError( - f"Could not resolve bindings[{index}].handoff.ref={handoff_ref!r}" - ) from exc - - binding_raw = raw.get("binding") - try: - binding_cls = _resolve_target(binding_raw) - except Exception as exc: - raise McpRunBindingHookError( - f"Could not resolve bindings[{index}].binding={binding_raw!r}. " - "Set agent_src to the agent checkout .../src (path-first; do not install " - "the agent into the platform venv)." - ) from exc - - executable_raw = raw.get("executable") - executable = Path(executable_raw).expanduser() if executable_raw else None - if executable is not None and not executable.is_file(): - raise McpRunBindingHookError(f"bindings[{index}].executable does not exist: {executable}") - - config_paths = _as_path_list(raw.get("config_paths") or raw.get("config_path")) - - self._entries.append( - { - "server": server, - "binding_cls": binding_cls, - "handoff_cls": handoff_cls, - "handoff_env": handoff_env, - "executable": executable.resolve() if executable is not None else None, - "config_paths": config_paths, - } - ) - - def prepare(self, config: Any, task: Any, evidence_dir: Path, workspace_dir: Path, session: Any) -> Any: - del workspace_dir - if not hasattr(config, "add_mcp_server"): - raise McpRunBindingHookError("Fabric config does not expose add_mcp_server; cannot rebind MCP.") - - prompt = task.agent_prompt() - parent = self._binding_parent or (evidence_dir / "mcp-bindings") - parent.mkdir(parents=True, exist_ok=True) - - started: list[dict[str, Any]] = [] - session.state["mcp_bindings"] = started - - try: - for entry in self._entries: - handoff = None - handoff_cls = entry["handoff_cls"] - handoff_env = entry["handoff_env"] - if handoff_cls is not None and handoff_env: - credential = os.environ.get(handoff_env) - if credential: - handoff = handoff_cls.start(credential, timeout_seconds=60.0) - - create_kwargs: dict[str, Any] = { - "credential_socket": handoff.socket_path if handoff is not None else None, - "credential_token": handoff.token if handoff is not None else None, - } - if entry["executable"] is not None: - create_kwargs["executable"] = entry["executable"] - config_paths: list[Path] = entry["config_paths"] - if config_paths: - create_kwargs["config_paths"] = config_paths - create_kwargs["config_path"] = config_paths[0] - - try: - binding = entry["binding_cls"].create( - prompt, - parent, - **_filter_kwargs(entry["binding_cls"].create, create_kwargs), - ) - except Exception: - if handoff is not None: - handoff.close() - raise - - # Register before rebinding so prepare failures can still cleanup. - started.append({"server": entry["server"], "binding": binding, "handoff": handoff}) - preserved = _server_snapshot(config, entry["server"]) - config = config.add_mcp_server( - entry["server"], - url=str(binding.mcp_command), - **preserved, - ) - except Exception: - self.cleanup(session) - raise - - return config - - def after_success(self, task: Any, result: Any, session: Any) -> dict[str, Any] | None: - del task, result - started = session.state.get("mcp_bindings") or [] - if not started: - raise McpRunBindingHookError("mcp bindings missing after Fabric.run") - - mcp_bindings: dict[str, Any] = {} - first_result: Any = None - for item in started: - server = item["server"] - binding = item["binding"] - audit = _verify_binding(binding) - entry_extras: dict[str, Any] = {} - mapping = _audit_mapping(audit) - if mapping is not None: - entry_extras["audit"] = mapping - payload = _result_payload(audit) - if payload is not None: - entry_extras["result"] = payload - if first_result is None: - first_result = payload - mcp_bindings[server] = entry_extras - - extras: dict[str, Any] = {"mcp_bindings": mcp_bindings} - # Deprecated alias for one release β€” FabricAgentRuntime historically read this key. - if first_result is not None: - extras["analyzer_analysis"] = first_result - return extras - - def cleanup(self, session: Any) -> None: - started: list[dict[str, Any]] = list(session.state.pop("mcp_bindings", []) or []) - for item in reversed(started): - binding = item.get("binding") - handoff = item.get("handoff") - server = item.get("server") - try: - if binding is not None: - binding.cleanup() - except Exception: - logger.exception("Failed to cleanup MCP binding for %s", server) - try: - if handoff is not None: - handoff.close() - except Exception: - logger.exception("Failed to close MCP handoff for %s", server) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/image.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/image.py deleted file mode 100644 index 6d218c8589..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/image.py +++ /dev/null @@ -1,173 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Build-if-missing provisioning for the Fabric sandbox image. - -``FabricContainerRuntime`` needs a container image with Fabric + its harness runtimes. Rather than -make callers hand-write a Dockerfile, the SDK owns the recipe (:mod:`sandbox.Dockerfile`, a -multi-stage build) and provisions the image opaquely: :func:`ensure_fabric_image` returns a usable -image tag, building it only when it isn't already present locally. This mirrors the -``ensure_task_image`` build-if-missing pattern (``docker image inspect`` β†’ ``docker build``). - -The tag is content-addressed on the recipe + selected extras, so a recipe change produces a new tag -(cache-bust) and an unchanged recipe reuses the cached image. The Fabric source is private/native -(no public wheel), so the build needs a local NeMo-Fabric checkout β€” resolved from ``fabric_repo`` / -``$NEMO_FABRIC_REPO`` / ``~/workspace/NeMo-Fabric``. Only the maturin build inputs are staged into the -context (not the whole repo), and the multi-stage build keeps the source and Rust toolchain out of the -final image. - -This is the local-Docker provisioning path. The intended evolution is a remote image registry as a -cache: :func:`ensure_fabric_image` keeps the same "return a usable tag" contract, its body swapping -local build for a registry pull (build-and-push on miss). - -DEPENDENCY (as of July 2026): the multi-stage image installs the ``nemo-fabric`` wheel and discards -the source, so Fabric must be able to resolve built-in adapters *from the installed distribution*. -That only works on NeMo-Fabric's ``installed-adapter-discovery`` branch (which bundles the adapters -under ``python/src/nemo_fabric/adapters`` and adds ``AdapterDescriptorSource::Installed``). On today's -``main`` the wheel ships no adapter descriptors, so a wheel-only image cannot resolve e.g. -``nvidia.fabric.hermes``. Once that lands on ``main``, switch to installing the top-level -``adapters/*`` packages explicitly here instead of relying on the branch's packaging. -""" - -from __future__ import annotations - -import hashlib -import logging -import os -import shutil -import subprocess -import tempfile -from pathlib import Path - -logger = logging.getLogger(__name__) - -# ``localhost/`` prefix so Docker treats it as an explicit local registry and does NOT qualify the tag -# to ``docker.io/…`` β€” this image is built locally and never pushed to Docker Hub. -DEFAULT_FABRIC_IMAGE_REPO = "localhost/nemo-evaluator/fabric-sandbox" -FABRIC_REPO_ENV = "NEMO_FABRIC_REPO" -_DEFAULT_FABRIC_REPO = Path.home() / "workspace" / "NeMo-Fabric" -_DOCKERFILE = Path(__file__).with_name("sandbox.Dockerfile") - -# Bound the docker subprocess calls so an unresponsive daemon fails fast instead of hanging the runtime. -# ``inspect`` is near-instant; the build compiles nemo-fabric (minutes), so it gets a generous ceiling. -_INSPECT_TIMEOUT_S = 30 -_BUILD_TIMEOUT_S = 3600 - -#: Harness runtime deps baked into the (single, harness-agnostic) Fabric image. The native -#: ``nemo-fabric`` build plus *all* built-in adapter descriptors are always present, so the CLI can -#: resolve any built-in harness; these extras add the per-harness *runtime* deps (``hermes`` β†’ -#: ``hermes-agent``; ``relay`` β†’ the ATIF exporter). Codex additionally needs node + the codex CLI + -#: the nemo-relay gateway binary and is not provisioned yet (see AALGO-321); append it here when ready. -_EXTRAS: tuple[str, ...] = ("hermes", "relay") - -#: Paths under the NeMo-Fabric checkout that the maturin build actually needs. Staging only these -#: (rather than the whole repo) keeps the build context small; the multi-stage build keeps them out -#: of the final image entirely. -_BUILD_SOURCE_PATHS = ("Cargo.toml", "Cargo.lock", "pyproject.toml", "README.md", "crates", "python") - - -class FabricImageError(RuntimeError): - """Raised when the Fabric sandbox image cannot be provisioned.""" - - -def _extras_arg() -> str: - return ",".join(_EXTRAS) - - -def fabric_image_tag(*, repo: str = DEFAULT_FABRIC_IMAGE_REPO) -> str: - """Content-addressed tag for the harness-agnostic Fabric image: ``:``. - - Not keyed by harness: one Fabric install + the bundled adapters runs any built-in harness, so the - image is the same regardless of which harness a task's config selects. - """ - recipe = _DOCKERFILE.read_bytes() + _extras_arg().encode("utf-8") - return f"{repo}:{hashlib.sha256(recipe).hexdigest()[:12]}" - - -def image_exists(tag: str, *, docker_bin: str = "docker") -> bool: - """Whether an image tag is present in the local Docker image store. - - Raises :class:`FabricImageError` when the Docker daemon is unreachable, so a stopped/misconfigured - daemon surfaces as a clear error instead of masquerading as "image absent" and triggering a build - that then also fails confusingly. - """ - try: - result = subprocess.run( - [docker_bin, "image", "inspect", tag], capture_output=True, check=False, timeout=_INSPECT_TIMEOUT_S - ) - except subprocess.TimeoutExpired as exc: - raise FabricImageError( - f"`docker image inspect` timed out after {_INSPECT_TIMEOUT_S}s (daemon unresponsive?)" - ) from exc - if result.returncode == 0: - return True - stderr = result.stderr.decode("utf-8", errors="replace") - if "cannot connect to the docker daemon" in stderr.lower(): - raise FabricImageError(f"cannot reach the Docker daemon (is it running?): {stderr.strip()}") - logger.debug("Fabric image %s not present in local store", tag) - return False - - -def _resolve_fabric_repo(fabric_repo: str | Path | None) -> Path: - default = Path(os.environ.get(FABRIC_REPO_ENV, _DEFAULT_FABRIC_REPO)) - repo = (Path(fabric_repo) if fabric_repo is not None else default).expanduser() - if not (repo / "pyproject.toml").is_file(): - raise FabricImageError( - f"NeMo-Fabric source not found at {repo}. The Fabric image is built from source " - f"(no public wheel); set {FABRIC_REPO_ENV} or pass fabric_repo to point at a checkout." - ) - return repo - - -def _stage_source(repo: Path, dest: Path) -> None: - """Copy only the maturin build inputs from ``repo`` into ``dest`` (not the whole checkout).""" - dest.mkdir(parents=True, exist_ok=True) - for name in _BUILD_SOURCE_PATHS: - src = repo / name - if src.is_dir(): - shutil.copytree(src, dest / name, ignore=shutil.ignore_patterns("target", "__pycache__", "*.whl")) - elif src.is_file(): - shutil.copy2(src, dest / name) - else: - raise FabricImageError(f"expected Fabric build input {name!r} not found under {repo}") - - -def ensure_fabric_image( - *, - fabric_repo: str | Path | None = None, - docker_bin: str = "docker", - force_build: bool = False, -) -> str: - """Return a usable Fabric image tag, building it only if not already present. - - One harness-agnostic image serves every built-in harness. Idempotent and content-addressed: an - unchanged recipe reuses the cached image; a changed recipe yields a new tag. Builds from a staged - copy of the local NeMo-Fabric source (build inputs only). - """ - tag = fabric_image_tag() - if not force_build and image_exists(tag, docker_bin=docker_bin): - logger.debug("Fabric image %s already present; skipping build.", tag) - return tag - - repo = _resolve_fabric_repo(fabric_repo) - logger.info( - "Building Fabric image (first build compiles nemo-fabric; this can take minutes)...", - extra=dict(tag=tag, repo=repo), - ) - with tempfile.TemporaryDirectory(prefix="nemo-fabric-image-") as ctx_dir: - ctx = Path(ctx_dir) - _stage_source(repo, ctx / "nemo-fabric") - shutil.copy2(_DOCKERFILE, ctx / "Dockerfile") - try: - subprocess.run( - [docker_bin, "build", "--build-arg", f"EXTRAS={_extras_arg()}", "-t", tag, str(ctx)], - check=True, - env={**os.environ, "DOCKER_BUILDKIT": "1"}, - timeout=_BUILD_TIMEOUT_S, - ) - except subprocess.TimeoutExpired as exc: - raise FabricImageError(f"docker build timed out after {_BUILD_TIMEOUT_S}s for {tag}") from exc - except subprocess.CalledProcessError as exc: - raise FabricImageError(f"docker build failed for {tag}: {exc}") from exc - logger.info("Built Fabric image %s.", tag, extra=dict(tag=tag)) - return tag diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py deleted file mode 100644 index 8195b9f2fe..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py +++ /dev/null @@ -1,823 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""NeMo Fabric-backed agent-eval runtime. - -``FabricAgentRuntime`` drives an agent harness (Codex, Hermes, ...) through the -NeMo Fabric Python SDK and adapts each normalized Fabric ``RunResult`` into an -:class:`AgentEvalTrial`. The harness is chosen by the supplied Fabric config's -``harness.adapter_id`` (never inferred from a model); an optional ``model`` slug -is applied as the config's default model, mirroring Fabric's own Harbor integration. - -Per-task settings (workspace, model, trajectory capture) are composed directly onto -a copy of the supplied config via the SDK's config helpers (``model_copy`` + -``enable_relay`` + ``environment``). Fabric removed profile overlays in 0.1.0rc2 β€” -``FabricConfig`` rejects a ``profiles`` key and ``Fabric.run`` takes no ``profiles`` -argument β€” so a run is described by exactly one complete typed config, and the -evaluator-owned per-task settings are authoritative simply by being applied last. - -Every task runs in its own fresh workspace: the runtime seeds it from -``inputs['files']`` (a no-op when there are none), runs the harness in it (via -``environment.workspace``), and exposes its final file tree as ``workspace`` -filesystem evidence, so workspace-reading metrics score a Fabric trial alongside -the ATIF trajectory. Any ``environment.workspace`` set in the supplied config is -overridden per task. - -``nemo_fabric`` is an optional native dependency: its types are imported for -annotations under ``TYPE_CHECKING`` and the package is loaded lazily at runtime, -so this module stays importable without it. -""" - -from __future__ import annotations - -import asyncio -import copy -import json -import logging -import shutil -from collections.abc import Mapping, Sequence -from datetime import UTC, datetime -from pathlib import Path -from typing import TYPE_CHECKING, Any -from uuid import uuid4 - -from nemo_platform.beta.evaluator.agent_eval.runtimes.fabric import _common -from nemo_platform.beta.evaluator.agent_eval.runtimes.fabric.hooks import FabricTaskRunHook, FabricTaskRunSession -from nemo_platform.beta.evaluator.agent_eval.runtimes.fabric.skills import ( - SKILL_MODE_CODEX_SKILLS_DIR, - AgentSkill, - SkillMode, - SkillProvenance, - SkillSet, - install_skills, - resolve_skill_mode, -) -from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask -from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput, RunnerInfo -from nemo_platform.beta.evaluator.agent_eval.workspace_seeds import SEED_FILES_INPUT_KEY, seed_workspace -from nemo_platform.beta.evaluator.values.atif import FinalMetrics -from nemo_platform.beta.evaluator.values.evidence import ( - EVIDENCE_FORMAT_ATIF, - EVIDENCE_TRACE, - CandidateEvidence, - EvidenceDescriptor, -) -from pydantic import JsonValue, ValidationError - -if TYPE_CHECKING: - # Annotations use nemo_fabric's real types (single source of truth). nemo_fabric is an optional - # native package not yet in our locked dependency set, so it is imported for typing only and - # loaded lazily at runtime (see ``run_tasks``). Drop the ty:ignore once nemo-fabric is a - # resolvable dependency and the checker can see it. - from nemo_fabric import ( # ty: ignore[unresolved-import] - Fabric, - FabricConfig, - RelayObservabilityConfig, - RunOutput, - RunResult, - ) - -DEFAULT_FABRIC_TIMEOUT_S = 600 -_RUNTIME_NAME = "fabric" -_MISSING_FABRIC_MSG = "FabricAgentRuntime requires the `nemo-fabric` package (native NeMo Fabric SDK)." -_MISSING_RELAY_MSG = ( - "FabricAgentRuntime trajectory capture requires the `nemo-relay` package " - "(install `nemo-fabric[relay]`), or set capture_trajectory=False." -) - -logger = logging.getLogger(__name__) - -# Evidence-dir layout for trajectory capture. These subdir names are our own local layout β€” we create -# them and hand them to Fabric/Relay, so they are not derived from either library. -_RELAY_SUBDIR = "relay" -_ARTIFACTS_SUBDIR = "artifacts" -# Per-task workspace: where seed files are staged and where the harness reads/writes. We -# create it, point Fabric's ``environment.workspace`` at it, and expose it as ``workspace`` evidence. -_WORKSPACE_SUBDIR = "workspace" -# Per-task skill staging dir (native injection): the skill's files are resolved here and the staged -# root is added to the task config's ``skills.paths``. For codex self-injection the skill lands in the -# workspace instead (no path added). -_SKILL_SUBDIR = "skill" -# Sentinel skill path attached only to probe Fabric's capability planner for the selected adapter's -# skills routing (see ``_resolve_skill_mode``). Never staged and need not exist on disk β€” the planner -# just reports how it would route a skill for this adapter. -_SKILL_PROBE_PATH = "nemo-eval-skill-capability-probe" -# Evidence key + descriptor kind for the staged workspace, consumed by the -# workspace-reading metrics. -_WORKSPACE_EVIDENCE_KEY = "workspace" -_WORKSPACE_EVIDENCE_KIND = "filesystem" -# File-exporter output names we choose for the Relay ATIF/ATOF trajectory (Relay accepts these as inputs). -_ATIF_FILENAME_TEMPLATE = "trajectory-{session_id}.atif.json" -_ATOF_FILENAME = "events.atof.jsonl" -# ``kind`` Fabric stamps on the promoted Relay ATIF artifact; used to surface it as trace evidence. -_ATIF_ARTIFACT_KIND = "atif" - - -class FabricAgentRuntime: - """AgentTaskRunner that generates trials by running tasks through NeMo Fabric. - - The harness is selected entirely by ``config["harness"]["adapter_id"]``. Across harnesses the - config shape differs mainly in that ``adapter_id``, optional input/output schemas, and any - harness-specific ``harness.settings``. Fabric owns each adapter's execution mechanism. See - ``examples/fabric_harness_runtimes.py`` for full Codex and Hermes config examples. - """ - - def __init__( - self, - *, - config: Mapping[str, Any], - model: str | None = None, - base_dir: str | Path | None = None, - work_root: str | Path | None = None, - timeout_s: int = DEFAULT_FABRIC_TIMEOUT_S, - capture_trajectory: bool = True, - trajectory_extra: Mapping[str, Any] | None = None, - runtime_name: str = _RUNTIME_NAME, - skills: Sequence[AgentSkill] | None = None, - task_hook: FabricTaskRunHook | None = None, - ) -> None: - self._config = config - self._model = model - self._base_dir = Path(base_dir).expanduser() if base_dir is not None else None - self._work_root = Path(work_root).expanduser() if work_root is not None else None - self._timeout_s = timeout_s - self._capture_trajectory = capture_trajectory - self._trajectory_extra = dict(trajectory_extra) if trajectory_extra else None - self._runtime_name = runtime_name - self._skill_set = SkillSet(tuple(skills or ())) - self._task_hook = task_hook - - def with_skills(self, skills: Sequence[AgentSkill]) -> FabricAgentRuntime: - """Return a copy of this runtime with ``skills`` *added* to its skill set; ``self`` is not modified. - - Additive and chainable: ``rt.with_skills([a]).with_skills([b])`` injects both a and b. Lets an A/B - eval derive a treated runtime from a skill-free baseline (``baseline.with_skills(the_skills)``) so - the two arms differ in exactly the injected skills. Skill names must be unique across the combined - set β€” two bundles claiming the same ``/`` would collide β€” so re-adding a skill already - present raises. A shallow copy suffices β€” the shared fields are immutable config/paths. - """ - clone = copy.copy(self) - clone._skill_set = self._skill_set.with_skills(skills) - return clone - - def with_skill(self, skill: AgentSkill) -> FabricAgentRuntime: - """Return a copy of this runtime with ``skill`` *added*; ``self`` is not modified. - - Thin wrapper over :meth:`with_skills` for the common single-skill case; equally chainable - (``rt.with_skill(a).with_skill(b)`` injects both). - """ - return self.with_skills([skill]) - - def _adapter_id(self) -> str: - """Harness adapter selected by the Fabric config (empty when unset).""" - harness = self._config.get("harness") if isinstance(self._config, Mapping) else None - adapter_id = harness.get("adapter_id") if isinstance(harness, Mapping) else None - return str(adapter_id) if adapter_id is not None else "" - - def _effective_model(self) -> str | None: - """The model a run will actually use, mirroring :meth:`_compose_config`'s precedence. - - ``_compose_config`` only overwrites the config's default model when ``self._model`` is set, so - a model supplied purely through ``config`` is what runs. Reporting ``self._model`` alone would - record ``None`` for those runs, giving two runs with *different* models identical provenance β€” - the one thing this metadata exists to prevent. - """ - if self._model: - return self._model - models = self._config.get("models") if isinstance(self._config, Mapping) else None - default = models.get("default") if isinstance(models, Mapping) else None - model = default.get("model") if isinstance(default, Mapping) else getattr(default, "model", None) - return str(model) if model is not None else None - - def runner_info(self) -> RunnerInfo: - """Identify this runner and the Fabric settings that shape its results.""" - return RunnerInfo( - name=self._runtime_name, - kind="runner", - config={ - "model": self._effective_model(), - "timeout_s": self._timeout_s, - "adapter_id": self._adapter_id(), - "skills": [skill.name for skill in self._skill_set.skills], - # Off means no relay/ATIF exporter, so the run captures no trajectory evidence β€” a - # metric that scores trajectories sees something different. - "capture_trajectory": self._capture_trajectory, - }, - ) - - async def run_tasks( - self, - tasks: Sequence[AgentEvalTask], - config: AgentEvalRunConfig | None = None, - ) -> Sequence[AgentEvalTrial]: - try: - # nemo_fabric ships a native (pyo3) core and is an optional dependency, so it is imported - # lazily here rather than at module load. - from nemo_fabric import Fabric, FabricConfig # ty: ignore[unresolved-import] - except ImportError as exc: - raise RuntimeError(_MISSING_FABRIC_MSG) from exc - - resolved_config = config or AgentEvalRunConfig() - # Assign a run id once per run so two runs (e.g. an A/B baseline vs. skilled variant) written - # under the same work_root/output_dir land in distinct, non-colliding evidence trees. Callers - # that set run_id keep their identifier. - if resolved_config.run_id is None: - resolved_config = resolved_config.model_copy(update={"run_id": _new_run_id()}) - agent_config = FabricConfig.from_mapping(self._config) - # Fail fast (once) if trajectory capture is requested but the nemo-relay gateway isn't - # importable, rather than failing every task the same way inside the per-task guard. - if self._capture_trajectory: - try: - import nemo_relay.observability # noqa: F401 # ty: ignore[unresolved-import] - except ImportError as exc: - raise RuntimeError(_MISSING_RELAY_MSG) from exc - # ``Fabric`` (formerly ``FabricClient``) is a lightweight, reusable facade β€” not a lifecycle - # context manager β€” so it is created once and reused across tasks with no cleanup. - client = Fabric() - - # Resolve once how a skill would reach this harness (the adapter is constant across tasks) by - # asking Fabric's own capability planner, so any adapter that declares native skills support β€” ours - # or an end-user's β€” is picked up automatically instead of via a hardcoded allow-list. Fail fast - # rather than silently run a skill-free trial mislabeled as "with skill", which would corrupt an - # A/B comparison. Only touched when a skill is set, so the no-skill path is unaffected. - skill_mode: SkillMode | None = None - if self._skill_set.skills: - skill_mode = self._resolve_skill_mode(client, agent_config) - if skill_mode is None: - adapter_id = agent_config.harness.adapter_id - raise RuntimeError( - f"FabricAgentRuntime received one or more skills but adapter {adapter_id!r} has no known " - "skill-injection strategy: Fabric does not route skills to it natively and it is not a " - "codex harness. Use a skills-native or codex harness, or drop the skills." - ) - - semaphore = asyncio.Semaphore(resolved_config.parallelism) - - async def run_one(index: int, task: AgentEvalTask) -> AgentEvalTrial: - async with semaphore: - return await self._run_task(client, agent_config, index, task, resolved_config, skill_mode) - - return await asyncio.gather(*(run_one(index, task) for index, task in enumerate(tasks))) - - def _resolve_skill_mode(self, client: Fabric, agent_config: FabricConfig) -> SkillMode | None: - """Ask Fabric how a skill would reach the selected harness, or ``None`` if it can't. - - Probes Fabric's capability planner: plan a copy of the config with a sentinel skill path attached - (it need not exist on disk) and read how the adapter routes skills. Querying the authoritative - source at runtime means adapters that declare native skills support β€” ours or an end-user's β€” are - detected without a hardcoded list. See :func:`~...skills.resolve_skill_mode`. - """ - probe_config = agent_config.model_copy(deep=True) - probe_config.add_skill_path(_SKILL_PROBE_PATH) - plan = client.plan(probe_config, base_dir=self._base_dir) - return resolve_skill_mode( - capability_plan=plan.capability_plan, - adapter_id=agent_config.harness.adapter_id, - ) - - async def _run_task( - self, - client: Fabric, - agent_config: FabricConfig, - index: int, - task: AgentEvalTask, - config: AgentEvalRunConfig, - skill_mode: SkillMode | None, - ) -> AgentEvalTrial: - # nemo_fabric is already imported+validated in ``run_tasks``; this is a cached sys.modules - # lookup, not a re-load, so the types are used where they're constructed instead of threaded down. - from nemo_fabric import RunRequest # ty: ignore[unresolved-import] - - evidence_dir = self._evidence_dir(index, task, config) - evidence_dir.mkdir(parents=True, exist_ok=True) - - # Every task runs in its own fresh workspace: seed any ``inputs['files']`` into it (a no-op when - # there are none), point the harness at it, and expose it as ``workspace`` filesystem evidence β€” - # a uniform per-task dir that maps cleanly onto a per-task container volume later. Seeding runs - # inside the guarded block so a bad seed (a path escaping the workspace, an unresolvable fileset) - # fails just this task, not the whole run; it is synchronous and may block (a fileset handler - # downloads), so it is offloaded off the shared event loop. - workspace_dir = evidence_dir / _WORKSPACE_SUBDIR - workspace_dir.mkdir(parents=True, exist_ok=True) - skill_provenances: list[SkillProvenance] = [] - hook_session = FabricTaskRunSession() - hook_extras: dict[str, Any] | None = None - try: - # Stage seed files into the workspace for their on-disk side effect; the prompt is the task - # instruction only, so the returned paths are unused. - await asyncio.to_thread(seed_workspace, workspace_dir, task.inputs.get(SEED_FILES_INPUT_KEY)) - - # Inject the skill set (if any) for this task. A native harness gets each staged bundle added - # to the config's ``skills.paths``; codex self-injection stages each bundle into the - # workspace and adds no path. One provenance per skill is stamped on the trial for the A/B - # diff. Blocking file I/O, off the event loop. - skill_paths: list[str] = [] - if self._skill_set.skills and skill_mode is not None: - installation = await asyncio.to_thread( - install_skills, - skills=self._skill_set.skills, - adapter_id=agent_config.harness.adapter_id, - mode=skill_mode, - workspace_dir=workspace_dir, - skill_stage_dir=(evidence_dir / _SKILL_SUBDIR).resolve(), - ) - skill_provenances = installation.provenances - skill_paths = installation.skill_paths - - # Everything the run needs lives in one typed config: Fabric no longer layers profile - # overlays, so the per-task workspace/model/trajectory settings are composed on last and are - # authoritative by construction. ``add_skill_path`` appends, so config-declared skills survive. - task_config = self._compose_config(agent_config, evidence_dir, workspace_dir, task=task) - for skill_path in skill_paths: - task_config.add_skill_path(skill_path) - - if self._task_hook is not None: - task_config = self._task_hook.prepare( - config=task_config, - task=task, - evidence_dir=evidence_dir, - workspace_dir=workspace_dir, - session=hook_session, - ) - - result = await asyncio.wait_for( - # ``Fabric.run`` folds the per-invocation input + request id into a ``RunRequest``. - client.run( - task_config, - base_dir=self._base_dir, - request=RunRequest(input=task.agent_prompt(), request_id=task.id), - ), - timeout=self._timeout_s, - ) - # Always try to harvest MCP binding results. Hermes often ends with - # ``completed=false`` / empty finals after a successful tool call; the binding - # audit is still the authoritative analyzer output for scoring. - if self._task_hook is not None: - try: - hook_extras = self._task_hook.after_success(task=task, result=result, session=hook_session) - except Exception as exc: # noqa: BLE001 - binding harvest must not abort the batch - logger.warning("Fabric task hook after_success failed: %s", exc) - if result.status == "succeeded": - raise - hook_extras = None - except TimeoutError as exc: - return self._failed_trial( - task, evidence_dir, exc, extra_metadata=self._failed_metadata(skill_provenances, evidence_dir) - ) - except Exception as exc: # noqa: BLE001 - a task failure must not abort the whole run - return self._failed_trial( - task, evidence_dir, exc, extra_metadata=self._failed_metadata(skill_provenances, evidence_dir) - ) - finally: - if self._task_hook is not None: - try: - self._task_hook.cleanup(session=hook_session) - except Exception: # noqa: BLE001 - hook cleanup must not mask the trial outcome - pass - # Codex self-injection staged each bundle *inside* the workspace so the harness could discover - # it. Remove them once the run is over (it is already captured in the trajectory) so the injected - # files don't linger in the durable workspace and, on any path that exposes it as filesystem - # evidence, read as agent output and skew workspace-reading metrics. In ``finally`` so a - # timed-out or errored run cleans up too, not just the success path. Best-effort per - # ``_remove_injected_bundle``; a no-op for native mode and when nothing was staged. - if skill_mode == SKILL_MODE_CODEX_SKILLS_DIR: - for provenance in skill_provenances: - await asyncio.to_thread(_remove_injected_bundle, workspace_dir, provenance["location"]) - - return self._to_trial( - task, - result, - evidence_dir, - workspace_dir, - skill_provenances=skill_provenances, - hook_extras=hook_extras, - ) - - @staticmethod - def _skill_metadata(provenances: list[SkillProvenance]) -> dict[str, Any]: - """Trial-metadata fields describing the injected skill set (the A/B provenance). - - ``skills`` is the full list of injected-skill provenances (empty = baseline). ``skill`` keeps the - historical single-provenance field β€” the lone provenance for a one-skill run, else ``None`` β€” so - single-skill consumers (e.g. ``SkillUsedMetric``) and existing trials keep working unchanged. - """ - return {"skill": provenances[0] if len(provenances) == 1 else None, "skills": provenances} - - @staticmethod - def _failed_metadata(provenances: list[SkillProvenance], evidence_dir: Path) -> dict[str, Any]: - """Trial metadata for a timed-out/errored task: skill provenance plus whatever tokens Relay flushed. - - Timeouts never reach ``_to_trial``, and there is no ``RunResult`` here, so the trajectory is read - straight from the relay dir β€” these are the long, expensive rows the token count matters most for. - """ - return { - **FabricAgentRuntime._skill_metadata(provenances), - **_atif_token_metadata(_relay_atif_path(evidence_dir)), - } - - def _to_trial( - self, - task: AgentEvalTask, - result: RunResult, - evidence_dir: Path, - workspace_dir: Path, - skill_provenances: list[SkillProvenance] | None = None, - hook_extras: Mapping[str, Any] | None = None, - ) -> AgentEvalTrial: - # Persist the full normalized Fabric result so graders (and debugging) can see the raw - # envelope, and expose it as an evidence descriptor. - result_path = evidence_dir / "fabric_result.json" - result_path.write_text(json.dumps(result.to_mapping(), indent=2, default=str), encoding="utf-8") - - extras = dict(hook_extras) if hook_extras else {} - base_metadata: dict[str, Any] = { - "runtime": self._runtime_name, - "harness": result.harness, - "adapter_id": result.adapter_id, - "adapter_kind": result.adapter_kind, - "invocation_id": result.invocation_id, - "agent_model": self._model, - # Skill provenance (name + content hash + injection mode) for the A/B diff. - **self._skill_metadata(skill_provenances or []), - **extras, - # Token usage from the Relay ATIF trajectory; Fabric's RunResult carries no usage of its - # own. Merged last so a hook extra can't shadow it. - **_atif_token_metadata(_atif_artifact_path(result)), - } - - if result.status != "succeeded": - # Hermes may report a non-success final message after a successful MCP tool - # call. Prefer the binding audit result over a hard fail when present. - binding_result = _first_mcp_binding_result(extras) - analysis = binding_result if binding_result is not None else extras.get("analyzer_analysis") - if analysis is not None: - base_metadata = { - **base_metadata, - "fabric_status": result.status, - "recovered_from_mcp_binding": True, - } - return AgentEvalTrial( - id=f"{task.id}:fabric", - task_id=task.id, - status=AgentEvalTrialStatus.COMPLETED, - output=AgentOutput( - output_text=json.dumps(analysis, default=str), - response=_normalize_output(result.output), - metadata={**base_metadata, "evidence_dir": str(evidence_dir)}, - ), - evidence=self._evidence(result, result_path, workspace_dir), - metadata={**base_metadata, "generated": True, "agent_ok": True}, - ) - return self._failed_trial(task, evidence_dir, _result_error(result), extra_metadata=base_metadata) - - # Fabric wraps the output in a ``RunOutput`` mapping (RunOutput response contract, #52), - # which is not itself a JSON value; normalize it to a plain mapping so it round-trips through the - # trial's ``JsonValue``-typed response. - output = _normalize_output(result.output) - # Author / mcp_run_binding hooks may attach a structured result. Prefer that when the - # harness returns an empty final message after a successful tool call. - output_text = _extract_output_text(output) - if not output_text or not str(output_text).strip(): - binding_result = _first_mcp_binding_result(extras) - analysis = binding_result if binding_result is not None else extras.get("analyzer_analysis") - if analysis is not None: - output_text = json.dumps(analysis, default=str) - return AgentEvalTrial( - id=f"{task.id}:fabric", - task_id=task.id, - status=AgentEvalTrialStatus.COMPLETED, - output=AgentOutput( - output_text=output_text, - response=output, - metadata={**base_metadata, "evidence_dir": str(evidence_dir)}, - ), - evidence=self._evidence(result, result_path, workspace_dir), - # AgentPhaseSuccessMetric reads agent_ok to score whether the agent phase finished cleanly - # (an explicit bool, not just trial status). - metadata={**base_metadata, "generated": True, "agent_ok": True}, - ) - - def _evidence(self, result: RunResult, result_path: Path, workspace_dir: Path) -> CandidateEvidence: - # The workspace is a host directory the harness ran in, so its final file tree is available on - # disk β€” expose it as filesystem evidence so workspace-reading metrics can score a Fabric trial. - descriptors: dict[str, EvidenceDescriptor] = { - "result": EvidenceDescriptor(kind="json", format="json", ref=str(result_path)), - _WORKSPACE_EVIDENCE_KEY: EvidenceDescriptor(kind=_WORKSPACE_EVIDENCE_KIND, ref=str(workspace_dir)), - } - for artifact in result.artifacts.artifacts: - descriptors[artifact.name] = EvidenceDescriptor( - kind=artifact.kind or "file", - ref=str(artifact.path), - metadata={"media_type": artifact.media_type}, - ) - # Surface the Relay ATIF trajectory under the standard trace evidence key so graders - # that consume a normalized trajectory find it. - if artifact.kind == _ATIF_ARTIFACT_KIND: - descriptors[EVIDENCE_TRACE] = EvidenceDescriptor( - kind=EVIDENCE_TRACE, - format=EVIDENCE_FORMAT_ATIF, - ref=str(artifact.path), - ) - return CandidateEvidence( - descriptors=descriptors, - metadata={ - "runtime": self._runtime_name, - "harness": result.harness, - "telemetry": [ - {"provider": ref.provider, "kind": ref.kind, "uri": ref.uri, "trace_id": ref.trace_id} - for ref in result.telemetry - ], - "events": [{"kind": event.kind, "message": event.message} for event in result.events], - }, - ) - - def _failed_trial( - self, - task: AgentEvalTask, - evidence_dir: Path, - error: Exception | Mapping[str, Any], - extra_metadata: Mapping[str, Any] | None = None, - ) -> AgentEvalTrial: - if isinstance(error, Mapping): - error_type = str(error.get("code") or error.get("stage") or "FabricError") - error_message = str(error.get("message") or error) - else: - error_type = error.__class__.__name__ - error_message = str(error) - error_path = evidence_dir / "error.json" - error_path.write_text(json.dumps({"error_type": error_type, "error": error_message}) + "\n", encoding="utf-8") - return AgentEvalTrial( - id=f"{task.id}:fabric", - task_id=task.id, - status=AgentEvalTrialStatus.FAILED, - output=None, - evidence=CandidateEvidence( - descriptors={"error": EvidenceDescriptor(kind="error", format="json", ref=str(error_path))}, - metadata={"runtime": self._runtime_name}, - ), - metadata={ - **(dict(extra_metadata) if extra_metadata else {}), - "runtime": self._runtime_name, - "agent_ok": False, - "error_type": error_type, - "error": error_message, - }, - ) - - def _compose_config( - self, - agent_config: FabricConfig, - evidence_dir: Path, - workspace_dir: Path, - task: AgentEvalTask, - ) -> FabricConfig: - # nemo_fabric is already imported+validated in ``run_tasks``; this is a cached sys.modules - # lookup, not a re-load, so the type is used where it's constructed instead of threaded down. - from nemo_fabric import EnvironmentConfig, ModelConfig # ty: ignore[unresolved-import] - - # Copy the base config and apply this task's workspace, model, and trajectory settings directly - # onto it. These land last, so they override anything the supplied config declared. - cfg = agent_config.model_copy(deep=True) - - # Point the harness at this task's staged workspace (the codex-cli adapter resolves its cwd from - # it). ``provider="local"`` is required by the native planner. Any config-supplied - # environment.workspace is overridden per task. - environment = cfg.environment or EnvironmentConfig(provider="local") - environment.provider = environment.provider or "local" - environment.workspace = str(workspace_dir.resolve()) - cfg.environment = environment - - # Apply the model as the config's default (mirrors nemo_fabric.integrations.harbor). - if self._model: - provider = self._model.split("/", maxsplit=1)[0] if "/" in self._model else "openai" - cfg.models["default"] = ModelConfig(provider=provider, model=self._model) - - if self._capture_trajectory: - # Enable Relay's ATIF/ATOF file exporter under this task's durable evidence dir, and pin the - # Fabric artifact root so the promoted ``trajectory-*.atif.json`` persists. Requires the - # ``nemo-relay`` gateway on PATH in the runtime. Stamp the task id (and any caller - # ``trajectory_extra``) onto ATIF ``extra`` so optimizer trials can join traces to rows. - relay_dir = evidence_dir / _RELAY_SUBDIR - artifacts_dir = evidence_dir / _ARTIFACTS_SUBDIR - relay_dir.mkdir(parents=True, exist_ok=True) - artifacts_dir.mkdir(parents=True, exist_ok=True) - row_extra = {"nemo.optimizer.row_id": task.id} if task.id else None - cfg.enable_relay( - output_dir=str(relay_dir), - observability=self._relay_config(relay_dir, extra=row_extra), - ) - cfg.runtime.artifacts = str(artifacts_dir) - cfg.environment.artifacts = str(artifacts_dir) - - return cfg - - def _relay_config( - self, - relay_dir: Path, - extra: Mapping[str, Any] | None = None, - ) -> RelayObservabilityConfig: - # The ATIF/ATOF observability config is built from Fabric's own typed relay-config objects so - # Fabric owns the schema (no hand-maintained dict that silently drifts when Fabric changes it), - # mirroring nemo_fabric's own Harbor integration. It is handed straight to ``enable_relay`` via - # its ``observability=`` parameter β€” the SDK only configures ATIF/ATOF observability, so it needs - # neither a generic ``components`` list nor the legacy component-wrapped shape. nemo_fabric is - # already imported+validated in ``run_tasks``, so this is a cached sys.modules lookup. - from nemo_fabric import ( # ty: ignore[unresolved-import] - RelayAtifConfig, - RelayAtofConfig, - RelayAtofFileSinkConfig, - RelayObservabilityConfig, - ) - - relay_dir_str = str(relay_dir) - atif_extra: dict[str, Any] | None = None - if self._trajectory_extra or extra: - atif_extra = {**(self._trajectory_extra or {}), **(dict(extra) if extra else {})} - return RelayObservabilityConfig( - atif=RelayAtifConfig( - enabled=True, - output_directory=relay_dir_str, - filename_template=_ATIF_FILENAME_TEMPLATE, - agent_name=self._runtime_name, - agent_version=_common.FABRIC_AGENT_VERSION, - extra=atif_extra, - ), - atof=RelayAtofConfig( - enabled=True, - sinks=[ - RelayAtofFileSinkConfig( - output_directory=relay_dir_str, - filename=_ATOF_FILENAME, - mode="overwrite", - ) - ], - ), - ) - - def _evidence_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> Path: - root = self._work_root - if root is None: - root = (config.work_dir or Path.cwd()) / "evidence" / "fabric" - # The run id isolates this run's evidence from other runs sharing the same root (A/B baseline - # vs. skilled); run_tasks always populates it, so the fallback only guards a direct call. - run_id = config.run_id or _new_run_id() - safe_task_id = _safe_path_name(task.id) - task_dir = f"{index:06d}-{safe_task_id}" if safe_task_id else f"task-{index:06d}" - return Path(root) / _safe_path_name(run_id) / task_dir - - -def _remove_injected_bundle(workspace_dir: Path, location: str) -> None: - """Remove the Codex-injected skill subtree from ``workspace_dir`` and prune emptied parents. - - ``location`` is workspace-relative (``.agents/skills/``). Best-effort: the skill was already - captured in the run's trajectory, so SkillUsedMetric (which reads the trace, not the workspace) is - unaffected, and any filesystem error here must not fail an otherwise-successful trial. - """ - workspace_root = workspace_dir.resolve() - injected = (workspace_dir / location).resolve() - # Guard against a location escaping the workspace (defensive; provenance is evaluator-authored). - if workspace_root not in injected.parents or not injected.exists(): - return - shutil.rmtree(injected, ignore_errors=True) - # Prune now-empty reserved parents (``.agents/skills``, ``.agents``) but never the workspace itself. - parent = injected.parent - while parent != workspace_root and parent.is_dir(): - try: - parent.rmdir() # only succeeds while empty - except OSError: - break - parent = parent.parent - - -def _normalize_output(output: RunOutput | JsonValue) -> JsonValue: - """Unwrap a Fabric ``RunResult.output`` into the plain JSON value the trial response stores. - - Newer Fabric wraps output in a ``RunOutput`` (the RunOutput response contract), which is a - ``Mapping``; copy it into a plain dict (equivalent to its ``to_mapping()``). Raw/older JSON outputs - are already JSON values and pass through unchanged. - """ - if isinstance(output, Mapping): - return dict(output) - return output - - -def _first_mcp_binding_result(extras: Mapping[str, Any]) -> Any | None: - """Return the first ``mcp_bindings..result`` payload, if any.""" - bindings = extras.get("mcp_bindings") - if not isinstance(bindings, Mapping): - return None - for entry in bindings.values(): - if isinstance(entry, Mapping) and "result" in entry: - return entry.get("result") - return None - - -def _extract_output_text(output: object) -> str | None: - """Pull the user-visible message out of a Fabric ``RunResult.output`` (JSON-shaped). - - Harness outputs vary; adapters commonly nest the final message under ``response`` (the codex-cli - adapter does). Prefer a string ``response``/``output_text``, else stringify the whole value. - """ - if output is None: - return None - if isinstance(output, str): - return output - if isinstance(output, Mapping): - for key in ("response", "output_text", "text", "message"): - value = output.get(key) - if isinstance(value, str): - return value - return json.dumps(output, default=str) - - -def _result_error(result: RunResult) -> Mapping[str, Any]: - error = result.error - if error is None: - return {"code": result.status, "message": "Fabric run did not succeed"} - return {"stage": error.stage, "code": error.code, "message": error.message} - - -def _atif_artifact_path(result: RunResult) -> Path | None: - """Path of the ATIF trajectory Fabric promoted as an artifact, if any.""" - for artifact in result.artifacts.artifacts: - if artifact.kind == _ATIF_ARTIFACT_KIND: - return Path(artifact.path) - return None - - -def _relay_atif_path(evidence_dir: Path) -> Path | None: - """Path of the Relay-written ATIF trajectory, used when no ``RunResult`` exists (timeout/error). - - Relay's filename template is per-session, so more than one file can land when subagents emit - their own sessions. Picking one under-reports and summing double-counts a root that already - aggregates, so anything other than a single match reports nothing rather than a wrong number. - """ - matches = sorted((evidence_dir / _RELAY_SUBDIR).glob(_ATIF_FILENAME_TEMPLATE.format(session_id="*"))) - if len(matches) == 1: - return matches[0] - if matches: - logger.warning("Fabric token capture: %d ATIF trajectories under %s; skipping", len(matches), evidence_dir) - return None - - -def _atif_token_metadata(path: Path | None) -> dict[str, int]: - """Project an ATIF trajectory's token totals onto the trial-metadata ``TOKEN_KEYS``. - - Each token field is resolved on its own: the trajectory-level ``final_metrics`` aggregate when it - reports that field, else the sum of the matching per-step ``metrics``. Every ``final_metrics`` - field is optional, so a block carrying only ``total_steps`` or a cost β€” or one that fails to - validate β€” must not suppress counts the steps do carry. That partial shape is likeliest on the - timeout path, where the trajectory was flushed mid-run and the counts matter most. - - ``total_tokens`` and ``cache_creation_tokens`` have no ATIF source and stay unset β€” Intake - recomputes the total. A missing or unreadable trajectory yields ``{}``: an absent token count - must not fail the trial. - """ - if path is None: - return {} - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except (OSError, ValueError) as exc: - logger.warning("Fabric token capture: unreadable ATIF trajectory %s (%s)", path, exc) - return {} - if not isinstance(payload, Mapping): - return {} - - totals = FinalMetrics() - final_metrics = payload.get("final_metrics") - if isinstance(final_metrics, Mapping): - try: - totals = FinalMetrics.model_validate(final_metrics) - except ValidationError as exc: - logger.warning("Fabric token capture: invalid final_metrics in %s (%s)", path, exc) - - captured = { - "prompt_tokens": (totals.total_prompt_tokens, "prompt_tokens"), - "completion_tokens": (totals.total_completion_tokens, "completion_tokens"), - "cache_read_tokens": (totals.total_cached_tokens, "cached_tokens"), - } - resolved = { - key: total if total is not None else _sum_step_metric(payload, step_key) - for key, (total, step_key) in captured.items() - } - return {key: value for key, value in resolved.items() if value is not None} - - -def _sum_step_metric(payload: Mapping[str, Any], key: str) -> int | None: - """Sum one per-step ATIF metric across the trajectory, or ``None`` when no step reported it.""" - total: int | None = None - for step in payload.get("steps") or []: - metrics = step.get("metrics") if isinstance(step, Mapping) else None - value = metrics.get(key) if isinstance(metrics, Mapping) else None - if isinstance(value, int) and not isinstance(value, bool): - total = value if total is None else total + value - return total - - -def _safe_path_name(value: str) -> str: - return "".join(char if char.isalnum() or char in "._-" else "-" for char in value).strip(".-")[:120] - - -def _new_run_id() -> str: - timestamp = datetime.now(UTC).strftime("%Y%m%d%H%M%S%f") - return f"fabric-{timestamp}-{uuid4().hex[:8]}" diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/sandbox.Dockerfile b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/sandbox.Dockerfile deleted file mode 100644 index 8b393d2059..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/sandbox.Dockerfile +++ /dev/null @@ -1,51 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# -# Multi-stage image for FabricContainerRuntime. The builder compiles nemo-fabric (maturin/Rust) into -# an isolated venv AND builds Fabric's own `fabric` CLI (the runtime execs `fabric run` to kick off -# the harness). The final stage copies only the venv + the CLI binary + the built-in adapters β€” no -# source tree and no Rust toolchain. Harness extras are selected via the EXTRAS build arg. -ARG PYTHON_VERSION=3.12 - -FROM python:${PYTHON_VERSION}-slim-bookworm AS builder -ARG EXTRAS=hermes,relay -RUN apt-get update \ - && apt-get install -y --no-install-recommends build-essential curl git pkg-config libssl-dev \ - && rm -rf /var/lib/apt/lists/* -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs -o /tmp/rustup-init.sh \ - && sh /tmp/rustup-init.sh -y --profile minimal \ - && rm -f /tmp/rustup-init.sh -ENV PATH=/root/.cargo/bin:$PATH -RUN python -m venv /opt/venv -ENV PATH=/opt/venv/bin:$PATH -# Only the maturin build inputs are in the context (see image._stage_source): the native nemo-fabric -# extension is compiled here and the harness/relay wheels are pulled from PyPI β€” into the venv only. -COPY nemo-fabric /src -RUN pip install --no-cache-dir "/src[${EXTRAS}]" -# Fabric's own CLI (Rust). The runtime execs `fabric run --profile … --input-file …`, -# which prints a normalized RunResult to stdout β€” so no in-image Python driver is needed. -RUN cargo build --release --manifest-path /src/Cargo.toml -p fabric-cli - -FROM python:${PYTHON_VERSION}-slim-bookworm AS runtime -COPY --from=builder /opt/venv /opt/venv -COPY --from=builder /src/target/release/fabric /usr/local/bin/fabric -# The CLI binary resolves built-in adapters from its compile-time repository path -# (CARGO_MANIFEST_DIR/../../python/src/nemo_fabric/adapters); ship just that dir to the baked path so a -# wheel-only image can resolve them. Depends on NeMo-Fabric's installed-adapter-discovery layout -# (see image.py); swap to installing the top-level adapters/* packages once that lands on main. -COPY --from=builder /src/python/src/nemo_fabric/adapters /src/python/src/nemo_fabric/adapters -# The CLI's baked path is the literal `/../../python/src/nemo_fabric/adapters`; the -# kernel needs `/src/crates/fabric-core` to exist to walk the `..`, even though nothing lives there. -RUN mkdir -p /src/crates/fabric-core -ENV PATH=/opt/venv/bin:$PATH -RUN python -c "from nemo_fabric import FabricClient" && fabric version -# Run agent-generated code as a non-root user: this sandbox execs `fabric run` over untrusted, -# agent-produced content, so dropping root narrows the blast radius of a container escape. Pre-create -# and own the fixed /in (seeded inputs) and /out (workspace + results) trees, since a non-root process -# cannot mkdir under / at exec time and the runtime creates /out/{workspace,relay,artifacts,logs} then. -RUN useradd --create-home --uid 1000 sandbox \ - && mkdir -p /in /out \ - && chown -R sandbox:sandbox /in /out -WORKDIR /out -USER sandbox diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/skills.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/skills.py deleted file mode 100644 index cfe5d7a71a..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/skills.py +++ /dev/null @@ -1,497 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Agent-skill injection for the Fabric agent-eval runtimes (PROTOTYPE). - -An *agent skill* is a directory following the `agentskills.io `_ -spec: a folder named ``/`` containing a required ``SKILL.md`` (YAML frontmatter with ``name`` + -``description``, then instructions) plus optional ``scripts/`` / ``references/`` / ``assets/``. We make -that bundle available to the harness before it runs a task so an A/B eval can score the same taskset -with and without the skill. The skill is a runtime-level knob: build one runtime with ``skill=None`` -and one with ``skill=`` over the same tasks, then diff the scores. - -An :class:`AgentSkill` points at a local skill directory; staging is an OS-level ``copytree`` (file -contents never pass through Python memory). The plugin resolves a platform fileset to a local -directory and constructs an ``AgentSkill`` from it β€” the SDK has no fileset concept of its own. - -How the skill reaches the harness depends on the selected Fabric adapter, and which mode applies is -decided by *querying Fabric's own capability planner at runtime* (:func:`resolve_skill_mode` over a -``RunPlan.capability_plan``), not a hardcoded adapter list β€” so it tracks whatever the installed -adapters declare, including end-user adapters we don't ship: - -* **Native** (:data:`SKILL_MODE_NATIVE`): the adapter advertises ``accepts: ["skills", ...]``, so - Fabric's planner routes skills to ``harness_native``. We stage the bundle into an isolated - ``/`` dir and add it to the config's ``skills.paths``; the adapter loads it (Hermes β†’ harness - ``skills.external_dirs``). As of nemo-fabric 0.1.0rc3 the hermes, claude AND **codex** adapters all - declare ``skills``, so this is the path every harness we ship currently takes. -* **Codex skills dir** (:data:`SKILL_MODE_CODEX_SKILLS_DIR`): a fallback for a codex-harness adapter - that does *not* accept the native skills config. The Codex CLI itself discovers agentskills bundles - from ``.agents/skills/`` in its working directory, so we place the bundle at - ``/.agents/skills//`` and let Codex find it β€” same discoverable-skill semantics as - native (cross-harness A/B stays apples-to-apples), no Fabric adapter change needed. - NOTE: the shipped codex adapter accepts ``skills`` today, so this branch is currently unreachable in - production and is exercised only by the fake-backed tests. It is kept for adapters (ours or an - end-user's) that route skills ``unsupported`` on a codex harness. - -If an adapter neither routes skills natively nor is a Codex harness, :func:`resolve_skill_mode` returns -``None`` and the runtime fails fast rather than silently running a skill-free trial. -""" - -from __future__ import annotations - -import hashlib -import re -import shutil -from collections.abc import Mapping, Sequence -from dataclasses import dataclass -from pathlib import Path -from typing import Literal, TypedDict - -from pydantic import BaseModel, ConfigDict, Field, field_validator - -#: Required entry document of an agentskills bundle. -PRIMARY_SKILL_DOC = "SKILL.md" -#: Directory Codex scans (relative to its working dir) for agentskills bundles. -CODEX_SKILLS_DIR = ".agents/skills" - -#: How an injected skill reaches the selected harness (resolved from Fabric's capability plan). The two -#: runtimes thread this value from :func:`resolve_skill_mode` down to :func:`install_skill` / -#: :func:`stage_skills_seed`, so a mistyped mode is a type error rather than a silent no-op. -SkillMode = Literal["native", "codex_skills_dir"] - -#: Skill reaches the harness via the native Fabric ``skills`` config (adapter accepts it). -SKILL_MODE_NATIVE: SkillMode = "native" -#: Skill is placed under ``/.agents/skills//`` for Codex to discover. -SKILL_MODE_CODEX_SKILLS_DIR: SkillMode = "codex_skills_dir" - -# agentskills.io name rule: 1-64 chars, lowercase alphanumeric + single interior hyphens. -_SKILL_NAME_RE = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$") -_MAX_NAME_LEN = 64 - -# Fabric capability-planner vocabulary (``RunPlan.capability_plan['routes']`` entries). A ``skills`` -# route with target ``harness_native`` means the selected adapter declared native skills support; the -# runtime plans a probe skill path and reads these to decide the injection mode (see resolve_skill_mode). -_SKILLS_ROUTE_KIND = "skills" -_SKILLS_TARGET_NATIVE = "harness_native" -# Fabric adapter id of Codex, which self-discovers ``.agents/skills/`` rather than -# accepting the native ``skills`` config. -_CODEX_ADAPTER_ID = "nvidia.fabric.codex" - - -class SkillInjectionError(ValueError): - """A skill could not be resolved, staged, or wired into the selected harness. - - Subclasses ``ValueError`` so the runtime's per-task error handling still catches it and fails - only that task. - """ - - -class AgentSkill(BaseModel): - """An agentskills.io bundle (a local directory) to make available to the agent before a task. - - ``name`` must satisfy the agentskills naming rule and is used as the staged bundle's directory name - (spec: the name matches the directory name). ``directory`` is the local skill directory, which must - contain a top-level ``SKILL.md``. - """ - - model_config = ConfigDict(extra="forbid") - - name: str = Field(description="agentskills skill name; also the bundle directory name and provenance id.") - directory: Path = Field(description="Local agentskills bundle directory (a SKILL.md at its root).") - - @field_validator("name") - @classmethod - def _valid_name(cls, value: str) -> str: - if len(value) > _MAX_NAME_LEN or not _SKILL_NAME_RE.match(value): - raise ValueError( - f"skill name {value!r} must be 1-{_MAX_NAME_LEN} chars, lowercase alphanumeric with " - "single interior hyphens (agentskills.io naming rule)" - ) - return value - - @classmethod - def from_directory(cls, directory: str | Path, *, name: str | None = None) -> AgentSkill: - """Build a skill from an on-disk agentskills bundle. ``name`` defaults to the directory basename.""" - root = Path(directory).expanduser().resolve() - if not (root / PRIMARY_SKILL_DOC).is_file(): - raise SkillInjectionError(f"skill directory {str(directory)!r} has no {PRIMARY_SKILL_DOC}") - return cls(name=name or root.name, directory=root) - - -class SkillProvenance(TypedDict): - """Which skill was injected into a trial and how; stamped into trial metadata for the A/B diff. - - A plain (JSON-serializable) dict so it drops straight into trial metadata. ``None`` in that slot - means the baseline (no skill). - """ - - name: str #: The skill's agentskills name. - hash: str #: sha256 over the staged bundle β€” attributes a score delta to an exact skill version. - mode: SkillMode #: How it was injected (:data:`SKILL_MODE_NATIVE` / :data:`SKILL_MODE_CODEX_SKILLS_DIR`). - adapter_id: str #: The harness adapter the skill was wired into. - location: str #: Where the bundle was staged (absolute for native, workspace-relative for codex). - - -@dataclass -class SkillInstallation: - """Result of installing a skill for one task. - - ``skill_paths`` are staged bundle roots the runtime hands to ``FabricConfig.add_skill_path`` (the - native branch emits one; the Codex branch emits none because placement in the workspace is the - delivery mechanism). ``provenance`` is stamped into trial metadata so the A/B comparison is - auditable. - """ - - skill_paths: list[str] - provenance: SkillProvenance - - -def native_skills_route(capability_plan: Mapping[str, object]) -> bool: - """Whether Fabric's capability planner routed skills to the harness natively. - - ``capability_plan`` is the ``RunPlan.capability_plan`` mapping from ``Fabric.plan(...)`` planned with - a skill path attached; its ``routes`` record each capability decision. A ``skills`` route with target - ``harness_native`` means the selected adapter declares ``accepts: ["skills", ...]`` and Fabric hands - the bundle to the harness itself. Any other outcome (``unsupported``, or no skills route) is False. - """ - routes = capability_plan.get("routes") - if not isinstance(routes, list): - return False - return any( - isinstance(route, Mapping) - and route.get("kind") == _SKILLS_ROUTE_KIND - and route.get("target") == _SKILLS_TARGET_NATIVE - for route in routes - ) - - -def resolve_skill_mode(*, capability_plan: Mapping[str, object], adapter_id: str) -> SkillMode | None: - """Resolve how a skill would reach the selected harness, or ``None`` if it can't. - - Driven by Fabric's own capability routing (queried at runtime via ``Fabric.plan``) rather than a - hardcoded adapter list, so it tracks whatever the installed adapters declare β€” including end-user - adapters we don't ship: - - * skills route natively (:func:`native_skills_route`) -> :data:`SKILL_MODE_NATIVE`; - * else a Codex harness (self-discovers ``.agents/skills/``) -> :data:`SKILL_MODE_CODEX_SKILLS_DIR`; - * else ``None`` -> the runtime fails fast rather than run a skill-free trial labeled "with skill". - """ - if native_skills_route(capability_plan): - return SKILL_MODE_NATIVE - if adapter_id.strip().lower() == _CODEX_ADAPTER_ID: - return SKILL_MODE_CODEX_SKILLS_DIR - return None - - -def install_skill( - *, - skill: AgentSkill, - adapter_id: str, - mode: SkillMode, - workspace_dir: Path, - skill_stage_dir: Path, -) -> SkillInstallation: - """Stage ``skill`` as a ``/`` bundle and wire it into the harness per ``mode``. - - Blocking file I/O β€” call via ``asyncio.to_thread`` from the async runtime. The bundle is always - namespaced under ``/`` so it never collides with task-seeded workspace-root files; the content - hash is computed over the staged bytes so provenance tracks the actual skill content. - - The native branch returns the staged root for ``FabricConfig.add_skill_path``, which appends to - whatever the base config already declares. Any preconfigured skills therefore survive injection - without this function having to re-list them. - """ - if mode == SKILL_MODE_NATIVE: - skill_root = skill_stage_dir / skill.name - _stage_bundle(skill.directory, skill_root, reserved=False) - return SkillInstallation( - skill_paths=[str(skill_root)], - provenance=_provenance(skill, _hash_directory(skill_root), mode, adapter_id, str(skill_root)), - ) - - if mode == SKILL_MODE_CODEX_SKILLS_DIR: - skill_root = workspace_dir / CODEX_SKILLS_DIR / skill.name - _stage_bundle(skill.directory, skill_root, reserved=True) - location = (Path(CODEX_SKILLS_DIR) / skill.name).as_posix() - return SkillInstallation( - skill_paths=[], - provenance=_provenance(skill, _hash_directory(skill_root), mode, adapter_id, location), - ) - - raise SkillInjectionError(f"unknown skill injection mode {mode!r} for adapter {adapter_id!r}") - - -@dataclass -class SkillsInstallation: - """Result of installing several skills for one task (see :func:`install_skills`). - - ``skill_paths`` is every staged native bundle root, in the given order, for the runtime to feed to - ``FabricConfig.add_skill_path``; the Codex branch emits none because workspace placement is the - delivery mechanism. ``provenances`` is one entry per skill, in the given order, stamped into trial - metadata so a multi-skill A/B comparison is auditable. - """ - - skill_paths: list[str] - provenances: list[SkillProvenance] - - -def require_unique_skill_names(skills: Sequence[AgentSkill]) -> None: - """Raise if two skills share a name β€” their ``/`` bundles would collide when staged. - - Each skill stages into its own ``/`` directory (native stage dir or ``.agents/skills/``), so a - repeated name would clobber (or fail to stage over) an earlier bundle. Checked up front so a - misconfigured runtime fails before any task runs, not mid-stage on the second collision. - """ - seen: set[str] = set() - duplicates: list[str] = [] - for skill in skills: - if skill.name in seen and skill.name not in duplicates: - duplicates.append(skill.name) - seen.add(skill.name) - if duplicates: - raise SkillInjectionError( - f"duplicate skill name(s) {duplicates}: each skill stages to its own '/' bundle, so " - "skill names must be unique within one runtime" - ) - - -@dataclass(frozen=True) -class SkillSet: - """Immutable, name-validated collection of :class:`AgentSkill`\\s shared by both Fabric runtimes. - - Centralizes the uniqueness check and clone-on-mutation pattern that - :class:`~...FabricAgentRuntime` and :class:`~...FabricContainerRuntime` would otherwise - duplicate: construction validates that skill names are unique; :meth:`with_skills` and - :meth:`with_skill` each return a new ``SkillSet`` without modifying ``self``. - """ - - skills: tuple[AgentSkill, ...] = () - - def __post_init__(self) -> None: - require_unique_skill_names(self.skills) - - def with_skills(self, skills: Sequence[AgentSkill]) -> SkillSet: - """Return a new ``SkillSet`` with ``skills`` appended; ``self`` is not modified.""" - return SkillSet((*self.skills, *skills)) - - def with_skill(self, skill: AgentSkill) -> SkillSet: - """Return a new ``SkillSet`` with ``skill`` appended; ``self`` is not modified.""" - return self.with_skills([skill]) - - -def install_skills( - *, - skills: Sequence[AgentSkill], - adapter_id: str, - mode: SkillMode, - workspace_dir: Path, - skill_stage_dir: Path, -) -> SkillsInstallation: - """Stage every skill in ``skills`` for one task and wire them all into the harness per ``mode``. - - Loops :func:`install_skill` β€” each skill stages into its own namespaced ``/`` bundle β€” and - collects the staged roots for the native mode. ``FabricConfig.add_skill_path`` appends and - de-duplicates, so every injected skill lands alongside whatever the base config already declared, - with no re-listing. Skill names must be unique (their ``/`` bundles would otherwise collide). - Blocking file I/O β€” call via ``asyncio.to_thread`` from the async runtime. - - Installation is all-or-nothing: if any skill fails to stage, the bundles already staged in this call - are rolled back before the error propagates, so a partial skill set never lingers on disk (the caller - raises before it ever sees provenances, so it cannot clean up itself). Only bundles this call staged - are removed, so a reserved-path collision can never delete a pre-existing task-seeded file. - """ - require_unique_skill_names(skills) - provenances: list[SkillProvenance] = [] - staged_roots: list[Path] = [] - try: - for skill in skills: - # Register the target BEFORE staging: install_skill can raise after it has already written - # files (a copytree failing partway, an unreadable file while hashing), and a root recorded - # only on success would leave that partial bundle behind. A target that already exists is - # never registered β€” in codex mode that is a task-seeded file install_skill refuses to - # clobber, and rolling it back would delete task input this call did not create. - stage_root = _skill_stage_root(skill, mode, workspace_dir, skill_stage_dir) - if not stage_root.exists(): - staged_roots.append(stage_root) - provenance = install_skill( - skill=skill, - adapter_id=adapter_id, - mode=mode, - workspace_dir=workspace_dir, - skill_stage_dir=skill_stage_dir, - ).provenance - provenances.append(provenance) - except Exception: - for root in staged_roots: - shutil.rmtree(root, ignore_errors=True) - raise - - skill_paths: list[str] = [] - if mode == SKILL_MODE_NATIVE: - # Each staged bundle root, order-preserved and de-duplicated (a native provenance's - # ``location`` is its absolute staged skill root). - skill_paths = list(dict.fromkeys(prov["location"] for prov in provenances)) - return SkillsInstallation(skill_paths=skill_paths, provenances=provenances) - - -def _skill_stage_root(skill: AgentSkill, mode: SkillMode, workspace_dir: Path, skill_stage_dir: Path) -> Path: - """Absolute on-disk root :func:`install_skill` would stage ``skill`` into, computed before staging. - - Mirrors install_skill's per-mode placement so :func:`install_skills` can register a rollback target - up front (an unknown mode raises there, not here; the returned path is simply never created, and - rolling back a path that does not exist is a no-op).""" - if mode == SKILL_MODE_NATIVE: - return skill_stage_dir / skill.name - return workspace_dir / CODEX_SKILLS_DIR / skill.name - - -def _render_skill_seed( - *, skill: AgentSkill, adapter_id: str, mode: SkillMode, workspace_dir: str, skills_dir: str -) -> tuple[dict[str, str], SkillProvenance]: - """Render one skill bundle into an in-sandbox ``{path: text}`` seed map + its provenance. - - The per-skill core of :func:`stage_skills_seed` (the containerized counterpart of :func:`install_skill`, - which ``copytree``\\ s onto host disk): the container has no host workspace, so the bundle is read into - memory as UTF-8 text and keyed at the harness's in-sandbox discovery path β€” native: ``/ - /``; codex: ``/.agents/skills//``. The content hash is over the source bundle - (matching :func:`install_skill`). The caller merges these into one seed set and, for native mode, a - single ``skills`` overlay β€” so no per-skill overlay is emitted here. - """ - bundle = _read_text_bundle(skill.directory) - skill_hash = _hash_directory(skill.directory) - if mode == SKILL_MODE_NATIVE: - skill_root = f"{skills_dir.rstrip('/')}/{skill.name}" - files = {f"{skill_root}/{rel}": text for rel, text in bundle.items()} - return files, _provenance(skill, skill_hash, mode, adapter_id, skill_root) - - if mode == SKILL_MODE_CODEX_SKILLS_DIR: - skill_root = f"{workspace_dir.rstrip('/')}/{CODEX_SKILLS_DIR}/{skill.name}" - files = {f"{skill_root}/{rel}": text for rel, text in bundle.items()} - location = f"{CODEX_SKILLS_DIR}/{skill.name}" - return files, _provenance(skill, skill_hash, mode, adapter_id, location) - - raise SkillInjectionError(f"unknown skill injection mode {mode!r} for adapter {adapter_id!r}") - - -def _read_text_bundle(directory: Path) -> dict[str, str]: - """Read an agentskills bundle into a ``{posix_relpath: text}`` map (requires a top-level ``SKILL.md``). - - Every file is decoded as UTF-8: the containerized seed set (``SandboxSpec.files``) is text-only, so a - binary file (e.g. an image under ``assets/``) raises here rather than silently corrupting the staged - bundle β€” the host :func:`install_skill` path (OS-level ``copytree``) handles binary bundles instead. - """ - src = directory.expanduser() - if not (src / PRIMARY_SKILL_DOC).is_file(): - raise SkillInjectionError(f"skill directory {str(directory)!r} has no {PRIMARY_SKILL_DOC}") - bundle: dict[str, str] = {} - for path in sorted(candidate for candidate in src.rglob("*") if candidate.is_file()): - rel = path.relative_to(src).as_posix() - try: - bundle[rel] = path.read_text(encoding="utf-8") - except UnicodeDecodeError as exc: - raise SkillInjectionError( - f"skill file {rel!r} is not UTF-8 text; containerized skill injection (via the sandbox " - "seed set) supports text bundles only" - ) from exc - return bundle - - -@dataclass -class SkillsSeed: - """Result of rendering several skills into one sandbox seed set (see :func:`stage_skills_seed`). - - The plural, containerized sibling of :class:`SkillsInstallation`: - - * ``files`` β€” the merged ``{absolute_in_sandbox_path: text}`` seed map for every staged bundle. - * ``skill_paths`` β€” every staged native bundle root, in order, for the runtime to merge into the - composed config's ``skills.paths``; the codex branch emits none. - * ``provenances`` β€” one entry per skill, in the given order, for the multi-skill A/B trial metadata. - """ - - files: dict[str, str] - skill_paths: list[str] - provenances: list[SkillProvenance] - - -def stage_skills_seed( - *, - skills: Sequence[AgentSkill], - adapter_id: str, - mode: SkillMode, - workspace_dir: str, - skills_dir: str, -) -> SkillsSeed: - """Render every skill in ``skills`` into one sandbox seed set for the container runtime. - - The plural, containerized sibling of :func:`install_skills`: renders each bundle (via - :func:`_render_skill_seed`) under its own ``/`` at the harness's in-sandbox discovery path and - collects the native in-sandbox roots. The caller merges those into the composed config's - ``skills.paths`` alongside whatever it already declared, so nothing has to be re-listed here. Skill - names must be unique β€” their ``/`` bundles would otherwise collide. No on-disk rollback is - needed (unlike :func:`install_skills`): the seed set is an in-memory map, so a failure to render any - skill just discards the accumulated map and raises, leaving nothing staged. - """ - require_unique_skill_names(skills) - files: dict[str, str] = {} - provenances: list[SkillProvenance] = [] - for skill in skills: - rendered, provenance = _render_skill_seed( - skill=skill, adapter_id=adapter_id, mode=mode, workspace_dir=workspace_dir, skills_dir=skills_dir - ) - files.update(rendered) - provenances.append(provenance) - - skill_paths: list[str] = [] - if mode == SKILL_MODE_NATIVE: - # Each staged bundle (a native provenance's ``location`` is its absolute in-sandbox skill - # root), order-preserved and de-duplicated. - skill_paths = list(dict.fromkeys(prov["location"] for prov in provenances)) - return SkillsSeed(files=files, skill_paths=skill_paths, provenances=provenances) - - -def _stage_bundle(directory: Path, skill_root: Path, *, reserved: bool) -> None: - """Stage the skill ``directory`` as an *exact* copy at ``skill_root`` (the ``/`` bundle dir). - - The staged bundle must reflect exactly the supplied directory, so provenance and behaviour track the - real content. ``reserved`` picks the collision policy for the destination: - - * ``reserved=False`` β€” the evaluator-owned native stage dir: recreate it, so a reused run id can't - leave a file that was since removed from the source bundle surviving in the stage. - * ``reserved=True`` β€” the Codex workspace path (``.agents/skills/``): refuse to clobber - pre-existing content there, since it can only be a task-seeded file colliding with the reserved - skill path. - """ - src = directory.expanduser() - if not (src / PRIMARY_SKILL_DOC).is_file(): - raise SkillInjectionError(f"skill directory {str(directory)!r} has no {PRIMARY_SKILL_DOC}") - if skill_root.exists(): - if reserved: - raise SkillInjectionError( - f"cannot stage skill into reserved path {str(skill_root)!r}: it already exists " - "(a task-seeded file collides with the injected skill bundle)" - ) - shutil.rmtree(skill_root) # evaluator-owned: recreate so the stage is an exact copy - skill_root.parent.mkdir(parents=True, exist_ok=True) - # OS-level copy β€” file contents never pass through Python memory. - shutil.copytree(src, skill_root) - - -def _provenance(skill: AgentSkill, skill_hash: str, mode: SkillMode, adapter_id: str, location: str) -> SkillProvenance: - return { - "name": skill.name, - "hash": skill_hash, - "mode": mode, - "adapter_id": adapter_id, - "location": location, - } - - -def _hash_directory(directory: Path) -> str: - """Stable sha256 over a directory's file tree (sorted relpath + contents).""" - digest = hashlib.sha256() - for path in sorted(path for path in directory.rglob("*") if path.is_file()): - digest.update(path.relative_to(directory).as_posix().encode("utf-8")) - digest.update(b"\0") - digest.update(path.read_bytes()) - digest.update(b"\0") - return digest.hexdigest() diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/__init__.py deleted file mode 100644 index 499d431d56..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/__init__.py +++ /dev/null @@ -1,84 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""NeMo Gym-backed :class:`AgentTaskRunner` for the agent-eval pipeline. - -Runs an *existing* NeMo Gym environment through its ``gym`` CLI and adapts the -rollout bundle into SDK :class:`AgentEvalTrial` objects, so an -:class:`AgentEvaluator` can score and report Gym runs through the same seam as -Harbor/Fabric/Model. Gym owns execution *and* scoring; this runtime imports an -existing Gym environment as-is β€” Gym is treated as a self-scoring engine and the -runtime only adapts its rollout bundle, it does not re-derive rewards. - -**Mapping** (Gym β†’ Evaluator): one Gym dataset β†’ one run; each distinct row β†’ -one :class:`AgentEvalTask` (id = content hash of the row); each attempt -(``_ng_rollout_index``) β†’ one :class:`AgentEvalTrial`; the per-attempt verifier -``reward`` β†’ a :class:`GymRewardMetric` score. ``num_repeats=R`` therefore yields -up to R trials per task. Row duplication is *not* a way to ask for repeated -attempts β€” ``num_repeats`` is (see :func:`discover_gym_tasks`). - -**Attribution** is by ``_ng_task_index``, which this runtime *assigns* rather -than infers. Gym only auto-assigns an index when a row doesn't already carry one -(``rollout_collection._preprocess_rows_from_config``), and its own fallback -dedup keys off the **raw jsonl line text** β€” a rule we cannot reproduce from -parsed rows. So instead of guessing, :meth:`GymAgentTaskRunner.run_tasks` -materializes a normalized dataset (one line per requested task, ``_ng_task_index`` -stamped explicitly) and feeds *that* to Gym. Gym echoes the index back on every -rollout record, giving a total, order-independent ``index β†’ task`` map. This also -means a caller can run a **subset** of tasks without Gym rolling out the rest. - -**Execution** is the two-step Gym flow (the one that reads a dataset directly -without triggering Gym's split-driven data-prep), preceded by a pre-flight: -``gym env validate`` merges the composed config and reports unset ``???`` values, -bad paths, and dangling cross-references without starting anything; then ``gym env -start`` brings up the resources-server + agent + model servers, and ``gym eval run ---no-serve --input `` collects rollouts against them. Both -commands receive the identical selection arguments, so what is validated is what -runs. The runtime shells out to the ``gym`` CLI on PATH, so this SDK never imports -``nemo_gym``. Subprocess -output is streamed to log files under the run's work dir *and* mirrored to this -module's logger at ``DEBUG``, so callers choose terminal visibility through -ordinary ``logging`` configuration. - -**Where Gym finds things.** NeMo Gym must be installed and its ``gym`` on PATH, -along with the target environment's own dependencies. Generally that means a -*separate* environment: Gym imports Ray at module load, and nemo-platform -excludes Ray by constraint over an unfixed CVE, so the two cannot share one. In a -job image the image owns PATH and this is unremarkable. There is deliberately no -config field naming a checkout, a venv, or a search root β€” these runner configs -become serialized job specs, and a local filesystem path means nothing on the -other side of that boundary. Environments themselves ship in the ``nemo-gym`` -wheel (``resources_servers`` and friends install beside ``nemo_gym``, configs and -example data included), so no checkout is needed to reach them. - -The subprocesses inherit this process's working directory, which is where Gym -looks for the gitignored ``env.yaml`` holding the collector's credentials before -falling back to its install root β€” so credentials never pass through this SDK. -Run from the directory holding that file; a Gym checkout there also has its -components take precedence, which is how you reach an environment the wheel does -not carry. - -**Boundaries**: the caller is responsible for a -Gym runtime whose deps are installed (each Gym env ships its own -``requirements.txt``), and for handing a *ready-to-run* dataset file (``--no-serve ---input`` bypasses Gym's prompt-templating/materialization). Service-side -provisioning (docker/k8s, Ray) is out of scope here β€” that is the plugin's job. - -A consequence of that bypass: an environment whose rows carry no rendered prompt -(``responses_create_params.input == []``, the prompt supplied by data-prep or by the -environment's own agent) is still supported β€” the row travels through this runtime intact -and the task simply has no ``inputs['instruction']``. See :func:`discover_gym_tasks`. -""" - -from nemo_platform.beta.evaluator.agent_eval.runtimes.gym.config import DEFAULT_REWARD_KEY, GymRuntimeConfig -from nemo_platform.beta.evaluator.agent_eval.runtimes.gym.dataset import discover_gym_tasks -from nemo_platform.beta.evaluator.agent_eval.runtimes.gym.runtime import GymAgentTaskRunner -from nemo_platform.beta.evaluator.metrics.runner_rewards import GymRewardMetric - -__all__ = [ - "DEFAULT_REWARD_KEY", - "GymAgentTaskRunner", - "GymRewardMetric", - "GymRuntimeConfig", - "discover_gym_tasks", -] diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/config.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/config.py deleted file mode 100644 index 200cd41884..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/config.py +++ /dev/null @@ -1,270 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Run configuration, and the Hydra grammar it is serialized into. - -Gym is a Hydra application, so every setting reaches it as an ``++dotted.path=value`` argument. -That grammar is typed and unforgiving β€” quoting rules live here alongside the config model they -serialize, so the two cannot drift. Redaction lives here too: these values are recorded as run -provenance, and the override map is a free-form escape hatch a caller can put a credential into. -""" - -from __future__ import annotations - -import logging -import re -from collections.abc import Mapping -from pathlib import Path -from typing import Any - -from pydantic import BaseModel, ConfigDict, Field - -logger = logging.getLogger(__name__) - - -DEFAULT_REWARD_KEY = "reward" -#: Gym's CLI, expected on PATH. Not configurable: these runner configs become serialized job specs, -#: and a path into somebody's venv is meaningless on the other side of that boundary. Note this name -#: is only ever *resolved*, never executed: :func:`_gym_executable` turns it into an absolute path -#: once, and that path is what the subprocesses run β€” so a child whose PATH differs from ours cannot -#: end up executing a different Gym. -_HYDRA_SUBDIR = "gym_hydra" -#: `gym env start`'s combined output, under the run's work dir. Named here because a *collection* -#: failure often has to point at it: the eval logs show the symptom, this shows the cause. -_SECRET_KEY_MARKERS = ("api_key", "apikey", "token", "secret", "password", "passwd", "credential") -#: Stand-in written in place of a redacted override value. -_REDACTED = "" - - -#: Dict keys Hydra reads back unchanged. Its ``dictKey`` rule accepts no quoting, so a key is -#: whatever the lexer makes of the bare text β€” this is deliberately narrower than what parses. -_HYDRA_DICT_KEY = re.compile(r"[A-Za-z_][A-Za-z0-9_.-]*\Z") -#: Bare words the lexer types rather than reading as text, so they cannot serve as string keys. -_HYDRA_KEY_LITERALS = frozenset({"true", "false", "null", "inf", "nan"}) - - -def _hydra_dict(value: Mapping[str, Any]) -> str: - """Render a mapping as a Hydra dict container, ``{key:value,...}``. - - Reached for a mapping nested inside a container β€” ``[{"b": 1}]`` β€” where there is no dotted path - to flatten onto, so the dict has to be spelled inline. Values recurse, so the typed spellings - below hold at any depth. - - Keys are emitted bare, because Hydra's ``dictKey`` rule has no quoted form: ``{'b':1}`` does not - parse at all. That leaves the key at the mercy of the lexer, which types it β€” ``{true:1}`` keys - on the boolean ``True``, ``{1.5:1}`` on a float β€” and rejects ``:``, ``,``, brackets, and quotes - outright. Anything outside the conservative shape above therefore raises here rather than - silently keying the config on something the caller did not write. - """ - rendered = [] - for key, item in value.items(): - if not isinstance(key, str) or not _HYDRA_DICT_KEY.match(key) or key.casefold() in _HYDRA_KEY_LITERALS: - raise ValueError( - f"Gym config override has dict key {key!r}, which Hydra's override grammar cannot " - "express as a string: keys are unquoted, so only a leading letter or underscore " - "followed by letters, digits, '_', '.', or '-' survives the round trip. Set this " - "key through the override path instead of nesting it inside a list." - ) - rendered.append(f"{key}:{_hydra_scalar(item)}") - return "{" + ",".join(rendered) + "}" - - -def _hydra_scalar(value: Any) -> str: - """Render a leaf value the way Hydra's override grammar reads it back. - - Hydra's grammar is typed, so an unquoted string is not necessarily a string: ``true`` parses as a - boolean, ``null`` as ``None``, ``1.5`` as a float, ``a,b`` as a *sweep*, and ``A[B`` fails to - parse outright. Strings are therefore always single-quoted, which round-trips every one of those - (verified against ``hydra.core.override_parser``). Interpolations survive quoting β€” the override - sets the literal text and OmegaConf resolves it on read β€” so ``${policy_base_url}`` still works. - - Only ``'`` is escaped. Hydra does **not** decode ``\\\\`` inside a quoted value: escaping - backslashes doubles them, so they are passed through raw. - - ``None`` and booleans get their own spellings, since ``str()`` would emit ``"None"``/``"True"`` - and Hydra reads those back as text. Containers recurse for the same reason: ``str()`` on a dict - emits Python's repr, whose quoted keys Hydra rejects outright. - """ - if value is None: - return "null" - if isinstance(value, bool): - return "true" if value else "false" - if isinstance(value, Mapping): - return _hydra_dict(value) - if isinstance(value, (list, tuple)): - return "[" + ",".join(_hydra_scalar(item) for item in value) + "]" - if isinstance(value, str): - # A trailing backslash would escape the closing quote and leave the value unterminated, and - # there is no spelling that avoids it β€” better to say so than to emit something unparseable. - if value.endswith("\\"): - raise ValueError( - f"Gym config override value {value!r} ends with a backslash, which Hydra's override " - "grammar cannot express: it escapes the closing quote." - ) - return "'" + value.replace("'", "\\'") + "'" - return str(value) - - -def _flatten_overrides(overrides: Mapping[str, Any], _prefix: str = "") -> list[str]: - """Flatten a nested override mapping into Hydra ``++dotted.path=value`` arguments. - - Callers describe overrides as structured data β€” ``{"a": {"b": 1}}`` β€” rather than as - pre-serialized Hydra strings, so the config survives being sent somewhere as JSON. Hydra itself - only speaks the flat form, so the translation happens here, at the point of invocation. - - ``++`` rather than ``+``: it sets a key whether or not it already exists, which is what an - override means. A bare ``+`` fails on a key the merged config already defines. - """ - arguments: list[str] = [] - for key, value in overrides.items(): - path = f"{_prefix}{key}" - # An empty mapping has no leaves to descend to, so recursing would drop the override - # entirely. It is still a value the caller asked to set: emit it as ``++path={}``, which - # clears the subtree. - if isinstance(value, Mapping) and value: - arguments.extend(_flatten_overrides(value, f"{path}.")) - else: - arguments.append(f"++{path}={_hydra_scalar(value)}") - return arguments - - -def _redact_hydra_params(overrides: Mapping[str, Any], _prefix: str = "") -> dict[str, Any]: - """Redact credential-looking values from overrides before they are recorded as provenance. - - ``hydra_params`` is a free-form escape hatch forwarded to Gym, so nothing stops a caller passing - ``{"model": {"api_key": "sk-..."}}``. ``RunnerInfo.config`` is persisted into the run bundle, so a - value that looks like a credential must not be written there. - - The *key* is always kept β€” knowing that a run overrode ``model.api_key`` is useful provenance; - knowing the value is a leak. Matching is on the full dotted path, so a marker anywhere in it - redacts, and nesting cannot hide a credential behind an innocuous leaf name. - - Lists are walked too, since a mapping inside one β€” ``{"models": [{"api_key": "sk-..."}]}`` β€” - reaches Gym just as a nested mapping does. The index contributes no path segment: what marks a - value as a credential is the key it sits under, not where in a list it happens to fall. - """ - redacted: dict[str, Any] = {} - for key, value in overrides.items(): - path = f"{_prefix}{key}" - if isinstance(value, Mapping): - redacted[key] = _redact_hydra_params(value, f"{path}.") - elif any(marker in path.casefold() for marker in _SECRET_KEY_MARKERS): - redacted[key] = _REDACTED - elif isinstance(value, (list, tuple)): - redacted[key] = [_redact_list_item(item, path) for item in value] - else: - redacted[key] = value - return redacted - - -def _redact_list_item(item: Any, path: str) -> Any: - """Redact inside one element of a list-valued override. See :func:`_redact_hydra_params`.""" - if isinstance(item, Mapping): - return _redact_hydra_params(item, f"{path}.") - if isinstance(item, (list, tuple)): - return [_redact_list_item(nested, path) for nested in item] - return item - - -def _selection_args(config: GymRuntimeConfig, work_dir: Path) -> list[str]: - """The environment/agent/model selection passed to Gym. - - Built once and handed verbatim to both ``gym env validate`` and ``gym env start``, so what is - validated is exactly what runs β€” a pre-flight against a different config would be worse than - none. - """ - selection = [ - "--config", - config.agent_config, - "--model-type", - config.model_type, - "--resources-server", - config.resources_server, - ] - if config.bind_resources_server: - # Composable (Pattern-A) agents leave resources_server.name unbound ('???'); bind it to the - # env we're running. Assumes the agent config's top-level key equals the agent name (the - # simple_agent convention) *and* that the resources-server is registered under the - # environment's own name β€” not universally true, so self-contained or differently-named - # servers set bind_resources_server=False and bind themselves via hydra_params. - selection.append( - f"+{config.agent}.responses_api_agents.{config.agent}.resources_server.name={config.resources_server}" - ) - selection.extend(_flatten_overrides(config.hydra_params)) - # Gym is a Hydra app, so each invocation writes a timestamped run directory β€” by default - # `outputs//