From eec6b56548ea84ec5b432769ecfc4146283a01b8 Mon Sep 17 00:00:00 2001 From: Matt Kornfield Date: Thu, 6 Aug 2026 18:33:03 +0000 Subject: [PATCH] build(sdk): stage extension packages at wheel build time Signed-off-by: Matt Kornfield --- .../fileset_filesystem_provider.py | 2 +- .../src/data_designer_nemo/filesystem.py | 2 +- .../src/data_designer_nemo/seed.py | 2 +- packages/filesets/pyproject.toml | 1 + packages/filesets/src/filesets/resources.py | 16 +- packages/models/pyproject.toml | 1 + packages/models/tests/test_client.py | 8 +- packages/nemo_evaluator_sdk/pyproject.toml | 1 + .../tests/agent_eval/test_evidence.py | 6 +- .../test_sandbox_compose_vendored_import.py | 24 +- packages/nemo_platform/pyproject.toml | 21 +- packages/nemo_platform_ext/pyproject.toml | 1 + .../cli/commands/api/files/__init__.py | 4 +- .../src/nemo_platform_ext/client/enhanced.py | 170 +- .../tests/client/test_client.py | 112 +- packages/nemo_platform_plugin/pyproject.toml | 23 + .../src/nemo_platform_plugin/commands.py | 2 +- .../nemo_platform_plugin/jobs/file_manager.py | 2 +- .../jobs/result_manager.py | 2 +- .../nemo_platform_plugin/nooa_model_client.py | 2 +- .../tests/test_commands.py | 2 +- .../nmp_common/src/nmp/common/auth/testing.py | 2 +- .../nmp_common/tests/sdk_factory/test_sdk.py | 14 +- .../nmp_testing/src/nmp/testing/client.py | 2 +- .../nemo-agents/src/nemo_agents_plugin/cli.py | 2 +- .../src/nemo_agents_plugin/leaderboard/cli.py | 2 +- .../nemo_agents_plugin/leaderboard/render.py | 2 +- .../src/nemo_agents_plugin/utils.py | 2 +- .../src/nemo_anonymizer_plugin/app/input.py | 2 +- .../sdk/resources.py | 2 +- .../src/nemo_evaluator/filesets.py | 2 +- .../src/nemo_experimentalist_plugin/client.py | 4 +- .../nemo-experimentalist/tests/test_client.py | 2 +- .../analyst/observability.py | 2 +- .../src/nemo_insights_plugin/client.py | 4 +- plugins/nemo-insights/testbed/export.py | 2 +- plugins/nemo-insights/testbed/ingest.py | 2 +- plugins/nemo-insights/tests/test_client.py | 2 +- .../api/v2/jobs/endpoints.py | 2 +- .../tasks/safe_synthesizer/__main__.py | 2 +- sdk/python/nemo-platform/hatch_build.py | 221 + sdk/python/nemo-platform/pyproject.toml | 42 +- .../nemo-platform/src/nemo_platform/_alias.py | 176 + .../src/nemo_platform/_client.py | 88 +- .../src/nemo_platform/auth/__init__.py | 15 +- .../src/nemo_platform/auth/device_flow.py | 304 - .../src/nemo_platform/auth/helpers.py | 224 - .../src/nemo_platform/auth/token_provider.py | 254 - .../nemo_platform/auth/workload_exchange.py | 207 - .../src/nemo_platform/beta/__init__.py | 4 + .../nemo_platform/beta/evaluator/__init__.py | 301 +- .../beta/evaluator/agent_eval/dashboard.py | 175 - .../beta/evaluator/agent_eval/evaluator.py | 783 -- .../beta/evaluator/agent_eval/metrics.py | 279 - .../beta/evaluator/agent_eval/persistence.py | 172 - .../beta/evaluator/agent_eval/results.py | 1331 -- .../agent_eval/runtimes/callable_runtime.py | 122 - .../agent_eval/runtimes/codex/runtime.py | 641 - .../agent_eval/runtimes/docker_sandbox.py | 366 - .../agent_eval/runtimes/environment.py | 178 - .../agent_eval/runtimes/fabric/_common.py | 158 - .../runtimes/fabric/container_runtime.py | 602 - .../runtimes/fabric/hook_loading.py | 141 - .../agent_eval/runtimes/fabric/hooks.py | 54 - .../runtimes/fabric/hooks_mcp_binding.py | 440 - .../agent_eval/runtimes/fabric/image.py | 173 - .../agent_eval/runtimes/fabric/runtime.py | 724 - .../runtimes/fabric/sandbox.Dockerfile | 51 - .../agent_eval/runtimes/fabric/skills.py | 497 - .../agent_eval/runtimes/harbor_runtime.py | 1413 -- .../agent_eval/runtimes/sandbox/api.py | 127 - .../agent_eval/runtimes/sandbox/base.py | 193 - .../sandbox/providers/_compose_cli.py | 485 - .../sandbox/providers/_compose_contracts.py | 87 - .../sandbox/providers/_compose_inspection.py | 226 - .../sandbox/providers/_compose_lifecycle.py | 333 - .../sandbox/providers/_compose_provider.py | 935 -- .../sandbox/providers/_compose_state.py | 116 - .../sandbox/providers/_compose_transfer.py | 380 - .../runtimes/sandbox/providers/compose.py | 39 - .../runtimes/sandbox/providers/docker.py | 265 - .../beta/evaluator/agent_eval/scores.py | 97 - .../beta/evaluator/agent_eval/tasks.py | 239 - .../beta/evaluator/agent_eval/trials.py | 248 - .../evaluator/agent_eval/workspace_seeds.py | 171 - .../beta/evaluator/agent_inference.py | 800 -- .../evaluator/agent_stream_translation.py | 76 - .../nemo_platform/beta/evaluator/constants.py | 4 - .../beta/evaluator/dataset_schemas/common.py | 177 - .../dataset_schemas/compatibility.py | 353 - .../evaluator/dataset_schemas/templates.py | 389 - .../beta/evaluator/datasets/__init__.py | 28 - .../beta/evaluator/datasets/loader.py | 441 - .../src/nemo_platform/beta/evaluator/enums.py | 64 - .../beta/evaluator/execution/_protocols.py | 20 - .../beta/evaluator/execution/backends/base.py | 92 - .../execution/backends/local/backend.py | 165 - .../execution/benchmark_execution.py | 694 - .../beta/evaluator/execution/config.py | 51 - .../beta/evaluator/execution/evaluator.py | 316 - .../beta/evaluator/execution/job_poll.py | 98 - .../evaluator/execution/metric_execution.py | 901 -- .../beta/evaluator/execution/pipeline.py | 64 - .../beta/evaluator/execution/runs.py | 143 - .../beta/evaluator/execution/samples.py | 43 - .../beta/evaluator/execution/scoring.py | 201 - .../beta/evaluator/execution/utils.py | 100 - .../beta/evaluator/execution/values.py | 53 - .../nemo_platform/beta/evaluator/inference.py | 456 - .../beta/evaluator/metrics/aggregation.py | 479 - .../beta/evaluator/metrics/bleu.py | 94 - .../beta/evaluator/metrics/exact_match.py | 52 - .../beta/evaluator/metrics/f1.py | 50 - .../beta/evaluator/metrics/hooks.py | 41 - .../beta/evaluator/metrics/llm_judge.py | 441 - .../evaluator/metrics/llm_judge_defaults.py | 26 - .../beta/evaluator/metrics/number_check.py | 95 - .../beta/evaluator/metrics/protocol.py | 115 - .../beta/evaluator/metrics/ragas/__init__.py | 41 - .../beta/evaluator/metrics/ragas/base.py | 588 - .../beta/evaluator/metrics/ragas/git_patch.py | 36 - .../beta/evaluator/metrics/ragas/imports.py | 192 - .../beta/evaluator/metrics/ragas/metrics.py | 183 - .../beta/evaluator/metrics/remote.py | 250 - .../beta/evaluator/metrics/resolution.py | 48 - .../beta/evaluator/metrics/rouge.py | 71 - .../beta/evaluator/metrics/string_check.py | 69 - .../evaluator/metrics/template_rendering.py | 141 - .../beta/evaluator/metrics/tool_calling.py | 172 - .../evaluator/metrics/tunable_rag_defaults.py | 91 - .../metrics/tunable_rag_evaluator.py | 238 - .../beta/evaluator/metrics/types.py | 64 - .../beta/evaluator/metrics/utils.py | 54 - .../beta/evaluator/resilience/api.py | 142 - .../beta/evaluator/resilience/classifier.py | 142 - .../beta/evaluator/resilience/config.py | 69 - .../beta/evaluator/resilience/errors.py | 117 - .../beta/evaluator/resilience/policy.py | 50 - .../beta/evaluator/resilience/scheduler.py | 459 - .../beta/evaluator/resilience/types.py | 93 - .../beta/evaluator/resolver_protocols.py | 29 - .../nemo_platform/beta/evaluator/resolvers.py | 69 - .../beta/evaluator/structured_output.py | 184 - .../nemo_platform/beta/evaluator/templates.py | 113 - .../beta/evaluator/values/__init__.py | 369 - .../beta/evaluator/values/agents.py | 138 - .../beta/evaluator/values/atif.py | 135 - .../beta/evaluator/values/common.py | 25 - .../beta/evaluator/values/dataset_schemas.py | 112 - .../beta/evaluator/values/datasets.py | 26 - .../beta/evaluator/values/evidence.py | 495 - .../evaluator/values/llm_judge_defaults.py | 80 - .../beta/evaluator/values/metrics.py | 588 - .../beta/evaluator/values/models.py | 179 - .../evaluator/values/multi_metric_results.py | 386 - .../beta/evaluator/values/params.py | 109 - .../beta/evaluator/values/protocol.py | 214 - .../beta/evaluator/values/results.py | 878 -- .../beta/evaluator/values/scores.py | 300 - .../src/nemo_platform/cli/__init__.py | 6 +- .../src/nemo_platform/cli/app.py | 345 - .../nemo_platform/cli/commands/__init__.py | 15 - .../cli/commands/api/__init__.py | 106 - .../cli/commands/api/adapters.py | 393 - .../cli/commands/api/experiments.py | 393 - .../cli/commands/api/files/__init__.py | 243 - .../cli/commands/api/files/filesets.py | 395 - .../cli/commands/api/files/otlp/__init__.py | 15 - .../cli/commands/api/files/otlp/logs.py | 129 - .../cli/commands/api/guardrail/__init__.py | 272 - .../cli/commands/api/guardrail/configs.py | 320 - .../cli/commands/api/iam/__init__.py | 15 - .../cli/commands/api/iam/role_bindings.py | 273 - .../cli/commands/api/inference/__init__.py | 80 - .../inference/deployment_configs/__init__.py | 429 - .../inference/deployment_configs/versions.py | 141 - .../api/inference/deployments/__init__.py | 579 - .../api/inference/deployments/versions.py | 150 - .../api/inference/gateway/__init__.py | 19 - .../commands/api/inference/gateway/model.py | 324 - .../api/inference/gateway/openai/__init__.py | 15 - .../inference/gateway/openai/v1/__init__.py | 15 - .../api/inference/gateway/openai/v1/models.py | 120 - .../api/inference/gateway/provider.py | 327 - .../cli/commands/api/inference/models.py | 120 - .../cli/commands/api/inference/prompts.py | 378 - .../cli/commands/api/inference/providers.py | 621 - .../commands/api/inference/virtual_models.py | 461 - .../cli/commands/api/intake/__init__.py | 25 - .../cli/commands/api/intake/annotations.py | 273 - .../commands/api/intake/evaluator_results.py | 276 - .../commands/api/intake/ingest/__init__.py | 21 - .../cli/commands/api/intake/ingest/atif.py | 127 - .../api/intake/ingest/chat_completions.py | 150 - .../api/intake/ingest/otlp/__init__.py | 15 - .../api/intake/ingest/otlp/v1/__init__.py | 15 - .../api/intake/ingest/otlp/v1/traces.py | 72 - .../cli/commands/api/intake/sessions.py | 50 - .../cli/commands/api/intake/spans/__init__.py | 197 - .../api/intake/spans/evaluator_results.py | 78 - .../cli/commands/api/intake/spans/groups.py | 156 - .../cli/commands/api/intake/traces.py | 168 - .../cli/commands/api/jobs/__init__.py | 603 - .../cli/commands/api/jobs/results.py | 220 - .../cli/commands/api/jobs/steps.py | 252 - .../cli/commands/api/jobs/tasks.py | 219 - .../cli/commands/api/models/__init__.py | 568 - .../cli/commands/api/models/adapters.py | 261 - .../cli/commands/api/projects.py | 341 - .../cli/commands/api/secrets/__init__.py | 326 - .../cli/commands/api/secrets/admin.py | 44 - .../cli/commands/api/workspaces/__init__.py | 356 - .../cli/commands/api/workspaces/members.py | 313 - .../src/nemo_platform/cli/commands/auth.py | 1045 -- .../src/nemo_platform/cli/commands/config.py | 331 - .../nemo_platform/cli/commands/config_help.py | 19 - .../src/nemo_platform/cli/commands/docs.py | 150 - .../cli/commands/manifest_registry.py | 184 - .../src/nemo_platform/cli/commands/plugins.py | 101 - .../cli/commands/quickstart/__init__.py | 15 - .../cli/commands/quickstart/cli.py | 1184 -- .../cli/commands/services/cli.py | 939 -- .../src/nemo_platform/cli/commands/setup.py | 2552 ---- .../cli/commands/skills/agents/claude.py | 33 - .../cli/commands/skills/agents/codex.py | 36 - .../cli/commands/skills/agents/cursor.py | 18 - .../cli/commands/skills/agents/opencode.py | 21 - .../nemo_platform/cli/commands/skills/base.py | 75 - .../nemo_platform/cli/commands/skills/cli.py | 336 - .../cli/commands/skills/installer.py | 59 - .../cli/commands/skills/registry.py | 408 - .../cli/commands/use_cases/__init__.py | 15 - .../cli/commands/use_cases/agent.py | 245 - .../cli/commands/use_cases/chat.py | 965 -- .../cli/commands/use_cases/wait.py | 151 - .../src/nemo_platform/cli/core/__init__.py | 8 - .../nemo_platform/cli/core/agent_helpers.py | 84 - .../src/nemo_platform/cli/core/api.py | 177 - .../nemo_platform/cli/core/autocomplete.py | 63 - .../nemo_platform/cli/core/code_generator.py | 306 - .../src/nemo_platform/cli/core/context.py | 211 - .../src/nemo_platform/cli/core/errors.py | 361 - .../src/nemo_platform/cli/core/formatters.py | 771 -- .../nemo_platform/cli/core/help_formatter.py | 950 -- .../src/nemo_platform/cli/core/lazy_load.py | 280 - .../src/nemo_platform/cli/core/logging.py | 24 - .../src/nemo_platform/cli/core/pagination.py | 261 - .../src/nemo_platform/cli/core/stdin_utils.py | 233 - .../src/nemo_platform/cli/core/streaming.py | 30 - .../nemo_platform/cli/core/table_config.py | 302 - .../cli/core/timestamp_formatter.py | 127 - .../src/nemo_platform/cli/core/types.py | 124 - .../src/nemo_platform/cli/core/waiters.py | 445 - .../src/nemo_platform/cli/docker_preflight.py | 147 - .../src/nemo_platform/cli/manifest.py | 108 - .../nemo_platform/cli/telemetry/__init__.py | 14 - .../src/nemo_platform/cli/telemetry/emit.py | 99 - .../src/nemo_platform/cli/telemetry/events.py | 97 - .../nemo_platform/cli/telemetry/handler.py | 440 - .../nemo_platform/cli/telemetry/runtime.py | 91 - .../nemo_platform/cli/telemetry/session.py | 157 - .../src/nemo_platform/client/__init__.py | 15 +- .../src/nemo_platform/client/factory.py | 726 - .../src/nemo_platform/client/tls.py | 16 - .../src/nemo_platform/config/README.md | 171 - .../src/nemo_platform/config/__init__.py | 23 +- .../src/nemo_platform/config/config.py | 648 - .../src/nemo_platform/config/models.py | 391 - .../src/nemo_platform/config/types.py | 7 - .../src/nemo_platform/filesets/__init__.py | 19 +- .../filesets/filesystem/callbacks.py | 229 - .../filesets/filesystem/filesystem.py | 1008 -- .../src/nemo_platform/filesets/resources.py | 1190 -- .../src/nemo_platform/local/__init__.py | 6 + .../src/nemo_platform/local/_service_child.py | 30 - .../src/nemo_platform/local/process.py | 859 -- .../src/nemo_platform/local/services.py | 747 - .../src/nemo_platform/local/transport.py | 154 - .../src/nemo_platform/models/__init__.py | 14 +- .../src/nemo_platform/models/resources.py | 1030 -- .../src/nemo_platform/quickstart/__init__.py | 99 +- .../src/nemo_platform/quickstart/_registry.py | 28 - .../src/nemo_platform/quickstart/cluster.py | 198 - .../src/nemo_platform/quickstart/config.py | 420 - .../src/nemo_platform/quickstart/container.py | 748 -- .../nemo_platform/quickstart/gpu_config.py | 111 - .../quickstart/platform_config.py | 82 - .../src/nemo_platform/quickstart/preflight.py | 452 - .../src/nemo_platform/quickstart/prompts.py | 418 - .../src/nemo_platform/quickstart/storage.py | 90 - .../nemo_platform/quickstart/validators.py | 267 - .../src/nemo_platform/skills/__init__.py | 28 +- .../nemo_platform/skills/inference/SKILL.md | 580 - .../skills/nemo-agent-config/SKILL.md | 327 - .../references/templates/agent.yaml | 72 - .../skills/nemo-build-agent/SKILL.md | 423 - .../references/templates/agent.yml | 51 - .../templates/data-designer-config.py | 80 - .../references/templates/eval-job.json | 7 - .../skills/nemo-evaluator/SKILL.md | 130 - .../references/benchmark-reproduction.md | 104 - .../references/metric-selection.md | 58 - .../references/sdk-execution.md | 250 - .../references/troubleshooting.md | 51 - .../skills/nemo-experiments-upload/SKILL.md | 201 - .../references/harbor-quickstart.md | 67 - .../references/troubleshooting.md | 46 - .../skills/nemo-explore/SKILL.md | 302 - .../nemo_platform/skills/nemo-files/SKILL.md | 87 - .../skills/nemo-guardrails/SKILL.md | 526 - .../nemo_platform/skills/nemo-intake/SKILL.md | 161 - .../nemo-intake/references/ingest-formats.md | 194 - .../skills/nemo-model-selection/SKILL.md | 500 - .../references/benchmark_cache.json | 11206 ---------------- .../skills/nemo-secrets/SKILL.md | 66 - .../skills/nemo-skill-selection/SKILL.md | 187 - .../nemo_platform/skills/nemo-spec/SKILL.md | 270 - .../references/templates/agent-spec.md | 135 - .../nemo_platform/skills/nemo-status/SKILL.md | 125 - .../skills/nemo-teardown/SKILL.md | 210 - .../skills/nemo-try-agent/SKILL.md | 163 - .../src/nemo_platform/ui/__init__.py | 15 +- .../src/nemo_platform/ui/output.py | 109 - .../src/nemo_platform/ui/prompts.py | 678 - .../vendored/nemo_platform_ext/__init__.py | 15 - .../nemo_platform_ext/auth/__init__.py | 15 - .../auth/test_device_flow.py | 479 - .../auth/test_token_provider.py | 509 - .../nemo_platform_ext/auth/test_utils.py | 243 - .../auth/test_workload_exchange.py | 258 - .../nemo_platform_ext/cli/__init__.py | 15 - .../cli/commands/__init__.py | 15 - .../cli/commands/conftest.py | 15 - .../cli/commands/skills/__init__.py | 15 - .../cli/commands/skills/agents/__init__.py | 15 - .../cli/commands/skills/agents/test_claude.py | 126 - .../cli/commands/skills/agents/test_codex.py | 165 - .../cli/commands/skills/agents/test_cursor.py | 67 - .../commands/skills/agents/test_opencode.py | 60 - .../cli/commands/skills/test_base.py | 38 - .../cli/commands/skills/test_cli.py | 387 - .../cli/commands/skills/test_installer.py | 139 - .../cli/commands/skills/test_registry.py | 422 - .../cli/commands/skills/test_skill_content.py | 142 - .../cli/commands/test_agent.py | 198 - .../cli/commands/test_auth.py | 1188 -- .../cli/commands/test_auth_password_grant.py | 80 - .../cli/commands/test_config.py | 424 - .../cli/commands/test_create_wait.py | 108 - .../cli/commands/test_plugins.py | 179 - .../cli/commands/test_services.py | 925 -- .../cli/commands/test_services_lifecycle.py | 742 - .../cli/commands/test_services_process.py | 1056 -- .../cli/commands/test_setup.py | 3575 ----- .../cli/commands/test_setup_cli.py | 118 - .../cli/commands/use_cases/__init__.py | 15 - .../cli/commands/use_cases/test_chat.py | 494 - .../nemo_platform_ext/cli/core/__init__.py | 15 - .../nemo_platform_ext/cli/core/test_api.py | 148 - .../cli/core/test_code_generator.py | 172 - .../cli/core/test_context.py | 175 - .../nemo_platform_ext/cli/core/test_errors.py | 544 - .../cli/core/test_formatters.py | 952 -- .../cli/core/test_help_formatter.py | 582 - .../cli/core/test_pagination.py | 357 - .../cli/core/test_stdin_utils.py | 208 - .../cli/core/test_table_config.py | 198 - .../cli/core/test_timestamp_formatter.py | 142 - .../cli/core/test_waiters.py | 278 - .../cli/integration/__init__.py | 15 - .../cli/integration/conftest.py | 124 - .../cli/integration/test_basic.py | 24 - .../cli/integration/test_cli_integration.py | 205 - .../cli/integration/test_error_handling.py | 26 - .../cli/integration/test_filesets.py | 328 - .../integration/test_format_combinations.py | 60 - .../cli/integration/test_stdin_integration.py | 143 - .../cli/telemetry/__init__.py | 15 - .../cli/telemetry/conftest.py | 23 - .../cli/telemetry/test_command_hook.py | 84 - .../cli/telemetry/test_emit.py | 208 - .../cli/telemetry/test_events.py | 75 - .../cli/telemetry/test_handler.py | 599 - .../cli/telemetry/test_job_events.py | 235 - .../cli/telemetry/test_onboarding_events.py | 377 - .../cli/telemetry/test_wire_contract_smoke.py | 179 - .../nemo_platform_ext/cli/test_agent_mode.py | 152 - .../nemo_platform_ext/cli/test_app.py | 600 - .../cli/test_docker_preflight.py | 209 - .../nemo_platform_ext/cli/test_docs.py | 129 - .../cli/test_docs_generator.py | 72 - .../vendored/nemo_platform_ext/cli/utils.py | 14 - .../nemo_platform_ext/client/__init__.py | 15 - .../nemo_platform_ext/client/test_client.py | 817 -- .../nemo_platform_ext/config/__init__.py | 15 - .../nemo_platform_ext/config/test_config.py | 1128 -- .../vendored/nemo_platform_ext/conftest.py | 36 - .../nemo_platform_ext/local/__init__.py | 15 - .../local/test_config_environment.py | 196 - .../local/test_daemon_lifecycle.py | 646 - .../local/test_health_child.py | 340 - .../local/test_port_socket.py | 165 - .../nemo_platform_ext/local/test_services.py | 1096 -- .../local/test_services_contract.py | 334 - .../local/test_sidecar_integration.py | 231 - .../nemo_platform_ext/local/test_transport.py | 167 - .../nemo_platform_ext/quickstart/__init__.py | 15 - .../quickstart/test_cluster_info_cli.py | 232 - .../quickstart/test_container.py | 769 -- .../quickstart/test_gpu_config.py | 103 - .../quickstart/test_preflight.py | 306 - .../quickstart/test_prompts.py | 306 - .../quickstart/test_quickstart_cli.py | 361 - .../quickstart/test_quickstart_config.py | 388 - .../vendored/nemo_platform_ext/ui/__init__.py | 15 - .../nemo_platform_ext/ui/test_prompts.py | 61 - .../ui/test_prompts_multiselect.py | 138 - .../overrides/nemo-platform/hatch_build.py | 221 + .../nemo-platform/src/nemo_platform/_alias.py | 176 + .../core/files/tests/integration/conftest.py | 4 +- .../test_huggingface_storage.py | 2 +- .../files/tests/integration/test_files_sdk.py | 4 +- .../integration/test_fileset_filesystem.py | 4 +- tools/lint/lint-sdk-vendored.sh | 13 +- .../cli_generator/overrides/files/download.py | 2 +- .../cli_generator/overrides/files/upload.py | 2 +- .../sdk/post_generation_update.py | 115 +- .../sdk/vendor/vendor_package.py | 200 +- .../tests/sdk/test_post_generation_update.py | 92 + .../tests/sdk/vendor/test_vendor_package.py | 259 +- uv.lock | 401 +- 431 files changed, 1997 insertions(+), 110920 deletions(-) create mode 100644 sdk/python/nemo-platform/hatch_build.py create mode 100644 sdk/python/nemo-platform/src/nemo_platform/_alias.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/auth/device_flow.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/auth/helpers.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/auth/token_provider.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/auth/workload_exchange.py create mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/__init__.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/dashboard.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/metrics.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/persistence.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/callable_runtime.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/codex/runtime.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/environment.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/_common.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/container_runtime.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/hook_loading.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/hooks.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/hooks_mcp_binding.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/image.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/sandbox.Dockerfile delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/skills.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/harbor_runtime.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/api.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/base.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/_compose_cli.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/_compose_contracts.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/_compose_inspection.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/_compose_lifecycle.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/_compose_provider.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/_compose_state.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/_compose_transfer.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/compose.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/docker.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/scores.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/tasks.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/trials.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/workspace_seeds.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_inference.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_stream_translation.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/constants.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/dataset_schemas/common.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/dataset_schemas/compatibility.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/dataset_schemas/templates.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/datasets/__init__.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/datasets/loader.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/enums.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/_protocols.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/backends/base.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/backends/local/backend.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/benchmark_execution.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/config.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/evaluator.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/job_poll.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/metric_execution.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/pipeline.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/runs.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/samples.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/scoring.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/utils.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/values.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/inference.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/aggregation.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/bleu.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/exact_match.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/f1.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/hooks.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/llm_judge.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/llm_judge_defaults.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/number_check.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/protocol.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/ragas/__init__.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/ragas/base.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/ragas/git_patch.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/ragas/imports.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/ragas/metrics.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/remote.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/resolution.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/rouge.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/string_check.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/template_rendering.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/tool_calling.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/tunable_rag_defaults.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/tunable_rag_evaluator.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/types.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/utils.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resilience/api.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resilience/classifier.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resilience/config.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resilience/errors.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resilience/policy.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resilience/scheduler.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resilience/types.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resolver_protocols.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resolvers.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/structured_output.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/templates.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/__init__.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/agents.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/atif.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/common.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/dataset_schemas.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/datasets.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/evidence.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/llm_judge_defaults.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/metrics.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/models.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/multi_metric_results.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/params.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/protocol.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/scores.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/app.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/__init__.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/__init__.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/adapters.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/experiments.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/files/__init__.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/files/filesets.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/files/otlp/__init__.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/files/otlp/logs.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/guardrail/__init__.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/guardrail/configs.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/iam/__init__.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/iam/role_bindings.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/inference/__init__.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/inference/deployment_configs/__init__.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/inference/deployment_configs/versions.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/inference/deployments/__init__.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/inference/deployments/versions.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/inference/gateway/__init__.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/inference/gateway/model.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/inference/gateway/openai/__init__.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/inference/gateway/openai/v1/__init__.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/inference/gateway/openai/v1/models.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/inference/gateway/provider.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/inference/models.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/inference/prompts.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/inference/providers.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/inference/virtual_models.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/__init__.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/annotations.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/evaluator_results.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/ingest/__init__.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/ingest/atif.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/ingest/chat_completions.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/ingest/otlp/__init__.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/ingest/otlp/v1/__init__.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/ingest/otlp/v1/traces.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/sessions.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/spans/__init__.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/spans/evaluator_results.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/spans/groups.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/traces.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/jobs/__init__.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/jobs/results.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/jobs/steps.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/jobs/tasks.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/models/__init__.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/models/adapters.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/projects.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/secrets/__init__.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/secrets/admin.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/workspaces/__init__.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/workspaces/members.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/auth.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/config.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/config_help.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/docs.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/manifest_registry.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/plugins.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/quickstart/__init__.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/quickstart/cli.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/services/cli.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/setup.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/skills/agents/claude.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/skills/agents/codex.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/skills/agents/cursor.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/skills/agents/opencode.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/skills/base.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/skills/cli.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/skills/installer.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/skills/registry.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/use_cases/__init__.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/use_cases/agent.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/use_cases/chat.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/use_cases/wait.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/core/__init__.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/core/agent_helpers.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/core/api.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/core/autocomplete.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/core/code_generator.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/core/context.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/core/errors.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/core/formatters.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/core/help_formatter.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/core/lazy_load.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/core/logging.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/core/pagination.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/core/stdin_utils.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/core/streaming.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/core/table_config.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/core/timestamp_formatter.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/core/types.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/core/waiters.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/docker_preflight.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/manifest.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/telemetry/__init__.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/telemetry/emit.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/telemetry/events.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/telemetry/handler.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/telemetry/runtime.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/telemetry/session.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/client/factory.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/client/tls.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/config/README.md delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/config/config.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/config/models.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/config/types.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/filesets/filesystem/callbacks.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/filesets/filesystem/filesystem.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/filesets/resources.py create mode 100644 sdk/python/nemo-platform/src/nemo_platform/local/__init__.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/local/_service_child.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/local/process.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/local/services.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/local/transport.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/models/resources.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/quickstart/_registry.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/quickstart/cluster.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/quickstart/config.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/quickstart/container.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/quickstart/gpu_config.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/quickstart/platform_config.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/quickstart/preflight.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/quickstart/prompts.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/quickstart/storage.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/quickstart/validators.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/skills/inference/SKILL.md delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/skills/nemo-agent-config/SKILL.md delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/skills/nemo-agent-config/references/templates/agent.yaml delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/skills/nemo-build-agent/SKILL.md delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/skills/nemo-build-agent/references/templates/agent.yml delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/skills/nemo-build-agent/references/templates/data-designer-config.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/skills/nemo-build-agent/references/templates/eval-job.json delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/skills/nemo-evaluator/SKILL.md delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/skills/nemo-evaluator/references/benchmark-reproduction.md delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/skills/nemo-evaluator/references/metric-selection.md delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/skills/nemo-evaluator/references/sdk-execution.md delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/skills/nemo-evaluator/references/troubleshooting.md delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/skills/nemo-experiments-upload/SKILL.md delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/skills/nemo-experiments-upload/references/harbor-quickstart.md delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/skills/nemo-experiments-upload/references/troubleshooting.md delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/skills/nemo-explore/SKILL.md delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/skills/nemo-files/SKILL.md delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/skills/nemo-guardrails/SKILL.md delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/skills/nemo-intake/SKILL.md delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/skills/nemo-intake/references/ingest-formats.md delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/skills/nemo-model-selection/SKILL.md delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/skills/nemo-model-selection/references/benchmark_cache.json delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/skills/nemo-secrets/SKILL.md delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/skills/nemo-skill-selection/SKILL.md delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/skills/nemo-spec/SKILL.md delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/skills/nemo-spec/references/templates/agent-spec.md delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/skills/nemo-status/SKILL.md delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/skills/nemo-teardown/SKILL.md delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/skills/nemo-try-agent/SKILL.md delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/ui/output.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/ui/prompts.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/__init__.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/auth/__init__.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/auth/test_device_flow.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/auth/test_token_provider.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/auth/test_utils.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/auth/test_workload_exchange.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/__init__.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/__init__.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/conftest.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/skills/__init__.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/skills/agents/__init__.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/skills/agents/test_claude.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/skills/agents/test_codex.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/skills/agents/test_cursor.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/skills/agents/test_opencode.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/skills/test_base.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/skills/test_cli.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/skills/test_installer.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/skills/test_registry.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/skills/test_skill_content.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_agent.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_auth.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_auth_password_grant.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_config.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_create_wait.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_plugins.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_services.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_services_lifecycle.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_services_process.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_setup.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_setup_cli.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/use_cases/__init__.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/use_cases/test_chat.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/core/__init__.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/core/test_api.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/core/test_code_generator.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/core/test_context.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/core/test_errors.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/core/test_formatters.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/core/test_help_formatter.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/core/test_pagination.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/core/test_stdin_utils.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/core/test_table_config.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/core/test_timestamp_formatter.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/core/test_waiters.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/integration/__init__.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/integration/conftest.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/integration/test_basic.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/integration/test_cli_integration.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/integration/test_error_handling.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/integration/test_filesets.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/integration/test_format_combinations.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/integration/test_stdin_integration.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/telemetry/__init__.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/telemetry/conftest.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/telemetry/test_command_hook.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/telemetry/test_emit.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/telemetry/test_events.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/telemetry/test_handler.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/telemetry/test_job_events.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/telemetry/test_onboarding_events.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/telemetry/test_wire_contract_smoke.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/test_agent_mode.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/test_app.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/test_docker_preflight.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/test_docs.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/test_docs_generator.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/utils.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/client/__init__.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/client/test_client.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/config/__init__.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/config/test_config.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/conftest.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/__init__.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/test_config_environment.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/test_daemon_lifecycle.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/test_health_child.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/test_port_socket.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/test_services.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/test_services_contract.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/test_sidecar_integration.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/test_transport.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/quickstart/__init__.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/quickstart/test_cluster_info_cli.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/quickstart/test_container.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/quickstart/test_gpu_config.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/quickstart/test_preflight.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/quickstart/test_prompts.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/quickstart/test_quickstart_cli.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/quickstart/test_quickstart_config.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/ui/__init__.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/ui/test_prompts.py delete mode 100644 sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/ui/test_prompts_multiselect.py create mode 100644 sdk/python/overrides/nemo-platform/hatch_build.py create mode 100644 sdk/python/overrides/nemo-platform/src/nemo_platform/_alias.py create mode 100644 tools/nemo-platform-sdk-tools/tests/sdk/test_post_generation_update.py diff --git a/packages/data_designer_nemo/src/data_designer_nemo/fileset_filesystem_provider.py b/packages/data_designer_nemo/src/data_designer_nemo/fileset_filesystem_provider.py index 3c418321b5..6ae662aba3 100644 --- a/packages/data_designer_nemo/src/data_designer_nemo/fileset_filesystem_provider.py +++ b/packages/data_designer_nemo/src/data_designer_nemo/fileset_filesystem_provider.py @@ -13,9 +13,9 @@ SeedReaderFileSystemContext, ) from data_designer_nemo.filesystem import make_filesystem +from filesets import FilesetFileSystem, FilesetPathError, build_fileset_ref, parse_fileset_ref from fsspec.implementations.dirfs import DirFileSystem from nemo_platform import AsyncNeMoPlatform, NeMoPlatform -from nemo_platform.filesets import FilesetFileSystem, FilesetPathError, build_fileset_ref, parse_fileset_ref from nemo_platform_plugin.files.client import AsyncFilesClient, FilesClient diff --git a/packages/data_designer_nemo/src/data_designer_nemo/filesystem.py b/packages/data_designer_nemo/src/data_designer_nemo/filesystem.py index 689e5c6d62..7968b59503 100644 --- a/packages/data_designer_nemo/src/data_designer_nemo/filesystem.py +++ b/packages/data_designer_nemo/src/data_designer_nemo/filesystem.py @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 from data_designer_nemo.sdk_translation import async_to_sync_sdk +from filesets import FilesetFileSystem from nemo_platform import AsyncNeMoPlatform, NeMoPlatform -from nemo_platform.filesets import FilesetFileSystem from nemo_platform_plugin.client.adapter import client_from_platform from nemo_platform_plugin.files.client import FilesClient diff --git a/packages/data_designer_nemo/src/data_designer_nemo/seed.py b/packages/data_designer_nemo/src/data_designer_nemo/seed.py index 8bd3a492c8..82fb956797 100644 --- a/packages/data_designer_nemo/src/data_designer_nemo/seed.py +++ b/packages/data_designer_nemo/src/data_designer_nemo/seed.py @@ -10,8 +10,8 @@ from data_designer_nemo.fileset_file_seed_source import FilesetFileSeedSource from data_designer_nemo.fileset_filesystem_provider import is_local_directory from data_designer_nemo.secret_resolver import validate_secret +from filesets import FilesetPathError, build_fileset_ref, parse_fileset_ref from nemo_platform import AsyncNeMoPlatform -from nemo_platform.filesets import FilesetPathError, build_fileset_ref, parse_fileset_ref from nemo_platform_plugin.client.adapter import client_from_platform from nemo_platform_plugin.client.errors import NotFoundError, PermissionDeniedError from nemo_platform_plugin.files.client import AsyncFilesClient diff --git a/packages/filesets/pyproject.toml b/packages/filesets/pyproject.toml index 9274914a4e..3ee6047145 100644 --- a/packages/filesets/pyproject.toml +++ b/packages/filesets/pyproject.toml @@ -31,6 +31,7 @@ packages = ["src/filesets"] [tool.vendor-package] package = "filesets" package_root = "packages/filesets" +sdk_include_mode = "source-package" target_sdk_module = "filesets" included_paths = [ "**/*.py", diff --git a/packages/filesets/src/filesets/resources.py b/packages/filesets/src/filesets/resources.py index de4b204ba0..1ac83b1729 100644 --- a/packages/filesets/src/filesets/resources.py +++ b/packages/filesets/src/filesets/resources.py @@ -12,10 +12,16 @@ from dataclasses import dataclass from functools import cached_property from pathlib import PurePath -from typing import Protocol, runtime_checkable +from typing import Any, Protocol, runtime_checkable from fsspec.callbacks import DEFAULT_CALLBACK, Callback from fsspec.core import has_magic +from nemo_platform.resources.files.files import ( + AsyncFilesResource as GeneratedAsyncFilesResource, +) +from nemo_platform.resources.files.files import ( + FilesResource as GeneratedFilesResource, +) from nemo_platform.resources.files.filesets import AsyncFilesetsResource, FilesetsResource from nemo_platform.resources.files.otlp.otlp import AsyncOtlpResource, OtlpResource from nemo_platform_plugin.files.client import AsyncFilesClient, FilesClient @@ -150,6 +156,7 @@ def __init__( # Retain the platform client so the generated fileset/otlp sub-resources # (which speak to the platform client, not the FilesClient) can be exposed. self._platform_client = client + self._generated_files = GeneratedFilesResource(client) self._async_client = async_files_client if files_client is not None: self._client = files_client @@ -163,6 +170,9 @@ def client(self) -> FilesClient: """Access the underlying FilesClient for direct API calls.""" return self._client + def __getattr__(self, name: str) -> Any: + return getattr(self._generated_files, name) + @cached_property def filesets(self) -> FilesetsResource: """Fileset entity CRUD (create/list/get/update/delete) via the generated SDK resource.""" @@ -693,6 +703,7 @@ def __init__(self, client, *, files_client: AsyncFilesClient | None = None) -> N # Retain the platform client so the generated fileset/otlp sub-resources # (which speak to the platform client, not the FilesClient) can be exposed. self._platform_client = client + self._generated_files = GeneratedAsyncFilesResource(client) if files_client is not None: self._client = files_client else: @@ -705,6 +716,9 @@ def client(self) -> AsyncFilesClient: """Access the underlying AsyncFilesClient for direct API calls.""" return self._client + def __getattr__(self, name: str) -> Any: + return getattr(self._generated_files, name) + @cached_property def filesets(self) -> AsyncFilesetsResource: """Fileset entity CRUD (create/list/get/update/delete) via the generated SDK resource.""" diff --git a/packages/models/pyproject.toml b/packages/models/pyproject.toml index 9f7c49c9d5..1f66650b7e 100644 --- a/packages/models/pyproject.toml +++ b/packages/models/pyproject.toml @@ -27,6 +27,7 @@ dev = [] [tool.vendor-package] package = "models" package_root = "packages/models" +sdk_include_mode = "source-package" target_sdk_module = "models" included_paths = [ "**/*.py", diff --git a/packages/models/tests/test_client.py b/packages/models/tests/test_client.py index f5d3ee849c..42fa9a71f4 100644 --- a/packages/models/tests/test_client.py +++ b/packages/models/tests/test_client.py @@ -449,8 +449,8 @@ def test_wait_for_openai_model_bounds_sleep_to_remaining_timeout(sdk): "get", side_effect=_not_found_error(), ), - patch("nemo_platform.models.resources.time.time", side_effect=clock.time), - patch("nemo_platform.models.resources.time.sleep", side_effect=clock.sleep), + patch("models.resources.time.time", side_effect=clock.time), + patch("models.resources.time.sleep", side_effect=clock.sleep), ): with pytest.raises(TimeoutError, match="OpenAI model ws/model-a not available"): sdk.models.wait_for_openai_model("model-a", workspace="ws", timeout=1.25, poll_interval=5) @@ -631,8 +631,8 @@ async def test_async_wait_for_openai_model_bounds_sleep_to_remaining_timeout(asy with ( patch.object(async_sdk.inference.gateway.openai.v1.models, "get", mock_get), - patch("nemo_platform.models.resources.time.time", side_effect=clock.time), - patch("nemo_platform.models.resources.asyncio.sleep", side_effect=clock.async_sleep), + patch("models.resources.time.time", side_effect=clock.time), + patch("models.resources.asyncio.sleep", side_effect=clock.async_sleep), ): with pytest.raises(TimeoutError, match="OpenAI model ws/model-a not available"): await async_sdk.models.wait_for_openai_model("model-a", workspace="ws", timeout=1.25, poll_interval=5) diff --git a/packages/nemo_evaluator_sdk/pyproject.toml b/packages/nemo_evaluator_sdk/pyproject.toml index 7441fe268f..037ae6108d 100644 --- a/packages/nemo_evaluator_sdk/pyproject.toml +++ b/packages/nemo_evaluator_sdk/pyproject.toml @@ -116,6 +116,7 @@ addopts = ["--import-mode=importlib"] [tool.vendor-package] package = "nemo_evaluator_sdk" package_root = "packages/nemo_evaluator_sdk" +sdk_include_mode = "source-package" source_module = "nemo_evaluator_sdk" target_sdk_module = "beta.evaluator" sdk_optional_dependencies_name = "nemo-evaluator-sdk" diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_evidence.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_evidence.py index ce42b78215..f6265d1078 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_evidence.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_evidence.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import asyncio +import importlib import json import os import time @@ -372,9 +373,10 @@ async def test_trace_handle_exposes_typed_tool_evidence_and_retains_modeled_fiel def test_observation_models_are_exported_from_source_and_vendored_values_packages() -> None: from nemo_evaluator_sdk.values import Observation as SourceObservation from nemo_evaluator_sdk.values import ObservationResult as SourceObservationResult - from nemo_platform.beta.evaluator.values import Observation as VendoredObservation - from nemo_platform.beta.evaluator.values import ObservationResult as VendoredObservationResult + vendored_values = importlib.import_module("nemo_platform.beta.evaluator.values") + VendoredObservation = vendored_values.Observation + VendoredObservationResult = vendored_values.ObservationResult assert SourceObservation.__name__ == VendoredObservation.__name__ == "Observation" assert SourceObservationResult.__name__ == VendoredObservationResult.__name__ == "ObservationResult" diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_sandbox_compose_vendored_import.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_sandbox_compose_vendored_import.py index bf0bb5e905..a77a56925c 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_sandbox_compose_vendored_import.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_sandbox_compose_vendored_import.py @@ -8,20 +8,16 @@ import importlib from pathlib import Path -from nemo_platform.beta.evaluator.agent_eval.runtimes.sandbox.providers.compose import ( - ComposeCleanupError, - ComposeCommandResult, - ComposeServiceTopology, - ComposeTeardownContext, - DockerComposeSandboxProvider, - ProgressCallback, - PullPolicy, - TeardownHook, -) - def test_vendored_compose_public_imports_are_constructible_without_docker(tmp_path: Path) -> None: """The vendored public Compose façade remains importable without Docker.""" + compose = importlib.import_module("nemo_platform.beta.evaluator.agent_eval.runtimes.sandbox.providers.compose") + ComposeCleanupError = compose.ComposeCleanupError + ComposeCommandResult = compose.ComposeCommandResult + ComposeServiceTopology = compose.ComposeServiceTopology + ComposeTeardownContext = compose.ComposeTeardownContext + DockerComposeSandboxProvider = compose.DockerComposeSandboxProvider + for cls in ( ComposeCleanupError, ComposeCommandResult, @@ -31,9 +27,9 @@ def test_vendored_compose_public_imports_are_constructible_without_docker(tmp_pa ): assert getattr(importlib.import_module(cls.__module__), cls.__name__) is cls - assert ProgressCallback is not None - assert PullPolicy is not None - assert TeardownHook is not None + assert compose.ProgressCallback is not None + assert compose.PullPolicy is not None + assert compose.TeardownHook is not None topology = ComposeServiceTopology("agent", frozenset({"agent"})) command_result = ComposeCommandResult(("docker", "compose", "ps"), 0, "", "") diff --git a/packages/nemo_platform/pyproject.toml b/packages/nemo_platform/pyproject.toml index b3c8887a56..b83a9721eb 100644 --- a/packages/nemo_platform/pyproject.toml +++ b/packages/nemo_platform/pyproject.toml @@ -126,6 +126,7 @@ files-service = [ "ngcsdk>=4.9.10", "duckdb>=1.1.3", "pandas>=1.5.3", + "filesets", "opentelemetry-proto>=1.28.2", "aioboto3>=15.5.0", "types-aioboto3[s3]>=15.5.0", @@ -348,24 +349,24 @@ nemo-platform-plugin = [ # Generated from [tool.bundle-package]; do not edit by hand. nemo-platform-sdk = [ - "httpx>=0.23.0, <1", - "pydantic>=2.0.0,<3", - "typing-extensions>=4.14, <5", - "anyio>=4.0.0,<5", - "distro>=1.7.0, <2", - "sniffio", "nemo-platform-plugin", "typer>=0.20.0", "rich>=13.7.1", "prompt_toolkit>=3.0.0", "requests>=2.31.0", + "pydantic>=2.0.0,<3", "pyyaml>=6.0.0", "docker>=7.0.0", "ngcsdk>=4.8.2", "nvidia-ml-py>=13.0.0", "psutil>=5.9.0", + "httpx>=0.23.0,<1", "openai", + "anyio>=4.0.0,<5", "fsspec>=2023.1.0", + "typing-extensions>=4.14, <5", + "distro>=1.7.0, <2", + "sniffio", ] # Generated from [tool.bundle-package]; do not edit by hand. @@ -459,9 +460,9 @@ services = [ "nmp-common", "pyleak>=0.1.0", "rich>=14.1.0", - "nemo-platform[platform-seed-service]", "nemo-platform[core-service]", "nemo-platform[studio-service]", + "nemo-platform[platform-seed-service]", "nemo-platform[intake-service]", "nemo-platform[hello-world-service]", "nemo-platform[guardrails-service]", @@ -592,6 +593,8 @@ safe-synthesizer = "nemo_safe_synthesizer_plugin.skills:get_skills_path" nemo-platform-sdk = { workspace = true } nemo-platform-ext = { workspace = true } nemo-evaluator-sdk = { workspace = true } +models = { workspace = true } +filesets = { workspace = true } nmp-build-tools = { workspace = true } [tool.hatch.version] @@ -628,6 +631,9 @@ only-include = ["_empty"] # Library packages nmp-common = { source = "../../packages/nmp_common/src/nmp/common", module = "nmp/common", inherit = { "entry-points" = ["nemo.*"] } } nemo-platform-plugin = { source = "../../packages/nemo_platform_plugin/src/nemo_platform_plugin", module = "nemo_platform_plugin" } +nemo-platform-ext = { source = "../../packages/nemo_platform_ext/src/nemo_platform_ext", module = "nemo_platform_ext", deps_group = "nemo-platform-sdk", force_include = { "../../../../docs" = "nemo_platform_ext/cli/docs" } } +models = { source = "../../packages/models/src/models", module = "models", deps_group = "nemo-platform-sdk" } +filesets = { source = "../../packages/filesets/src/filesets", module = "filesets", deps_group = "nemo-platform-sdk" } nemo-evaluator-sdk = { source = "../../packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk", module = "nemo_evaluator_sdk" } data-designer-nemo = { source = "../../packages/data_designer_nemo/src/data_designer_nemo", module = "data_designer_nemo" } nmp-platform-runner = { source = "../../packages/nmp_platform_runner/src/nmp/platform_runner", module = "nmp/platform_runner", deps_group = "services" } @@ -666,7 +672,6 @@ module = "nemo_platform" inherit."optional-dependencies" = true inherit.scripts = true inherit."entry-points" = ["nemo.*"] -force_include."../../../../../docs" = "nemo_platform/cli/docs" [tool.bundle-package.nemo-agents-example-calculator] source = "../../plugins/nemo-agents/examples/calculator-agent/src/calculator_agent" diff --git a/packages/nemo_platform_ext/pyproject.toml b/packages/nemo_platform_ext/pyproject.toml index 7136be3899..64c5bf5972 100644 --- a/packages/nemo_platform_ext/pyproject.toml +++ b/packages/nemo_platform_ext/pyproject.toml @@ -88,6 +88,7 @@ packages = ["src/nemo_platform_ext"] [tool.vendor-package] package = "nemo_platform_ext" package_root = "packages/nemo_platform_ext" +sdk_include_mode = "source-package" # Globs are evaluated per top-level module (e.g. `skills/`, `cli/`, `quickstart/`). # `**/*.md` recursively includes SKILL.md plus any companion markdown files # skills ship under `resources/` (e.g. notes, sub-docs, prompts). Add further diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/files/__init__.py b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/files/__init__.py index f5adedbda5..c4d8e3200f 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/files/__init__.py +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/files/__init__.py @@ -69,7 +69,7 @@ def upload_files( if workspace is None: workspace = client._get_workspace_path_param() - from nemo_platform.filesets import RichProgressCallback + from filesets import RichProgressCallback with RichProgressCallback(description="Uploading") as callback: if fileset is not None: @@ -136,7 +136,7 @@ def download_files( if workspace is None: workspace = client._get_workspace_path_param() - from nemo_platform.filesets import RichProgressCallback + from filesets import RichProgressCallback with RichProgressCallback(description="Downloading") as callback: client.files.download( diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/client/enhanced.py b/packages/nemo_platform_ext/src/nemo_platform_ext/client/enhanced.py index af6451b3ed..f8980ae6fe 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/client/enhanced.py +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/client/enhanced.py @@ -7,7 +7,7 @@ import os from pathlib import Path -from typing import Any, Mapping +from typing import Any, Mapping, Self import httpx from httpx import Timeout @@ -49,6 +49,20 @@ def _should_bootstrap_config( ) +def _copy_requires_bootstrap( + *, + config_path: Path | None, + context_name: str | None, + access_token: str | None, +) -> bool: + return ( + config_path is not None + or context_name is not None + or access_token is not None + or bool(os.environ.get(WORKLOAD_IDENTITY_TOKEN_FILE_ENVVAR)) + ) + + class NeMoPlatform(SyncAPIClient): def __init__( self, @@ -133,6 +147,9 @@ def __init__( http_client: Custom ``httpx.Client`` instance. When provided, the auth bootstrap is skipped entirely regardless of other parameters. """ + env_base_url = os.environ.get("NEMO_PLATFORM_BASE_URL") + bootstrap_base_url = base_url if base_url is not None else env_base_url + should_bootstrap = _should_bootstrap_config( http_client=http_client, base_url=base_url, @@ -146,7 +163,7 @@ def __init__( client_init_kwargs = build_client_init_kwargs( config_path=config_path, - base_url=base_url, + base_url=bootstrap_base_url, context_name=context_name, access_token=access_token, extra_headers=default_headers, @@ -155,9 +172,19 @@ def __init__( if workspace is None: workspace = client_init_kwargs.workspace default_headers = client_init_kwargs.default_headers + if client_init_kwargs.http_client is not None and not isinstance( + client_init_kwargs.http_client, httpx.Client + ): + raise TypeError("Expected httpx.Client from sync client factory") http_client = client_init_kwargs.http_client except Exception as e: - raise RuntimeError(f"NeMoPlatform client initialization failed: {e}") + raise RuntimeError(f"NeMoPlatform client initialization failed: {e}") from e + + if base_url is None: + base_url = bootstrap_base_url + + if base_url is None: + raise RuntimeError("NeMoPlatform client initialization failed: base_url is required") client_verify = client_verify_from_env() if http_client is None and client_verify is not True: @@ -194,6 +221,66 @@ def __getattr__(self, name: str) -> Any: self.__dict__[name] = instance return instance + def copy( + self, + *, + workspace: str | None = None, + base_url: str | httpx.URL | None = None, + inference_base_url: str | httpx.URL | None = None, + config_path: Path | None = None, + context_name: str | None = None, + access_token: str | None = None, + timeout: float | Timeout | None | NotGiven = not_given, + http_client: httpx.Client | None = None, + max_retries: int | NotGiven = not_given, + default_headers: Mapping[str, str] | None = None, + set_default_headers: Mapping[str, str] | None = None, + default_query: Mapping[str, object] | None = None, + set_default_query: Mapping[str, object] | None = None, + _extra_kwargs: Mapping[str, Any] = {}, + ) -> Self: + """ + Create a new client instance re-using the same options given to the current client with optional overriding. + """ + if default_headers is not None and set_default_headers is not None: + raise ValueError("The `default_headers` and `set_default_headers` arguments are mutually exclusive") + + if default_query is not None and set_default_query is not None: + raise ValueError("The `default_query` and `set_default_query` arguments are mutually exclusive") + + headers = self._custom_headers + if default_headers is not None: + headers = {**headers, **default_headers} + elif set_default_headers is not None: + headers = set_default_headers + + params = self._custom_query + if default_query is not None: + params = {**params, **default_query} + elif set_default_query is not None: + params = set_default_query + + if http_client is None and not _copy_requires_bootstrap( + config_path=config_path, + context_name=context_name, + access_token=access_token, + ): + http_client = self._client + return self.__class__( + workspace=workspace or self.workspace, + base_url=base_url or self.base_url, + inference_base_url=inference_base_url or self.inference_base_url, + config_path=config_path, + context_name=context_name, + access_token=access_token, + timeout=self.timeout if isinstance(timeout, NotGiven) else timeout, + http_client=http_client, + max_retries=self.max_retries if isinstance(max_retries, NotGiven) else max_retries, + default_headers=headers, + default_query=params, + **_extra_kwargs, + ) + class AsyncNeMoPlatform(AsyncAPIClient): # client options @@ -295,6 +382,9 @@ async def main() -> None: http_client: Custom ``httpx.AsyncClient`` instance. When provided, the auth bootstrap is skipped entirely regardless of other parameters. """ + env_base_url = os.environ.get("NEMO_PLATFORM_BASE_URL") + bootstrap_base_url = base_url if base_url is not None else env_base_url + should_bootstrap = _should_bootstrap_config( http_client=http_client, base_url=base_url, @@ -308,7 +398,7 @@ async def main() -> None: client_init_kwargs = build_async_client_init_kwargs( config_path=config_path, - base_url=base_url, + base_url=bootstrap_base_url, context_name=context_name, access_token=access_token, extra_headers=default_headers, @@ -317,9 +407,19 @@ async def main() -> None: if workspace is None: workspace = client_init_kwargs.workspace default_headers = client_init_kwargs.default_headers + if client_init_kwargs.http_client is not None and not isinstance( + client_init_kwargs.http_client, httpx.AsyncClient + ): + raise TypeError("Expected httpx.AsyncClient from async client factory") http_client = client_init_kwargs.http_client except Exception as e: - raise RuntimeError(f"NeMoPlatform client initialization failed: {e}") + raise RuntimeError(f"NeMoPlatform client initialization failed: {e}") from e + + if base_url is None: + base_url = bootstrap_base_url + + if base_url is None: + raise RuntimeError("NeMoPlatform client initialization failed: base_url is required") client_verify = client_verify_from_env() if http_client is None and client_verify is not True: @@ -358,3 +458,63 @@ def __getattr__(self, name: str) -> Any: instance = resource_cls(self) self.__dict__[name] = instance return instance + + def copy( + self, + *, + workspace: str | None = None, + base_url: str | httpx.URL | None = None, + inference_base_url: str | httpx.URL | None = None, + config_path: Path | None = None, + context_name: str | None = None, + access_token: str | None = None, + timeout: float | Timeout | None | NotGiven = not_given, + http_client: httpx.AsyncClient | None = None, + max_retries: int | NotGiven = not_given, + default_headers: Mapping[str, str] | None = None, + set_default_headers: Mapping[str, str] | None = None, + default_query: Mapping[str, object] | None = None, + set_default_query: Mapping[str, object] | None = None, + _extra_kwargs: Mapping[str, Any] = {}, + ) -> Self: + """ + Create a new client instance re-using the same options given to the current client with optional overriding. + """ + if default_headers is not None and set_default_headers is not None: + raise ValueError("The `default_headers` and `set_default_headers` arguments are mutually exclusive") + + if default_query is not None and set_default_query is not None: + raise ValueError("The `default_query` and `set_default_query` arguments are mutually exclusive") + + headers = self._custom_headers + if default_headers is not None: + headers = {**headers, **default_headers} + elif set_default_headers is not None: + headers = set_default_headers + + params = self._custom_query + if default_query is not None: + params = {**params, **default_query} + elif set_default_query is not None: + params = set_default_query + + if http_client is None and not _copy_requires_bootstrap( + config_path=config_path, + context_name=context_name, + access_token=access_token, + ): + http_client = self._client + return self.__class__( + workspace=workspace or self.workspace, + base_url=base_url or self.base_url, + inference_base_url=inference_base_url or self.inference_base_url, + config_path=config_path, + context_name=context_name, + access_token=access_token, + timeout=self.timeout if isinstance(timeout, NotGiven) else timeout, + http_client=http_client, + max_retries=self.max_retries if isinstance(max_retries, NotGiven) else max_retries, + default_headers=headers, + default_query=params, + **_extra_kwargs, + ) diff --git a/packages/nemo_platform_ext/tests/client/test_client.py b/packages/nemo_platform_ext/tests/client/test_client.py index 785b72c160..e4b79332f9 100644 --- a/packages/nemo_platform_ext/tests/client/test_client.py +++ b/packages/nemo_platform_ext/tests/client/test_client.py @@ -299,8 +299,8 @@ def test_exchanges_workload_identity_token_file(self, mock_exchange, _mock_disco assert mock_exchange.call_args.kwargs["scope"] == "openid email groups" @pytest.mark.asyncio - @patch("nemo_platform.client.factory.discover_nmp_config", return_value=_MOCK_WORKLOAD_NMP_CONFIG) - @patch("nemo_platform.auth.workload_exchange.token_exchange_grant") + @patch("nemo_platform_ext.client.factory.discover_nmp_config", return_value=_MOCK_WORKLOAD_NMP_CONFIG) + @patch("nemo_platform_ext.auth.workload_exchange.token_exchange_grant") async def test_async_exchanges_workload_identity_token_file_at_request_time( self, mock_exchange, _mock_discover, tmp_path, monkeypatch ): @@ -540,7 +540,7 @@ def test_refresh_grant_failure_surfaces_clear_error(self, mock_post, _mock_disco class TestClientConstructorBootstrapBypass: - @patch("nemo_platform.client.factory.build_client_init_kwargs") + @patch("nemo_platform_ext.client.factory.build_client_init_kwargs") def test_sync_constructor_with_base_url_skips_config_bootstrap(self, mock_build_client_kwargs): mock_build_client_kwargs.side_effect = AssertionError("bootstrap should not be called") @@ -553,8 +553,29 @@ def test_sync_constructor_with_base_url_skips_config_bootstrap(self, mock_build_ mock_build_client_kwargs.assert_not_called() + @patch("nemo_platform_ext.client.factory.build_client_init_kwargs") + def test_sync_constructor_env_base_url_still_bootstraps_when_base_url_omitted( + self, mock_build_client_kwargs, monkeypatch + ): + monkeypatch.setenv("NEMO_PLATFORM_BASE_URL", "http://env-host:8081") + mock_build_client_kwargs.return_value = MagicMock( + base_url="http://env-host:8081", + workspace="test-workspace", + default_headers=None, + http_client=None, + ) + + client = NeMoPlatform() + try: + assert str(client.base_url).rstrip("/") == "http://env-host:8081" + assert client.workspace == "test-workspace" + finally: + client.close() + + assert mock_build_client_kwargs.call_args.kwargs["base_url"] == "http://env-host:8081" + @patch("nemo_platform._client.DefaultHttpxClient") - @patch("nemo_platform.client.factory.build_client_init_kwargs") + @patch("nemo_platform_ext.client.factory.build_client_init_kwargs") def test_sync_constructor_direct_mode_uses_nemo_scoped_ca_bundle( self, mock_build_client_kwargs, mock_default_httpx_client, monkeypatch ): @@ -571,7 +592,27 @@ def test_sync_constructor_direct_mode_uses_nemo_scoped_ca_bundle( mock_build_client_kwargs.assert_not_called() mock_default_httpx_client.assert_called_once_with(verify="/tmp/nemo-ca.pem") - @patch("nemo_platform.client.factory.build_client_init_kwargs") + @patch("nemo_platform_ext.client.factory.build_client_init_kwargs") + def test_sync_copy_with_access_token_bootstraps_instead_of_reusing_http_client(self, mock_build_client_kwargs): + mock_build_client_kwargs.return_value = MagicMock( + base_url="http://override-host:8081", + workspace="test-workspace", + default_headers={"Authorization": "Bearer replacement-token"}, + http_client=None, + ) + + client = NeMoPlatform(base_url="http://original-host:8081", workspace="original-workspace") + original_http_client = client._client + copied = client.copy(access_token="replacement-token") + try: + assert copied._client is not original_http_client + finally: + copied.close() + client.close() + + assert mock_build_client_kwargs.call_args.kwargs["access_token"] == "replacement-token" + + @patch("nemo_platform_ext.client.factory.build_client_init_kwargs") def test_sync_constructor_with_workload_file_and_base_url_bootstraps(self, mock_build_client_kwargs, monkeypatch): monkeypatch.setenv(WORKLOAD_IDENTITY_TOKEN_FILE_ENVVAR, "/var/run/secrets/nemo-platform/workload/token") mock_build_client_kwargs.return_value = MagicMock( @@ -589,7 +630,7 @@ def test_sync_constructor_with_workload_file_and_base_url_bootstraps(self, mock_ assert mock_build_client_kwargs.call_args.kwargs["base_url"] == "http://override-host:8081" - @patch("nemo_platform.client.factory.build_client_init_kwargs") + @patch("nemo_platform_ext.client.factory.build_client_init_kwargs") def test_sync_constructor_passes_context_name_to_bootstrap(self, mock_build_client_kwargs): mock_build_client_kwargs.return_value = MagicMock( base_url="http://override-host:8081", @@ -609,9 +650,9 @@ def test_sync_constructor_passes_context_name_to_bootstrap(self, mock_build_clie def test_sync_constructor_rejects_legacy_context_argument(self): with pytest.raises(TypeError, match="unexpected keyword argument 'context'"): - NeMoPlatform(context="ctx-b") + NeMoPlatform(context="ctx-b") # ty: ignore[unknown-argument] - @patch("nemo_platform.client.factory.build_client_init_kwargs") + @patch("nemo_platform_ext.client.factory.build_client_init_kwargs") def test_sync_constructor_with_http_client_skips_config_bootstrap(self, mock_build_client_kwargs): mock_build_client_kwargs.side_effect = AssertionError("bootstrap should not be called") @@ -630,7 +671,7 @@ def test_sync_constructor_with_http_client_skips_config_bootstrap(self, mock_bui mock_build_client_kwargs.assert_not_called() @pytest.mark.asyncio - @patch("nemo_platform.client.factory.build_async_client_init_kwargs") + @patch("nemo_platform_ext.client.factory.build_async_client_init_kwargs") async def test_async_constructor_with_base_url_skips_config_bootstrap(self, mock_build_client_kwargs): mock_build_client_kwargs.side_effect = AssertionError("bootstrap should not be called") @@ -643,9 +684,31 @@ async def test_async_constructor_with_base_url_skips_config_bootstrap(self, mock mock_build_client_kwargs.assert_not_called() + @pytest.mark.asyncio + @patch("nemo_platform_ext.client.factory.build_async_client_init_kwargs") + async def test_async_constructor_env_base_url_still_bootstraps_when_base_url_omitted( + self, mock_build_client_kwargs, monkeypatch + ): + monkeypatch.setenv("NEMO_PLATFORM_BASE_URL", "http://env-host:8081") + mock_build_client_kwargs.return_value = MagicMock( + base_url="http://env-host:8081", + workspace="test-workspace", + default_headers=None, + http_client=None, + ) + + client = AsyncNeMoPlatform() + try: + assert str(client.base_url).rstrip("/") == "http://env-host:8081" + assert client.workspace == "test-workspace" + finally: + await client.close() + + assert mock_build_client_kwargs.call_args.kwargs["base_url"] == "http://env-host:8081" + @pytest.mark.asyncio @patch("nemo_platform._client.DefaultAsyncHttpxClient") - @patch("nemo_platform.client.factory.build_async_client_init_kwargs") + @patch("nemo_platform_ext.client.factory.build_async_client_init_kwargs") async def test_async_constructor_direct_mode_uses_nemo_scoped_ca_bundle( self, mock_build_client_kwargs, mock_default_httpx_client, monkeypatch ): @@ -663,7 +726,30 @@ async def test_async_constructor_direct_mode_uses_nemo_scoped_ca_bundle( mock_default_httpx_client.assert_called_once_with(verify="/tmp/nemo-ca.pem") @pytest.mark.asyncio - @patch("nemo_platform.client.factory.build_async_client_init_kwargs") + @patch("nemo_platform_ext.client.factory.build_async_client_init_kwargs") + async def test_async_copy_with_access_token_bootstraps_instead_of_reusing_http_client( + self, mock_build_client_kwargs + ): + mock_build_client_kwargs.return_value = MagicMock( + base_url="http://override-host:8081", + workspace="test-workspace", + default_headers={"Authorization": "Bearer replacement-token"}, + http_client=None, + ) + + client = AsyncNeMoPlatform(base_url="http://original-host:8081", workspace="original-workspace") + original_http_client = client._client + copied = client.copy(access_token="replacement-token") + try: + assert copied._client is not original_http_client + finally: + await copied.close() + await client.close() + + assert mock_build_client_kwargs.call_args.kwargs["access_token"] == "replacement-token" + + @pytest.mark.asyncio + @patch("nemo_platform_ext.client.factory.build_async_client_init_kwargs") async def test_async_constructor_with_workload_file_and_base_url_bootstraps( self, mock_build_client_kwargs, monkeypatch ): @@ -684,7 +770,7 @@ async def test_async_constructor_with_workload_file_and_base_url_bootstraps( assert mock_build_client_kwargs.call_args.kwargs["base_url"] == "http://override-host:8081" @pytest.mark.asyncio - @patch("nemo_platform.client.factory.build_async_client_init_kwargs") + @patch("nemo_platform_ext.client.factory.build_async_client_init_kwargs") async def test_async_constructor_with_http_client_skips_config_bootstrap(self, mock_build_client_kwargs): mock_build_client_kwargs.side_effect = AssertionError("bootstrap should not be called") @@ -703,7 +789,7 @@ async def test_async_constructor_with_http_client_skips_config_bootstrap(self, m mock_build_client_kwargs.assert_not_called() @pytest.mark.asyncio - @patch("nemo_platform.client.factory.build_async_client_init_kwargs") + @patch("nemo_platform_ext.client.factory.build_async_client_init_kwargs") async def test_async_constructor_passes_context_name_to_bootstrap(self, mock_build_client_kwargs): mock_build_client_kwargs.return_value = MagicMock( base_url="http://override-host:8081", diff --git a/packages/nemo_platform_plugin/pyproject.toml b/packages/nemo_platform_plugin/pyproject.toml index 57408c9e29..4b41058d09 100644 --- a/packages/nemo_platform_plugin/pyproject.toml +++ b/packages/nemo_platform_plugin/pyproject.toml @@ -42,6 +42,25 @@ Source = "https://github.com/NVIDIA-NeMo/nemo-platform/tree/main/packages/nemo_p Documentation = "https://github.com/NVIDIA-NeMo/nemo-platform/tree/main/packages/nemo_platform_plugin/src/nemo_platform_plugin/docs" [project.optional-dependencies] +# Generated from [tool.bundle-package]; do not edit by hand. +nemo-evaluator-sdk = [ + "pydantic>=2.10.6", + "jinja2>=3.1.6", + "jsonschema>=4.23.0", + "jsonpath-ng>=1.7.0", + "pyarrow>=19.0.1", + "pandas>=1.5.3", + "openai>=1.61.0", + "httpx>=0.27.0,<1", + "sacrebleu>=2.5.1", + "rouge_score==0.1.2", + "ragas==0.4.3", + "langchain-openai>=1.3.5", + "langchain-nvidia-ai-endpoints>=1.4.3,<2.0.0", + "nemo-relay>=0.6.0,<0.7", + "nemo-fabric>=0.1.1,<0.3.0", +] + # Generated from [tool.bundle-package]; do not edit by hand. nemo-platform-sdk = [ "httpx>=0.23.0, <1", @@ -83,3 +102,7 @@ packages = ["src/nemo_platform_plugin"] [tool.bundle-package] nemo-platform-sdk = { source = "../../sdk/python/nemo-platform/src/nemo_platform", module = "nemo_platform" } +nemo-platform-ext = { source = "../nemo_platform_ext/src/nemo_platform_ext", module = "nemo_platform_ext", deps_group = "nemo-platform-sdk", force_include = { "../../../../docs" = "nemo_platform_ext/cli/docs" } } +models = { source = "../models/src/models", module = "models", deps_group = "nemo-platform-sdk" } +filesets = { source = "../filesets/src/filesets", module = "filesets", deps_group = "nemo-platform-sdk" } +nemo-evaluator-sdk = { source = "../nemo_evaluator_sdk/src/nemo_evaluator_sdk", module = "nemo_evaluator_sdk" } diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/commands.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/commands.py index 295dd7e049..3b05ec72cd 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/commands.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/commands.py @@ -1303,7 +1303,7 @@ def _post_function_submit( def _resolve_cluster_name_to_base_url(cluster_name: str) -> str: """Resolve a configured cluster name to its base URL.""" - from nemo_platform.config.config import Config + from nemo_platform_ext.config.config import Config config = Config.load() for cluster in config.get_config_file().clusters: diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/jobs/file_manager.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/jobs/file_manager.py index effcf552e9..cd613d19c0 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/jobs/file_manager.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/jobs/file_manager.py @@ -11,8 +11,8 @@ import anyio import fsspec.asyn +from filesets import FilesetFileSystem, build_fileset_ref, parse_fileset_ref from nemo_platform import AsyncNeMoPlatform, NeMoPlatform -from nemo_platform.filesets import FilesetFileSystem, build_fileset_ref, parse_fileset_ref from nemo_platform_plugin.files.types import CreateFilesetRequest from nemo_platform_plugin.jobs.schemas import FileStorageType diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/jobs/result_manager.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/jobs/result_manager.py index 3ab12e2fc6..cac46051d9 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/jobs/result_manager.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/jobs/result_manager.py @@ -9,8 +9,8 @@ from pathlib import Path from typing import Generic, Literal, Type, TypeVar, overload +from filesets import parse_fileset_ref from nemo_platform import AsyncNeMoPlatform, NeMoPlatform -from nemo_platform.filesets import parse_fileset_ref from nemo_platform_plugin.client.adapter import client_from_platform from nemo_platform_plugin.client.errors import ConflictError as ClientConflictError from nemo_platform_plugin.client.errors import NemoClientError diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/nooa_model_client.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/nooa_model_client.py index db0cd22a77..2b677a3cad 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/nooa_model_client.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/nooa_model_client.py @@ -15,9 +15,9 @@ from dataclasses import dataclass from nemo_platform import AsyncNeMoPlatform -from nemo_platform.config import get_context from nemo_platform.types.inference import ModelProvider from nemo_platform.types.models import ModelEntity +from nemo_platform_ext.config import get_context from nooa.unifiedllm import CompletionClient, UnifiedLLM _PLACEHOLDER_API_KEY = "not-needed" diff --git a/packages/nemo_platform_plugin/tests/test_commands.py b/packages/nemo_platform_plugin/tests/test_commands.py index 534c63f8a1..e5d3ef707d 100644 --- a/packages/nemo_platform_plugin/tests/test_commands.py +++ b/packages/nemo_platform_plugin/tests/test_commands.py @@ -301,7 +301,7 @@ def get_config_file(self) -> SimpleNamespace: else: monkeypatch.setenv("NMP_BASE_URL", env_base_url) monkeypatch.setattr("nemo_platform_plugin.scheduler.NemoJobScheduler.submit_remote", _capture) - monkeypatch.setattr("nemo_platform.config.config.Config.load", lambda: _FakeConfig()) + monkeypatch.setattr("nemo_platform_ext.config.config.Config.load", lambda: _FakeConfig()) app = _app_with_jobs(_GreetJob) state = _State(context_base_url) diff --git a/packages/nmp_common/src/nmp/common/auth/testing.py b/packages/nmp_common/src/nmp/common/auth/testing.py index 696b18e7d3..55c748f47f 100644 --- a/packages/nmp_common/src/nmp/common/auth/testing.py +++ b/packages/nmp_common/src/nmp/common/auth/testing.py @@ -44,7 +44,7 @@ from typing import Any, Dict, Optional import httpx -from nemo_platform.auth.helpers import generate_unsigned_jwt as generate_unsigned_jwt_helper +from nemo_platform_ext.auth.helpers import generate_unsigned_jwt as generate_unsigned_jwt_helper # Some packages do not have respx as a dependency try: diff --git a/packages/nmp_common/tests/sdk_factory/test_sdk.py b/packages/nmp_common/tests/sdk_factory/test_sdk.py index e3df60674d..e981cb5f53 100644 --- a/packages/nmp_common/tests/sdk_factory/test_sdk.py +++ b/packages/nmp_common/tests/sdk_factory/test_sdk.py @@ -7,7 +7,7 @@ import httpx import pytest -from nemo_platform.auth.helpers import NMPOIDCConfig +from nemo_platform_ext.auth.helpers import NMPOIDCConfig from nemo_platform_plugin.client.constants import WORKLOAD_IDENTITY_TOKEN_FILE_ENVVAR from nmp.common.config import Configuration, PlatformConfig from nmp.common.http_clients import shared_async_http_client, shared_sync_http_client @@ -189,8 +189,10 @@ def token_exchange_grant(**kwargs): monkeypatch.setenv(WORKLOAD_IDENTITY_TOKEN_FILE_ENVVAR, str(subject_token_file)) monkeypatch.setenv("NMP_PRINCIPAL", json.dumps({"id": "creator@example.com", "email": "creator@example.com"})) monkeypatch.delenv("NMP_ACCESS_TOKEN", raising=False) - monkeypatch.setattr("nemo_platform.client.factory.discover_nmp_config", lambda _base_url: _workload_oidc_config()) - monkeypatch.setattr("nemo_platform.auth.workload_exchange.token_exchange_grant", token_exchange_grant) + monkeypatch.setattr( + "nemo_platform_ext.client.factory.discover_nmp_config", lambda _base_url: _workload_oidc_config() + ) + monkeypatch.setattr("nemo_platform_ext.auth.workload_exchange.token_exchange_grant", token_exchange_grant) sdk = get_platform_sdk() try: @@ -388,8 +390,10 @@ def token_exchange_grant(**kwargs): monkeypatch.setenv(WORKLOAD_IDENTITY_TOKEN_FILE_ENVVAR, str(subject_token_file)) monkeypatch.setenv("NMP_PRINCIPAL", json.dumps({"id": "creator@example.com", "email": "creator@example.com"})) monkeypatch.delenv("NMP_ACCESS_TOKEN", raising=False) - monkeypatch.setattr("nemo_platform.client.factory.discover_nmp_config", lambda _base_url: _workload_oidc_config()) - monkeypatch.setattr("nemo_platform.auth.workload_exchange.token_exchange_grant", token_exchange_grant) + monkeypatch.setattr( + "nemo_platform_ext.client.factory.discover_nmp_config", lambda _base_url: _workload_oidc_config() + ) + monkeypatch.setattr("nemo_platform_ext.auth.workload_exchange.token_exchange_grant", token_exchange_grant) sdk = get_task_sdk(as_service="customizer") try: diff --git a/packages/nmp_testing/src/nmp/testing/client.py b/packages/nmp_testing/src/nmp/testing/client.py index 8d7433a846..5113ba6e25 100644 --- a/packages/nmp_testing/src/nmp/testing/client.py +++ b/packages/nmp_testing/src/nmp/testing/client.py @@ -176,7 +176,7 @@ def _create_svc( def _install_asgi_files_resource(sdk: NeMoPlatform, async_http_client: httpx.AsyncClient) -> None: """Route sync SDK file uploads through the in-process test app.""" - from nemo_platform.filesets.resources import FilesResource + from filesets.resources import FilesResource from nemo_platform_plugin.client.adapter import client_from_platform from nemo_platform_plugin.files.client import AsyncFilesClient, FilesClient diff --git a/plugins/nemo-agents/src/nemo_agents_plugin/cli.py b/plugins/nemo-agents/src/nemo_agents_plugin/cli.py index b6899ef7e2..bc870f44fc 100644 --- a/plugins/nemo-agents/src/nemo_agents_plugin/cli.py +++ b/plugins/nemo-agents/src/nemo_agents_plugin/cli.py @@ -71,7 +71,7 @@ ) from nemo_agents_plugin.leaderboard.cli import register_leaderboard_commands from nemo_agents_plugin.usage.cli import register_usage_commands -from nemo_platform.cli.core.formatters import Column, format_output +from nemo_platform_ext.cli.core.formatters import Column, format_output from nemo_platform_plugin.cli import NemoCLI from nemo_platform_plugin.cli_errors import print_http_request_error, print_http_status_error from nemo_platform_plugin.cli_progress import request_progress diff --git a/plugins/nemo-agents/src/nemo_agents_plugin/leaderboard/cli.py b/plugins/nemo-agents/src/nemo_agents_plugin/leaderboard/cli.py index b8b7bd0d5f..e16104d0b2 100644 --- a/plugins/nemo-agents/src/nemo_agents_plugin/leaderboard/cli.py +++ b/plugins/nemo-agents/src/nemo_agents_plugin/leaderboard/cli.py @@ -24,7 +24,7 @@ from nemo_agents_plugin.leaderboard.rank import rank_entries from nemo_agents_plugin.leaderboard.render import render_entries from nemo_agents_plugin.leaderboard.types import AgentLeaderboardEntry -from nemo_platform.cli.core.help_formatter import create_typer_app +from nemo_platform_ext.cli.core.help_formatter import create_typer_app def register_leaderboard_commands(app: typer.Typer) -> None: diff --git a/plugins/nemo-agents/src/nemo_agents_plugin/leaderboard/render.py b/plugins/nemo-agents/src/nemo_agents_plugin/leaderboard/render.py index d507a3cf77..08bee0a931 100644 --- a/plugins/nemo-agents/src/nemo_agents_plugin/leaderboard/render.py +++ b/plugins/nemo-agents/src/nemo_agents_plugin/leaderboard/render.py @@ -13,7 +13,7 @@ from io import StringIO from nemo_agents_plugin.leaderboard.types import AgentLeaderboard, AgentLeaderboardEntry -from nemo_platform.cli.core.help_formatter import _get_terminal_width +from nemo_platform_ext.cli.core.help_formatter import _get_terminal_width from rich.console import Console from rich.table import Table diff --git a/plugins/nemo-agents/src/nemo_agents_plugin/utils.py b/plugins/nemo-agents/src/nemo_agents_plugin/utils.py index bc2c8e2ffb..eae83701c7 100644 --- a/plugins/nemo-agents/src/nemo_agents_plugin/utils.py +++ b/plugins/nemo-agents/src/nemo_agents_plugin/utils.py @@ -195,7 +195,7 @@ def get_internal_base_url() -> str | None: def get_default_model() -> str | None: """Return the default model for the platform from the SDK context.""" - from nemo_platform.config import get_context + from nemo_platform_ext.config import get_context return get_context().default_model diff --git a/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/app/input.py b/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/app/input.py index 4d58885d2b..b9d48168ae 100644 --- a/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/app/input.py +++ b/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/app/input.py @@ -14,9 +14,9 @@ import anyio from anonymizer.config.anonymizer_config import AnonymizerInput +from filesets import FilesetPathError, build_fileset_ref, parse_fileset_ref from nemo_anonymizer_plugin.app.errors import AnonymizerInvalidConfigError from nemo_platform import AsyncNeMoPlatform, NeMoPlatform -from nemo_platform.filesets import FilesetPathError, build_fileset_ref, parse_fileset_ref from nemo_platform_plugin.jobs.file_manager import AsyncFilesetFileManager, FilesetFileManager, TmpDirPath from pydantic import BaseModel, Field, ValidationError diff --git a/plugins/nemo-data-designer/src/nemo_data_designer_plugin/sdk/resources.py b/plugins/nemo-data-designer/src/nemo_data_designer_plugin/sdk/resources.py index 4d4a61ed19..34c8207ec4 100644 --- a/plugins/nemo-data-designer/src/nemo_data_designer_plugin/sdk/resources.py +++ b/plugins/nemo-data-designer/src/nemo_data_designer_plugin/sdk/resources.py @@ -16,6 +16,7 @@ from data_designer.config.preview_results import PreviewResults from data_designer.config.utils.info import InterfaceInfo from data_designer.logging import RandomEmoji +from models.resources import AsyncModelsResource, ModelsResource from nemo_data_designer_plugin.functions._types import ( AnalysisFrame, DatasetFrame, @@ -42,7 +43,6 @@ validate_config_sync, ) from nemo_platform import AsyncNeMoPlatform, NeMoPlatform -from nemo_platform.models.resources import AsyncModelsResource, ModelsResource from nemo_platform.types.inference import ModelProvider as NMPModelProvider from nemo_platform_plugin.functions.frames import Done, Error, Heartbeat from nemo_platform_plugin.sdk import NemoPluginSDKResources diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/filesets.py b/plugins/nemo-evaluator/src/nemo_evaluator/filesets.py index 44ffe32cff..d7925b23dd 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/filesets.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/filesets.py @@ -9,8 +9,8 @@ from pathlib import Path import fsspec.asyn +from filesets import FilesetFileSystem from nemo_platform import AsyncNeMoPlatform, NeMoPlatform -from nemo_platform.filesets import FilesetFileSystem from nemo_platform_plugin.client.adapter import client_from_platform from nemo_platform_plugin.files.client import AsyncFilesClient, FilesClient from pydantic import Field, RootModel diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/client.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/client.py index cbaa1b8e07..31deb18ee4 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/client.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/client.py @@ -20,8 +20,8 @@ from urllib.parse import urlparse from nemo_platform import AsyncNeMoPlatform -from nemo_platform.auth.helpers import discover_nmp_config -from nemo_platform.config.config import Config +from nemo_platform_ext.auth.helpers import discover_nmp_config +from nemo_platform_ext.config.config import Config # Loopback hosts are served by an unauthenticated local platform; attaching # (and refreshing) OAuth tokens there is both unnecessary and a failure mode diff --git a/plugins/nemo-experimentalist/tests/test_client.py b/plugins/nemo-experimentalist/tests/test_client.py index 7c43e2bc58..d5362b2cd0 100644 --- a/plugins/nemo-experimentalist/tests/test_client.py +++ b/plugins/nemo-experimentalist/tests/test_client.py @@ -5,7 +5,7 @@ import pytest from nemo_experimentalist_plugin.client import make_client -from nemo_platform.auth.helpers import NMPOIDCConfig +from nemo_platform_ext.auth.helpers import NMPOIDCConfig REMOTE_URL = "https://nemo-platform.example.com" diff --git a/plugins/nemo-insights/src/nemo_insights_plugin/analyst/observability.py b/plugins/nemo-insights/src/nemo_insights_plugin/analyst/observability.py index 6c60f05f2f..e562a7bd9a 100644 --- a/plugins/nemo-insights/src/nemo_insights_plugin/analyst/observability.py +++ b/plugins/nemo-insights/src/nemo_insights_plugin/analyst/observability.py @@ -8,7 +8,7 @@ from uuid import uuid4 from nemo_insights_plugin.client import LOOPBACK_HOSTS -from nemo_platform.config.config import Config +from nemo_platform_ext.config.config import Config from nooa.tracing import enable_tracing, exporters, flush_traces, set_session ANALYST_OBSERVABILITY_ENV = "NEMO_INSIGHTS_ANALYST_OBSERVABILITY" diff --git a/plugins/nemo-insights/src/nemo_insights_plugin/client.py b/plugins/nemo-insights/src/nemo_insights_plugin/client.py index 777ad04b06..1f4c32e8dd 100644 --- a/plugins/nemo-insights/src/nemo_insights_plugin/client.py +++ b/plugins/nemo-insights/src/nemo_insights_plugin/client.py @@ -20,8 +20,8 @@ from urllib.parse import urlparse from nemo_platform import AsyncNeMoPlatform -from nemo_platform.auth.helpers import discover_nmp_config -from nemo_platform.config.config import Config +from nemo_platform_ext.auth.helpers import discover_nmp_config +from nemo_platform_ext.config.config import Config # Loopback hosts are served by an unauthenticated local platform; attaching # (and refreshing) OAuth tokens there is both unnecessary and a failure mode diff --git a/plugins/nemo-insights/testbed/export.py b/plugins/nemo-insights/testbed/export.py index 46138364b6..d5ed676623 100644 --- a/plugins/nemo-insights/testbed/export.py +++ b/plugins/nemo-insights/testbed/export.py @@ -21,7 +21,7 @@ from urllib.parse import urlparse from nemo_platform import AsyncNeMoPlatform -from nemo_platform.config.config import Config +from nemo_platform_ext.config.config import Config EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc) PAGE_SIZE = 200 # generous pages: drain-all in few round-trips diff --git a/plugins/nemo-insights/testbed/ingest.py b/plugins/nemo-insights/testbed/ingest.py index 164eef8eee..2543c47a77 100644 --- a/plugins/nemo-insights/testbed/ingest.py +++ b/plugins/nemo-insights/testbed/ingest.py @@ -11,7 +11,7 @@ import httpx from nemo_platform import NeMoPlatform -from nemo_platform.config.config import Config +from nemo_platform_ext.config.config import Config _LOOPBACK_HOSTS = frozenset({"localhost", "127.0.0.1", "::1", "0.0.0.0"}) diff --git a/plugins/nemo-insights/tests/test_client.py b/plugins/nemo-insights/tests/test_client.py index 4be5bb9762..dd42937c4e 100644 --- a/plugins/nemo-insights/tests/test_client.py +++ b/plugins/nemo-insights/tests/test_client.py @@ -4,7 +4,7 @@ from unittest.mock import MagicMock, patch from nemo_insights_plugin.client import make_client -from nemo_platform.auth.helpers import NMPOIDCConfig +from nemo_platform_ext.auth.helpers import NMPOIDCConfig REMOTE_URL = "https://nemo-platform.example.com" diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/api/v2/jobs/endpoints.py b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/api/v2/jobs/endpoints.py index 78b4fc3a45..48fd82f8e1 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/api/v2/jobs/endpoints.py +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/api/v2/jobs/endpoints.py @@ -11,8 +11,8 @@ from typing import Any from urllib.parse import urlparse +from filesets import FilesetPathError, parse_fileset_ref from nemo_platform import AsyncNeMoPlatform, NotFoundError, PermissionDeniedError -from nemo_platform.filesets import FilesetPathError, parse_fileset_ref from nemo_platform_plugin.authz import AuthzScope from nemo_platform_plugin.client.adapter import client_from_platform from nemo_platform_plugin.client.errors import NotFoundError as ClientNotFoundError diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/tasks/safe_synthesizer/__main__.py b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/tasks/safe_synthesizer/__main__.py index d3e03b51e1..0d263fcfc6 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/tasks/safe_synthesizer/__main__.py +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/tasks/safe_synthesizer/__main__.py @@ -25,8 +25,8 @@ import pandas as pd from datasets import Dataset, DatasetDict, load_dataset +from filesets import parse_fileset_ref from nemo_platform import NeMoPlatform -from nemo_platform.filesets import parse_fileset_ref from nemo_platform_plugin.client.adapter import client_from_platform from nemo_platform_plugin.config import get_platform_config from nemo_platform_plugin.jobs.client import JobsClient diff --git a/sdk/python/nemo-platform/hatch_build.py b/sdk/python/nemo-platform/hatch_build.py new file mode 100644 index 0000000000..52ff76325b --- /dev/null +++ b/sdk/python/nemo-platform/hatch_build.py @@ -0,0 +1,221 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# ruff: noqa: I001 + +from __future__ import annotations + +import collections.abc +import shutil +import tempfile +from pathlib import Path + +from hatchling.builders.hooks.plugin.interface import BuildHookInterface + +GENERATED_INIT_FILE = """# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +""" + + +class SourcePackage: + def __init__(self, *, source: str, target: str, include: tuple[str, ...]) -> None: + self.source = source + self.target = target + self.include = include + + +class CustomBuildHook(BuildHookInterface): + """Stage source SDK extensions into SDK build artifacts.""" + + def initialize(self, version: str, build_data: dict[str, object]) -> None: + if version == "editable": + # Editable installs run against the repo/workspace sources, where + # these packages are already importable as workspace packages. + return + + packages = _source_packages(self.config) + project_root = Path(self.root).resolve() + source_base = _find_source_base(project_root, packages) + self._stage_tmp = tempfile.TemporaryDirectory(prefix="nmp-sdk-stage-") + stage_root = Path(self._stage_tmp.name) + + force_include = _force_include(build_data) + if self.target_name == "sdist": + _stage_sdist_sources(source_base, stage_root, force_include, packages) + patched_pyproject = _write_sdist_pyproject(project_root, stage_root, self.metadata.version) + _replace_force_include_target(force_include, source=patched_pyproject, target="pyproject.toml") + else: + _stage_wheel_sources(source_base, stage_root, force_include, packages) + + build_data["force_include"] = force_include + + def finalize(self, _version: str, _build_data: dict[str, object], _artifact_path: str) -> None: + stage_tmp = getattr(self, "_stage_tmp", None) + if stage_tmp is not None: + stage_tmp.cleanup() + + +def _source_packages(config: collections.abc.Mapping[str, object]) -> tuple[SourcePackage, ...]: + packages = [] + for entry in _config_entries(config, "source-packages"): + packages.append( + SourcePackage( + source=_required_string(entry, "source", "source-packages"), + target=_required_string(entry, "target", "source-packages"), + include=_include_patterns(entry), + ) + ) + return tuple(packages) + + +def _force_include(build_data: collections.abc.Mapping[str, object]) -> dict[str, str]: + existing_force_include = build_data.get("force_include") + if not isinstance(existing_force_include, dict): + return {} + return {str(source): str(target) for source, target in existing_force_include.items()} + + +def _find_source_base(project_root: Path, packages: tuple[SourcePackage, ...]) -> Path: + """Find the root containing the configured source package paths. + + Monorepo builds run from ``sdk/python/nemo-platform`` while wheels built + from an sdist run from the extracted sdist root. Walking upward supports + both layouts without hard-coding a fixed number of parent directories. + """ + for candidate in (project_root, *project_root.parents): + if all((candidate / package.source).is_dir() for package in packages): + return candidate + + missing = ", ".join(package.source for package in packages) + raise FileNotFoundError(f"Could not find SDK source package roots from {project_root}: {missing}") + + +def _stage_wheel_sources( + source_base: Path, + stage_root: Path, + force_include: dict[str, str], + packages: tuple[SourcePackage, ...], +) -> None: + for package in packages: + source_root = source_base / package.source + package_stage = stage_root / package.target + _copy_included_paths(source_root, package_stage, package.include) + _ensure_init_files(package_stage) + force_include[str(package_stage)] = package.target + + +def _stage_sdist_sources( + source_base: Path, + stage_root: Path, + force_include: dict[str, str], + packages: tuple[SourcePackage, ...], +) -> None: + for package in packages: + source_root = source_base / package.source + package_stage = stage_root / package.source + _copy_included_paths(source_root, package_stage, package.include) + force_include[str(package_stage)] = package.source + + +def _write_sdist_pyproject(project_root: Path, stage_root: Path, version: str) -> Path: + """Write a self-contained sdist pyproject. + + The monorepo SDK pyproject uses ``nmp-build-tools`` from the uv workspace + for dynamic versioning. An sdist is outside that workspace, so wheel builds + from the sdist need static version metadata and no workspace-only build + dependency. + """ + source = project_root / "pyproject.toml" + destination = stage_root / "pyproject.toml" + destination.write_text(_sdist_pyproject(source.read_text(encoding="utf-8"), version), encoding="utf-8") + return destination + + +def _sdist_pyproject(content: str, version: str) -> str: + content = content.replace('dynamic = ["readme", "version"]', f'dynamic = ["readme"]\nversion = "{version}"') + content = content.replace('"hatch-fancy-pypi-readme", "nmp-build-tools"', '"hatch-fancy-pypi-readme"') + content = "\n".join(line for line in content.splitlines() if not _is_nmp_build_tools_workspace_source(line)) + content = _remove_toml_section(content, "[tool.hatch.version]") + return f"{content.rstrip()}\n" + + +def _is_nmp_build_tools_workspace_source(line: str) -> bool: + return line.strip().replace(" ", "") == "nmp-build-tools={workspace=true}" + + +def _remove_toml_section(content: str, section_header: str) -> str: + lines = content.splitlines() + output = [] + skipping = False + + for line in lines: + stripped = line.strip() + if stripped == section_header: + skipping = True + continue + if skipping and stripped.startswith("[") and stripped.endswith("]"): + skipping = False + if not skipping: + output.append(line) + + return "\n".join(output) + + +def _replace_force_include_target(force_include: dict[str, str], *, source: Path, target: str) -> None: + for existing_source, existing_target in tuple(force_include.items()): + if existing_target == target: + del force_include[existing_source] + force_include[str(source)] = target + + +def _config_entries( + config: collections.abc.Mapping[str, object], key: str +) -> tuple[collections.abc.Mapping[str, object], ...]: + raw_entries = config.get(key, []) + if not isinstance(raw_entries, list): + raise TypeError(f"`{key}` must be an array") + + entries = [] + for entry in raw_entries: + if not isinstance(entry, dict): + raise TypeError(f"`{key}` entries must be tables") + entries.append(entry) + return tuple(entries) + + +def _required_string(entry: collections.abc.Mapping[str, object], key: str, section: str) -> str: + value = entry.get(key) + if not isinstance(value, str) or not value: + raise TypeError(f"`{section}` entries must define a non-empty `{key}` string") + return value + + +def _include_patterns(entry: collections.abc.Mapping[str, object]) -> tuple[str, ...]: + raw_patterns = entry.get("include", ["**/*.py"]) + if not isinstance(raw_patterns, list) or any( + not isinstance(pattern, str) or not pattern for pattern in raw_patterns + ): + raise TypeError("`source-packages` entries must define `include` as an array of non-empty strings") + return tuple(raw_patterns) + + +def _copy_included_paths(source_root: Path, target_root: Path, patterns: tuple[str, ...]) -> None: + if not source_root.is_dir(): + raise FileNotFoundError(f"Source package path does not exist: {source_root}") + + seen: set[Path] = set() + for pattern in patterns: + for source_file in source_root.glob(pattern): + if not source_file.is_file() or source_file in seen: + continue + seen.add(source_file) + relative_path = source_file.relative_to(source_root) + target_file = target_root / relative_path + target_file.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source_file, target_file) + + +def _ensure_init_files(package_root: Path) -> None: + for directory in (package_root, *(path for path in package_root.rglob("*") if path.is_dir())): + init_file = directory / "__init__.py" + if not init_file.exists(): + init_file.write_text(GENERATED_INIT_FILE, encoding="utf-8") diff --git a/sdk/python/nemo-platform/pyproject.toml b/sdk/python/nemo-platform/pyproject.toml index a8abf0ac54..8c6d0e5872 100644 --- a/sdk/python/nemo-platform/pyproject.toml +++ b/sdk/python/nemo-platform/pyproject.toml @@ -118,8 +118,26 @@ include = [ [tool.hatch.build.targets.wheel] packages = ["src/nemo_platform"] -[tool.hatch.build.targets.wheel.force-include] -"../../../docs" = "nemo_platform/cli/docs" +[tool.hatch.build.targets.wheel.hooks.custom] +path = "hatch_build.py" + +[[tool.hatch.build.targets.wheel.hooks.custom.source-packages]] +source = "packages/nemo_platform_ext/src/nemo_platform_ext" +target = "nemo_platform_ext" +include = ["**/*.py", "skills/**/*.md", "skills/**/*.yaml", "skills/**/*.yml", "skills/**/*.json"] + +[[tool.hatch.build.targets.wheel.hooks.custom.source-packages]] +source = "packages/models/src/models" +target = "models" + +[[tool.hatch.build.targets.wheel.hooks.custom.source-packages]] +source = "packages/filesets/src/filesets" +target = "filesets" + +[[tool.hatch.build.targets.wheel.hooks.custom.source-packages]] +source = "packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk" +target = "nemo_evaluator_sdk" +include = ["**/*.py", "agent_eval/runtimes/fabric/sandbox.Dockerfile"] [tool.hatch.build.targets.sdist] # Basically everything except hidden files/directories (such as .github, .devcontainers, .python-version, etc) include = [ @@ -135,6 +153,26 @@ include = [ "tests/*", ] +[tool.hatch.build.targets.sdist.hooks.custom] +path = "hatch_build.py" + +[[tool.hatch.build.targets.sdist.hooks.custom.source-packages]] +source = "packages/nemo_platform_ext/src/nemo_platform_ext" +target = "nemo_platform_ext" +include = ["**/*.py", "skills/**/*.md", "skills/**/*.yaml", "skills/**/*.yml", "skills/**/*.json"] + +[[tool.hatch.build.targets.sdist.hooks.custom.source-packages]] +source = "packages/models/src/models" +target = "models" + +[[tool.hatch.build.targets.sdist.hooks.custom.source-packages]] +source = "packages/filesets/src/filesets" +target = "filesets" + +[[tool.hatch.build.targets.sdist.hooks.custom.source-packages]] +source = "packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk" +target = "nemo_evaluator_sdk" +include = ["**/*.py", "agent_eval/runtimes/fabric/sandbox.Dockerfile"] [tool.hatch.metadata.hooks.fancy-pypi-readme] content-type = "text/markdown" diff --git a/sdk/python/nemo-platform/src/nemo_platform/_alias.py b/sdk/python/nemo-platform/src/nemo_platform/_alias.py new file mode 100644 index 0000000000..630a682e5f --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/_alias.py @@ -0,0 +1,176 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# ruff: noqa: I001 - the generated SDK and workspace use different import-order settings. + +from __future__ import annotations + +import sys +from importlib import import_module, util +from importlib.abc import Loader, MetaPathFinder +from importlib.machinery import ModuleSpec +from types import ModuleType +from typing import Any + + +class _AliasLoader(Loader): + def __init__(self, alias_name: str, target_name: str) -> None: + self._alias_name = alias_name + self._target_name = target_name + + def create_module(self, spec: ModuleSpec) -> ModuleType: + del spec + target = import_module(self._target_name) + module = ModuleType(self._alias_name, target.__doc__) + _populate_alias_namespace(module.__dict__, self._alias_name, self._target_name, target) + return module + + def exec_module(self, module: ModuleType) -> None: + del module + return None + + +class _AliasFinder(MetaPathFinder): + def __init__(self) -> None: + self._aliases: dict[str, str] = {} + + def add_alias(self, alias_name: str, target_name: str) -> None: + self._aliases[alias_name] = target_name + + def find_spec( + self, + fullname: str, + _path: object | None = None, + _target: ModuleType | None = None, + ) -> ModuleSpec | None: + target_name = self._target_for(fullname) + if target_name is None: + return None + + target_spec = util.find_spec(target_name) + if target_spec is None: + return None + + is_package = target_spec.submodule_search_locations is not None + spec = ModuleSpec( + fullname, + _AliasLoader(fullname, target_name), + origin=target_spec.origin, + is_package=is_package, + ) + spec.cached = target_spec.cached + spec.has_location = target_spec.has_location + if is_package: + spec.submodule_search_locations = target_spec.submodule_search_locations + return spec + + def _target_for(self, fullname: str) -> str | None: + for alias_name, target_name in sorted(self._aliases.items(), key=lambda item: len(item[0]), reverse=True): + if fullname == alias_name: + return target_name + prefix = f"{alias_name}." + if fullname.startswith(prefix): + suffix = fullname[len(alias_name) :] + return f"{target_name}{suffix}" + return None + + +_FINDER: _AliasFinder | None = None + +_MODULE_METADATA_NAMES = frozenset( + { + "__builtins__", + "__cached__", + "__dir__", + "__doc__", + "__file__", + "__getattr__", + "__loader__", + "__name__", + "__package__", + "__path__", + "__spec__", + } +) + + +def _module_alias_name(value: ModuleType, alias_name: str, target_name: str) -> str | None: + module_name = value.__name__ + if module_name == target_name: + return alias_name + + target_prefix = f"{target_name}." + if module_name.startswith(target_prefix): + return f"{alias_name}{module_name[len(target_name) :]}" + + return None + + +def _alias_value(value: Any, alias_name: str, target_name: str) -> Any: + if not isinstance(value, ModuleType): + return value + + alias_module_name = _module_alias_name(value, alias_name, target_name) + if alias_module_name is None: + return value + + if alias_module_name == alias_name: + return sys.modules.get(alias_name, value) + + return import_module(alias_module_name) + + +def _populate_alias_namespace( + namespace: dict[str, Any], + alias_name: str, + target_name: str, + target: ModuleType, +) -> None: + namespace["__doc__"] = target.__doc__ + + target_path = getattr(target, "__path__", None) + if target_path is not None: + namespace["__path__"] = list(target_path) + + for name, value in target.__dict__.items(): + if name in _MODULE_METADATA_NAMES: + continue + if isinstance(value, ModuleType) and _module_alias_name(value, alias_name, target_name) is not None: + continue + namespace.setdefault(name, value) + + def __getattr__(name: str) -> Any: + value = _alias_value(getattr(target, name), alias_name, target_name) + namespace[name] = value + return value + + def __dir__() -> list[str]: + return sorted({*namespace, *dir(target)}) + + namespace["__getattr__"] = __getattr__ + namespace["__dir__"] = __dir__ + + +def alias_package(target_name: str, namespace: dict[str, Any]) -> ModuleType: + """Expose a native source package through a ``nemo_platform`` package path. + + Tiny generated ``__init__.py`` files call this from legacy SDK locations + such as ``nemo_platform.filesets``. The finder below maps submodule imports + like ``nemo_platform.filesets.resources`` to the real staged package path + (``filesets.resources``), so runtime resolution works without copying the + package tree into ``nemo_platform``. + """ + alias_name = str(namespace["__name__"]) + target = import_module(target_name) + _alias_finder().add_alias(alias_name, target_name) + _populate_alias_namespace(namespace, alias_name, target_name, target) + return target + + +def _alias_finder() -> _AliasFinder: + global _FINDER + + if _FINDER is None: + _FINDER = _AliasFinder() + sys.meta_path.insert(0, _FINDER) + return _FINDER diff --git a/sdk/python/nemo-platform/src/nemo_platform/_client.py b/sdk/python/nemo-platform/src/nemo_platform/_client.py index 9c5761f202..856213b1ee 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/_client.py +++ b/sdk/python/nemo-platform/src/nemo_platform/_client.py @@ -49,9 +49,9 @@ AsyncAPIClient, ) from nemo_platform._base_client import DefaultAsyncHttpxClient, DefaultHttpxClient -from nemo_platform.client.tls import client_verify_from_env from nemo_platform_plugin.client.constants import WORKLOAD_IDENTITY_TOKEN_FILE_ENVVAR from pathlib import Path +from nemo_platform_ext.client.tls import client_verify_from_env if TYPE_CHECKING: from .resources import ( @@ -124,6 +124,20 @@ def _should_bootstrap_config( ) +def _copy_requires_bootstrap( + *, + config_path: Path | None, + context_name: str | None, + access_token: str | None, +) -> bool: + return ( + config_path is not None + or context_name is not None + or access_token is not None + or bool(os.environ.get(WORKLOAD_IDENTITY_TOKEN_FILE_ENVVAR)) + ) + + class NeMoPlatform(SyncAPIClient): # client options workspace: str | None @@ -210,6 +224,9 @@ def __init__( http_client: Custom ``httpx.Client`` instance. When provided, the auth bootstrap is skipped entirely regardless of other parameters. """ + env_base_url = os.environ.get("NEMO_PLATFORM_BASE_URL") + bootstrap_base_url = base_url if base_url is not None else env_base_url + should_bootstrap = _should_bootstrap_config( http_client=http_client, base_url=base_url, @@ -219,11 +236,11 @@ def __init__( ) if should_bootstrap: try: - from nemo_platform.client.factory import build_client_init_kwargs + from nemo_platform_ext.client.factory import build_client_init_kwargs client_init_kwargs = build_client_init_kwargs( config_path=config_path, - base_url=base_url, + base_url=bootstrap_base_url, context_name=context_name, access_token=access_token, extra_headers=default_headers, @@ -232,9 +249,19 @@ def __init__( if workspace is None: workspace = client_init_kwargs.workspace default_headers = client_init_kwargs.default_headers + if client_init_kwargs.http_client is not None and not isinstance( + client_init_kwargs.http_client, httpx.Client + ): + raise TypeError("Expected httpx.Client from sync client factory") http_client = client_init_kwargs.http_client except Exception as e: - raise RuntimeError(f"NeMoPlatform client initialization failed: {e}") + raise RuntimeError(f"NeMoPlatform client initialization failed: {e}") from e + + if base_url is None: + base_url = bootstrap_base_url + + if base_url is None: + raise RuntimeError("NeMoPlatform client initialization failed: base_url is required") client_verify = client_verify_from_env() if http_client is None and client_verify is not True: @@ -379,6 +406,10 @@ def copy( *, workspace: str | None = None, base_url: str | httpx.URL | None = None, + inference_base_url: str | httpx.URL | None = None, + config_path: Path | None = None, + context_name: str | None = None, + access_token: str | None = None, timeout: float | Timeout | None | NotGiven = not_given, http_client: httpx.Client | None = None, max_retries: int | NotGiven = not_given, @@ -409,13 +440,22 @@ def copy( elif set_default_query is not None: params = set_default_query - http_client = http_client or self._client + if http_client is None and not _copy_requires_bootstrap( + config_path=config_path, + context_name=context_name, + access_token=access_token, + ): + http_client = self._client return self.__class__( workspace=workspace or self.workspace, base_url=base_url or self.base_url, + inference_base_url=inference_base_url or self.inference_base_url, + config_path=config_path, + context_name=context_name, + access_token=access_token, timeout=self.timeout if isinstance(timeout, NotGiven) else timeout, http_client=http_client, - max_retries=max_retries if is_given(max_retries) else self.max_retries, + max_retries=self.max_retries if isinstance(max_retries, NotGiven) else max_retries, default_headers=headers, default_query=params, **_extra_kwargs, @@ -583,6 +623,9 @@ async def main() -> None: http_client: Custom ``httpx.AsyncClient`` instance. When provided, the auth bootstrap is skipped entirely regardless of other parameters. """ + env_base_url = os.environ.get("NEMO_PLATFORM_BASE_URL") + bootstrap_base_url = base_url if base_url is not None else env_base_url + should_bootstrap = _should_bootstrap_config( http_client=http_client, base_url=base_url, @@ -592,11 +635,11 @@ async def main() -> None: ) if should_bootstrap: try: - from nemo_platform.client.factory import build_async_client_init_kwargs + from nemo_platform_ext.client.factory import build_async_client_init_kwargs client_init_kwargs = build_async_client_init_kwargs( config_path=config_path, - base_url=base_url, + base_url=bootstrap_base_url, context_name=context_name, access_token=access_token, extra_headers=default_headers, @@ -605,9 +648,19 @@ async def main() -> None: if workspace is None: workspace = client_init_kwargs.workspace default_headers = client_init_kwargs.default_headers + if client_init_kwargs.http_client is not None and not isinstance( + client_init_kwargs.http_client, httpx.AsyncClient + ): + raise TypeError("Expected httpx.AsyncClient from async client factory") http_client = client_init_kwargs.http_client except Exception as e: - raise RuntimeError(f"NeMoPlatform client initialization failed: {e}") + raise RuntimeError(f"NeMoPlatform client initialization failed: {e}") from e + + if base_url is None: + base_url = bootstrap_base_url + + if base_url is None: + raise RuntimeError("NeMoPlatform client initialization failed: base_url is required") client_verify = client_verify_from_env() if http_client is None and client_verify is not True: @@ -755,6 +808,10 @@ def copy( *, workspace: str | None = None, base_url: str | httpx.URL | None = None, + inference_base_url: str | httpx.URL | None = None, + config_path: Path | None = None, + context_name: str | None = None, + access_token: str | None = None, timeout: float | Timeout | None | NotGiven = not_given, http_client: httpx.AsyncClient | None = None, max_retries: int | NotGiven = not_given, @@ -785,13 +842,22 @@ def copy( elif set_default_query is not None: params = set_default_query - http_client = http_client or self._client + if http_client is None and not _copy_requires_bootstrap( + config_path=config_path, + context_name=context_name, + access_token=access_token, + ): + http_client = self._client return self.__class__( workspace=workspace or self.workspace, base_url=base_url or self.base_url, + inference_base_url=inference_base_url or self.inference_base_url, + config_path=config_path, + context_name=context_name, + access_token=access_token, timeout=self.timeout if isinstance(timeout, NotGiven) else timeout, http_client=http_client, - max_retries=max_retries if is_given(max_retries) else self.max_retries, + max_retries=self.max_retries if isinstance(max_retries, NotGiven) else max_retries, default_headers=headers, default_query=params, **_extra_kwargs, diff --git a/sdk/python/nemo-platform/src/nemo_platform/auth/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/auth/__init__.py index 1275d78dff..12a873240d 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/auth/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/auth/__init__.py @@ -1,15 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +from nemo_platform._alias import alias_package as _alias_package + +_alias_package("nemo_platform_ext.auth", globals()) diff --git a/sdk/python/nemo-platform/src/nemo_platform/auth/device_flow.py b/sdk/python/nemo-platform/src/nemo_platform/auth/device_flow.py deleted file mode 100644 index 06a62e1874..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/auth/device_flow.py +++ /dev/null @@ -1,304 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""OAuth 2.0 Device Authorization Flow (RFC 8628) implementation.""" - -import asyncio -import time -import webbrowser -from dataclasses import dataclass - -import httpx -from rich.console import Console -from rich.panel import Panel - -from nemo_platform.auth.token_provider import refresh_token_grant -from nemo_platform.client.tls import client_verify_from_env - -console = Console() - - -async def _async_pause(seconds: float) -> None: - await asyncio.sleep(seconds) - - -@dataclass -class DeviceCodeResponse: - """Response from device authorization endpoint.""" - - device_code: str - user_code: str - verification_uri: str - verification_uri_complete: str | None - expires_in: int - interval: int - - -@dataclass -class TokenResponse: - """Response from token endpoint.""" - - access_token: str - id_token: str | None # ID token (JWT) - refresh_token: str | None - token_type: str - expires_in: int - scope: str | None - - @property - def token_for_nmp(self) -> str: - """Return the token to use for NeMo Platform authentication.""" - return self.access_token - - -class DeviceFlowError(Exception): - """Device flow authentication error.""" - - pass - - -class DeviceFlow: - """OAuth 2.0 Device Authorization Flow client.""" - - def __init__( - self, - device_authorization_endpoint: str, - token_endpoint: str, - client_id: str, - scope: str = "openid email profile", - ): - self.device_authorization_endpoint = device_authorization_endpoint - self.token_endpoint = token_endpoint - self.client_id = client_id - self.scope = scope - - async def start_device_authorization(self) -> DeviceCodeResponse: - """Start the device authorization flow.""" - async with httpx.AsyncClient(verify=client_verify_from_env()) as client: - response = await client.post( - self.device_authorization_endpoint, - data={ - "client_id": self.client_id, - "scope": self.scope, - }, - timeout=30.0, - ) - response.raise_for_status() - data = response.json() - - return DeviceCodeResponse( - device_code=data["device_code"], - user_code=data["user_code"], - verification_uri=data["verification_uri"], - verification_uri_complete=data.get("verification_uri_complete"), - expires_in=data["expires_in"], - interval=data.get("interval", 5), - ) - - async def poll_for_token( - self, - device_code: str, - interval: int, - expires_in: int, - ) -> TokenResponse: - """Poll the token endpoint until authorization is complete.""" - start_time = time.time() - - async with httpx.AsyncClient(verify=client_verify_from_env()) as client: - while time.time() - start_time < expires_in: - await _async_pause(interval) - - response = await client.post( - self.token_endpoint, - data={ - "grant_type": "urn:ietf:params:oauth:grant-type:device_code", - "client_id": self.client_id, - "device_code": device_code, - "scope": self.scope, # Casdoor doesn't propagate scope from DeviceAuthCache - }, - timeout=30.0, - ) - - if response.status_code == 200: - data = response.json() - return TokenResponse( - access_token=data["access_token"], - id_token=data.get("id_token"), # Capture ID token if present - refresh_token=data.get("refresh_token"), - token_type=data.get("token_type", "Bearer"), - expires_in=data.get("expires_in", 3600), - scope=data.get("scope"), - ) - - error_data = response.json() - error = error_data.get("error") - - if error == "authorization_pending": - continue - elif error == "slow_down": - interval += 5 - continue - elif error == "expired_token": - raise DeviceFlowError("Authorization request expired") - elif error == "access_denied": - raise DeviceFlowError("User denied authorization") - else: - raise DeviceFlowError(f"Token request failed: {error}") - - raise DeviceFlowError("Authorization timed out") - - -async def authenticate_with_device_flow( - device_authorization_endpoint: str, - token_endpoint: str, - client_id: str, - scope: str = "openid email profile", - open_browser: bool = True, -) -> TokenResponse: - """Perform OAuth device flow authentication. - - Args: - device_authorization_endpoint: URL for device authorization - token_endpoint: URL for token exchange - client_id: OAuth client ID - scope: OAuth scopes to request - open_browser: Whether to automatically open the browser - - Returns: - TokenResponse with access and refresh tokens - """ - flow = DeviceFlow( - device_authorization_endpoint=device_authorization_endpoint, - token_endpoint=token_endpoint, - client_id=client_id, - scope=scope, - ) - - # Start device authorization - device_response = await flow.start_device_authorization() - - # Display user code and instructions - console.print() - console.print( - Panel( - f"[bold cyan]Visit:[/] {device_response.verification_uri}\n" - f"[bold cyan]Enter code:[/] [bold yellow]{device_response.user_code}[/]", - title="Authorization Required", - border_style="cyan", - ) - ) - - # Optionally open browser - if open_browser and device_response.verification_uri_complete: - console.print("\n[dim]Opening browser...[/]") - webbrowser.open(device_response.verification_uri_complete) - elif open_browser: - console.print("\n[dim]Opening browser...[/]") - webbrowser.open(device_response.verification_uri) - - console.print("\n[dim]Waiting for authorization...[/]") - - # Poll for token - token_response = await flow.poll_for_token( - device_code=device_response.device_code, - interval=device_response.interval, - expires_in=device_response.expires_in, - ) - - console.print("[green]Authorization successful![/]") - - return token_response - - -async def refresh_access_token( - token_endpoint: str, - client_id: str, - refresh_token: str, - scope: str | None = None, -) -> TokenResponse: - """ - Refresh an access token using a refresh token. - - Args: - token_endpoint: The OAuth token endpoint URL. - client_id: The OAuth client ID. - refresh_token: The refresh token from a previous authentication. - scope: OAuth scopes to request (required for some IdPs like Azure AD). - - Returns: - TokenResponse with new access_token (and possibly new refresh_token). - - Raises: - DeviceFlowError: If token refresh fails. - """ - try: - data = await asyncio.to_thread( - refresh_token_grant, - token_endpoint, - client_id, - refresh_token, - scope=scope, - ) - except RuntimeError as e: - raise DeviceFlowError(str(e)) from e - - return TokenResponse( - access_token=data["access_token"], - id_token=data.get("id_token"), - refresh_token=data.get("refresh_token"), # May be rotated - token_type=data.get("token_type", "Bearer"), - expires_in=data.get("expires_in", 3600), - scope=data.get("scope"), - ) - - -def authenticate_with_password_grant( - token_endpoint: str, - client_id: str, - username: str, - password: str, - scope: str = "openid profile email", -) -> TokenResponse: - """Obtain tokens using the Resource Owner Password Credentials grant (RFC 6749). - - Use this for non-interactive environments (e.g. CI) where no browser is available. - The IdP must have the password grant enabled for the application. - - Args: - token_endpoint: The OAuth token endpoint URL. - client_id: The OAuth client ID. - username: Resource owner username (e.g. testuser or built-in/admin). - password: Resource owner password. - scope: OAuth scopes to request. - - Returns: - TokenResponse with access_token and optional refresh_token. - - Raises: - DeviceFlowError: If the token request fails. - """ - data = { - "grant_type": "password", - "client_id": client_id, - "username": username, - "password": password, - "scope": scope, - } - with httpx.Client(verify=client_verify_from_env()) as client: - response = client.post(token_endpoint, data=data, timeout=30.0) - - if response.status_code != 200: - error_data = response.json() if response.headers.get("content-type", "").startswith("application/json") else {} - error = error_data.get("error", "unknown_error") - error_description = error_data.get("error_description", response.text) - raise DeviceFlowError(f"Token request failed: {error} - {error_description}") - - resp_data = response.json() - return TokenResponse( - access_token=resp_data["access_token"], - id_token=resp_data.get("id_token"), - refresh_token=resp_data.get("refresh_token"), - token_type=resp_data.get("token_type", "Bearer"), - expires_in=resp_data.get("expires_in", 3600), - scope=resp_data.get("scope"), - ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/auth/helpers.py b/sdk/python/nemo-platform/src/nemo_platform/auth/helpers.py deleted file mode 100644 index c3ff84c17a..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/auth/helpers.py +++ /dev/null @@ -1,224 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this code except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Auth helpers for NeMo Platform CLI (scope normalization, JWT decode, scope validation).""" - -from __future__ import annotations - -import base64 -import json -import time -from dataclasses import dataclass -from typing import Any - -import httpx - -from nemo_platform.client.tls import client_verify_from_env - -DEFAULT_OAUTH_SCOPES = "openid profile email offline_access" - - -class AuthError(Exception): - """Authentication-related error (CLI auth commands).""" - - pass - - -def normalize_scope_prefix(prefix: str | None) -> str: - """Normalize scope prefix to ensure it ends with a separator. - - Azure AD scope URIs require a '/' between the app ID and scope name. - This handles misconfigured clusters that omit the trailing slash. - - Args: - prefix: The scope prefix from cluster configuration (may be None or empty) - - Returns: - Empty string if prefix is None/empty, otherwise prefix with trailing '/' - """ - if not prefix: - return "" - return prefix if prefix.endswith("/") else f"{prefix}/" - - -def scope_short(scope: str, scope_prefix: str) -> str: - """Return scope in short form for comparison (strip prefix if present).""" - if scope_prefix and scope.startswith(scope_prefix): - return scope[len(scope_prefix) :] - return scope - - -def _decode_jwt_segment(token: str, index: int) -> dict[str, Any]: - try: - parts = token.split(".") - if len(parts) != 3: - return {} - payload = parts[index] - payload += "=" * (-len(payload) % 4) - decoded = base64.urlsafe_b64decode(payload) - data = json.loads(decoded) - return data if isinstance(data, dict) else {} - except Exception: - return {} - - -def decode_jwt_header(token: str) -> dict[str, Any]: - """Decode JWT header without verification.""" - return _decode_jwt_segment(token, 0) - - -def decode_jwt_claims(token: str) -> dict[str, Any]: - """Decode JWT claims without verification (for display purposes only).""" - return _decode_jwt_segment(token, 1) - - -def is_unsigned_jwt(token: str) -> bool: - """Return True when JWT uses ``alg=none``.""" - header = decode_jwt_header(token) - return str(header.get("alg", "")).lower() == "none" - - -def _base64url_encode_json(payload: dict[str, Any]) -> str: - encoded = json.dumps(payload, separators=(",", ":")).encode("utf-8") - return base64.urlsafe_b64encode(encoded).rstrip(b"=").decode("ascii") - - -def generate_unsigned_jwt( - principal_id: str, - *, - email: str | None = None, - groups: list[str] | None = None, - scopes: list[str] | None = None, - expires_in_seconds: int | None = 3600, - issued_at: int | None = None, - audience: str | None = None, - issuer: str | None = None, - extra_claims: dict[str, Any] | None = None, -) -> str: - """Generate an unsigned JWT (`alg=none`) for local development and testing.""" - now = issued_at if issued_at is not None else int(time.time()) - claims: dict[str, Any] = { - "sub": principal_id, - "iat": now, - } - - if email: - claims["email"] = email - if groups: - claims["groups"] = groups - if scopes: - claims["scope"] = " ".join(scopes) - if expires_in_seconds is not None: - claims["exp"] = now + expires_in_seconds - if audience: - claims["aud"] = audience - if issuer: - claims["iss"] = issuer - if extra_claims: - claims.update(extra_claims) - - header_segment = _base64url_encode_json({"alg": "none", "typ": "JWT"}) - claims_segment = _base64url_encode_json(claims) - return f"{header_segment}.{claims_segment}." - - -@dataclass(frozen=True) -class NMPOIDCConfig: - """OIDC configuration discovered from the NeMo Platform.""" - - auth_enabled: bool - issuer: str | None = None - client_id: str | None = None - token_endpoint: str | None = None - device_authorization_endpoint: str | None = None - default_scopes: str = DEFAULT_OAUTH_SCOPES - scope_prefix: str | None = None - workload_token_exchange_enabled: bool = False - workload_client_id: str | None = None - workload_token_endpoint: str | None = None - workload_audience: str | None = None - workload_scope: str | None = None - - -def discover_nmp_config(base_url: str, timeout: float = 10.0) -> NMPOIDCConfig: - """Fetch OIDC configuration from the NeMo Platform auth discovery endpoint.""" - response = httpx.get( - f"{base_url.rstrip('/')}/apis/auth/discovery", - timeout=timeout, - verify=client_verify_from_env(), - ) - response.raise_for_status() - data = response.json() - - oidc = data.get("oidc") or {} - return NMPOIDCConfig( - auth_enabled=data.get("auth_enabled", False), - issuer=oidc.get("issuer"), - client_id=oidc.get("client_id"), - token_endpoint=oidc.get("token_endpoint"), - device_authorization_endpoint=oidc.get("device_authorization_endpoint"), - default_scopes=oidc.get("default_scopes", DEFAULT_OAUTH_SCOPES), - scope_prefix=oidc.get("scope_prefix"), - workload_token_exchange_enabled=oidc.get("workload_token_exchange_enabled", False), - workload_client_id=oidc.get("workload_client_id"), - workload_token_endpoint=oidc.get("workload_token_endpoint"), - workload_audience=oidc.get("workload_audience"), - workload_scope=oidc.get("workload_scope"), - ) - - -def build_effective_scope(requested_scopes: str, scope_prefix: str | None) -> str: - """Prepend scope_prefix to custom scopes (those with ':' or ending with '.default').""" - prefix = normalize_scope_prefix(scope_prefix) - if not prefix: - return requested_scopes - expanded = [] - for s in requested_scopes.split(): - if ":" in s or s.endswith(".default"): - expanded.append(f"{prefix}{s}") - else: - expanded.append(s) - return " ".join(expanded) - - -def validate_requested_scopes_granted( - effective_scope: str, - granted_scopes: list[str], - scope_prefix: str, -) -> None: - """Validate that requested platform scopes appear in granted scopes; raise AuthError if not. - - Compares in short form so IdPs (e.g. Azure AD) that return scp as "platform:read" - match requested "api://nmp/platform:read". - """ - requested_platform = {s for s in effective_scope.split() if ":" in s} - requested_short = {scope_short(s, scope_prefix) for s in requested_platform} - granted_set = set(granted_scopes) - granted_short = {scope_short(s, scope_prefix) for s in granted_set} - missing_short = requested_short - granted_short - if not missing_short: - return - full_missing = sorted(s for s in requested_platform if scope_short(s, scope_prefix) in missing_short) - hint = "" - if scope_prefix and "api://" in scope_prefix: - hint = ( - "\nHint: For Azure AD, add the scopes in the app registration (Expose an API) and grant " - "admin consent. See tools/auth/azure/README.md." - ) - raise AuthError( - f"Token is missing requested scopes: {' '.join(full_missing)}.\n" - "The identity provider did not grant the requested scopes. " - "Check IdP configuration." + hint - ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/auth/token_provider.py b/sdk/python/nemo-platform/src/nemo_platform/auth/token_provider.py deleted file mode 100644 index 947e0a87d6..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/auth/token_provider.py +++ /dev/null @@ -1,254 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Token provider with automatic refresh for NeMo Platform SDK authentication.""" - -import asyncio -import json -import logging -import threading -import time -from collections.abc import Callable -from contextlib import AbstractContextManager, nullcontext -from dataclasses import dataclass, field - -import httpx -from typing_extensions import Self - -from nemo_platform.auth.helpers import decode_jwt_claims -from nemo_platform.client.tls import client_verify_from_env - -logger = logging.getLogger(__name__) - -# Refresh proactively when less than this many seconds remain before expiry. -DEFAULT_REFRESH_MARGIN_SECONDS = 60 - - -class TokenRefreshError(RuntimeError): - """Structured error raised for OAuth refresh_token grant failures.""" - - def __init__(self, *, error: str, error_description: str) -> None: - self.error = error - self.error_description = error_description - super().__init__(f"Token refresh failed: {error} - {error_description}") - - -def _validate_expires_in(expires_in: object) -> int | float | None: - if isinstance(expires_in, bool): - return None - return expires_in if isinstance(expires_in, int | float) else None - - -def refresh_token_grant( - token_endpoint: str, - client_id: str, - refresh_token: str, - *, - scope: str | None = None, - timeout: float = 30.0, -) -> dict: - """Execute OAuth refresh_token grant and return token response JSON.""" - data: dict[str, str] = { - "grant_type": "refresh_token", - "client_id": client_id, - "refresh_token": refresh_token, - } - if scope: - data["scope"] = scope - - response = httpx.post(token_endpoint, data=data, timeout=timeout, verify=client_verify_from_env()) - - if response.status_code != 200: - error_data: dict[str, str] = {} - if response.headers.get("content-type", "").startswith("application/json"): - try: - error_data = response.json() - except (json.JSONDecodeError, ValueError): - error_data = {} - error = error_data.get("error", "unknown_error") - error_description = error_data.get("error_description", response.text) - raise TokenRefreshError(error=error, error_description=error_description) - - return response.json() - - -@dataclass -class TokenSet: - """A pair of access + refresh tokens with expiry metadata.""" - - access_token: str - refresh_token: str | None = None - expires_at: float | None = None - - @staticmethod - def from_access_token( - access_token: str, - refresh_token: str | None = None, - expires_in: object = None, - ) -> Self: - """Create a TokenSet, extracting expiry from the JWT's `exp` claim.""" - expires_at = None - claims = decode_jwt_claims(access_token) - if claims: - expires_at = claims.get("exp") - validated_expires_in = _validate_expires_in(expires_in) - if expires_at is None and validated_expires_in is not None: - expires_at = time.time() + float(validated_expires_in) - return TokenSet( - access_token=access_token, - refresh_token=refresh_token, - expires_at=float(expires_at) if expires_at is not None else None, - ) - - def is_expired(self, margin_seconds: float = DEFAULT_REFRESH_MARGIN_SECONDS) -> bool: - """Check if the access token is expired or about to expire.""" - if self.expires_at is None: - return False - return time.time() >= (self.expires_at - margin_seconds) - - -@dataclass -class OIDCTokenProvider: - """Provides access tokens with automatic refresh via the OAuth2 refresh_token grant. - - This is the core component for SDK-level token management. It: - - Holds the current access + refresh tokens - - Proactively refreshes the access token before it expires - - Is thread-safe (uses a lock for concurrent access) - - Optionally persists refreshed tokens via a callback - - Args: - token_endpoint: The OAuth2 token endpoint URL. - client_id: The OAuth2 client ID. - tokens: The current token set. - refresh_margin_seconds: Seconds before expiry to proactively refresh. - load_tokens: Optional callback to reload tokens from a shared store (e.g. - config file) before refresh attempts. - refresh_lock: Optional context manager factory for serializing refresh - transactions across processes. - on_tokens_refreshed: Optional callback invoked with the new ``TokenSet`` - after a successful refresh. Use this to persist tokens (e.g. write - them back to ``~/.config/nmp/config.yaml``). - """ - - token_endpoint: str - client_id: str - tokens: TokenSet = field(default_factory=lambda: TokenSet(access_token="")) - refresh_margin_seconds: float = DEFAULT_REFRESH_MARGIN_SECONDS - refresh_scope: str | None = None - load_tokens: Callable[[], TokenSet | None] | None = None - refresh_lock: Callable[[], AbstractContextManager[None]] | None = None - on_tokens_refreshed: Callable[[TokenSet], None] | None = None - _lock: threading.Lock = field(default_factory=threading.Lock, repr=False) - - def get_access_token(self) -> str: - """Return a valid access token, refreshing if necessary.""" - with self._lock: - if self.tokens.is_expired(self.refresh_margin_seconds): - self._refresh() - return self.tokens.access_token - - async def get_access_token_async(self) -> str: - """Return a valid access token in async contexts. - - Runs refresh logic in a worker thread so token refresh does not block the - event loop. - """ - return await asyncio.to_thread(self.get_access_token) - - def reload_tokens(self) -> bool: - """Reload tokens from a shared store, if configured.""" - with self._lock: - return self._reload_tokens_from_source() - - def _reload_tokens_from_source(self) -> bool: - if self.load_tokens is None: - return False - - try: - loaded_tokens = self.load_tokens() - except Exception: - logger.warning("Failed to reload shared tokens", exc_info=True) - return False - - if loaded_tokens is None or loaded_tokens == self.tokens: - return False - - self.tokens = loaded_tokens - logger.debug("Reloaded shared tokens (expires_at=%s)", self.tokens.expires_at) - return True - - def _refresh(self, *, force: bool = False) -> None: - """Refresh the access token using the refresh_token grant. - - Raises: - RuntimeError: If no refresh token is available or the refresh request fails. - """ - lock_context = self.refresh_lock() if self.refresh_lock is not None else nullcontext() - with lock_context: - self._reload_tokens_from_source() - if not force and not self.tokens.is_expired(self.refresh_margin_seconds): - return - - if not self.tokens.refresh_token: - raise RuntimeError( - "Access token has expired and no refresh token is available. " - "Re-authenticate with `nemo auth login` to obtain new tokens." - ) - - logger.debug("Refreshing access token via %s", self.token_endpoint) - - token_data: dict - try: - token_data = refresh_token_grant( - token_endpoint=self.token_endpoint, - client_id=self.client_id, - refresh_token=self.tokens.refresh_token, - scope=self.refresh_scope, - ) - except TokenRefreshError as exc: - if exc.error != "invalid_grant": - raise - - if not self._reload_tokens_from_source(): - raise - - if not force and not self.tokens.is_expired(self.refresh_margin_seconds): - logger.debug("Recovered from invalid_grant with shared tokens") - return - - if not self.tokens.refresh_token: - raise RuntimeError( - "Access token has expired and no refresh token is available. " - "Re-authenticate with `nemo auth login` to obtain new tokens." - ) - - token_data = refresh_token_grant( - token_endpoint=self.token_endpoint, - client_id=self.client_id, - refresh_token=self.tokens.refresh_token, - scope=self.refresh_scope, - ) - - new_access_token = token_data["access_token"] - # The IdP may rotate the refresh token. - new_refresh_token = token_data.get("refresh_token", self.tokens.refresh_token) - - self.tokens = TokenSet.from_access_token( - new_access_token, - new_refresh_token, - expires_in=token_data.get("expires_in"), - ) - logger.debug("Access token refreshed successfully (expires_at=%s)", self.tokens.expires_at) - - if self.on_tokens_refreshed: - try: - self.on_tokens_refreshed(self.tokens) - except Exception: - logger.warning("Failed to persist refreshed tokens", exc_info=True) - - def force_refresh(self) -> str: - """Force a token refresh regardless of expiry. Returns the new access token.""" - with self._lock: - self._refresh(force=True) - return self.tokens.access_token diff --git a/sdk/python/nemo-platform/src/nemo_platform/auth/workload_exchange.py b/sdk/python/nemo-platform/src/nemo_platform/auth/workload_exchange.py deleted file mode 100644 index c54423fc01..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/auth/workload_exchange.py +++ /dev/null @@ -1,207 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Workload identity token exchange for SDK authentication.""" - -from __future__ import annotations - -import asyncio -import json -import logging -import math -import threading -from dataclasses import dataclass, field -from ipaddress import ip_address -from pathlib import Path -from urllib.parse import urlparse - -import httpx -from nemo_platform_plugin.client.constants import WORKLOAD_IDENTITY_TOKEN_FILE_ENVVAR - -from nemo_platform.auth.token_provider import DEFAULT_REFRESH_MARGIN_SECONDS, TokenSet -from nemo_platform.client.tls import client_verify_from_env - -logger = logging.getLogger(__name__) - -TOKEN_EXCHANGE_GRANT_TYPE = "urn:ietf:params:oauth:grant-type:token-exchange" -JWT_TOKEN_TYPE = "urn:ietf:params:oauth:token-type:jwt" -ACCESS_TOKEN_TYPE = "urn:ietf:params:oauth:token-type:access_token" - - -class WorkloadTokenExchangeError(RuntimeError): - """Structured error raised for RFC 8693 workload token exchange failures.""" - - def __init__(self, *, error: str, error_description: str) -> None: - self.error = error - self.error_description = error_description - super().__init__(f"Workload token exchange failed: {error} - {error_description}") - - -def read_subject_token_file(path: Path) -> str: - """Read a subject token from a workload identity token file.""" - try: - token = path.read_text(encoding="utf-8").strip() - except OSError as exc: - raise ValueError(f"Unable to read {WORKLOAD_IDENTITY_TOKEN_FILE_ENVVAR} at {path}: {exc}") from exc - if not token: - raise ValueError(f"{WORKLOAD_IDENTITY_TOKEN_FILE_ENVVAR} at {path} is empty") - return token - - -def _is_loopback_host(hostname: str | None) -> bool: - if hostname == "localhost": - return True - if hostname is None: - return False - try: - return ip_address(hostname).is_loopback - except ValueError: - return False - - -def _validate_token_endpoint(token_endpoint: str) -> None: - """Reject non-HTTPS token endpoints (except loopback for local dev).""" - parsed = urlparse(token_endpoint) - if parsed.scheme == "https": - return - if parsed.scheme == "http" and _is_loopback_host(parsed.hostname): - return - raise ValueError( - f"OIDC token endpoint must use HTTPS (got {token_endpoint!r}). " - "HTTP is only allowed for loopback addresses (localhost, 127.0.0.1, ::1)." - ) - - -def token_exchange_grant( - *, - token_endpoint: str, - client_id: str, - subject_token: str, - audience: str | None = None, - scope: str | None = None, - timeout: float = 30.0, -) -> dict[str, object]: - """Execute RFC 8693 token exchange and return token response JSON.""" - _validate_token_endpoint(token_endpoint) - data: dict[str, str] = { - "grant_type": TOKEN_EXCHANGE_GRANT_TYPE, - "client_id": client_id, - "subject_token": subject_token, - "subject_token_type": JWT_TOKEN_TYPE, - "requested_token_type": ACCESS_TOKEN_TYPE, - } - if audience: - data["audience"] = audience - if scope: - data["scope"] = scope - - response = httpx.post(token_endpoint, data=data, timeout=timeout, verify=client_verify_from_env()) - - if response.status_code != 200: - error_data: dict[str, object] = {} - if response.headers.get("content-type", "").startswith("application/json"): - error_data = _response_json_object( - response, - error_description="Token endpoint error response was not a JSON object", - ) - error = _response_string(error_data, "error", "unknown_error") - error_description = _response_string(error_data, "error_description", response.text) - raise WorkloadTokenExchangeError(error=error, error_description=error_description) - - token_data = _response_json_object( - response, - error_description="Token endpoint response was not a JSON object", - ) - _access_token_from_response(token_data) - return token_data - - -def _response_json_object(response: httpx.Response, *, error_description: str) -> dict[str, object]: - try: - payload = response.json() - except (json.JSONDecodeError, ValueError) as exc: - raise WorkloadTokenExchangeError(error="invalid_response", error_description=error_description) from exc - if not isinstance(payload, dict): - raise WorkloadTokenExchangeError(error="invalid_response", error_description=error_description) - return payload - - -def _response_string(payload: dict[str, object], key: str, default: str) -> str: - value = payload.get(key) - return value if isinstance(value, str) and value else default - - -def _access_token_from_response(token_data: dict[str, object]) -> str: - access_token = token_data.get("access_token") - if not isinstance(access_token, str) or not access_token.strip(): - raise WorkloadTokenExchangeError( - error="invalid_response", - error_description="Token endpoint response did not include a non-empty access_token", - ) - return access_token - - -def _expires_in_from_response(token_data: dict[str, object]) -> int | float | None: - expires_in = token_data.get("expires_in") - if isinstance(expires_in, bool): - return None - return expires_in if isinstance(expires_in, int | float) else None - - -@dataclass -class WorkloadTokenExchangeProvider: - """Provides access tokens by exchanging a workload identity subject token file.""" - - token_endpoint: str - client_id: str - subject_token_file: Path - audience: str | None = None - scope: str | None = None - refresh_margin_seconds: float = DEFAULT_REFRESH_MARGIN_SECONDS - tokens: TokenSet = field(default_factory=lambda: TokenSet(access_token="")) - _lock: threading.Lock = field(default_factory=threading.Lock, repr=False) - - def get_access_token(self) -> str: - """Return a valid access token, exchanging the current subject token if needed.""" - with self._lock: - if not self.tokens.access_token or self.tokens.is_expired(self.refresh_margin_seconds): - self._exchange() - return self.tokens.access_token - - async def get_access_token_async(self) -> str: - """Return a valid access token in async contexts.""" - return await asyncio.to_thread(self.get_access_token) - - def _exchange(self) -> None: - subject_token = read_subject_token_file(self.subject_token_file) - logger.debug("Exchanging workload identity token via %s", self.token_endpoint) - token_data = token_exchange_grant( - token_endpoint=self.token_endpoint, - client_id=self.client_id, - subject_token=subject_token, - audience=self.audience, - scope=self.scope, - ) - access_token = _access_token_from_response(token_data) - try: - tokens = TokenSet.from_access_token( - access_token, - refresh_token=None, - expires_in=_expires_in_from_response(token_data), - ) - except (OverflowError, TypeError, ValueError) as exc: - raise WorkloadTokenExchangeError( - error="invalid_response", - error_description="Token endpoint response did not include a usable access_token lifetime", - ) from exc - if tokens.expires_at is None or not math.isfinite(tokens.expires_at): - raise WorkloadTokenExchangeError( - error="invalid_response", - error_description="Token endpoint response did not include a usable access_token lifetime", - ) - if tokens.is_expired(0): - raise WorkloadTokenExchangeError( - error="invalid_response", - error_description="Token endpoint response returned an expired access_token", - ) - self.tokens = tokens diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/beta/__init__.py new file mode 100644 index 0000000000..8b2527cba3 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Beta SDK extensions.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/__init__.py index 114048d8af..2f271d337a 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/__init__.py @@ -1,301 +1,6 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""NeMo Evaluator SDK. +from nemo_platform._alias import alias_package as _alias_package -The public surface resolves lazily (PEP 562). Importing this package must not drag in the -execution/backend or metric stack: importing any submodule runs this module first, so eager -re-exports made ``import nemo_platform.beta.evaluator.agent_eval.runtimes.harbor_runtime`` — all the -optimizer needs — cost ~1400 modules (openai, sacrebleu, zstandard, ...) instead of ~485, and -turned every one of those transitive packages into an evaluation-time failure mode for the -SDK-backed evaluator. - -Add a new re-export to ``_LAZY_ATTRS``, the ``TYPE_CHECKING`` block and ``__all__`` — never as a -module-level import. ``tests/test_lazy_public_api.py`` locks the boundary in. -""" - -# ruff: noqa: I001 - the vendored SDK mirror uses different import-order settings. - -from importlib import import_module as _import_module -from importlib.metadata import PackageNotFoundError as _PackageNotFoundError -from importlib.metadata import version as _package_version -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - # Annotations and static analysis only; these must never execute at run time. Listing the - # names in ``__all__`` is what marks them as re-exports for ruff and the type checkers. - # - # AGENTS.md ("Python Style notes") says not to import types under TYPE_CHECKING and to use a - # regular import "when possible". A regular import is exactly what this module exists to - # remove, so the exception is deliberate: these names are re-exports, not annotations, and - # every one of them resolves for real through ``__getattr__`` below. - from nemo_platform.beta.evaluator.agent_stream_translation import ( - AgentStreamTranslation, - AgentStreamTranslationContext, - AgentStreamTranslator, - SseFrame, - ) - from nemo_platform.beta.evaluator.datasets import DatasetLoadError, load_dataset, load_dataset_as_dicts - from nemo_platform.beta.evaluator.execution.backends.local.backend import LocalBackend - from nemo_platform.beta.evaluator.execution.evaluator import Evaluator - from nemo_platform.beta.evaluator.execution.values import ( - EvaluationError, - EvaluationPhase, - ) - from nemo_platform.beta.evaluator.metrics.bleu import BLEUMetric - from nemo_platform.beta.evaluator.metrics.exact_match import ExactMatchMetric - from nemo_platform.beta.evaluator.metrics.f1 import F1Metric - from nemo_platform.beta.evaluator.metrics.llm_judge import LLMJudgeMetric - from nemo_platform.beta.evaluator.metrics.number_check import NumberCheckMetric - from nemo_platform.beta.evaluator.metrics.protocol import ( - Metric, - MetricTypeName, - validate_metric_result, - ) - from nemo_platform.beta.evaluator.metrics.remote import NemoAgentToolkitRemoteMetric, RemoteMetric - from nemo_platform.beta.evaluator.metrics.rouge import ROUGEMetric - from nemo_platform.beta.evaluator.metrics.string_check import StringCheckMetric - from nemo_platform.beta.evaluator.metrics.tool_calling import ToolCallingMetric - from nemo_platform.beta.evaluator.metrics.tunable_rag_evaluator import TunableRagEvaluatorMetric - from nemo_platform.beta.evaluator.resolver_protocols import ModelResolver, SecretResolver - from nemo_platform.beta.evaluator.resolvers import LocalModelResolver, LocalSecretResolver - from nemo_platform.beta.evaluator.structured_output import ( - InferenceFn, - InferenceStructuredOutput, - StructuredOutput, - StructuredOutputMode, - default_structured_output_mode, - detect_structured_output_mode, - ) - from nemo_platform.beta.evaluator.values import ( - Agent, - AgentBase, - BooleanValue, - CandidateOutput, - ContinuousScore, - BenchmarkEvaluationResult, - DatasetRow, - DatasetRows, - DiscreteScore, - EvaluationResult, - FieldMapping, - InferenceParams, - JSONScoreParser, - Label, - MetricDescriptor, - MetricDiagnostic, - MetricInput, - MetricOutput, - MetricOutputSpec, - MetricResult, - Model, - ModelRef, - GenericAgent, - NatAgentConfig, - NemoAgentToolkitAgent, - RangeScore, - ReasoningParams, - RemoteScore, - RubricScore, - RunConfig, - RunConfigOnline, - RunConfigOnlineModel, - SecretRef, - ) - - -def _resolve_version() -> str: - """Report the version of whichever distribution actually shipped this code. - - ``nemo-evaluator-sdk`` is not published on its own — this package is also vendored into the - ``nemo-platform`` wheel as ``nemo_platform.beta.evaluator``. There the SDK distribution does - not exist, so resolving only that name reported ``"0.0.0"`` unconditionally and any telemetry - or support log that read it got a useless constant. - """ - for distribution in ("nemo-evaluator-sdk", "nemo-platform"): - try: - return _package_version(distribution) - except _PackageNotFoundError: - continue - return "0.0.0" - - -version = _resolve_version() - -# Re-exported name -> the submodule that defines it, relative to this package. Relative on -# purpose: the vendoring tool mirrors this file into nemo_platform.beta.evaluator by rewriting -# module paths, and a relative name has nothing to rewrite, so the mirror is correct by -# construction. Mirrors the TYPE_CHECKING block above, in the same order. -_LAZY_ATTRS: dict[str, str] = { - "AgentStreamTranslation": ".agent_stream_translation", - "AgentStreamTranslationContext": ".agent_stream_translation", - "AgentStreamTranslator": ".agent_stream_translation", - "SseFrame": ".agent_stream_translation", - "DatasetLoadError": ".datasets", - "load_dataset": ".datasets", - "load_dataset_as_dicts": ".datasets", - "LocalBackend": ".execution.backends.local.backend", - "Evaluator": ".execution.evaluator", - "EvaluationError": ".execution.values", - "EvaluationPhase": ".execution.values", - "BLEUMetric": ".metrics.bleu", - "ExactMatchMetric": ".metrics.exact_match", - "F1Metric": ".metrics.f1", - "LLMJudgeMetric": ".metrics.llm_judge", - "NumberCheckMetric": ".metrics.number_check", - "Metric": ".metrics.protocol", - "MetricTypeName": ".metrics.protocol", - "validate_metric_result": ".metrics.protocol", - "NemoAgentToolkitRemoteMetric": ".metrics.remote", - "RemoteMetric": ".metrics.remote", - "ROUGEMetric": ".metrics.rouge", - "StringCheckMetric": ".metrics.string_check", - "ToolCallingMetric": ".metrics.tool_calling", - "TunableRagEvaluatorMetric": ".metrics.tunable_rag_evaluator", - "ModelResolver": ".resolver_protocols", - "SecretResolver": ".resolver_protocols", - "LocalModelResolver": ".resolvers", - "LocalSecretResolver": ".resolvers", - "InferenceFn": ".structured_output", - "InferenceStructuredOutput": ".structured_output", - "StructuredOutput": ".structured_output", - "StructuredOutputMode": ".structured_output", - "default_structured_output_mode": ".structured_output", - "detect_structured_output_mode": ".structured_output", - "Agent": ".values", - "AgentBase": ".values", - "BooleanValue": ".values", - "CandidateOutput": ".values", - "ContinuousScore": ".values", - "BenchmarkEvaluationResult": ".values", - "DatasetRow": ".values", - "DatasetRows": ".values", - "DiscreteScore": ".values", - "EvaluationResult": ".values", - "FieldMapping": ".values", - "InferenceParams": ".values", - "JSONScoreParser": ".values", - "Label": ".values", - "MetricDescriptor": ".values", - "MetricDiagnostic": ".values", - "MetricInput": ".values", - "MetricOutput": ".values", - "MetricOutputSpec": ".values", - "MetricResult": ".values", - "Model": ".values", - "ModelRef": ".values", - "GenericAgent": ".values", - "NatAgentConfig": ".values", - "NemoAgentToolkitAgent": ".values", - "RangeScore": ".values", - "ReasoningParams": ".values", - "RemoteScore": ".values", - "RubricScore": ".values", - "RunConfig": ".values", - "RunConfigOnline": ".values", - "RunConfigOnlineModel": ".values", - "SecretRef": ".values", -} - -__all__ = [ - "BLEUMetric", - "Agent", - "AgentBase", - "EvaluationError", - "EvaluationPhase", - "DatasetLoadError", - "DatasetRows", - "RunConfig", - "RunConfigOnline", - "RunConfigOnlineModel", - "BenchmarkEvaluationResult", - "EvaluationResult", - "Evaluator", - "ExactMatchMetric", - "F1Metric", - "FieldMapping", - "InferenceParams", - "InferenceFn", - "InferenceStructuredOutput", - "JSONScoreParser", - "Metric", - "MetricTypeName", - "MetricDescriptor", - "MetricDiagnostic", - "MetricInput", - "MetricOutput", - "MetricOutputSpec", - "MetricResult", - "LLMJudgeMetric", - "BooleanValue", - "CandidateOutput", - "ContinuousScore", - "DatasetRow", - "DiscreteScore", - "Label", - "LocalBackend", - "LocalModelResolver", - "LocalSecretResolver", - "Model", - "ModelRef", - "GenericAgent", - "ModelResolver", - "NatAgentConfig", - "NemoAgentToolkitAgent", - "AgentStreamTranslation", - "AgentStreamTranslationContext", - "AgentStreamTranslator", - "NemoAgentToolkitRemoteMetric", - "NumberCheckMetric", - "RangeScore", - "ReasoningParams", - "RemoteMetric", - "RemoteScore", - "ROUGEMetric", - "RubricScore", - "SecretRef", - "SecretResolver", - "SseFrame", - "StringCheckMetric", - "StructuredOutput", - "StructuredOutputMode", - "ToolCallingMetric", - "TunableRagEvaluatorMetric", - "default_structured_output_mode", - "detect_structured_output_mode", - "load_dataset", - "load_dataset_as_dicts", - "validate_metric_result", - "version", -] - - -def __getattr__(name: str) -> object: - """Import the submodule that defines ``name`` on first access (PEP 562). - - An *unknown* name raises ``AttributeError``, which is required: ``from pkg import sub`` only - falls back to importing a submodule when attribute lookup raises ``AttributeError``. - - A *known* name whose submodule fails to import propagates that ``ImportError`` unchanged, and - that is deliberate — ``ModuleNotFoundError: No module named 'sacrebleu'`` is far more useful - than an ``AttributeError`` claiming ``BLEUMetric`` does not exist. The consequence is that - ``hasattr(nemo_evaluator_sdk, name)`` raises rather than returning ``False`` when a name's - dependencies are not installed, since ``hasattr`` only swallows ``AttributeError``. To probe - for an optional part of the surface, catch ``ImportError`` around the access instead of using - ``hasattr``; to probe only for name membership, test against ``__all__``. - """ - submodule = _LAZY_ATTRS.get(name) - if submodule is None: - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") - value = getattr(_import_module(submodule, __name__), name) - globals()[name] = value # cache, so later lookups skip __getattr__ entirely - return value - - -def __dir__() -> list[str]: - # The declared surface plus any submodule the caller has already imported. Everything this - # module needs for its own machinery is imported under a leading underscore so the filter - # below keeps it out of autocomplete and inspect.getmembers without a name-by-name denylist; - # ``TYPE_CHECKING`` is the one exception, kept unaliased so type checkers still recognise it. - public = {name for name in globals() if not name.startswith("_")} - {"TYPE_CHECKING"} - return sorted(set(__all__) | public) +_alias_package("nemo_evaluator_sdk", globals()) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/dashboard.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/dashboard.py deleted file mode 100644 index 4c72c455dc..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/dashboard.py +++ /dev/null @@ -1,175 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Small HTML dashboard for standalone agent-eval result bundles.""" - -from __future__ import annotations - -import html -import json -from pathlib import Path -from typing import Any - -from nemo_platform.beta.evaluator.agent_eval.results import AgentEvalResult -from nemo_platform.beta.evaluator.agent_eval.scores import AgentEvalTaskScore -from nemo_platform.beta.evaluator.values.results import AggregateScalarScore, AggregateScore -from pydantic import BaseModel - - -def write_dashboard(result: AgentEvalResult, output_path: str | Path) -> Path: - """Write an HTML dashboard and return its path.""" - path = Path(output_path) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(render_dashboard(result), encoding="utf-8") - return path - - -def render_dashboard(result: AgentEvalResult) -> str: - """Render a compact generic report for metric outputs.""" - return f""" - - - - - Agent Eval Report - - - -
-

Agent Eval Report

-
Run {_e(result.run_id)} · {_e(result.summary.task_count)} tasks · {_e(result.summary.trial_count)} trials
-
-
-
-
Tasks{_e(result.summary.task_count)}
-
Trials{_e(result.summary.trial_count)}
-
Metric Scores{_e(result.summary.score_count)}
-
-

Metric Rollups

- {_metric_rollups(result)} -

Scores

- {_score_table(result.scores)} -
- - -""" - - -def _metric_rollups(result: AgentEvalResult) -> str: - aggregated = result.summary.scores.scores - if not aggregated: - return '

No numeric metric outputs to summarize.

' - rows: list[str] = [] - for score in sorted(aggregated, key=lambda item: item.name): - rows.append( - "" - f"{_e(score.name)}" - f"{_format_score(_headline_value(score))}" - f"{_format_score(_median(score))}" - f"{_format_score(score.sample_std_dev)}" - f"{_count(score.count)}" - f"{_e(score.nan_count)}" - "" - ) - return ( - "" - "" + "".join(rows) + "
NameValueMedianStd devCountNaN
" - ) - - -def _headline_value(score: AggregateScore) -> float | None: - """The one number to show: a scalar's ``value``, otherwise the mean of the distribution. - - A scalar score has no mean — rendering the column straight off ``score.mean`` would leave every - runner-imported figure blank in the table where it is the only thing worth reading. - """ - return score.value if isinstance(score, AggregateScalarScore) else score.mean - - -def _median(score: AggregateScore) -> float | None: - """The median, whether it arrived as a field or only inside a percentile distribution. - - Reading `percentiles.p50` alone would blank the column for every imported aggregate: a backend that - reports a median without a full distribution (Gym does) sets `median` and nothing else, which is the - case the field was added for. Natively computed scores populate both, identically. - """ - if score.median is not None: - return score.median - percentiles = getattr(score, "percentiles", None) - return percentiles.p50 if percentiles is not None else None - - -def _count(count: int | None) -> str: - """Sample size, or an em dash when the producer didn't report one (imported aggregates). - - Tests for None specifically: a genuine 0 means every sample was NaN, which is worth seeing. - """ - return "—" if count is None else _e(count) - - -def _score_table(scores: list[AgentEvalTaskScore]) -> str: - if not scores: - return '

No metric scores.

' - rows = [ - "" - f"{_e(score.task_id)}" - f"{_e(score.trial_id)}" - f"{_e(score.metric_type)}" - f"{_outputs(score)}" - "" - for score in scores - ] - return ( - "" - + "".join(rows) - + "
TaskTrialMetricOutputs
" - ) - - -def _outputs(score: AgentEvalTaskScore) -> str: - chunks = [] - for output in score.outputs: - chunks.append( - f'
{_e(output.name)}
{_e(_jsonish(output.value))}
' - ) - return '
' + "".join(chunks) + "
" - - -def _jsonish(value: Any) -> str: - if isinstance(value, BaseModel): - value = value.model_dump(mode="json") - try: - return json.dumps(value, indent=2, sort_keys=True) - except (TypeError, ValueError): - # ValueError covers circular references; fall back to a plain string rather than crash rendering. - return str(value) - - -def _format_score(value: float | None) -> str: - if value is None: - return "n/a" - return f"{value:.3f}" - - -def _e(value: object) -> str: - return html.escape(str(value), quote=True) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py deleted file mode 100644 index bae3b15c29..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py +++ /dev/null @@ -1,783 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Standalone agent evaluation orchestration.""" - -# ruff: noqa: I001 - the vendored SDK mirror uses different import-order settings. - -from __future__ import annotations - -import asyncio -import uuid -from collections import defaultdict -from collections.abc import Awaitable, Callable, Sequence -from datetime import UTC, datetime -from importlib.metadata import PackageNotFoundError -from importlib.metadata import version as package_version -from logging import getLogger -from pathlib import Path -from typing import Any, cast, overload -from urllib.parse import urlparse - -import httpx -import nemo_platform.beta.evaluator.inference as inference -from nemo_platform.beta.evaluator.agent_eval.results import AgentEvalResult, AgentEvalSummary, RunMetadata -from nemo_platform.beta.evaluator.agent_eval.scores import ( - AgentEvalDiagnostic, - AgentEvalDiagnosticSeverity, - AgentEvalScoreStatus, - AgentEvalTaskScore, - TRIAL_STATUS_DETAIL, -) -from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask -from nemo_platform.beta.evaluator.agent_eval.trials import ( - AgentEvalTarget, - AgentEvalTrial, - AgentEvalTrialStatus, - AgentOutput, - AgentTaskRunner, - RunAggregationsProvider, - RunnerInfo, -) -from nemo_platform.beta.evaluator.agent_inference import ( - AgentInferenceContext, - AgentInferenceFn, - AgentInferenceFnFactory, - make_agent_inference_fn, - new_agent_inference_client, -) -from nemo_platform.beta.evaluator.execution.metric_execution import generate_online_sample, run_sync -from nemo_platform.beta.evaluator.execution.samples import build_metric_input -from nemo_platform.beta.evaluator.inference import InferenceFn -from nemo_platform.beta.evaluator.metrics.protocol import Metric, validate_metric_result -from nemo_platform.beta.evaluator.metrics.utils import metric_type_name -from nemo_platform.beta.evaluator.values import ( - Agent, - AgentBase, - GenericAgent, - Model, - RunConfig, - RunConfigOnline, - RunConfigOnlineModel, -) -from nemo_platform.beta.evaluator.values.results import AggregateScore -from nemo_platform.beta.evaluator.values.evidence import ( - EVIDENCE_FORMAT_JSON, - EVIDENCE_TRACE, - CandidateEvidence, - EvidenceDescriptor, -) -from openai import AsyncOpenAI - -log = getLogger(__name__) - -_SAMPLE_KEYS_EXCLUDED_FROM_OUTPUT_METADATA = frozenset( - { - "evidence", - "invocation_metadata", - "invocation_status", - "output_text", - "response", - "trajectory", - } -) - - -class AgentEvaluator: - """Run stored-trial or live-target agent evaluations. - - The online inference seam (an optional ``inference_fn``, transport ``client``, and - ``default_headers``) is injected on the evaluator instance rather than the run config, - because these are runtime transport concerns rather than declarative run settings. A - single ``inference_fn``/``client`` pair serves both model and agent targets; leave them - unset to let the evaluator build a default client for the resolved target type. - """ - - @overload - def __init__( - self, - *, - inference_fn: InferenceFn | AgentInferenceFn | None = None, - agent_inference_fn_factory: None = None, - client: AsyncOpenAI | httpx.AsyncClient | None = None, - default_headers: dict[str, str] | None = None, - ) -> None: ... - - @overload - def __init__( - self, - *, - inference_fn: None = None, - agent_inference_fn_factory: AgentInferenceFnFactory, - client: AsyncOpenAI | httpx.AsyncClient | None = None, - default_headers: dict[str, str] | None = None, - ) -> None: ... - - def __init__( - self, - *, - inference_fn: InferenceFn | AgentInferenceFn | None = None, - agent_inference_fn_factory: AgentInferenceFnFactory | None = None, - client: AsyncOpenAI | httpx.AsyncClient | None = None, - default_headers: dict[str, str] | None = None, - ) -> None: - """Configure runtime dependencies for live target generation. - - Args: - inference_fn: Optional model or agent inference override. When omitted, the - evaluator selects the default implementation for the target type. - agent_inference_fn_factory: Optional per-task factory for agent inference. - The evaluator supplies persistence and invocation identity through an - :class:`AgentInferenceContext`. - client: Optional transport client matching the target type: ``AsyncOpenAI`` for - models or ``httpx.AsyncClient`` for agents. - default_headers: Additional HTTP headers forwarded to live inference requests. - """ - if inference_fn is not None and agent_inference_fn_factory is not None: - raise ValueError("provide either inference_fn or agent_inference_fn_factory, not both") - self.inference_fn = inference_fn - self.agent_inference_fn_factory = agent_inference_fn_factory - self.client = client - self.default_headers = default_headers - - async def run( - self, - *, - tasks: Sequence[AgentEvalTask], - trials: Sequence[AgentEvalTrial] | None = None, - target: AgentEvalTarget | None = None, - config: AgentEvalRunConfig | None = None, - ) -> AgentEvalResult: - """Evaluate imported trials or generate live trials before scoring. - - Exactly one of ``trials`` or ``target`` must be provided. - """ - resolved_config = config or AgentEvalRunConfig() - task_list = list(tasks) - if not task_list: - raise ValueError("at least one task is required") - - run_id = resolved_config.run_id or _new_run_id() - runtime_config = resolved_config.model_copy(update={"run_id": run_id}) - started_at = datetime.now(UTC) - - # Branch on which seam was supplied so the type checker can narrow ``target`` to a - # concrete ``AgentEvalTarget`` without a cast. - if trials is not None: - if target is not None: - raise ValueError("provide exactly one of trials or target") - trial_list = list(trials) - elif target is not None: - trial_list = await self._generate_trials(tasks=task_list, target=target, config=runtime_config) - else: - raise ValueError("provide exactly one of trials or target") - scores = await self._score_trials( - tasks=task_list, - trials=trial_list, - config=runtime_config, - run_id=run_id, - ) - runner_scores = _collect_runner_aggregate_scores(target) if target is not None else [] - finished_at = datetime.now(UTC) - metadata = RunMetadata( - labels=dict(runtime_config.labels), - target=_describe_target(target, runtime_config.params), - started_at=started_at, - finished_at=finished_at, - duration_sec=(finished_at - started_at).total_seconds(), - sdk_version=_sdk_version(), - ) - result = AgentEvalResult( - run_id=run_id, - tasks=task_list, - trials=trial_list, - scores=scores, - summary=AgentEvalSummary.from_scores(scores, tasks=task_list, extra_scores=runner_scores), - metadata=metadata, - work_dir=runtime_config.work_dir, - ) - - return result - - def run_sync( - self, - *, - tasks: Sequence[AgentEvalTask], - trials: Sequence[AgentEvalTrial] | None = None, - target: AgentEvalTarget | None = None, - config: AgentEvalRunConfig | None = None, - ) -> AgentEvalResult: - """Synchronous bridge for :meth:`run`.""" - return run_sync(lambda: self.run(tasks=tasks, trials=trials, target=target, config=config)) - - async def _score_trials( - self, - *, - tasks: list[AgentEvalTask], - trials: list[AgentEvalTrial], - config: AgentEvalRunConfig, - run_id: str, - ) -> list[AgentEvalTaskScore]: - tasks_by_id = {task.id: task for task in tasks} - task_index_by_id = {task.id: index for index, task in enumerate(tasks)} - trials_by_task: dict[str, list[AgentEvalTrial]] = defaultdict(list) - for trial in trials: - if trial.task_id not in tasks_by_id: - raise ValueError(f"trial {trial.id!r} references unknown task {trial.task_id!r}") - trials_by_task[trial.task_id].append(trial) - - # Fail loudly when a task produced no trial. Imported trials or an AgentTaskRunner may omit a - # task entirely; without this an incomplete run would look successful aside from lower summary - # counts. (A richer alternative is to emit a "missing trial" failed score per metric.) - tasks_without_trials = [task.id for task in tasks if not trials_by_task.get(task.id)] - if tasks_without_trials: - raise ValueError(f"no trials produced for tasks: {sorted(tasks_without_trials)}") - - for task in tasks: - if not task.metrics: - raise ValueError(f"task {task.id!r} does not declare any metrics") - - semaphore = asyncio.Semaphore(config.parallelism) - - async def guarded_score(task: AgentEvalTask, trial: AgentEvalTrial, metric: Metric) -> AgentEvalTaskScore: - async with semaphore: - row_index = task_index_by_id[task.id] - if trial.status == AgentEvalTrialStatus.FAILED: - return _failed_metric_score( - run_id=run_id, - task=task, - trial=trial, - metric=metric, - row_index=row_index, - diagnostic=AgentEvalDiagnostic( - severity=AgentEvalDiagnosticSeverity.ERROR, - message=f"trial {trial.id!r} is failed", - source=metric_type_name(metric), - # The key pass@k reads to tell "the agent produced nothing" (a failed - # attempt) from "the metric raised" (an unusable measurement). - details={TRIAL_STATUS_DETAIL: trial.status.value}, - ), - ) - try: - return await _score_metric( - run_id=run_id, - task=task, - trial=trial, - metric=metric, - row_index=row_index, - ) - except Exception as exc: - if config.fail_fast: - raise - log.warning( - "metric %s failed for trial %r (task %r): %s", - metric_type_name(metric), - trial.id, - task.id, - exc, - ) - return _failed_metric_score( - run_id=run_id, - task=task, - trial=trial, - metric=metric, - row_index=row_index, - diagnostic=_exception_diagnostic(exc, metric_type_name(metric)), - ) - - return await asyncio.gather( - *[ - guarded_score(task, trial, metric) - for task in tasks - for trial in trials_by_task.get(task.id, []) - for metric in task.metrics - ] - ) - - async def _generate_trials( - self, - *, - tasks: list[AgentEvalTask], - target: AgentEvalTarget, - config: AgentEvalRunConfig, - ) -> list[AgentEvalTrial]: - if isinstance(target, AgentTaskRunner): - return list(await target.run_tasks(tasks, config=config)) - if not isinstance(target, (Model, AgentBase)): - raise NotImplementedError(f"unsupported agent-eval target type: {type(target).__name__}") - - params = _resolve_live_params(config, target) - prompt_template = config.prompt_template or _default_prompt_template(target) - semaphore = asyncio.Semaphore(params.parallelism) - - # Use the injected transport client when provided; otherwise build a default for the - # resolved target type and close it when generation finishes. - client = self.client - close_client: Callable[[], Awaitable[Any]] | None = None - if client is None and self.inference_fn is None: - if isinstance(target, Model): - client = inference.new_inference_client(target) - close_client = client.close - else: - client = new_agent_inference_client() - close_client = client.aclose - - try: - # When config.params.ignore_request_failure is set, convert a failed generation request - # into a FAILED trial (which the scorer turns into failed metric scores) instead of - # aborting the whole run. This matches the existing online-evaluator contract. - async def generate_one(index: int, task: AgentEvalTask) -> AgentEvalTrial: - async with semaphore: - # Keep evaluator-owned runtime identity separate from task inputs. - # ``_generate_sample`` exposes these values to request templates under - # ``agent_eval``. For agent targets, the same values are supplied to the - # inference factory so stream translators and evidence can carry stable - # evaluation identifiers without coupling them to this evaluator. - agent_eval_context = { - "run_id": config.run_id, - "task_id": task.id, - "invocation_id": f"{config.run_id}:{task.id}:{target.name}", - } - evidence_dir = ( - _task_evidence_dir(Path(config.work_dir), index=index, task_id=task.id) - if config.work_dir is not None and isinstance(target, AgentBase) - else None - ) - resolved_inference_fn = self.inference_fn - if isinstance(target, AgentBase) and resolved_inference_fn is None: - factory = self.agent_inference_fn_factory or make_agent_inference_fn - resolved_inference_fn = factory( - AgentInferenceContext( - evidence_dir=evidence_dir, - metadata=agent_eval_context, - ) - ) - try: - sample = await _generate_sample( - target=target, - row=_task_row(task), - index=index, - prompt_template=prompt_template, - params=params, - inference_fn=resolved_inference_fn, - client=client, - default_headers=self.default_headers, - agent_eval_context=agent_eval_context, - ) - except Exception as exc: - if params.ignore_request_failure: - return _failed_generation_trial(task, target, exc) - raise - return _trial_from_sample(task, target, sample) - - return await asyncio.gather(*(generate_one(index, task) for index, task in enumerate(tasks))) - finally: - if close_client is not None: - await close_client() - - -async def _generate_sample( - *, - target: Model | Agent, - row: dict[str, Any], - index: int, - prompt_template: str | dict[str, Any], - params: RunConfigOnline | RunConfigOnlineModel, - inference_fn: InferenceFn | AgentInferenceFn | None, - client: AsyncOpenAI | httpx.AsyncClient | None, - default_headers: dict[str, str] | None, - agent_eval_context: dict[str, Any], -) -> dict[str, Any]: - # InferenceFn and AgentInferenceFn are callable protocols, so isinstance cannot discriminate - # the injected fn; narrow it per target type with a cast (matching execution/benchmark_execution). - # The transport client is a real class union, so isinstance narrowing is enough there. - if isinstance(target, Model): - model_params = cast(RunConfigOnlineModel, params) - preprocess_hooks, postprocess_hooks = inference.new_hooks(model_params, model_format=target.format) - model_inference_fn = ( - cast(InferenceFn, inference_fn) if inference_fn is not None else inference.make_inference_request - ) - return await generate_online_sample( - target=target, - row=row, - index=index, - prompt_template=prompt_template, - params=model_params, - inference_fn=model_inference_fn, - client=client if isinstance(client, AsyncOpenAI) else None, - preprocess_hooks=preprocess_hooks, - postprocess_hooks=postprocess_hooks, - default_headers=default_headers, - template_context={"agent_eval": agent_eval_context}, - ) - - if inference_fn is None: - raise TypeError("expected AgentInferenceFn for Agent target") - agent_inference_fn = cast(AgentInferenceFn, inference_fn) - return await generate_online_sample( - target=target, - row=row, - index=index, - prompt_template=prompt_template, - params=params, - inference_fn=agent_inference_fn, - client=client if isinstance(client, httpx.AsyncClient) else None, - default_headers=default_headers, - template_context={"agent_eval": agent_eval_context}, - ) - - -def _trial_from_sample(task: AgentEvalTask, target: Model | Agent, sample: dict[str, Any]) -> AgentEvalTrial: - output_text = sample.get("output_text") - if not (isinstance(output_text, str) and output_text.strip()): - # Reasoning models that exhaust the token budget can return only - # `reasoning_content` with empty `content`. Fall back to that text so the - # trial stays scorable instead of being dropped as empty output. - output_text = _reasoning_content_fallback(sample.get("response")) - evidence = sample.get("evidence") - if evidence is not None and not isinstance(evidence, CandidateEvidence): - evidence = CandidateEvidence.model_validate(evidence) - - # Evidence precedence: - # - trajectory exists: merge it without replacing a typed trace. - # - no trajectory, but typed evidence exists: preserve that evidence unchanged. - # - neither exists: synthesize the fallback trace. - if "trajectory" in sample: - trace = EvidenceDescriptor(kind=EVIDENCE_TRACE, format=EVIDENCE_FORMAT_JSON, data=sample["trajectory"]) - descriptors = dict(evidence.descriptors) if evidence is not None else {} - descriptors.setdefault(EVIDENCE_TRACE, trace) - evidence = CandidateEvidence( - descriptors=descriptors, - metadata=dict(evidence.metadata) if evidence is not None else {}, - ) - elif evidence is None: - evidence = CandidateEvidence( - descriptors={ - EVIDENCE_TRACE: EvidenceDescriptor( - kind=EVIDENCE_TRACE, - format=EVIDENCE_FORMAT_JSON, - data={"task_id": task.id, "target": target.name}, - ) - } - ) - - status_value = sample.get("invocation_status", AgentEvalTrialStatus.COMPLETED.value) - status = AgentEvalTrialStatus(status_value) - invocation_metadata = sample.get("invocation_metadata") - if not isinstance(invocation_metadata, dict): - invocation_metadata = {} - - return AgentEvalTrial( - id=f"{task.id}:{target.name}", - task_id=task.id, - status=status, - output=AgentOutput( - output_text=output_text if isinstance(output_text, str) else None, - response=sample.get("response"), - metadata={ - **invocation_metadata, - **{ - key: value for key, value in sample.items() if key not in _SAMPLE_KEYS_EXCLUDED_FROM_OUTPUT_METADATA - }, - }, - ), - evidence=evidence, - metadata={ - **invocation_metadata, - "model_id": target.name, - "target_name": target.name, - "generated": True, - }, - ) - - -def _reasoning_content_fallback(response: Any) -> str | None: - if not isinstance(response, dict): - return None - choices = response.get("choices") - if not isinstance(choices, list): - return None - for choice in choices: - message = choice.get("message") if isinstance(choice, dict) else None - if not isinstance(message, dict): - continue - reasoning = message.get("reasoning_content") - if isinstance(reasoning, str) and reasoning.strip(): - return reasoning - return None - - -def _failed_generation_trial(task: AgentEvalTask, target: Model | Agent, exc: Exception) -> AgentEvalTrial: - return AgentEvalTrial( - id=f"{task.id}:{target.name}", - task_id=task.id, - status=AgentEvalTrialStatus.FAILED, - output=None, - evidence=CandidateEvidence( - descriptors={ - "error": EvidenceDescriptor( - kind="error", - data={"error_type": exc.__class__.__name__, "error": str(exc)}, - ) - } - ), - metadata={ - "model_id": target.name, - "target_name": target.name, - "generated": True, - "error_type": exc.__class__.__name__, - "error": str(exc), - }, - ) - - -async def _score_metric( - *, - run_id: str, - task: AgentEvalTask, - trial: AgentEvalTrial, - metric: Metric, - row_index: int, -) -> AgentEvalTaskScore: - output_spec = metric.output_spec() - metric_result = validate_metric_result( - await metric.compute_scores(build_metric_input(_metric_row(task, trial), _trial_sample(trial), row_index)), - output_spec, - ) - metric_type = metric_type_name(metric) - return AgentEvalTaskScore( - id=_score_id(run_id, task.id, trial.id, metric_type), - run_id=run_id, - task_id=task.id, - trial_id=trial.id, - metric_type=metric_type, - status=AgentEvalScoreStatus.COMPLETED, - outputs=metric_result.outputs, - # Persist the metric's own diagnostics (e.g. per-criterion judge verdicts) — the failure path - # already records diagnostics; the success path dropped them. - diagnostics=[ - AgentEvalDiagnostic( - severity=AgentEvalDiagnosticSeverity.INFO, - message=diagnostic.message, - source=metric_type, - details=diagnostic.details or {}, - ) - for diagnostic in metric_result.diagnostics - ], - metadata={ - "row_index": row_index, - "trial_metadata": trial.metadata, - }, - ) - - -def _failed_metric_score( - *, - run_id: str, - task: AgentEvalTask, - trial: AgentEvalTrial, - metric: Metric, - row_index: int, - diagnostic: AgentEvalDiagnostic, -) -> AgentEvalTaskScore: - metric_type = metric_type_name(metric) - return AgentEvalTaskScore( - id=_score_id(run_id, task.id, trial.id, metric_type), - run_id=run_id, - task_id=task.id, - trial_id=trial.id, - metric_type=metric_type, - status=AgentEvalScoreStatus.FAILED, - outputs=[], - diagnostics=[diagnostic], - metadata={ - "row_index": row_index, - "trial_metadata": trial.metadata, - }, - ) - - -def _exception_diagnostic(exc: Exception, metric_type: str) -> AgentEvalDiagnostic: - return AgentEvalDiagnostic( - severity=AgentEvalDiagnosticSeverity.ERROR, - message=str(exc) or exc.__class__.__name__, - source=metric_type, - details={"exception_type": exc.__class__.__name__}, - ) - - -def _score_id(run_id: str, task_id: str, trial_id: str, metric_type: str) -> str: - return f"{run_id}:{task_id}:{trial_id}:{metric_type}" - - -def _trial_sample(trial: AgentEvalTrial) -> dict[str, Any]: - if trial.output is None: - return {} - sample: dict[str, Any] = { - **trial.metadata, - **trial.output.metadata, - } - if trial.output.output_text is not None: - sample["output_text"] = trial.output.output_text - if trial.output.response is not None: - sample["response"] = trial.output.response - if trial.evidence is not None: - sample["evidence"] = trial.evidence - return sample - - -def _resolve_live_params( - config: AgentEvalRunConfig, - target: Model | Agent, -) -> RunConfigOnline | RunConfigOnlineModel: - params = config.params - if isinstance(target, Model): - if params is None: - return RunConfigOnlineModel(parallelism=config.parallelism) - if isinstance(params, RunConfigOnlineModel): - return params - if isinstance(params, RunConfigOnline): - return RunConfigOnlineModel(**params.model_dump(mode="python")) - if isinstance(params, RunConfig): - return RunConfigOnlineModel(**params.model_dump(mode="python")) - - if params is None: - return RunConfigOnline(parallelism=config.parallelism) - if isinstance(params, RunConfigOnlineModel): - return RunConfigOnline( - **params.model_dump( - mode="python", - exclude={"inference", "system_prompt", "reasoning", "structured_output"}, - ) - ) - if isinstance(params, RunConfigOnline): - return params - return RunConfigOnline(**params.model_dump(mode="python")) - - -def _default_prompt_template(target: Model | Agent) -> dict[str, Any] | str: - # Every default renders against the single canonical task input, ``instruction`` (see - # ``AgentEvalTask.agent_prompt``); no other input key is special. - if isinstance(target, GenericAgent): - # A generic HTTP agent defines its own request entirely through its `body` template, which - # renders against the task inputs (e.g. `{{ instruction }}`). Pass the task row through - # unchanged so `body` — not a chat/completions assumption — shapes the payload. See - # `_resolve_http_agent_invocation`, which renders `body` against this request. - return "{{ item }}" - if isinstance(target, Model) and _is_completions_endpoint(target.url): - return {"prompt": "{{item.instruction}}"} - return {"messages": [{"role": "user", "content": "{{item.instruction}}"}]} - - -def _task_row(task: AgentEvalTask) -> dict[str, Any]: - # The task inputs verbatim, plus the task id. `instruction` is the single canonical input the - # target is prompted with (see `AgentEvalTask.agent_prompt` and `_default_prompt_template`); no - # input key is synthesized or aliased here. - return {**task.inputs, "task_id": task.id} - - -def _metric_row(task: AgentEvalTask, trial: AgentEvalTrial) -> dict[str, Any]: - return { - "task": { - "id": task.id, - "intent": task.intent, - "metadata": task.metadata, - }, - "inputs": task.inputs, - # Grader-only ground truth: available to metrics here but never seeded into the agent's - # workspace (see AgentEvalTask.reference), so a metric can grade against held-out artifacts. - "reference": task.reference, - "trial": { - "id": trial.id, - "task_id": trial.task_id, - "status": trial.status.value, - "metadata": trial.metadata, - }, - } - - -def _is_completions_endpoint(url: str) -> bool: - path = urlparse(url).path.rstrip("/") - return path.endswith("/completions") and not path.endswith("/chat/completions") - - -def _sdk_version() -> str | None: - try: - return package_version("nemo-evaluator-sdk") - except PackageNotFoundError: # pragma: no cover - only when running from an uninstalled tree - return None - - -def _describe_target( - target: AgentEvalTarget | None, - params: RunConfig | RunConfigOnline | RunConfigOnlineModel | None = None, -) -> RunnerInfo: - """Identify what produced the trials, for the run's provenance. - - Runners identify themselves via the required :meth:`AgentTaskRunner.runner_info`; trials supplied - directly have no runner. - - Models and agents are described by name *and* the settings they were invoked with — the endpoint - ``url``, plus the whole ``params`` object (temperature, max_tokens, reasoning effort, system prompt, - retries, ...). A name alone is not an identity: the same model name served from two different URLs, - or at two different temperatures, would otherwise record identical provenance. ``params`` is dumped - whole rather than cherry-picked, because a filtered subset is what bites you later when the omitted - field turns out to be the one that mattered. It carries no credentials — ``Model.api_key_secret`` is - a reference on the model, and ``default_headers`` is excluded from serialization. - """ - if target is None: - return RunnerInfo(name="imported", kind="imported") - if isinstance(target, (Model, AgentBase)): - config: dict[str, Any] = {"url": getattr(target, "url", None)} - if params is not None: - config["params"] = params.model_dump(mode="json", exclude_none=True) - return RunnerInfo(name=target.name, kind="model" if isinstance(target, Model) else "agent", config=config) - return target.runner_info() - - -def _collect_runner_aggregate_scores(target: object) -> list[AggregateScore]: - """The typed subset of a runner's own aggregations, for merging into ``summary.scores``. - - A runner that maps its numbers onto aggregate scores namespaces them under ``runner..``, so - they sit alongside the SDK's own without being mistaken for them. That namespace is *enforced*, not - merely documented: ``RunAggregationsProvider`` is a public extension point, ``summary.scores`` is a - flat list, and a third-party runner returning ``gym_reward.reward`` would not overwrite the SDK's - own aggregate but sit next to it under the same name, leaving any lookup to pick one arbitrarily. - - Offending entries are dropped with a warning rather than raised on. This runs *after* ``run_tasks``, - so raising would sink a completed run — potentially hours of collection — over a naming bug, while - the numbers themselves remain in the runner's own files inside the bundle. - """ - if not isinstance(target, RunAggregationsProvider): - return [] - runner_info = getattr(target, "runner_info", None) # structurally optional: the protocol is a companion - runner_name = runner_info().name if callable(runner_info) else None - prefix = f"runner.{runner_name}." if runner_name else "runner." - collected: list[AggregateScore] = [] - for score in target.run_aggregate_scores(): - if not score.name.startswith(prefix): - log.warning( - "Dropping runner-contributed aggregate %r: RunAggregationsProvider names must be " - "namespaced %r so an imported figure is never mistaken for one the SDK computed.", - score.name, - prefix, - ) - continue - collected.append(score) - return collected - - -def _new_run_id() -> str: - timestamp = datetime.now(UTC).strftime("%Y%m%d%H%M%S") - return f"agent-eval-{timestamp}-{uuid.uuid4().hex[:8]}" - - -def _task_evidence_dir(output_dir: Path, *, index: int, task_id: str) -> Path: - safe_task_id = _safe_path_component(task_id) - task_dir = f"{index:06d}-{safe_task_id}" if safe_task_id else f"task-{index:06d}" - return output_dir / "evidence" / task_dir - - -def _safe_path_component(value: str) -> str: - sanitized = "".join(char if char.isalnum() or char in "-_." else "-" for char in value) - return sanitized.strip("-_.")[:120] diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/metrics.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/metrics.py deleted file mode 100644 index 51896c22cc..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/metrics.py +++ /dev/null @@ -1,279 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Reusable agent-eval metrics and the typed view over trial measurements. - -Two complementary pieces, both keyed off ``AgentEvalTrial``: - -* Metrics (scorers) — ``AgentPhaseSuccessMetric`` reads the agent-phase outcome - stamped on trial metadata; ``EvidencePresenceMetric`` is a genuine - *metric-over-evidence* that scores by inspecting ``candidate.evidence`` (a - filesystem evidence handle) rather than trusting a verifier's stamped reward. -* ``TrialMeasurements`` — the single documented place that names the loose - metadata keys gating/reporting read, applying the fallbacks (``duration_ms`` → - ``runtime_sec``, ``passed`` → ``reward``). -""" - -from __future__ import annotations - -import json -import logging -from collections.abc import Mapping -from typing import Any - -from nemo_platform.beta.evaluator.agent_eval.trials import EVIDENCE_FINAL_STATE -from nemo_platform.beta.evaluator.metrics.protocol import ( - CandidateOutput, - MetricInput, - MetricOutput, - MetricOutputSpec, - MetricResult, -) -from nemo_platform.beta.evaluator.values.atif import Trajectory -from nemo_platform.beta.evaluator.values.evidence import EVIDENCE_TRACE -from pydantic import BaseModel, ConfigDict, ValidationError - -logger = logging.getLogger(__name__) - -# Token-measurement keys carried on trial metadata (and in result.json["metrics"]). -TOKEN_KEYS: tuple[str, ...] = ( - "prompt_tokens", - "completion_tokens", - "total_tokens", - "cache_creation_tokens", - "cache_read_tokens", -) - - -class AgentPhaseSuccessMetric: - """Emit ``True`` when the agent phase exited successfully, else ``False``. - - The metric ``type`` is overridable via the ``metric_type`` class attribute so - callers can namespace it; the output name stays ``agent_phase_success`` (which - gating reads as a reward signal — ``True``/``False`` coerces to ``1.0``/``0.0``). - """ - - metric_type: str = "agent_phase_success" - - @property - def type(self) -> str: - return self.metric_type - - def output_spec(self) -> list[MetricOutputSpec]: - return [MetricOutputSpec.boolean("agent_phase_success")] - - async def compute_scores(self, input: MetricInput) -> MetricResult: - # Only an explicit boolean counts as success; a stray truthy string - # (e.g. "false") must not mark a failed trial as passed. - raw_agent_ok = input.candidate.metadata.get("agent_ok") - agent_ok = raw_agent_ok if isinstance(raw_agent_ok, bool) else False - return MetricResult(outputs=[MetricOutput(name="agent_phase_success", value=agent_ok)]) - - -class EvidencePresenceMetric: - """Emit ``True`` when a named filesystem evidence directory exists (and is non-empty). - - Reads ``candidate.evidence`` directly — the canonical metric-over-evidence - pattern — so the result reflects what the agent actually produced on disk, - not a reward stamped into metadata by a verifier. - """ - - def __init__( - self, - *, - evidence_name: str = EVIDENCE_FINAL_STATE, - output_name: str = "evidence_present", - require_non_empty: bool = True, - ) -> None: - self._evidence_name = evidence_name - self._output_name = output_name - self._require_non_empty = require_non_empty - - @property - def type(self) -> str: - return "evidence_presence" - - def output_spec(self) -> list[MetricOutputSpec]: - return [MetricOutputSpec.boolean(self._output_name)] - - async def compute_scores(self, input: MetricInput) -> MetricResult: - present = False - evidence = input.candidate.evidence - if evidence is not None and evidence.get(self._evidence_name) is not None: - try: - handle = await evidence.filesystem(self._evidence_name) - if await handle.exists(): - present = bool(await handle.iter_paths(recursive=True)) if self._require_non_empty else True - except (KeyError, ValueError) as exc: - logger.warning( - "EvidencePresenceMetric scored False: could not resolve evidence %r for output %r: %s", - self._evidence_name, - self._output_name, - exc, - ) - return MetricResult(outputs=[MetricOutput(name=self._output_name, value=present)]) - - -class SkillUsedMetric: - """Emit ``skill_present`` and ``skill_used`` so an eval can flag a failure to use an injected skill. - - * ``skill_present`` — ``True`` when one or more skills were injected into the trial. Reads - the ``"skills"`` metadata key a skill-aware runtime stamps — a list of provenance dicts - (``{"name", "hash", "mode", "adapter_id", "location", ...}``, see ``fabric.skills.SkillProvenance``). - Baseline trials carry an empty list. - * ``skill_used`` — best-effort ``True`` when the agent referenced *any* injected skill in its ATIF - trajectory. It matches each skill's staged ``location`` (a specific, low-false-positive path - signal — e.g. a read of ``.agents/skills//SKILL.md``) against tool-call names/arguments, - step messages, reasoning, and observations. A bare skill-*name* match is intentionally NOT - counted (the name commonly appears in the task prompt), so ``skill_present=True, skill_used=False`` - flags a *likely* failure to use the skill. - - Limitation: an absent trajectory reference cannot fully distinguish "not used" from "used without - leaving a filesystem trace" — strongest for codex-style filesystem discovery, weaker for in-context - skill loading. Authoritative usage detection via harness skill-activation events is a follow-up. - With no skill present, both outputs are ``False``. - """ - - metric_type: str = "skill_used" - OUTPUT_PRESENT: str = "skill_present" - OUTPUT_USED: str = "skill_used" - # Metadata key skill-aware runtimes stamp the provenance list under (matches the fabric runtime). - _SKILLS_KEY: str = "skills" - - def __init__(self, *, trace_evidence: str = EVIDENCE_TRACE) -> None: - self._trace_evidence = trace_evidence - - @property - def type(self) -> str: - return self.metric_type - - def output_spec(self) -> list[MetricOutputSpec]: - return [ - MetricOutputSpec.boolean(self.OUTPUT_PRESENT), - MetricOutputSpec.boolean(self.OUTPUT_USED), - ] - - async def compute_scores(self, input: MetricInput) -> MetricResult: - provenances = self._extract_provenances(input.candidate.metadata) - present = bool(provenances) - used = await self._any_skill_used(input.candidate, provenances) if present else False - return MetricResult( - outputs=[ - MetricOutput(name=self.OUTPUT_PRESENT, value=present), - MetricOutput(name=self.OUTPUT_USED, value=used), - ] - ) - - def _extract_provenances(self, metadata: Mapping[str, Any]) -> list[Mapping[str, Any]]: - skills = metadata.get(self._SKILLS_KEY) - if isinstance(skills, list): - return [p for p in skills if isinstance(p, Mapping) and p] - return [] - - async def _any_skill_used(self, candidate: CandidateOutput, provenances: list[Mapping[str, Any]]) -> bool: - locations = [loc for p in provenances if isinstance(loc := p.get("location"), str) and loc] - if not locations: - return False - evidence = candidate.evidence - if evidence is None or evidence.get(self._trace_evidence) is None: - return False - try: - trajectory = await (await evidence.trace(self._trace_evidence)).trace() - except (KeyError, ValueError, ValidationError, OSError) as exc: - # Best-effort: a missing/malformed/invalid trajectory must score skill_used=False, not raise. - # ValidationError covers Trajectory.model_validate; OSError covers the underlying file read. - logger.warning( - "SkillUsedMetric scored skill_used=False: could not read trace %r: %s", self._trace_evidence, exc - ) - return False - return any(_trajectory_references(trajectory, loc) for loc in locations) - - -class TrialMeasurements(BaseModel): - """Numeric measurements projected from trial metadata. - - Reporting/gating consume it via :meth:`from_metadata`; producers keep writing - the same keys onto ``AgentEvalTrial.metadata``. - """ - - model_config = ConfigDict(extra="forbid") - - prompt_tokens: int | None = None - completion_tokens: int | None = None - total_tokens: int | None = None - cache_creation_tokens: int | None = None - cache_read_tokens: int | None = None - runtime_sec: float | None = None - reward: float | None = None - passed: bool | None = None - - @classmethod - def from_metadata(cls, metadata: Mapping[str, Any] | None) -> TrialMeasurements: - """Project loose trial metadata onto the typed contract. - - Applies the historical fallbacks so callers don't re-implement them: - ``runtime_sec`` falls back to ``duration_ms / 1000``; ``reward`` falls - back to ``1.0``/``0.0`` derived from ``passed`` when no explicit reward - is recorded. - """ - metadata = metadata or {} - - tokens = {key: _as_int(metadata.get(key)) for key in TOKEN_KEYS} - passed = metadata.get("passed") - passed = bool(passed) if isinstance(passed, bool) else None - - return cls( - **tokens, - runtime_sec=_runtime_sec(metadata), - reward=_reward(metadata, passed), - passed=passed, - ) - - -def _trajectory_references(trajectory: Trajectory, needle: str) -> bool: - """Whether ``needle`` appears anywhere an agent action could reference the skill. - - Scans each step's message, reasoning, tool calls (name + arguments), and observation results. - """ - for step in trajectory.steps: - if needle in step.message or (step.reasoning_content is not None and needle in step.reasoning_content): - return True - for call in step.tool_calls or []: - if needle in call.function_name: - return True - if call.arguments is not None and needle in json.dumps(call.arguments, default=str): - return True - if step.observation is not None: - for result in step.observation.results: - if result.content is not None and needle in json.dumps(result.content, default=str): - return True - return False - - -def _as_int(value: Any) -> int | None: - # bool is an int subclass; never treat True/False as a token count. - if isinstance(value, bool): - return None - return value if isinstance(value, int) else None - - -def _runtime_sec(metadata: Mapping[str, Any]) -> float | None: - runtime_sec = metadata.get("runtime_sec") - if isinstance(runtime_sec, int | float) and not isinstance(runtime_sec, bool): - return float(runtime_sec) - duration_ms = metadata.get("duration_ms") - if isinstance(duration_ms, int | float) and not isinstance(duration_ms, bool): - return float(duration_ms) / 1000.0 - return None - - -def _reward(metadata: Mapping[str, Any], passed: bool | None) -> float | None: - reward = metadata.get("reward") - if reward is not None: - try: - return float(reward) - except (TypeError, ValueError): - return None - if passed is not None: - return 1.0 if passed else 0.0 - return None diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/persistence.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/persistence.py deleted file mode 100644 index 0f8d158c97..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/persistence.py +++ /dev/null @@ -1,172 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Persistence helpers for standalone agent-eval result bundles.""" - -from __future__ import annotations - -import json -from collections.abc import Iterator, Sequence -from pathlib import Path -from typing import Any - -from nemo_platform.beta.evaluator.agent_eval.dashboard import write_dashboard -from nemo_platform.beta.evaluator.agent_eval.results import AgentEvalResult, BundleLocation -from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial -from pydantic import BaseModel - -#: Filename of the rendered HTML dashboard inside a bundle. -DASHBOARD_FILENAME = "report.html" - - -def persist_run( - result: AgentEvalResult, - output_dir: str | Path, - *, - write_html_dashboard: bool = True, -) -> BundleLocation: - """Write a completed run to a bundle at ``output_dir`` and report where it landed. - - Explicit rather than a side effect of :meth:`AgentEvaluator.run`: computing an evaluation and - storing one are different decisions, and folding them together is what forced the result object to - carry paths it could not know at construction time. (Same reasoning as ``publish_to_intake``.) - - Set ``write_html_dashboard=False`` to skip rendering ``report.html`` — the dashboard is written - here so the manifest can record it in a single pass. - """ - path = Path(output_dir) - path.mkdir(parents=True, exist_ok=True) - - # Render first so the manifest below can name it; the dashboard reads only the run's own contents. - dashboard_path = write_dashboard(result, path / DASHBOARD_FILENAME) if write_html_dashboard else None - - _write_json(path / "metadata.json", result.metadata) - _write_jsonl(path / "tasks.jsonl", result.tasks) - _write_trials(path / "trials.jsonl", result.trials, base=path) - _write_jsonl(path / "scores.jsonl", result.scores) - _write_json(path / "summary.json", result.summary) - - location = BundleLocation(output_dir=path, dashboard_path=dashboard_path) - _write_json(path / "run.json", _run_manifest(result, location)) - return location - - -def _run_manifest(result: AgentEvalResult, location: BundleLocation) -> dict[str, Any]: - return { - "run_id": result.run_id, - "output_dir": str(location.output_dir), - "dashboard_path": str(location.dashboard_path) if location.dashboard_path is not None else None, - "artifacts": { - "metadata": "metadata.json", - "tasks": "tasks.jsonl", - "trials": "trials.jsonl", - "scores": "scores.jsonl", - "summary": "summary.json", - }, - } - - -def _write_json(path: Path, value: BaseModel | dict[str, Any]) -> None: - if isinstance(value, BaseModel): - payload = value.model_dump(mode="json") - else: - payload = value - path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") - - -def _write_jsonl(path: Path, rows: Sequence[BaseModel]) -> None: - # Stream row-by-row instead of joining the whole payload in memory first. - with path.open("w", encoding="utf-8") as handle: - for row in rows: - handle.write(json.dumps(row.model_dump(mode="json"), sort_keys=True)) - handle.write("\n") - - -def _write_trials(path: Path, trials: Sequence[BaseModel], *, base: Path) -> None: - """Write trials, rewriting evidence refs bundle-relative so the bundle is self-contained. - - A trial's evidence lives under the bundle (``/evidence/...``); storing the ref relative to - the bundle (rather than the launch CWD) means a moved or copied bundle re-scores without any path fixups. - Refs that point outside the bundle (rare) are left verbatim. - """ - resolved_base = base.resolve() - with path.open("w", encoding="utf-8") as handle: - row: dict[str, Any] - for trial in trials: - row = trial.model_dump(mode="json") - for descriptor in ((row.get("evidence") or {}).get("descriptors") or {}).values(): - descriptor["ref"] = _relativize_ref(descriptor.get("ref"), resolved_base) - handle.write(json.dumps(row, sort_keys=True)) - handle.write("\n") - - -def _relativize_ref(ref: str | None, base: Path) -> str | None: - """Make an evidence ref relative to the bundle dir when it lives under it; else leave it verbatim.""" - if not ref: - return ref - try: - return Path(ref).resolve().relative_to(base).as_posix() - except ValueError: - return ref # evidence written outside the bundle — cannot relativize - - -def read_trials(run_dir: str | Path) -> list[AgentEvalTrial]: - """Hydrate the persisted trials of a run bundle — the inverse of the ``trials.jsonl`` ``persist_run`` writes. - - Each row is loaded back into an ``AgentEvalTrial`` with its evidence pointing at the on-disk - deliverables, so a stored run can be **re-scored** — ``AgentEvaluator().run(tasks=…, trials=…)`` with - fresh metrics/judge — without re-running the agent. Evidence refs are resolved relative to ``run_dir`` - when the stored (launch-relative) ref no longer resolves, so a moved or copied bundle still works. - """ - directory = Path(run_dir) - trials: list[AgentEvalTrial] = [] - for row in _read_jsonl(directory / "trials.jsonl"): - for descriptor in ((row.get("evidence") or {}).get("descriptors") or {}).values(): - descriptor["ref"] = _resolve_evidence_ref(directory, descriptor.get("ref")) - trials.append(AgentEvalTrial.model_validate(row)) - return trials - - -def _resolve_evidence_ref(run_dir: Path, ref: str | None) -> str | None: - """Resolve a persisted evidence ref against the bundle dir. - - Self-contained bundles store refs relative to the bundle (see ``_write_trials``), so those resolve - directly under ``run_dir``. Falls back for still-valid absolute refs, and for moved bundles / legacy - absolute refs (rebuilt under ``run_dir`` from the ``evidence/`` tail). - """ - if not ref: - return ref - base = run_dir.resolve() - candidate = Path(ref) - if not candidate.is_absolute(): - rebuilt = run_dir / candidate - if _resolves_within(base, rebuilt): - return str(rebuilt) - elif candidate.exists(): - return ref - parts = candidate.parts - if "evidence" in parts: - rebuilt = run_dir / Path(*parts[parts.index("evidence") :]) - if _resolves_within(base, rebuilt): - return str(rebuilt) - return ref - - -def _resolves_within(base: Path, path: Path) -> bool: - """Whether ``path`` exists and stays inside ``base`` after resolving — no ``..``/symlink escape. - - Rebuilt refs are joined onto ``run_dir``; a bundle is designed to be moved/copied, so a ref with ``..`` - (or a symlink) must not be allowed to point the hydrated evidence outside the bundle it was loaded from. - """ - resolved = path.resolve() - if not resolved.exists(): - return False - return resolved == base or base in resolved.parents - - -def _read_jsonl(path: Path) -> Iterator[dict[str, Any]]: - with path.open(encoding="utf-8") as handle: - for line in handle: - stripped = line.strip() - if stripped: - yield json.loads(stripped) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py deleted file mode 100644 index 568ff27bee..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py +++ /dev/null @@ -1,1331 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Aggregated summary, coverage, and the root result for a completed agent evaluation.""" - -from __future__ import annotations - -import json -import math -from collections.abc import Mapping, Sequence -from datetime import datetime -from enum import Enum -from pathlib import Path -from typing import Any - -from nemo_platform.beta.evaluator.agent_eval.scores import ( - AgentEvalDiagnosticSeverity, - AgentEvalScoreStatus, - AgentEvalTaskScore, - is_trial_failure, -) -from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalTask, SemanticReducer, ViewSignal -from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, RunnerInfo -from nemo_platform.beta.evaluator.metrics.aggregation import compute_percentiles -from nemo_platform.beta.evaluator.metrics.protocol import MetricOutput -from nemo_platform.beta.evaluator.metrics.utils import metric_type_name -from nemo_platform.beta.evaluator.values.protocol import BooleanValue, ContinuousScore, DiscreteScore, Label -from nemo_platform.beta.evaluator.values.results import ( - AggregatedMetricResult, - AggregateRangeScore, - AggregateScore, - ResultView, - flatten_dict, - format_table, - serialize_value, - summary_aggregate_record, -) -from pydantic import BaseModel, ConfigDict, Field, field_serializer, model_validator - -#: Metric-output value schemas retained in the ordered per-task value mapping. Broader than -#: :data:`_PASS_AT_K_VALUE_SCHEMAS` on purpose: a :class:`TrialMetricValue` is per-trial evidence, so a -#: count or a judge's label is worth keeping even though neither is a "did it pass?" signal. -_TASK_METRIC_VALUE_SCHEMAS = (ContinuousScore, DiscreteScore, BooleanValue, Label) - -#: Metric-output value schemas eligible for pass@k (a per-trial "did it pass?" signal). Labels, -#: discrete/count outputs, and free models (e.g. token measurements) are excluded. -_PASS_AT_K_VALUE_SCHEMAS = (ContinuousScore, BooleanValue) - -#: Score value at or above which a trial counts as a pass for pass@k. Full credit — pass@k answers -#: "did the agent solve the task", so partial credit is not a pass. Deliberately not configurable: -#: it's a reporting-time interpretation, and making it tunable would yield pass@k numbers that look -#: comparable across runs but aren't. -_PASS_VALUE = 1.0 - - -class AgentEvalMetricOutputCoverage(BaseModel): - """Coverage counts for one metric output across scored trials.""" - - model_config = ConfigDict(extra="forbid") - - total: int = Field(default=0, description="Total scores considered for this metric output.") - scored: int = Field(default=0, description="Scores that produced this output successfully.") - failed: int = Field(default=0, description="Scores where the metric failed to run.") - missing: int = Field(default=0, description="Scores where the output was expected but absent.") - - -#: Tokens :class:`TrialMetricValue` escapes non-finite floats as, and the floats they decode to. -#: Strict JSON has no literal for these, so they travel as strings -- which is the whole reason the -#: record carries ``value_type``: without it, a label that happens to read "NaN" is the same three -#: bytes as a real NaN. -_SPECIAL_FLOAT_TOKENS_MAP: dict[str, float] = { - "NaN": float("nan"), - "Infinity": float("inf"), - "-Infinity": float("-inf"), -} - - -def _escape_special_float(value: float) -> str: - """The token :data:`_SPECIAL_FLOAT_TOKENS_MAP` decodes back to ``value``. - - Looked up rather than spelled out a second time, so the encode and decode directions cannot - drift apart. NaN needs :func:`math.isnan` rather than equality: it is the one float that does - not equal itself, so a lookup keyed by value would miss it. - """ - for token, decoded in _SPECIAL_FLOAT_TOKENS_MAP.items(): - if decoded == value or (math.isnan(decoded) and math.isnan(value)): - return token - raise ValueError(f"{value!r} is a finite float and needs no escape") - - -class TrialMetricValueType(str, Enum): - """What kind of value one trial recorded under one metric output. - - Deliberately coarser than the declared value schemas: JSON already round-trips int, float and - bool distinctly, so a ``continuous``/``discrete``/``boolean`` split would restate what the payload - already says and give a reader two sources of truth for one fact. The only thing JSON cannot - carry is whether a string is a number's escape or a label, and that is exactly what this - discriminates. - """ - - NUMBER = "number" - LABEL = "label" - MISSING = "missing" - - -class TrialMetricValue(BaseModel): - """One trial's measured value under one metric output: which trial made it, and what it measured. - - Values keep the type the metric produced them in -- a count stays an int, a flag stays a bool, a - judge's verdict stays the string it was -- because this is one trial's measurement, not a mean or - other aggregate. Look up the matching trial by ``trial_id`` (in ``result.trials`` or - ``trials.jsonl``); do not assume list index lines up across metric outputs. Read it through - :func:`numeric_metric_values` when you intend to do arithmetic. - - Frozen because these records are handed out by reference from the summary: a consumer rescaling - values in place (Gym reports reward on 0-100 where we use 0-1) would otherwise rewrite the run's - own results, and a later persist would save the rewrite. - """ - - model_config = ConfigDict(extra="forbid", frozen=True) - - trial_id: str = Field( - description=( - "Identifier of the trial that produced this value. Joins to AgentEvalTrial.id " - "(trials.jsonl) and AgentEvalTaskScore.trial_id (scores.jsonl)." - ) - ) - # The default is never observed: `_derive_value_type` runs before validation and always supplies - # one. It exists so callers can write TrialMetricValue(trial_id=..., value=...) without - # restating what the value already says -- the type checker reads the signature, not the validator. - value_type: TrialMetricValueType = Field( - default=TrialMetricValueType.MISSING, - description=( - "Which kind of value this record holds: 'number' (float, int or bool), 'label' (a " - "categorical string), or 'missing' (the trial failed before it could be measured). " - "Always present in serialized output, because non-finite floats are escaped as strings " - "-- without it a genuine label reading 'NaN' and a real NaN are the same three bytes. " - "Derived from 'value' when omitted, so hand-built records and bundles written before " - "this field existed both load." - ), - ) - value: float | int | bool | str | None = Field( - description=( - "What the metric output measured, in the type the metric produced it in -- a number, a " - "label, or None when the trial failed before it could be measured: a trial that did " - "not pass. Required rather than defaulted: None is a load-bearing signal pass@k counts " - "as not passing, so an omitted value must not quietly become one. None never means " - "'no value of this kind'; that is what value_type is for." - ), - ) - - @model_validator(mode="before") - @classmethod - def _derive_value_type(cls, data: Any) -> Any: - """Fill in ``value_type`` when absent, and decode the escaped-float form when present. - - Runs *before* the union so the escape is undone while the discriminator is still readable: - afterwards pydantic's smart mode has already committed ``"NaN"`` to ``str``, and the record - would be a label whatever the type said. - """ - if not isinstance(data, Mapping): - return data - value = data.get("value") - declared = data.get("value_type") - - if declared is None: - # No discriminator: a hand-built record, or a bundle written before this field existed. - # The old encoding gave a string exactly one meaning -- the escape -- so honour that - # rather than reading a pre-widening NaN as the label "NaN". A label that genuinely - # reads "NaN" must therefore name its value_type explicitly. - if isinstance(value, str): - if value in _SPECIAL_FLOAT_TOKENS_MAP: - return { - **data, - "value_type": TrialMetricValueType.NUMBER, - "value": _SPECIAL_FLOAT_TOKENS_MAP[value], - } - return {**data, "value_type": TrialMetricValueType.LABEL} - return { - **data, - "value_type": TrialMetricValueType.MISSING if value is None else TrialMetricValueType.NUMBER, - } - - if TrialMetricValueType(declared) is TrialMetricValueType.NUMBER and isinstance(value, str): - decoded = _SPECIAL_FLOAT_TOKENS_MAP.get(value) - if decoded is None: - raise ValueError( - f"value_type='number' but value {value!r} is not one of the escaped-float tokens " - f"{sorted(_SPECIAL_FLOAT_TOKENS_MAP)}; a categorical value must declare value_type='label'" - ) - return {**data, "value": decoded} - return data - - @model_validator(mode="after") - def _value_matches_its_type(self) -> TrialMetricValue: - """Re-narrow the union, so a record cannot claim one kind and carry another.""" - if self.value_type is TrialMetricValueType.MISSING: - if self.value is not None: - raise ValueError("value_type='missing' requires value None (a trial that died before measurement)") - elif self.value_type is TrialMetricValueType.LABEL: - if not isinstance(self.value, str): - raise ValueError(f"value_type='label' requires a string value, got {type(self.value).__name__}") - elif not isinstance(self.value, bool | int | float): - raise ValueError(f"value_type='number' requires a numeric value, got {type(self.value).__name__}") - return self - - @field_serializer("value") - def serialize_nan(self, value: float | int | bool | str | None) -> float | int | bool | str | None: - """Escape non-finite floats as strings, so ``summary.json`` stays strict JSON. - - A metric may legitimately score a trial NaN, and this is the first summary field to carry - a raw metric value rather than a filtered aggregate. ``json.dumps`` would write a bare ``NaN`` - or ``Infinity`` token, which is valid Python but not valid JSON, so any strict reader of - ``summary.json`` would reject the whole bundle. ``value_type`` says which of these strings is - an escape and which is a label, so the round trip is lossless in both directions. - - This is deliberately *wider* than :meth:`MetricOutput.serialize_nan`, which escapes NaN only - and has no decoding validator -- an infinite value reaches ``scores.jsonl`` as ``null``. - Collapsing the SDK's several non-finite-float escapes into one pair belongs in ``values/``. - """ - if isinstance(value, float) and not math.isfinite(value): - return _escape_special_float(value) - return value - - -#: One task's recorded values, keyed ``"."``. Named because the nesting is -#: otherwise spelled out at every producer, consumer and local that touches it, and because the key -#: format is the part a reader cannot infer from ``dict[str, ...]``. -TrialValuesByMetric = dict[str, list[TrialMetricValue]] - - -class PerTaskOutcome(BaseModel): - """Every trial's value at one task under one metric output.""" - - model_config = ConfigDict(extra="forbid") - - metric_name: str = Field(description="'.', e.g. 'gym_reward.reward'.") - trials: list[TrialMetricValue] = Field( - description="Values in trial order. A value of None is a trial that died before scoring." - ) - - -class PerTaskOutcomes(BaseModel): - """One task's values across every metric output that measured it.""" - - model_config = ConfigDict(extra="forbid") - - task_id: str = Field(description="The task these outcomes belong to.") - outcomes: list[PerTaskOutcome] = Field(description="One entry per metric output, sorted by metric_name.") - - -class AgentEvalSummary(BaseModel): - """Aggregated scores, coverage, per-task metric values, and run counts for an agent-eval run.""" - - model_config = ConfigDict(extra="forbid") - - scores: AggregatedMetricResult = Field( - default_factory=lambda: AggregatedMetricResult(scores=[]), - description=( - "Aggregated statistics (mean/min/max/std_dev/nan_count) per metric output, named " - "'.', plus per-semantic-view rollups named 'view.'. " - "Failed or missing scores are surfaced as nan_count." - ), - examples=[ - # Emission order is real: metric outputs, then views, then pass@k. Note that pass@k - # counts *tasks* (4) where the metric output counts *trials* (10). - { - "scores": [ - { - "name": "harbor_reward.reward", - "score_type": "range", - "count": 10, - "nan_count": 2, - "mean": 0.6, - "min": 0.0, - "max": 1.0, - "std_dev": 0.4899, - }, - { - "name": "view.legal_quality", - "score_type": "range", - "count": 8, - "nan_count": 4, - "mean": 0.7375, - "min": 0.1, - "max": 1.0, - "std_dev": 0.3674, - }, - { - "name": "harbor_reward.reward.pass@1", - "score_type": "range", - "count": 4, - "nan_count": 1, - "mean": 0.5, - "min": 0.0, - "max": 1.0, - "std_dev": 0.3727, - }, - { - "name": "harbor_reward.reward.pass@2", - "score_type": "range", - "count": 4, - "nan_count": 1, - "mean": 0.6667, - "min": 0.0, - "max": 1.0, - "std_dev": 0.4082, - }, - ] - }, - # A separate run, because a runner's imported figures cannot co-occur with another - # runner's metrics. Scalars carry `value` and no distribution, and no `count` when the - # backend reports a figure without the sample size behind it. - { - "scores": [ - { - "name": "gym_reward.reward", - "score_type": "range", - "count": 20, - "nan_count": 0, - "mean": 0.65, - "min": 0.0, - "max": 1.0, - "std_dev": 0.477, - }, - {"name": "runner.gym.pass@1/accuracy", "score_type": "scalar", "nan_count": 0, "value": 0.68}, - ] - }, - ], - ) - metric_coverage: dict[str, dict[str, AgentEvalMetricOutputCoverage]] = Field( - default_factory=dict, - description="Per-metric, per-output coverage counts (total/scored/failed/missing).", - examples=[ - # Same 12 trials under two metrics, which is what distinguishes a low mean from low - # coverage. The two dead trials fail every metric; the judge failed once more on its own, - # and once completed without emitting its output at all (missing, not failed). - { - "harbor_reward": {"reward": {"total": 12, "scored": 10, "failed": 2, "missing": 0}}, - "rubric_judge": {"criteria_pass_rate": {"total": 12, "scored": 8, "failed": 3, "missing": 1}}, - } - ], - ) - task_metric_values: dict[str, TrialValuesByMetric] = Field( - default_factory=dict, - description=( - "Per task, the values each '.' measured, in trial order. Each " - "record names the trial that produced it, so values join across keys -- and out to " - "trials.jsonl and scores.jsonl -- by trial_id. Values keep the type the metric produced " - "them in: a count stays an int, a flag stays a bool, a judge's verdict stays a label -- " - "read them through numeric_metric_values() before doing arithmetic. A failed trial has " - "value None: a trial that did not pass. An unmeasured trial (metric failed, output " - "absent) has no entry at all, so each key's list is independent: align by trial_id, " - "never by position. An empty list means nothing was measured, including a task that " - "produced no trial." - ), - examples=[ - { - "contract-review-msa-indemnity": { - "harbor_reward.reward": [ - {"trial_id": "contract-review-msa-indemnity__k3f9wq2", "value_type": "number", "value": 1.0}, - {"trial_id": "contract-review-msa-indemnity__t7m2xb4", "value_type": "number", "value": 0.0}, - {"trial_id": "contract-review-msa-indemnity__9jr4vd1", "value_type": "number", "value": 1.0}, - ], - # A count stays an int, and t7m2xb4's judge verdict is kept as a label -- neither - # is pass@k-eligible, but both are per-trial evidence worth recording. - "steps.count": [ - {"trial_id": "contract-review-msa-indemnity__k3f9wq2", "value_type": "number", "value": 14}, - {"trial_id": "contract-review-msa-indemnity__t7m2xb4", "value_type": "number", "value": 31}, - {"trial_id": "contract-review-msa-indemnity__9jr4vd1", "value_type": "number", "value": 12}, - ], - # t7m2xb4 is absent here rather than null: its judge timed out, so that trial - # went unmeasured. Index 1 is therefore a different trial in each of these lists. - "rubric_judge.criteria_pass_rate": [ - {"trial_id": "contract-review-msa-indemnity__k3f9wq2", "value_type": "number", "value": 0.75}, - {"trial_id": "contract-review-msa-indemnity__9jr4vd1", "value_type": "number", "value": 1.0}, - ], - }, - "nda-scope-carveouts": { - # p2hn8sc died in the sandbox, so it is 'missing' in every key: a trial that - # happened and did not pass, as opposed to one that was never measured. - "harbor_reward.reward": [ - {"trial_id": "nda-scope-carveouts__p2hn8sc", "value_type": "missing", "value": None}, - {"trial_id": "nda-scope-carveouts__w5db3qy", "value_type": "number", "value": 1.0}, - {"trial_id": "nda-scope-carveouts__z8kt1nf", "value_type": "number", "value": 0.0}, - ], - "rubric_judge.verdict": [ - {"trial_id": "nda-scope-carveouts__p2hn8sc", "value_type": "missing", "value": None}, - {"trial_id": "nda-scope-carveouts__w5db3qy", "value_type": "label", "value": "compliant"}, - {"trial_id": "nda-scope-carveouts__z8kt1nf", "value_type": "label", "value": "overbroad"}, - ], - }, - # Requested, but the runner returned no trial for it: keys declared, nothing measured. - "merger-hsr-filing-threshold": { - "harbor_reward.reward": [], - "rubric_judge.verdict": [], - }, - } - ], - ) - task_count: int = Field(default=0, description="Number of tasks represented in the run.") - trial_count: int = Field(default=0, description="Number of distinct trials scored.") - score_count: int = Field(default=0, description="Total number of metric scores.") - - @property - def scores_by_name(self) -> Mapping[str, AggregateScore]: - """Aggregates keyed by name — see :attr:`AggregatedMetricResult.scores_by_name`.""" - return self.scores.scores_by_name - - def score(self, name: str) -> AggregateScore: - """Return the aggregate named ``name`` — see :meth:`AggregatedMetricResult.score`. - - Exists so callers needn't know the aggregates sit one level down, behind a field whose name - differs from the summary's own accessor by a single character. - """ - return self.scores.score(name) - - def task_outcomes(self, metric_name: str | None = None) -> list[PerTaskOutcomes]: - """:attr:`task_metric_values` as models that name their own keys, sorted by task then metric. - - A read-time *view*, not the wire format. The field itself stays a nested dict because it is - persisted per run: repeating "task_id"/"metric_name" on every row would grow ``summary.json`` - for no new information, and lookup by task and metric stays O(1). Reach for this when you - want a typed object to pass around or to hand to a template. - - ``metric_name`` narrows to one ``"."``, which is what a report over a - single metric wants:: - - summary.task_outcomes() -> every task, every metric output - summary.task_outcomes("gym_reward.reward") -> every task that metric measured - - A task the named metric never measured is **dropped**, not returned empty: it was scored by - a different metric, so reporting it as unmeasured would invent missing coverage. A task that - declared the metric but produced no usable value is different - it keeps its entry with an - empty ``trials`` list, because there the coverage really is missing. That is the same - distinction :attr:`task_metric_values` draws by having a key at all. - """ - outcomes_by_task = [ - ( - task_id, - [ - PerTaskOutcome(metric_name=key, trials=list(records)) - for key, records in sorted(by_key.items()) - if metric_name is None or key == metric_name - ], - ) - for task_id, by_key in sorted(self.task_metric_values.items()) - ] - return [ - PerTaskOutcomes(task_id=task_id, outcomes=outcomes) - for task_id, outcomes in outcomes_by_task - if metric_name is None or outcomes - ] - - @staticmethod - def from_scores( - scores: Sequence[AgentEvalTaskScore], - *, - tasks: Sequence[AgentEvalTask] | None = None, - extra_scores: Sequence[AggregateScore] = (), - ) -> AgentEvalSummary: - """Build aggregated scores, task values, and coverage for a set of metric scores. - - ``extra_scores`` are already-aggregated scores contributed by the runner (namespaced - ``runner..``), merged in so a backend's own figures are addressable the same way as ours. - """ - task_list = list(tasks) if tasks is not None else None - task_metric_values = _task_metric_values(scores, task_list) - return AgentEvalSummary( - scores=_aggregate_scores( - scores, - task_list, - extra_scores, - task_metric_values=task_metric_values, - ), - metric_coverage=_metric_coverage(scores, task_list), - task_metric_values=task_metric_values, - task_count=len(task_list) if task_list is not None else len({score.task_id for score in scores}), - trial_count=len({score.trial_id for score in scores}), - score_count=len(scores), - ) - - -class RunMetadata(BaseModel): - """Provenance for a run: what was evaluated, by what, and when. - - Answers "what produced this result?" — previously improvised by callers inside an untyped - ``benchmark`` dict. ``labels`` remains free-form for caller-specific tags, but the fields that - every run has are typed. - """ - - model_config = ConfigDict(extra="forbid") - - labels: dict[str, str] = Field( - default_factory=dict, - description="Caller-supplied tags for this run (e.g. benchmark, mode, backend). Free-form by design.", - ) - target: RunnerInfo | None = Field( - default=None, - description="Identity of the runner/model/agent that produced the trials; None for imported trials.", - ) - started_at: datetime | None = Field(default=None, description="UTC timestamp when the run began.") - finished_at: datetime | None = Field(default=None, description="UTC timestamp when scoring completed.") - duration_sec: float | None = Field(default=None, description="Wall-clock seconds from start to finish.") - sdk_version: str | None = Field(default=None, description="nemo-evaluator-sdk version that produced the run.") - - -class BundleLocation(BaseModel): - """Where a run was written, returned by :meth:`AgentEvalResult.persist`. - - Kept off :class:`AgentEvalResult` because it is not a property of the evaluation — it is the - outcome of choosing to store it. Holding one means the bundle exists, so there is no optional to - re-check; a run that was never persisted simply has no ``BundleLocation``. - """ - - model_config = ConfigDict(extra="forbid", frozen=True) - - output_dir: Path = Field(description="Directory the run bundle was written to.") - dashboard_path: Path | None = Field( - default=None, - description="Path to the rendered HTML dashboard, or None when dashboard writing was disabled.", - ) - - -class AgentEvalResult(BaseModel): - """Root result for a completed agent evaluation: tasks, trials, scores, and summary. - - Describes the evaluation and nothing else — storing it is a separate decision, made by calling - :meth:`persist`. Because the result carries no paths, it never holds a location that was unknown - when it was constructed, and nothing has to mutate it after the fact. - """ - - model_config = ConfigDict(extra="forbid") - - run_id: str = Field(description="Identifier of this run.") - tasks: list[AgentEvalTask] = Field(description="Immutable task definitions evaluated in this run.") - trials: list[AgentEvalTrial] = Field(description="Trials produced or imported for the run.") - scores: list[AgentEvalTaskScore] = Field(description="Metric scores computed for the trials.") - summary: AgentEvalSummary = Field(description="Derived rollups and coverage computed for the run.") - metadata: RunMetadata = Field( - default_factory=RunMetadata, - description="Run provenance: labels, target identity, timings, SDK version.", - ) - work_dir: Path | None = Field( - default=None, - description="Directory the run worked in, where its runtimes wrote trial evidence. Known " - "before the run starts (it comes from the run config), so unlike a bundle location it is " - "never attached after the fact. None for a purely in-memory run.", - ) - - def persist(self, output_dir: str | Path | None = None, *, write_dashboard: bool = True) -> BundleLocation: - """Write this run to a bundle and return where it landed. - - Deliberately a call rather than something ``AgentEvaluator.run`` does for you: computing an - evaluation and storing one are separate decisions (the same reasoning as ``publish_to_intake``). - - Defaults to :attr:`work_dir`, which is the directory the trials' evidence already lives under — - so the bundle is self-contained and survives being moved. Passing a different ``output_dir`` - leaves those evidence references pointing back at the original directory. That is supported (a - re-scored run may reference an earlier run's deliverables) but the resulting bundle only - resolves while the original directory is still there. - - Set ``write_dashboard=False`` to skip rendering ``report.html``. - """ - # Imported here rather than at module scope: persistence imports this module for the types it - # writes, so a top-level import would be circular. - from nemo_platform.beta.evaluator.agent_eval.persistence import persist_run - - target = output_dir if output_dir is not None else self.work_dir - if target is None: - raise ValueError( - "this run has no work_dir to persist into (it ran in memory); pass an explicit " - "output_dir, or set work_dir on the AgentEvalRunConfig so evidence and bundle share " - "a directory" - ) - return persist_run(self, target, write_html_dashboard=write_dashboard) - - def to_records(self, view: ResultView = "rows") -> list[dict[str, Any]]: - """Convert this run into flat dictionaries for export or inspection. - - ``view="rows"`` yields one record per metric score — the agent-eval analogue of the dataset - path's row. The fan-out is preserved rather than collapsed: ``task_id`` and ``trial_id`` are - columns, so a consumer can still group by task, which is what pass@k depends on. - - ``view="aggregate"`` matches the dataset path exactly — percentiles flattened, histograms - kept as JSON strings so the view stays tabular. - - Args: - view: Output projection, either ``"rows"`` or ``"aggregate"``. - - Returns: - Flat record dictionaries for downstream table/dataframe conversion. - - Raises: - ValueError: If ``view`` is unsupported. - """ - if view == "rows": - return [_score_record(score) for score in self.scores] - - if view == "aggregate": - records: list[dict[str, Any]] = [] - for score in self.summary.scores.scores: - record: dict[str, Any] = {} - for key, value in score.model_dump(mode="json").items(): - if key == "percentiles" and isinstance(value, dict): - flatten_dict("percentiles", value, record) - elif key == "histogram" and value is not None: - # Histograms stay as JSON strings so aggregate views remain tabular instead - # of expanding variable-width nested columns. - record[key] = json.dumps(value, sort_keys=True) - else: - record[key] = value - records.append(record) - return records - - raise ValueError(f"Unsupported view {view!r}. Expected 'rows' or 'aggregate'.") - - def to_table(self, view: ResultView = "rows"): - """Convert records into a ``pyarrow.Table``. - - Args: - view: Output projection, either ``"rows"`` or ``"aggregate"``. - - Columns are unioned across every record before the table is built. ``pa.Table.from_pylist`` - takes its schema from the first record alone, and in a row view ``error`` and - ``diagnostics.*`` appear only on failed scores — so a run whose first score succeeded would - otherwise export a table with the failure columns silently missing. ``to_pandas`` already - unions keys, and the two should not disagree about what a run contains. - - Args: - view: Output projection, either ``"rows"`` or ``"aggregate"``. - - Returns: - Table built from ``to_records(view=view)``. - """ - import pyarrow as pa - - records = self.to_records(view=view) - # dict-of-None preserves first-appearance order, matching how format_table derives columns. - columns = {key: None for record in records for key in record} - return pa.Table.from_pylist([{key: record.get(key) for key in columns} for record in records]) - - def to_pandas(self, view: ResultView = "rows"): - """Convert records into a pandas ``DataFrame``. - - Args: - view: Output projection, either ``"rows"`` or ``"aggregate"``. - - Returns: - DataFrame built from ``to_records(view=view)``. - """ - import pandas as pd - - return pd.DataFrame.from_records(self.to_records(view=view)) - - def format_summary(self, max_rows: int = 10, *, max_error_rows: int | None = None) -> str: - """Render a human-readable summary with aggregates and a score preview. - - Args: - max_rows: Maximum number of score records included in the preview. - max_error_rows: Maximum number of failed scores included in the error-details section. - Defaults to ``max_rows``. - - Returns: - Multi-line summary string suitable for terminal/notebook display. - """ - if max_error_rows is None: - max_error_rows = max_rows - aggregate_records = [summary_aggregate_record(score) for score in self.summary.scores.scores] - preview = [_score_preview_record(score) for score in self.scores[:max_rows]] - parts = [ - _agent_eval_summary_header(self), - "", - "Aggregate scores", - format_table(aggregate_records), - ] - if preview: - parts.extend( - [ - "", - f"Score preview (first {len(preview)} of {len(self.scores)})", - format_table(preview), - ] - ) - parts.extend(_format_score_errors(self.scores, max_error_rows=max_error_rows)) - return "\n".join(parts) - - def print_summary(self, max_rows: int = 10, *, max_error_rows: int | None = None) -> None: - """Print ``format_summary`` output. - - Args: - max_rows: Maximum number of score records included in the preview. - max_error_rows: Maximum number of failed scores included in the error-details section. - Defaults to ``max_rows``. - """ - print(self.format_summary(max_rows=max_rows, max_error_rows=max_error_rows)) - - def __str__(self) -> str: - """Return the default compact summary representation. - - Returns: - Summary string with up to five preview scores. - """ - return self.format_summary(max_rows=5) - - -def _score_error_text(score: AgentEvalTaskScore) -> str | None: - """Join the error-severity diagnostic messages for a score, or None when it has none.""" - messages = [ - diagnostic.message - for diagnostic in score.diagnostics - if diagnostic.severity is AgentEvalDiagnosticSeverity.ERROR - ] - return "; ".join(messages) if messages else None - - -def _score_diagnostics_columns(score: AgentEvalTaskScore) -> dict[str, str]: - """JSON-encoded diagnostic columns, keyed ``diagnostics.``. - - Encoded as compact JSON for the same reason the dataset path does it: diagnostics have a - metric-defined shape, and exports stay flat only if that shape is a string. - """ - if not score.diagnostics: - return {} - return { - f"diagnostics.{score.metric_type}": json.dumps( - [serialize_value(diagnostic) for diagnostic in score.diagnostics], sort_keys=True - ) - } - - -def _score_preview_record(score: AgentEvalTaskScore) -> dict[str, Any]: - """Identity and status columns shared by the row export and the summary preview.""" - record: dict[str, Any] = { - "task_id": score.task_id, - "trial_id": score.trial_id, - "metric_type": score.metric_type, - "status": score.status.value, - } - for output in score.outputs: - record[f"output.{output.name}"] = serialize_value(output.value) - return record - - -def _score_record(score: AgentEvalTaskScore) -> dict[str, Any]: - """Full export record for one score: identity, preview columns, error text, and diagnostics. - - Carries ``id``, ``run_id``, and ``metadata`` that the summary preview leaves out. An export is - the thing a caller joins, concatenates, and keeps: ``id`` is what a row is addressable by, - ``run_id`` keeps a frame self-describing once several runs are stacked into one, and - ``metadata`` is caller-supplied — dropping it silently discards data the SDK never owned. The - preview stays narrow because it is read on a terminal, the same split the dataset path makes - between ``to_records`` and ``summary_row_base_record``. - """ - record: dict[str, Any] = {"id": score.id, "run_id": score.run_id} - record.update(_score_preview_record(score)) - if error_text := _score_error_text(score): - record["error"] = error_text - record.update(_score_diagnostics_columns(score)) - # Flattened rather than JSON-encoded: metadata is free-form but usually shallow and scalar, so - # dotted columns keep it queryable. Diagnostics get the JSON treatment instead because their - # shape is metric-defined and variable-width. - flatten_dict("metadata", serialize_value(score.metadata), record) - return record - - -def _agent_eval_summary_header(result: AgentEvalResult) -> str: - """Build the header line, mirroring the shape :func:`summary_header` produces for row results. - - The counts differ because the units do — a run has tasks, trials, and scores where the dataset - path has rows — but the ``Name(field=value, ...)`` shape is the same, and a status the run never - produced is left out, matching what that header does with its zero counts. - - Statuses are counted by tallying the scores present, so an absent status simply never becomes a - key; there is no zero to filter out. - """ - status_counts: dict[str, int] = {} - for score in result.scores: - status_counts[score.status.value] = status_counts.get(score.status.value, 0) + 1 - fields = [ - f"tasks={len(result.tasks)}", - f"trials={len(result.trials)}", - f"scores={len(result.scores)}", - f"aggregate_scores={len(result.summary.scores.scores)}", - ] - fields.extend(f"{status}={count}" for status, count in sorted(status_counts.items())) - return f"AgentEvalResult({', '.join(fields)})" - - -def _format_score_errors( - scores: Sequence[AgentEvalTaskScore], - *, - max_error_rows: int | None, -) -> list[str]: - """Render the failed-score detail section, separating a failed trial from a failed metric. - - Both arrive as ``FAILED``, but they mean different things to a reader: a failed trial is one - the agent is answerable for, a failed metric is a measurement that never happened. The - dataset path has no equivalent distinction to make, so this section is agent-eval's own rather - than a reuse of :func:`format_error_details`. - """ - failed = [score for score in scores if score.status is AgentEvalScoreStatus.FAILED] - if not failed: - return [] - - # max(0, ...) guards a negative limit, which slicing would otherwise read as an offset from the - # end: failed[:-2] shows all but the last two rather than none. An over-large limit needs no - # guard, since a slice past the end is simply the whole list. Mirrors format_error_details. - shown_limit = len(failed) if max_error_rows is None else max(0, max_error_rows) - shown = failed[:shown_limit] - parts = ["", f"Error details ({len(shown)} of {len(failed)} failed scores)"] - for score in shown: - kind = "failed trial" if is_trial_failure(score) else "failed metric" - parts.extend(["", f"[{score.task_id} / {score.trial_id} / {score.metric_type}] {kind}"]) - parts.append(_score_error_text(score) or "(no error-severity diagnostic recorded)") - if len(shown) < len(failed): - parts.extend(["", f"... {len(failed) - len(shown)} more failed scores omitted"]) - return parts - - -def _aggregate_scores( - scores: Sequence[AgentEvalTaskScore], - tasks: Sequence[AgentEvalTask] | None, - extra_scores: Sequence[AggregateScore] = (), - *, - task_metric_values: dict[str, TrialValuesByMetric], -) -> AggregatedMetricResult: - """Aggregate per-metric-output, per-semantic-view, and task-level pass@k values into range scores. - - Each metric output becomes a score named ``.``, each semantic view - ``view.``, and each score-like output additionally yields ``..pass@k`` - task-level rollups. Failed and missing scores are surfaced as ``nan_count`` so coverage is visible - alongside the statistics. ``extra_scores`` (runner-contributed, ``runner.``-namespaced) are appended - as-is. - """ - aggregated: list[AggregateScore] = [] - - output_names = _metric_output_names(scores, tasks) - for metric_type, names in sorted(output_names.items()): - metric_records = [score for score in scores if score.metric_type == metric_type] - total = len(metric_records) - for output_name in names: - values: list[float] = [] - for score in metric_records: - value = None - # PARTIAL scores can still emit valid per-output values; include them so - # stats agree with coverage (which counts non-FAILED outputs as scored). - # Outputs actually missing on a PARTIAL score stay None -> counted as nan. - if score.status in (AgentEvalScoreStatus.COMPLETED, AgentEvalScoreStatus.PARTIAL): - output = _score_output(score, output_name) - value = _numeric_value(output) if output is not None else None - if value is not None: - values.append(value) - aggregated.append(_aggregate_range_score(f"{metric_type}.{output_name}", values, total)) - - for view_name, (values, total) in sorted(_semantic_view_values(scores, tasks).items()): - aggregated.append(_aggregate_range_score(f"view.{view_name}", values, total)) - - # Required rather than recomputed here: the summary needs the same mapping, and deriving it - # twice is what this rewiring exists to stop. The one caller builds it once and shares it. - aggregated.extend(_task_pass_at_k_scores(task_metric_values, tasks)) - aggregated.extend(extra_scores) - - return AggregatedMetricResult(scores=aggregated) - - -def metric_values(records: Sequence[TrialMetricValue]) -> list[float | int | bool | str | None]: - """The bare per-trial values, for consumers reading records without caring which trial made them. - - Preserves order, cardinality, type, and the None-versus-absent distinction exactly as recorded. - Reach for :func:`numeric_metric_values` before doing arithmetic: this list may hold labels, and - ``value >= 1.0`` raises on one. - """ - return [record.value for record in records] - - -def numeric_metric_values(records: Sequence[TrialMetricValue]) -> list[float | None]: - """The values that can be compared and averaged, as floats, for consumers doing arithmetic. - - A number becomes a float (a bool becomes 1.0/0.0, matching how a pass/fail flag has always been - read). A dead trial stays ``None`` -- it is a trial that definitively did not pass, and - dropping it would let a crashed rollout flatter the agent. - - A label is **dropped**, not zeroed. A categorical verdict says nothing about whether the agent - solved the task, so it is an unmeasured trial rather than a failed one -- the same reading this - module gives a metric that raised (see :func:`is_trial_failure`). Charging it as a failure would - misattribute a measurement problem to the agent, and counting it as a pass is not defined. - - A label can land under a score-like key: :func:`validate_metric_result` coerces and discards, so - a metric declaring a continuous score may still return the string ``"0.9"``, and an output one - task never declared may be score-like on another. This is where that stops being arithmetic. - """ - values: list[float | None] = [] - for record in records: - value = record.value - if value is None: - values.append(None) - elif isinstance(value, bool | int | float): - # bool first: it is a subclass of int, and False must become 0.0 rather than be dropped. - values.append(float(value)) - return values - - -def _pass_at_k(n: int, c: int, k: int) -> float: - """Unbiased pass@k estimator (Chen et al., 2021): ``1 - C(n-c, k) / C(n, k)``. - - The probability that at least one of ``k`` samples drawn without replacement from ``n`` trials - (``c`` of them passing) is a pass. Caller guarantees ``1 <= k <= n``. - """ - if n - c < k: - return 1.0 - product = 1.0 - for i in range(n - c + 1, n + 1): - product *= 1.0 - k / i - return 1.0 - product - - -def _scorelike_outputs(tasks: Sequence[AgentEvalTask] | None) -> set[tuple[str, str]]: - """``(metric_type, output_name)`` pairs whose declared value is a score (continuous or boolean). - - pass@k is only meaningful for a per-trial pass/fail signal, so labels, discrete/count outputs, - and free models (e.g. token measurements) are excluded. Needs task metric specs; with no tasks - the set is empty and pass@k is skipped. - """ - scorelike: set[tuple[str, str]] = set() - if tasks is None: - return scorelike - for task in tasks: - for metric in task.metrics: - metric_type = metric_type_name(metric) - for spec in metric.output_spec(): - if issubclass(spec.value_schema, _PASS_AT_K_VALUE_SCHEMAS): - scorelike.add((metric_type, spec.name)) - return scorelike - - -def _task_metric_values( - scores: Sequence[AgentEvalTaskScore], - tasks: Sequence[AgentEvalTask] | None, -) -> dict[str, TrialValuesByMetric]: - """Ordered per-trial records per task, keyed ``.``. - - ``task-a`` declares ``reward.score`` (continuous), ``steps.count`` (discrete) and - ``usage.prompt_tokens`` (a free model) and runs four trials:: - - in t0 reward 1.0 steps 5 usage 1200 - t1 reward steps 9 usage 1300 # the judge died, not the agent - t2 # every metric fails as a trial failure - t3 reward 0.0 steps 7 usage 1100 - - out {"task-a": {"reward.score": [(t0, 1.0), (t2, None), (t3, 0.0)], - "steps.count": [(t0, 5), (t1, 9), (t2, None), (t3, 7)]}} - - (shown as ``(trial_id, value)`` pairs; each is an :class:`TrialMetricValue`) - - ``usage.prompt_tokens`` is absent because its declared schema is not in - :data:`_TASK_METRIC_VALUE_SCHEMAS`; t1 is missing from ``reward.score`` but present in - ``steps.count``; t2 is ``None`` in both. ``steps.count`` keeps its ints -- values are recorded in - the type the metric produced them in, not flattened to float. - - Which keys a task gets: - - - declared by its metric spec under :data:`_TASK_METRIC_VALUE_SCHEMAS` -> kept - - declared under any other schema -> dropped, even when the emitted value is numeric, so a - ``MetricOutputSpec.model("prompt_tokens", TokenCount)`` measurement never becomes a key - - undeclared, but some score emitted a recordable value for it -> kept - - ``tasks is None`` -> no specs to filter against, so every recordable output observed is kept - - What each score contributes to its key, in trial order: - - - failed trial (:func:`is_trial_failure`) -> value ``None``, a trial that did not pass - - failed metric, or the output absent -> no entry; the trial is unmeasured, not unsuccessful - - a value a metric can emit (number, bool or label) -> that value, in its own type - - anything else (a dict, a list, a literal null) -> no entry; see :func:`_native_value` - - pass@k needs that asymmetry, and it is why a list is indexed by surviving measurement rather than - by trial: above, index 1 is t2 under ``reward.score`` but t1 under ``steps.count``. Every entry - therefore names its trial, and ``trial_id`` — not position — is what joins two keys of one task, - or joins out to ``trials.jsonl`` and ``scores.jsonl``. Ids are recorded as the runner reported - them and are never deduplicated: two records sharing an id stay two records, so a runner that - reuses one costs pass@k nothing. - """ - output_keys: dict[str, set[tuple[str, str]]] = {} - # Outputs a task declared under a schema this mapping does not retain. Tracked so an emitted - # numeric value cannot add back what that task's spec filter just excluded, and carrying the task - # id because tasks in one run need not declare the same output under the same schema. - excluded: set[tuple[str, str, str]] = set() - if tasks is not None: - for task in tasks: - task_keys = output_keys.setdefault(task.id, set()) - for metric in task.metrics: - metric_type = metric_type_name(metric) - for spec in metric.output_spec(): - if issubclass(spec.value_schema, _TASK_METRIC_VALUE_SCHEMAS): - task_keys.add((metric_type, spec.name)) - else: - excluded.add((task.id, metric_type, spec.name)) - - # Materialized once: the key set has to be settled before any record can be filed (a trial - # failure reaches every key of its metric, including keys only a later score reveals), and - # `scores` is walked exactly once so a one-shot sequence still works. - ordered = list(scores) - for score in ordered: - # setdefault, not add: a task whose every score failed still earns an entry, so it reads as - # measured-and-empty rather than absent. - task_keys = output_keys.setdefault(score.task_id, set()) - if score.status in (AgentEvalScoreStatus.COMPLETED, AgentEvalScoreStatus.PARTIAL): - for output in score.outputs: - if (score.task_id, score.metric_type, output.name) in excluded: - continue - if _native_value(output) is not None: - task_keys.add((score.metric_type, output.name)) - - # Key set settled, so the records fill in score order -- which is what puts each key's list in - # trial order. - outputs_by_task_metric: dict[tuple[str, str], list[str]] = {} - by_task: dict[str, TrialValuesByMetric] = {} - for task_id, keys in output_keys.items(): - ordered_keys = sorted(keys) - by_task[task_id] = {f"{metric_type}.{name}": [] for metric_type, name in ordered_keys} - for metric_type, name in ordered_keys: - outputs_by_task_metric.setdefault((task_id, metric_type), []).append(name) - - for score in ordered: - output_names = outputs_by_task_metric.get((score.task_id, score.metric_type)) - if not output_names: - continue - task_values = by_task[score.task_id] - if is_trial_failure(score): - for name in output_names: - task_values[f"{score.metric_type}.{name}"].append( - TrialMetricValue(trial_id=score.trial_id, value_type=TrialMetricValueType.MISSING, value=None) - ) - continue - if score.status not in (AgentEvalScoreStatus.COMPLETED, AgentEvalScoreStatus.PARTIAL): - continue - # Indexed once per score rather than rescanned per output, and first-wins on a duplicate name - # to match :func:`_score_output`. - outputs: dict[str, MetricOutput] = {} - for output in score.outputs: - outputs.setdefault(output.name, output) - for name in output_names: - output = outputs.get(name) - payload = _native_value(output) if output is not None else None - if payload is not None: - value_type, value = payload - task_values[f"{score.metric_type}.{name}"].append( - TrialMetricValue(trial_id=score.trial_id, value_type=value_type, value=value) - ) - return by_task - - -def _task_pass_at_k_scores( - task_metric_values: dict[str, TrialValuesByMetric], - tasks: Sequence[AgentEvalTask] | None, -) -> list[AggregateScore]: - """Task-level pass@k over the R trials per task, aggregated across tasks (uniform for any runner). - - For each score-like metric output, group trials by task, count trials ``n`` and passes ``c`` - (value ``>= _PASS_VALUE``), then emit ``..pass@k`` for ``k`` in ``1..max(n)`` as - the across-task mean of the unbiased per-task estimator (over tasks with at least ``k`` trials). - ``pass@1`` equals the macro per-task pass rate, i.e. the task-level mean. - - **A failed trial did not pass.** It counts toward ``n`` and never toward ``c``: an agent that - solved a task once and crashed once did not go one-for-one. A failed *metric* is different — it - leaves the trial unmeasured rather than unsuccessful, so it stays out of ``n`` entirely rather - than being charged to the agent (see :func:`is_trial_failure`). Tasks left with no usable value at - all drop out of the estimate and are reported as ``nan_count``, uniform across ``k``, so a shrinking - denominator is never silent. (Tasks excluded from a given ``k`` merely for having fewer than ``k`` - trials are *not* counted there — that is the estimator working as defined, not missing data.) - - "No usable value" includes a task that was never scored at all: it declares the metric, holds an - empty value list, and lands in ``nan_count`` like any other unmeasured task. That is the same - missing coverage whether the trial died or was never produced, and excluding it would report - pass@k over a denominator quietly smaller than the task set asked for. - - Note this is reachable only through :meth:`AgentEvalSummary.from_scores` called directly with a - task list wider than the scores — a caller re-aggregating a subset, say. A full run cannot get - here: :meth:`AgentEvaluator._score_trials` refuses to score at all when a task produced no trial, - so a runner that drops one fails the run rather than reporting it as missing coverage. - """ - scorelike = _scorelike_outputs(tasks) - if not scorelike: - return [] - aggregated: list[AggregateScore] = [] - for metric_type, output_name in sorted(scorelike): - key = f"{metric_type}.{output_name}" - values_by_task = [ - numeric_metric_values(outputs[key]) for outputs in task_metric_values.values() if key in outputs - ] - measured = [values for values in values_by_task if values] - if not measured: - continue - # Empty value lists stay in nan_count (via total); for each k, mean the unbiased - # estimator over tasks with n >= k (None / < full credit do not count as passes). - unmeasured = sum(not values for values in values_by_task) - # (n, c) per task, counted once: neither depends on k, so counting inside the k loop would - # re-walk every task's values max_n times over. - counts = [ - (len(values), sum(value is not None and value >= _PASS_VALUE for value in values)) for values in measured - ] - max_n = max(n for n, _ in counts) - for k in range(1, max_n + 1): - per_task = [_pass_at_k(n, c, k) for n, c in counts if n >= k] - if per_task: - aggregated.append(_aggregate_range_score(f"{key}.pass@{k}", per_task, len(per_task) + unmeasured)) - return aggregated - - -def _aggregate_range_score(name: str, values: list[float], total: int) -> AggregateRangeScore: - finite = [value for value in values if math.isfinite(value)] - count = len(finite) - nan_count = max(total - count, 0) - if not finite: - return AggregateRangeScore(name=name, count=0, nan_count=nan_count) - total_sum = sum(finite) - mean = total_sum / count - # Report both conventions explicitly rather than picking one: the population figures describe the - # values actually evaluated, the sample figures estimate the process they were drawn from (which is - # what repeated trials over one task are sampling). Sample stats are undefined for a single value. - sum_sq_dev = sum((value - mean) ** 2 for value in finite) - variance = sum_sq_dev / count - sample_variance = sum_sq_dev / (count - 1) if count > 1 else None - percentiles = compute_percentiles(sorted(finite)) - return AggregateRangeScore( - name=name, - count=count, - nan_count=nan_count, - sum=total_sum, - mean=mean, - min=min(finite), - max=max(finite), - variance=variance, - std_dev=math.sqrt(variance), - sample_variance=sample_variance, - sample_std_dev=math.sqrt(sample_variance) if sample_variance is not None else None, - # Reuse the deterministic-metric percentile helper so agent-eval and metric aggregation report - # the same distribution the same way. - percentiles=percentiles, - # Surfaced alongside the other basic stats so `median` means the same thing whether a score - # was computed here or imported from a backend that reports one without a full distribution. - median=percentiles.p50, - ) - - -def _metric_coverage( - scores: Sequence[AgentEvalTaskScore], - tasks: Sequence[AgentEvalTask] | None, -) -> dict[str, dict[str, AgentEvalMetricOutputCoverage]]: - output_names = _metric_output_names(scores, tasks) - coverage: dict[str, dict[str, AgentEvalMetricOutputCoverage]] = {} - for metric_type, names in sorted(output_names.items()): - metric_records = [score for score in scores if score.metric_type == metric_type] - metric_coverage: dict[str, AgentEvalMetricOutputCoverage] = {} - for output_name in names: - total = len(metric_records) - failed = sum(1 for score in metric_records if score.status == AgentEvalScoreStatus.FAILED) - scored = sum( - 1 - for score in metric_records - if score.status != AgentEvalScoreStatus.FAILED - and any(output.name == output_name for output in score.outputs) - ) - metric_coverage[output_name] = AgentEvalMetricOutputCoverage( - total=total, - scored=scored, - failed=failed, - missing=max(total - scored - failed, 0), - ) - coverage[metric_type] = metric_coverage - return coverage - - -def _metric_output_names( - scores: Sequence[AgentEvalTaskScore], - tasks: Sequence[AgentEvalTask] | None, -) -> dict[str, list[str]]: - names: dict[str, set[str]] = {} - if tasks is not None: - for task in tasks: - for metric in task.metrics: - metric_type = metric_type_name(metric) - for output in metric.output_spec(): - names.setdefault(metric_type, set()).add(output.name) - - for score in scores: - for output in score.outputs: - names.setdefault(score.metric_type, set()).add(output.name) - return {metric_type: sorted(output_names) for metric_type, output_names in names.items()} - - -def _semantic_view_values( - scores: Sequence[AgentEvalTaskScore], - tasks: Sequence[AgentEvalTask] | None, -) -> dict[str, tuple[list[float], int]]: - """Return reduced view values and the number of attempted reductions per view. - - The integer in each tuple is the total number of trial/view reductions - attempted (the denominator for nan_count); the list holds the values that - reduced successfully. - """ - if tasks is None: - return {} - - tasks_by_id = {task.id: task for task in tasks} - # Match the stats path: PARTIAL scores may carry usable signal outputs. Missing - # signals still skip the view reduction below, so admitting PARTIAL is safe. - score_by_key = { - (score.task_id, score.trial_id, score.metric_type): score - for score in scores - if score.status in (AgentEvalScoreStatus.COMPLETED, AgentEvalScoreStatus.PARTIAL) - } - trials_by_task: dict[str, set[str]] = {} - for score in scores: - trials_by_task.setdefault(score.task_id, set()).add(score.trial_id) - - values_by_view: dict[str, list[float]] = {} - totals_by_view: dict[str, int] = {} - for task_id, trial_ids in trials_by_task.items(): - task = tasks_by_id.get(task_id) - if task is None: - continue - for trial_id in trial_ids: - for view_name, view in task.views.items(): - totals_by_view[view_name] = totals_by_view.get(view_name, 0) + 1 - signal_values: list[float] = [] - for signal in view.signals: - score = score_by_key.get((task_id, trial_id, signal.metric)) - output = _score_output(score, signal.output) if score is not None else None - value = _semantic_value(output) if output is not None else None - if value is None: - signal_values = [] - break - signal_values.append(value) - if not signal_values: - continue - reduced = _reduce_semantic_view(view.reducer, signal_values, view.signals) - if reduced is not None: - values_by_view.setdefault(view_name, []).append(reduced) - - return {view_name: (values_by_view.get(view_name, []), total) for view_name, total in totals_by_view.items()} - - -def _score_output(score: AgentEvalTaskScore | None, output_name: str) -> MetricOutput | None: - if score is None: - return None - for output in score.outputs: - if output.name == output_name: - return output - return None - - -def _reduce_semantic_view( - reducer: SemanticReducer, - values: list[float], - signals: list[ViewSignal], -) -> float | None: - if reducer == SemanticReducer.SINGLE: - return values[0] - if reducer == SemanticReducer.ALL: - return min(values) - if reducer == SemanticReducer.ANY: - return max(values) - if reducer == SemanticReducer.MEAN: - return mean_numeric(values) - weights = [signal.weight if signal.weight is not None else 1.0 for signal in signals] - denominator = sum(weights) - if denominator == 0: - return None - return sum(value * weight for value, weight in zip(values, weights, strict=True)) / denominator - - -def _numeric_value(output: MetricOutput) -> float | None: - value = output.value - if isinstance(value, bool): - return None - if isinstance(value, int | float): - return float(value) - if isinstance(value, BaseModel): - root = getattr(value, "root", None) - if isinstance(root, bool): - return None - if isinstance(root, int | float): - return float(root) - return None - - -def _native_value(output: MetricOutput) -> tuple[TrialMetricValueType, float | int | bool | str] | None: - """The payload for one metric output, in the type the metric produced it in. - - The *preserving* counterpart to :func:`_semantic_value`, which projects to a float because its - callers (aggregate stats, semantic views) do arithmetic. A :class:`TrialMetricValue` is not - arithmetic: it is the per-trial evidence a reader looks up by ``trial_id`` in ``result.trials`` - or ``trials.jsonl``, so a count stays an int, a flag stays a bool, and a judge's verdict stays - the string it was. - - Returns ``None`` -- "nothing a trial can record" -- rather than a value, so an output holding - a dict, a list, or a literal null stays *absent* from the value list. That is not the same as - the ``None`` a dead trial records, and conflating the two would charge pass@k a trial the - agent never made. - """ - value = output.value - if isinstance(value, BaseModel): - value = getattr(value, "root", None) - # bool first: it is a subclass of int, and it is a pass/fail signal rather than a measurement. - if isinstance(value, bool | int | float): - return (TrialMetricValueType.NUMBER, value) - if isinstance(value, str): - return (TrialMetricValueType.LABEL, value) - return None - - -def _semantic_value(output: MetricOutput) -> float | None: - """:func:`_native_value` projected to a float, for the callers that do arithmetic. - - The two answer different questions -- preserve versus interpret -- but they must agree on what a - metric value *is*, so the RootModel unwrap and the "what counts as numeric" rule live in - :func:`_native_value` alone. A label projects to ``None``: a view or aggregate cannot average it. - """ - payload = _native_value(output) - if payload is None or payload[0] is not TrialMetricValueType.NUMBER: - return None - return float(payload[1]) - - -def mean_numeric(values: list[float]) -> float | None: - """Return the mean of finite numeric values, ignoring missing and NaN.""" - finite = [value for value in values if math.isfinite(value)] - if not finite: - return None - return sum(finite) / len(finite) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/callable_runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/callable_runtime.py deleted file mode 100644 index d6af7e43be..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/callable_runtime.py +++ /dev/null @@ -1,122 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Minimal, dependency-light AgentTaskRunner backed by a user-supplied callable.""" - -from __future__ import annotations - -import asyncio -from collections.abc import Awaitable, Callable, Sequence -from dataclasses import dataclass, field -from typing import Any - -from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask -from nemo_platform.beta.evaluator.agent_eval.trials import ( - AgentEvalTrial, - AgentEvalTrialStatus, - AgentOutput, - RunnerInfo, - callable_identity, -) -from nemo_platform.beta.evaluator.values.evidence import CandidateEvidence - - -@dataclass(slots=True) -class TrialDraft: - """What an agent callable returns for one task: final output plus optional evidence. - - The runtime wraps this into a completed :class:`AgentEvalTrial`. Returning a - :class:`AgentOutput` or a plain string is also accepted as shorthand. - """ - - output: AgentOutput - evidence: CandidateEvidence | None = None - metadata: dict[str, Any] = field(default_factory=dict) - - -AgentTaskFn = Callable[[AgentEvalTask], Awaitable[TrialDraft | AgentOutput | str]] - - -class CallableAgentTaskRunner: - """Smallest possible :class:`AgentTaskRunner`: delegate each task to an async callable. - - The callable receives an :class:`AgentEvalTask` and returns the agent's final output as - a :class:`TrialDraft`, an :class:`AgentOutput`, or a plain string. This runtime adds only - what the ``AgentTaskRunner`` contract needs: bounded concurrency, stable trial ids, and - failure capture (an exception becomes a ``FAILED`` trial instead of aborting the batch). - It requires no Docker or external agent SDK, so it doubles as a reference for richer - runtimes and as the seam an ``AgentEvaluator`` drives via ``run(target=runner)``. - """ - - def __init__( - self, - agent_fn: AgentTaskFn, - *, - parallelism: int | None = None, - trial_id_suffix: str = "trial", - ) -> None: - self._agent_fn = agent_fn - self._parallelism = parallelism - self._trial_id_suffix = trial_id_suffix - - def runner_info(self) -> RunnerInfo: - """Identify this runner; the agent callable itself is the result-shaping detail.""" - return RunnerInfo( - name="callable", - kind="runner", - config={ - "agent_fn": callable_identity(self._agent_fn), - "parallelism": self._parallelism, - }, - ) - - async def run_tasks( - self, - tasks: Sequence[AgentEvalTask], - config: AgentEvalRunConfig | None = None, - ) -> list[AgentEvalTrial]: - """Run every task through the callable and return one trial per task, in order.""" - parallelism = self._parallelism if self._parallelism is not None else (config.parallelism if config else 4) - semaphore = asyncio.Semaphore(max(1, parallelism)) - - async def run_one(task: AgentEvalTask) -> AgentEvalTrial: - async with semaphore: - try: - result = await self._agent_fn(task) - except Exception as exc: # noqa: BLE001 - surfaced as a FAILED trial, not a crash - return self._failed_trial(task, exc) - return self._completed_trial(task, result) - - return list(await asyncio.gather(*(run_one(task) for task in tasks))) - - def _trial_id(self, task: AgentEvalTask) -> str: - return f"{task.id}:{self._trial_id_suffix}" - - def _completed_trial(self, task: AgentEvalTask, result: TrialDraft | AgentOutput | str) -> AgentEvalTrial: - draft = _as_trial_draft(result) - return AgentEvalTrial( - id=self._trial_id(task), - task_id=task.id, - status=AgentEvalTrialStatus.COMPLETED, - output=draft.output, - evidence=draft.evidence, - metadata=draft.metadata, - ) - - def _failed_trial(self, task: AgentEvalTask, exc: Exception) -> AgentEvalTrial: - return AgentEvalTrial( - id=self._trial_id(task), - task_id=task.id, - status=AgentEvalTrialStatus.FAILED, - metadata={"error": f"{type(exc).__name__}: {exc}"}, - ) - - -def _as_trial_draft(result: TrialDraft | AgentOutput | str) -> TrialDraft: - if isinstance(result, TrialDraft): - return result - if isinstance(result, AgentOutput): - return TrialDraft(output=result) - if isinstance(result, str): - return TrialDraft(output=AgentOutput(output_text=result)) - raise TypeError(f"agent callable must return TrialDraft, AgentOutput, or str; got {type(result).__name__}") diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/codex/runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/codex/runtime.py deleted file mode 100644 index a66e4c99a9..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/codex/runtime.py +++ /dev/null @@ -1,641 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Codex-backed agent-eval runtimes.""" - -# ruff: noqa: I001, T201 - the vendored SDK mirror uses different import-order and print settings. - -from __future__ import annotations - -import asyncio -import contextlib -import json -import os -import shlex -import shutil -import stat -import subprocess -import tempfile -from collections.abc import Awaitable, Callable, Mapping, Sequence -from enum import StrEnum -from pathlib import Path -from typing import Any - -from nemo_platform.beta.evaluator.agent_eval.runtimes.docker_sandbox import DockerSandboxAgentRuntime -from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask -from nemo_platform.beta.evaluator.agent_eval.trials import ( - AgentEvalTrial, - AgentEvalTrialStatus, - AgentOutput, - RunnerInfo, - callable_identity, -) -from nemo_platform.beta.evaluator.agent_eval.workspace_seeds import SEED_FILES_INPUT_KEY, seed_workspace -from nemo_platform.beta.evaluator.values.evidence import CandidateEvidence, EvidenceDescriptor - -#: Wall-clock ceiling for a single task's Codex CLI invocation — one ``process.communicate()`` covering -#: the agent's whole run on that task, not a per-request or per-turn limit. Tasks run independently, so -#: this is not a budget for the evaluation as a whole. On expiry the process is terminated and the task -#: is recorded as a failed trial; it does not abort the run. -DEFAULT_CODEX_TIMEOUT_S = 600 -DEFAULT_CODEX_DOCKER_MODEL = "gpt-5.4" -DEFAULT_CODEX_DOCKER_CLI_IMAGE = "node:22-alpine" -DEFAULT_CODEX_DOCKER_CLI_PACKAGE = "@openai/codex@0.137.0" -ProcessFactory = Callable[..., Awaitable[Any]] - - -class RuntimeChoice(StrEnum): - """Which Codex execution mode the caller wants.""" - - DOCKER = "docker" - LOCAL = "local" - - -class EffectiveCodexRuntime(StrEnum): - """The concrete runtime chosen for a :class:`RuntimeChoice` + environment.""" - - DOCKER_SANDBOX = "docker_sandbox" - DOCKER_CLI = "docker_cli" - LOCAL_CLI = "local_cli" - - -#: Builds the prompt handed to Codex on stdin for a task. Swap it to change how a task is framed -#: (e.g. a benchmark-specific preamble); the default presents the task and invites workspace edits. -CodexPromptBuilder = Callable[[AgentEvalTask], str] - - -class CodexCliAgentRuntime: - """AgentTaskRunner that uses the locally installed Codex CLI credentials.""" - - def __init__( - self, - *, - model: str | None = None, - work_root: str | Path | None = None, - codex_bin: str = "codex", - timeout_s: int = DEFAULT_CODEX_TIMEOUT_S, - prompt_builder: CodexPromptBuilder | None = None, - process_factory: ProcessFactory | None = None, - runtime_name: str = "codex_cli", - ) -> None: - self._model = model - self._work_root = Path(work_root).expanduser() if work_root is not None else None - self._codex_bin = codex_bin - self._timeout_s = timeout_s - self._prompt_builder = prompt_builder or AgentEvalTask.agent_prompt - self._process_factory = process_factory or asyncio.create_subprocess_exec - self._runtime_name = runtime_name - - def runner_info(self) -> RunnerInfo: - """Identify this runner and the Codex CLI settings that shape its results. - - Uses ``runtime_name``, which subclasses already set (the Docker variant reports - ``codex_docker_cli``) and which trials are stamped with, so provenance agrees with them. - """ - return RunnerInfo( - name=self._runtime_name, - kind="runner", - config={ - "model": self._model, - "timeout_s": self._timeout_s, - "codex_bin": self._codex_bin, - "prompt_builder": callable_identity(self._prompt_builder), - }, - ) - - async def run_tasks( - self, - tasks: Sequence[AgentEvalTask], - config: AgentEvalRunConfig | None = None, - ) -> Sequence[AgentEvalTrial]: - if shutil.which(self._codex_bin) is None: - raise RuntimeError(f"Codex CLI executable {self._codex_bin!r} was not found on PATH") - - resolved_config = config or AgentEvalRunConfig() - semaphore = asyncio.Semaphore(resolved_config.parallelism) - - async def run_one(index: int, task: AgentEvalTask) -> AgentEvalTrial: - async with semaphore: - return await self._run_task(index, task, resolved_config) - - return await asyncio.gather(*(run_one(index, task) for index, task in enumerate(tasks))) - - async def _run_task(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> AgentEvalTrial: - evidence_dir = self._evidence_dir(index, task, config) - workspace_dir = evidence_dir / "workspace" - - try: - # The task directory is mounted into Docker, but its private parent is not. Keeping that - # parent host-owned and 0700 preserves the local confidentiality boundary even when a - # container is interrupted before its recursive cleanup completes. - _ensure_private_directory(evidence_dir.parent) - _ensure_private_directory(evidence_dir) - _ensure_private_directory(workspace_dir) - except Exception as exc: - # The path that failed setup is not safe to use for artifact persistence. In particular, - # writing through a rejected evidence-directory symlink would escape the private tree. - return _failed_codex_trial(task, None, exc, runtime_name=self._runtime_name) - - prompt_path = evidence_dir / "prompt.txt" - task_path = evidence_dir / "task.json" - stdout_path = evidence_dir / "stdout.jsonl" - stderr_path = evidence_dir / "stderr.txt" - final_output_path = evidence_dir / "final_output.txt" - - # Persist the task for debugging, but never the grader-only fields: the docker variant mounts - # this evidence dir into the sandbox (danger-full-access), so serializing `intent` (desired - # behavior) or `reference` (held-out ground truth) here would let the agent read them back out - # of `/evidence/task.json` — the same reward-hacking leak the intent-free prompt closes. - try: - _write_private_text(task_path, task.model_dump_json(indent=2, exclude={"intent", "reference"})) - except Exception as exc: - return _failed_codex_trial(task, evidence_dir, exc, runtime_name=self._runtime_name) - - command = self._command(workspace_dir=workspace_dir, final_output_path=final_output_path) - process: Any | None = None - try: - # Seed inside the guarded block so a bad seed (e.g. a path escaping the workspace) fails - # just this task rather than aborting the whole run. Offload to a worker thread: seeding is - # synchronous (a handler may do blocking I/O, e.g. the plugin's fileset download), and this - # runs on the event loop shared by every concurrent task, so a blocking seed would stall them all. - seeded_files = await asyncio.to_thread(seed_workspace, workspace_dir, task.inputs.get(SEED_FILES_INPUT_KEY)) - # Build the prompt after seeding and inside the guarded block: an instruction-less task - # raises here, failing just this task instead of aborting the run (and seeding wins if both). - prompt = self._prompt_builder(task) - _write_private_text(prompt_path, prompt) - process = await self._process_factory( - *command, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - if process is None: - raise RuntimeError("process factory failed to create a process") - stdout, stderr = await asyncio.wait_for( - process.communicate(prompt.encode("utf-8")), - timeout=self._timeout_s, - ) - except TimeoutError as exc: - await _terminate_process(process) - return _failed_codex_trial(task, evidence_dir, exc, runtime_name=self._runtime_name) - except Exception as exc: - return _failed_codex_trial(task, evidence_dir, exc, runtime_name=self._runtime_name) - - stdout_text = _decode_process_output(stdout) - stderr_text = _decode_process_output(stderr) - artifact_persistence_error: str | None = None - try: - _write_private_text(stdout_path, stdout_text) - _write_private_text(stderr_path, stderr_text) - except Exception as exc: - artifact_persistence_error = f"{exc.__class__.__name__}: {exc}" - - permission_cleanup_error: str | None = None - try: - self._validate_artifact_permissions(evidence_dir) - except Exception as exc: - permission_cleanup_error = f"{exc.__class__.__name__}: {exc}" - - if process.returncode != 0: - return _failed_codex_trial( - task, - evidence_dir, - RuntimeError(f"codex exec exited with status {process.returncode}: {stderr_text.strip()}"), - runtime_name=self._runtime_name, - permission_cleanup_error=permission_cleanup_error, - artifact_persistence_error=artifact_persistence_error, - ) - - if artifact_persistence_error is not None: - return _failed_codex_trial( - task, - evidence_dir, - RuntimeError(f"failed to persist Codex evidence: {artifact_persistence_error}"), - runtime_name=self._runtime_name, - permission_cleanup_error=permission_cleanup_error, - artifact_persistence_error=artifact_persistence_error, - ) - if permission_cleanup_error is not None: - return _failed_codex_trial( - task, - evidence_dir, - PermissionError(f"Codex evidence permission normalization failed: {permission_cleanup_error}"), - runtime_name=self._runtime_name, - permission_cleanup_error=permission_cleanup_error, - ) - - try: - output_text = _read_private_final_output(final_output_path, fallback=stdout_text) - except Exception as exc: - return _failed_codex_trial(task, evidence_dir, exc, runtime_name=self._runtime_name) - return AgentEvalTrial( - id=f"{task.id}:codex", - task_id=task.id, - status=AgentEvalTrialStatus.COMPLETED, - output=AgentOutput( - output_text=output_text, - metadata={ - "runtime": self._runtime_name, - "agent": "codex", - "agent_model": self._model, - "evidence_dir": str(evidence_dir), - }, - ), - evidence=CandidateEvidence( - descriptors={ - "workspace": EvidenceDescriptor(kind="filesystem", ref=str(workspace_dir)), - "prompt": EvidenceDescriptor(kind="text", format="txt", ref=str(prompt_path)), - "task": EvidenceDescriptor(kind="json", format="json", ref=str(task_path)), - "stdout": EvidenceDescriptor(kind="codex_stdout", format="jsonl", ref=str(stdout_path)), - "stderr": EvidenceDescriptor(kind="text", format="txt", ref=str(stderr_path)), - "final_output": EvidenceDescriptor(kind="text", format="txt", ref=str(final_output_path)), - }, - metadata={"runtime": self._runtime_name, "agent": "codex"}, - ), - metadata={ - "runtime": self._runtime_name, - "agent": "codex", - "agent_model": self._model, - "agent_ok": True, - "seeded_files": seeded_files, - "generated": True, - }, - ) - - def _command(self, *, workspace_dir: Path, final_output_path: Path) -> list[str]: - command = [ - self._codex_bin, - "exec", - "--skip-git-repo-check", - "--ephemeral", - "--ignore-user-config", - "--sandbox", - "workspace-write", - "--cd", - str(workspace_dir), - "--output-last-message", - str(final_output_path), - "--json", - ] - if self._model is not None: - command.extend(["--model", self._model]) - command.append("-") - return command - - def _validate_artifact_permissions(self, evidence_dir: Path) -> None: - """Validate runtime-specific artifact postconditions after the process exits.""" - - def _evidence_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> Path: - root = self._work_root - if root is None: - root = (config.work_dir or Path.cwd()) / "evidence" / "codex" - safe_task_id = _safe_path_name(task.id) - task_dir = f"{index:06d}-{safe_task_id}" if safe_task_id else f"task-{index:06d}" - return Path(root) / task_dir - - -class CodexDockerCliAgentRuntime(CodexCliAgentRuntime): - """AgentTaskRunner that runs Codex CLI inside a Docker container.""" - - def __init__( - self, - *, - model: str | None = None, - work_root: str | Path | None = None, - docker_bin: str = "docker", - image: str = DEFAULT_CODEX_DOCKER_CLI_IMAGE, - codex_package: str = DEFAULT_CODEX_DOCKER_CLI_PACKAGE, - auth_path: str | Path | None = None, - timeout_s: int = DEFAULT_CODEX_TIMEOUT_S, - prompt_builder: CodexPromptBuilder | None = None, - process_factory: ProcessFactory | None = None, - ) -> None: - super().__init__( - model=model, - work_root=work_root, - timeout_s=timeout_s, - prompt_builder=prompt_builder, - process_factory=process_factory, - runtime_name="codex_docker_cli", - ) - self._docker_bin = docker_bin - self._image = image - self._codex_package = codex_package - self._auth_path = ( - Path(auth_path).expanduser() if auth_path is not None else Path.home() / ".codex" / "auth.json" - ) - - async def run_tasks( - self, - tasks: Sequence[AgentEvalTask], - config: AgentEvalRunConfig | None = None, - ) -> Sequence[AgentEvalTrial]: - if shutil.which(self._docker_bin) is None: - raise RuntimeError(f"Docker executable {self._docker_bin!r} was not found on PATH") - if not self._auth_path.exists(): - raise RuntimeError( - f"Codex auth file was not found at {self._auth_path}. Run `codex login` or use OPENAI_API_KEY " - "so --runtime docker can use DockerSandboxAgentRuntime." - ) - - resolved_config = config or AgentEvalRunConfig() - semaphore = asyncio.Semaphore(resolved_config.parallelism) - - async def run_one(index: int, task: AgentEvalTask) -> AgentEvalTrial: - async with semaphore: - return await self._run_task(index, task, resolved_config) - - return await asyncio.gather(*(run_one(index, task) for index, task in enumerate(tasks))) - - def _command(self, *, workspace_dir: Path, final_output_path: Path) -> list[str]: - evidence_dir = final_output_path.parent - inner_command = [ - "npx", - "-y", - self._codex_package, - "exec", - "--skip-git-repo-check", - "--ephemeral", - "--sandbox", - "danger-full-access", - "--cd", - "/workspace", - "--output-last-message", - "/evidence/final_output.txt", - "--json", - ] - if self._model is not None: - inner_command.extend(["--model", self._model]) - inner_command.append("-") - # Codex intentionally runs as root: the container mounts its auth under /root and coding tasks - # may need to install tools. Repair the bind-mounted trees before Docker returns so the host can - # score and persist every artifact the agent created without widening access to other host users. - # Capture the bind mount's owner as seen inside this container before Codex runs: raw host UID/GID - # values are not portable across Docker Desktop and rootless user-namespace mappings. Keep Codex - # failures authoritative; only surface the required chmod status when Codex itself succeeded. - shell_command = ( - "host_owner=\"$(stat -c '%u:%g' /evidence 2>/dev/null)\" || true; " - f"{shlex.join(inner_command)}; " - "codex_status=$?; " - 'if [ -n "$host_owner" ]; then ' - 'chown -R "$host_owner" /workspace /evidence 2>/dev/null || true; ' - "fi; " - "chmod -R u+rwX,go-rwx /workspace /evidence; " - "permissions_status=$?; " - 'if [ "$codex_status" -ne 0 ]; then exit "$codex_status"; fi; ' - 'exit "$permissions_status"' - ) - return [ - self._docker_bin, - "run", - "--rm", - "-i", - "-e", - "PYTHONDONTWRITEBYTECODE=1", - "-v", - f"{self._auth_path.resolve()}:/root/.codex/auth.json:ro", - "-v", - f"{workspace_dir.resolve()}:/workspace", - "-v", - f"{evidence_dir.resolve()}:/evidence", - self._image, - "sh", - "-lc", - shell_command, - ] - - def _validate_artifact_permissions(self, evidence_dir: Path) -> None: - _validate_private_tree(evidence_dir) - - -def resolve_codex_runtime( - *, - runtime: RuntimeChoice, - model: str | None, - output_dir: Path, - env: Mapping[str, str] = os.environ, - prompt_builder: CodexPromptBuilder | None = None, -) -> tuple[CodexCliAgentRuntime | CodexDockerCliAgentRuntime | DockerSandboxAgentRuntime, EffectiveCodexRuntime]: - """Pick and construct a Codex runtime for a run-mode + environment. - - ``local`` runs the on-PATH Codex CLI. ``docker`` prefers the OpenAI-Agents ``DockerSandbox`` when - ``OPENAI_API_KEY`` is an OpenAI platform secret (``sk-...``) and otherwise falls back to the - containerized Codex CLI (which mounts ``~/.codex/auth.json``). ``prompt_builder`` is threaded into - the CLI runtimes; the sandbox runtime does its own prompting. Returns the runtime plus the - :class:`EffectiveCodexRuntime` actually chosen so callers can label/report it. - """ - effective_runtime = _resolve_codex_runtime(runtime, env) - if effective_runtime == EffectiveCodexRuntime.LOCAL_CLI: - return ( - CodexCliAgentRuntime( - model=model, - work_root=output_dir / "evidence" / "codex", - prompt_builder=prompt_builder, - ), - effective_runtime, - ) - if effective_runtime == EffectiveCodexRuntime.DOCKER_CLI: - return ( - CodexDockerCliAgentRuntime( - model=model, - work_root=output_dir / "evidence" / "codex-docker", - prompt_builder=prompt_builder, - ), - effective_runtime, - ) - if effective_runtime == EffectiveCodexRuntime.DOCKER_SANDBOX: - return DockerSandboxAgentRuntime(model=model or DEFAULT_CODEX_DOCKER_MODEL), effective_runtime - raise ValueError(f"unsupported Codex runtime {runtime!r}") - - -def _resolve_codex_runtime(runtime: RuntimeChoice, env: Mapping[str, str] = os.environ) -> EffectiveCodexRuntime: - if runtime == RuntimeChoice.LOCAL: - return EffectiveCodexRuntime.LOCAL_CLI - if runtime == RuntimeChoice.DOCKER: - if _openai_sdk_secret_key_is_set(env): - return EffectiveCodexRuntime.DOCKER_SANDBOX - return EffectiveCodexRuntime.DOCKER_CLI - raise ValueError(f"unsupported Codex runtime {runtime!r}") - - -def _openai_sdk_secret_key_is_set(env: Mapping[str, str] = os.environ) -> bool: - return env.get("OPENAI_API_KEY", "").strip().startswith("sk-") - - -def list_codex_agent_models(*, codex_bin: str = "codex") -> list[dict[str, Any]]: - """Return visible Codex model descriptors from the local Codex CLI.""" - if shutil.which(codex_bin) is None: - raise RuntimeError(f"Codex CLI executable {codex_bin!r} was not found on PATH") - result = subprocess.run( - [codex_bin, "debug", "models"], - check=True, - capture_output=True, - text=True, - ) - payload = json.loads(result.stdout) - models = payload.get("models") - if not isinstance(models, list): - raise RuntimeError("Codex model catalog did not contain a models list") - visible = [model for model in models if isinstance(model, dict) and model.get("visibility") == "list"] - return sorted(visible, key=lambda model: int(model.get("priority") or 0), reverse=True) - - -def print_codex_agent_models(*, codex_bin: str = "codex") -> None: - """Print local Codex model slugs and display names.""" - for model in list_codex_agent_models(codex_bin=codex_bin): - slug = model.get("slug") - if not isinstance(slug, str): - continue - display_name = model.get("display_name") - if isinstance(display_name, str) and display_name != slug: - print(f"{slug}\t{display_name}") - else: - print(slug) - - -def _failed_codex_trial( - task: AgentEvalTask, - evidence_dir: Path | None, - exc: Exception, - *, - runtime_name: str = "codex_cli", - permission_cleanup_error: str | None = None, - artifact_persistence_error: str | None = None, -) -> AgentEvalTrial: - evidence: CandidateEvidence | None = None - error_artifact_error: str | None = None - if evidence_dir is not None: - error_path = evidence_dir / "error.json" - try: - _write_private_text( - error_path, json.dumps({"error_type": exc.__class__.__name__, "error": str(exc)}) + "\n" - ) - except Exception as artifact_exc: - error_artifact_error = f"{artifact_exc.__class__.__name__}: {artifact_exc}" - else: - evidence = CandidateEvidence( - descriptors={"error": EvidenceDescriptor(kind="error", format="json", ref=str(error_path))}, - metadata={"runtime": runtime_name, "agent": "codex"}, - ) - - metadata: dict[str, Any] = { - "runtime": runtime_name, - "agent": "codex", - "agent_ok": False, - "error_type": exc.__class__.__name__, - "error": str(exc), - } - if permission_cleanup_error is not None: - metadata["permission_cleanup_error"] = permission_cleanup_error - if artifact_persistence_error is not None: - metadata["artifact_persistence_error"] = artifact_persistence_error - if error_artifact_error is not None: - metadata["error_artifact_error"] = error_artifact_error - return AgentEvalTrial( - id=f"{task.id}:codex", - task_id=task.id, - status=AgentEvalTrialStatus.FAILED, - output=None, - evidence=evidence, - metadata=metadata, - ) - - -def _ensure_private_directory(path: Path) -> None: - """Create or repair a host-owned directory without following a leaf symlink.""" - path.mkdir(mode=0o700, parents=True, exist_ok=True) - descriptor = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) - try: - path_stat = os.fstat(descriptor) - if path_stat.st_uid != os.getuid(): - raise PermissionError(f"directory is not owned by the invoking host user: {path}") - os.fchmod(descriptor, 0o700) - finally: - os.close(descriptor) - - -def _write_private_text(path: Path, content: str) -> None: - """Atomically publish a host-created evidence artifact with owner-only access.""" - descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent) - temporary_path = Path(temporary_name) - try: - os.fchmod(descriptor, 0o600) - temporary_file = os.fdopen(descriptor, "w", encoding="utf-8") - descriptor = -1 - with temporary_file: - temporary_file.write(content) - os.replace(temporary_path, path) - finally: - if descriptor != -1: - with contextlib.suppress(OSError): - os.close(descriptor) - temporary_path.unlink(missing_ok=True) - - -def _read_private_final_output(path: Path, *, fallback: str) -> str: - """Read a regular agent-created final output without following it, then republish it privately.""" - try: - descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) - except FileNotFoundError: - _write_private_text(path, fallback) - return fallback - - try: - if not stat.S_ISREG(os.fstat(descriptor).st_mode): - raise PermissionError(f"final output is not a regular file: {path}") - with os.fdopen(descriptor, "r", encoding="utf-8") as output_file: - descriptor = -1 - output_text = output_file.read() - finally: - if descriptor != -1: - os.close(descriptor) - - _write_private_text(path, output_text) - return output_text - - -def _validate_private_tree(root: Path) -> None: - """Require a host-owned, owner-only tree without following agent-created symlinks.""" - expected_uid = os.getuid() - pending = [root] - while pending: - path = pending.pop() - path_stat = path.lstat() - if stat.S_ISLNK(path_stat.st_mode): - continue - if path_stat.st_uid != expected_uid: - raise PermissionError(f"artifact is not owned by the invoking host user: {path}") - - mode = stat.S_IMODE(path_stat.st_mode) - if mode & 0o077: - raise PermissionError(f"artifact grants group or other access: {path} ({mode:o})") - if stat.S_ISDIR(path_stat.st_mode): - if mode & 0o700 != 0o700: - raise PermissionError(f"directory is not owner-readable, writable, and traversable: {path} ({mode:o})") - with os.scandir(path) as entries: - pending.extend(Path(entry.path) for entry in entries) - elif stat.S_ISREG(path_stat.st_mode): - if mode & 0o600 != 0o600: - raise PermissionError(f"file is not owner-readable and writable: {path} ({mode:o})") - else: - raise PermissionError(f"artifact is not a regular file or directory: {path}") - - -async def _terminate_process(process: Any | None) -> None: - if process is None or process.returncode is not None: - return - process.kill() - with contextlib.suppress(Exception): - await process.wait() - - -def _decode_process_output(value: bytes | str | None) -> str: - if value is None: - return "" - if isinstance(value, str): - return value - return value.decode("utf-8", errors="replace") - - -def _safe_path_name(value: str) -> str: - return "".join(char if char.isalnum() or char in "._-" else "-" for char in value).strip(".-")[:120] diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py deleted file mode 100644 index bdf1cc979e..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py +++ /dev/null @@ -1,366 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Docker-backed sandbox runtime for agent-eval trials.""" - -from __future__ import annotations - -import asyncio -import contextlib -import inspect -import json -import re -import shutil -import tarfile -import tempfile -from collections.abc import Awaitable, Callable, Sequence -from dataclasses import dataclass -from datetime import UTC, datetime -from pathlib import Path -from typing import Any -from uuid import uuid4 - -from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask -from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput, RunnerInfo -from nemo_platform.beta.evaluator.values.evidence import CandidateEvidence, EvidenceDescriptor -from pydantic_core import to_jsonable_python - -DEFAULT_INSTRUCTIONS = ( - "Complete the task inside the sandbox workspace. Inspect the provided task files, " - "write any durable artifacts under output/, and return a concise final answer." -) -_RUNTIME_NAME = "docker_sandbox" -_SAFE_NAME_PATTERN = re.compile(r"[^A-Za-z0-9_.-]+") - - -@dataclass(frozen=True) -class SandboxSDK: - """Loaded OpenAI Agents SDK symbols used by the runtime.""" - - Runner: Any - RunConfig: Any - SandboxRunConfig: Any - Manifest: Any - SandboxAgent: Any - DockerSandboxClient: Any - DockerSandboxClientOptions: Any - File: Any - Dir: Any - LocalDir: Any - DEFAULT_PYTHON_SANDBOX_IMAGE: str - docker_from_env: Callable[[], Any] - - -def _load_agents_sdk() -> SandboxSDK: - try: - # The OpenAI Agents SDK is imported only when this Docker runtime is actually used, so it - # is absent from the default type-checking environment. - from agents.run import RunConfig # ty: ignore[unresolved-import] - from agents.sandbox import Manifest, SandboxAgent, SandboxRunConfig # ty: ignore[unresolved-import] - from agents.sandbox.config import DEFAULT_PYTHON_SANDBOX_IMAGE # ty: ignore[unresolved-import] - from agents.sandbox.entries import Dir, File, LocalDir # ty: ignore[unresolved-import] - from agents.sandbox.sandboxes.docker import ( # ty: ignore[unresolved-import] - DockerSandboxClient, - DockerSandboxClientOptions, - ) - - from agents import Runner # ty: ignore[unresolved-import] - from docker import from_env as docker_from_env - except ImportError as exc: - # Audience split is in the error text: SDK extras are not propagated into the - # vendored nemo_platform.beta.evaluator mirror. - raise RuntimeError( - "DockerSandboxAgentRuntime requires the openai-agents[docker] Python packages. " - "Standalone SDK: pip install 'nemo-evaluator-sdk[agent-runtimes]'. " - "Vendored nemo-platform.beta.evaluator (no SDK extras): " - "pip install 'openai-agents[docker]'" - ) from exc - - return SandboxSDK( - Runner=Runner, - RunConfig=RunConfig, - SandboxRunConfig=SandboxRunConfig, - Manifest=Manifest, - SandboxAgent=SandboxAgent, - DockerSandboxClient=DockerSandboxClient, - DockerSandboxClientOptions=DockerSandboxClientOptions, - File=File, - Dir=Dir, - LocalDir=LocalDir, - DEFAULT_PYTHON_SANDBOX_IMAGE=DEFAULT_PYTHON_SANDBOX_IMAGE, - docker_from_env=docker_from_env, - ) - - -class DockerSandboxAgentRuntime: - """Generate agent-eval trials by running a SandboxAgent in Docker per task.""" - - def __init__( - self, - *, - model: str | None = None, - instructions: str | None = None, - image: str | None = None, - work_root: Path | None = None, - timeout_s: float | None = None, - agent_factory: Callable[..., Any] | None = None, - sandbox_client_factory: Callable[[], Any] | None = None, - runner: Any | None = None, - ) -> None: - self._model = model - self._instructions = instructions or DEFAULT_INSTRUCTIONS - self._image = image - self._work_root = work_root - self._timeout_s = timeout_s - self._agent_factory = agent_factory - self._sandbox_client_factory = sandbox_client_factory - self._runner = runner - - def runner_info(self) -> RunnerInfo: - """Identify this runner and the sandbox settings that shape its results.""" - return RunnerInfo( - name="docker_sandbox", - kind="runner", - config={ - "model": self._model, - "image": self._image, - "timeout_s": self._timeout_s, - "instructions": self._instructions, - }, - ) - - async def run_tasks( - self, - tasks: Sequence[AgentEvalTask], - config: AgentEvalRunConfig | None = None, - ) -> Sequence[AgentEvalTrial]: - resolved_config = config or AgentEvalRunConfig() - if resolved_config.run_id is None: - resolved_config = resolved_config.model_copy(update={"run_id": _new_runtime_run_id()}) - sdk = _load_agents_sdk() - semaphore = asyncio.Semaphore(resolved_config.parallelism) - - async def run_one(index: int, task: AgentEvalTask) -> AgentEvalTrial: - async with semaphore: - return await self._run_task(index, task, resolved_config, sdk) - - return await asyncio.gather(*(run_one(index, task) for index, task in enumerate(tasks))) - - async def _run_task( - self, - index: int, - task: AgentEvalTask, - config: AgentEvalRunConfig, - sdk: SandboxSDK, - ) -> AgentEvalTrial: - evidence_dir = self._evidence_dir(index, task, config) - evidence_dir.mkdir(parents=True, exist_ok=True) - client = self._build_client(sdk) - sandbox = None - - try: - # Build the prompt inside the guarded block: an instruction-less task raises here and fails - # just this task rather than aborting the whole run. - prompt = task.agent_prompt() - manifest = self._build_manifest(task, sdk) - agent = self._build_agent(manifest, sdk) - sandbox = await client.create( - manifest=manifest, - options=sdk.DockerSandboxClientOptions(image=self._image or sdk.DEFAULT_PYTHON_SANDBOX_IMAGE), - ) - async with sandbox: - result = await self._run_agent(agent, prompt, sandbox, sdk) - return await self._completed_trial(task, result, sandbox, evidence_dir) - except Exception as exc: - return self._failed_trial(task, exc, evidence_dir) - finally: - if sandbox is not None: - with contextlib.suppress(Exception): - await client.delete(sandbox) - - def _build_manifest(self, task: AgentEvalTask, sdk: SandboxSDK) -> Any: - # Seed only the agent-facing projection of the task: the prompt (its instruction) plus any - # declared workspace files. We deliberately do NOT serialize the task object into the - # workspace — nothing in the runtime consumes it, and dumping the whole DTO would expose - # grader-only fields (e.g. ``reference`` held-out ground truth) to the agent. - entries: dict[str, Any] = { - "instruction.md": sdk.File(content=task.agent_prompt().encode("utf-8")), - "output": sdk.Dir(), - } - workspace_dir = task.inputs.get("workspace_dir") - if workspace_dir is not None: - entries["workspace"] = sdk.LocalDir(src=_validated_workspace_dir(workspace_dir)) - return sdk.Manifest(root="/workspace", entries=entries) - - def _build_agent(self, manifest: Any, sdk: SandboxSDK) -> Any: - agent_factory = self._agent_factory or sdk.SandboxAgent - kwargs = { - "name": "NeMo Agent Eval Docker Sandbox Runtime", - "instructions": self._instructions, - "default_manifest": manifest, - } - if self._model is not None: - kwargs["model"] = self._model - return agent_factory(**kwargs) - - def _build_client(self, sdk: SandboxSDK) -> Any: - if self._sandbox_client_factory is not None: - return self._sandbox_client_factory() - return sdk.DockerSandboxClient(sdk.docker_from_env()) - - async def _run_agent(self, agent: Any, prompt: str, sandbox: Any, sdk: SandboxSDK) -> Any: - runner = self._runner or sdk.Runner - run = runner.run( - agent, - prompt, - run_config=sdk.RunConfig(sandbox=sdk.SandboxRunConfig(session=sandbox)), - ) - if self._timeout_s is not None: - return await asyncio.wait_for(_maybe_await(run), timeout=self._timeout_s) - return await _maybe_await(run) - - async def _completed_trial( - self, - task: AgentEvalTask, - result: Any, - sandbox: Any, - evidence_dir: Path, - ) -> AgentEvalTrial: - final_output = getattr(result, "final_output", None) - final_output_text = "" if final_output is None else str(final_output) - - final_output_path = evidence_dir / "final_output.txt" - run_items_path = evidence_dir / "run_items.json" - raw_responses_path = evidence_dir / "raw_responses.json" - workspace_tar_path = evidence_dir / "workspace.tar" - final_state_dir = evidence_dir / "final_state" - - final_output_path.write_text(final_output_text, encoding="utf-8") - _write_json(run_items_path, _jsonable(getattr(result, "new_items", []))) - _write_json(raw_responses_path, _jsonable(getattr(result, "raw_responses", []))) - await _persist_workspace(sandbox, workspace_tar_path, final_state_dir) - - return AgentEvalTrial( - id=f"{task.id}:docker-sandbox", - task_id=task.id, - status=AgentEvalTrialStatus.COMPLETED, - output=AgentOutput( - output_text=final_output_text, - response={"final_output": final_output_text}, - metadata={ - "runtime": _RUNTIME_NAME, - "evidence_dir": str(evidence_dir), - }, - ), - evidence=CandidateEvidence( - descriptors={ - "final_state": EvidenceDescriptor(kind="filesystem", ref=str(final_state_dir)), - "workspace_archive": EvidenceDescriptor(kind="archive", format="tar", ref=str(workspace_tar_path)), - "run_items": EvidenceDescriptor(kind="run_items", format="json", ref=str(run_items_path)), - "raw_responses": EvidenceDescriptor( - kind="raw_responses", format="json", ref=str(raw_responses_path) - ), - "final_output": EvidenceDescriptor(kind="text", format="txt", ref=str(final_output_path)), - }, - metadata={"runtime": _RUNTIME_NAME, "sandbox_backend": "docker"}, - ), - metadata={"runtime": _RUNTIME_NAME, "generated": True}, - ) - - def _failed_trial(self, task: AgentEvalTask, exc: Exception, evidence_dir: Path) -> AgentEvalTrial: - error_path = evidence_dir / "error.json" - _write_json( - error_path, - { - "error_type": exc.__class__.__name__, - "error": str(exc), - }, - ) - return AgentEvalTrial( - id=f"{task.id}:docker-sandbox", - task_id=task.id, - status=AgentEvalTrialStatus.FAILED, - output=None, - evidence=CandidateEvidence( - descriptors={ - "error": EvidenceDescriptor(kind="error", format="json", ref=str(error_path)), - }, - metadata={"runtime": _RUNTIME_NAME, "sandbox_backend": "docker"}, - ), - metadata={ - "runtime": _RUNTIME_NAME, - "error_type": exc.__class__.__name__, - "error": str(exc), - }, - ) - - def _evidence_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> Path: - root = config.work_dir if config.work_dir is not None else self._work_root - if root is None: - root = Path(tempfile.gettempdir()) / "nemo-evaluator-agent-runtime" - run_id = config.run_id or _new_runtime_run_id() - safe_task_id = _safe_path_name(task.id) - task_name = f"{index:06d}-{safe_task_id}" if safe_task_id else f"task-{index:06d}" - return Path(root) / "agent-runtime" / run_id / task_name - - -def _validated_workspace_dir(workspace_dir: Any) -> Path: - if not isinstance(workspace_dir, (str, Path)): - raise ValueError(f"workspace_dir must be a path, got {type(workspace_dir).__name__}") - path = Path(workspace_dir).expanduser() - if not path.is_absolute(): - raise ValueError(f"workspace_dir must be an absolute path; got {workspace_dir!r}") - resolved = path.resolve() - if not resolved.is_dir(): - raise ValueError(f"workspace_dir does not exist or is not a directory: {resolved}") - return resolved - - -async def _maybe_await(value: Awaitable[Any] | Any) -> Any: - if inspect.isawaitable(value): - return await value - return value - - -async def _persist_workspace(sandbox: Any, workspace_tar_path: Path, final_state_dir: Path) -> None: - archive = await sandbox.persist_workspace() - try: - with workspace_tar_path.open("wb") as out: - shutil.copyfileobj(archive, out) - finally: - close = getattr(archive, "close", None) - if close is not None: - close() - - _extract_tar_safely(workspace_tar_path, final_state_dir) - - -def _extract_tar_safely(archive_path: Path, destination_root: Path) -> None: - if destination_root.exists(): - shutil.rmtree(destination_root) - destination_root.mkdir(parents=True, exist_ok=True) - - # The stdlib `data` filter (Python 3.12+) rejects absolute paths, parent-directory - # traversal, links, and special files, so we do not hand-roll those guards. - with tarfile.open(archive_path, "r:*") as archive: - archive.extractall(destination_root, filter="data") - - -def _write_json(path: Path, payload: Any) -> None: - path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") - - -def _jsonable(value: Any) -> Any: - # Normalize pydantic models, dataclasses, Paths, sets, etc. into JSON-safe values; - # `repr` is the last-resort fallback for anything still not serializable. - return to_jsonable_python(value, fallback=repr) - - -def _safe_path_name(value: str) -> str: - sanitized = _SAFE_NAME_PATTERN.sub("-", value).strip(".-") - return sanitized[:120] - - -def _new_runtime_run_id() -> str: - timestamp = datetime.now(UTC).strftime("%Y%m%d%H%M%S%f") - return f"agent-runtime-{timestamp}-{uuid4().hex[:8]}" diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/environment.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/environment.py deleted file mode 100644 index 94b50eb890..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/environment.py +++ /dev/null @@ -1,178 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Process/filesystem environment boundary for agent-eval runtimes. - -Sits *below* :class:`AgentTaskRunner` so a runtime needn't know whether the -agent/verifier run under Docker, locally, or another filesystem-backed sandbox. -It is a process/filesystem abstraction: :class:`EnvRunSpec`'s ``mounts``/ -``extra_args`` are filesystem hints that non-filesystem providers may ignore. -Handles route both roles through a single :meth:`AbstractEnvironmentHandle.run`. -""" - -from __future__ import annotations - -import abc -import asyncio -import os -import re -import subprocess -from collections.abc import Callable -from dataclasses import dataclass, field -from typing import Literal, Protocol, runtime_checkable - -from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask - -EnvRole = Literal["agent", "verifier"] -_SENSITIVE_MARKERS = ("KEY", "TOKEN", "SECRET", "PASSWORD") - - -def _redact_for_logging(cmd: list[str]) -> str: - """Scrub secret-looking values (``KEY=…`` tokens and ``--flag value`` pairs).""" - out: list[str] = [] - redact_next = False - for token in cmd: - if redact_next: - out.append("***REDACTED***") - redact_next = False - elif "=" in token: - left, right = token.split("=", 1) - sensitive = any(m in left.upper() for m in _SENSITIVE_MARKERS) - out.append(f"{left}=***REDACTED***" if sensitive else f"{left}={right}") - else: - normalized = token.lstrip("-").replace("-", "_").upper() - if token.startswith("-") and any(m in normalized for m in _SENSITIVE_MARKERS): - redact_next = True - out.append(token) - return " ".join(out) - - -def default_image_tag(task_id: str) -> str: - """Default task → image-tag mapping (callers may inject their own). - - Sanitizes ``task_id`` to a valid Docker image name so ids with spaces or - other unsupported characters don't fail the build/run. - """ - safe = re.sub(r"[^a-z0-9_.-]+", "-", task_id.lower()).strip(".-") - return f"{safe or 'task'}:latest" - - -@dataclass(frozen=True) -class EnvCommandResult: - """Outcome of running a single command inside a prepared environment.""" - - exit_code: int - timed_out: bool = False - - @property - def ok(self) -> bool: - return self.exit_code == 0 and not self.timed_out - - -@dataclass -class EnvRunSpec: - """How to execute one command inside an environment handle. - - ``mounts``/``extra_args`` are filesystem-environment hints (e.g. Docker bind - mounts and extra CLI args). Non-filesystem providers may ignore them. - """ - - command: list[str] - env: dict[str, str] = field(default_factory=dict) - mounts: list[tuple[str, str]] = field(default_factory=list) - workdir: str | None = None - timeout: int | None = None - extra_args: list[str] = field(default_factory=list) - - -@runtime_checkable -class AgentEnvironmentHandle(Protocol): - """A prepared, single-task environment that can run agent/verifier commands.""" - - async def run_agent(self, spec: EnvRunSpec) -> EnvCommandResult: ... - - async def run_verifier(self, spec: EnvRunSpec) -> EnvCommandResult: ... - - async def close(self) -> None: ... - - -@runtime_checkable -class AgentEnvironmentProvider(Protocol): - """Creates per-task environment handles. Pluggable: Docker now, others later.""" - - async def prepare( - self, - task: AgentEvalTask, - config: AgentEvalRunConfig | None = None, - ) -> AgentEnvironmentHandle: ... - - -class AbstractEnvironmentHandle(abc.ABC): - """Base handle that routes both roles through a single :meth:`run`. - - Concrete handles implement :meth:`run`; ``run_agent``/``run_verifier`` are - role-specialized wrappers so the duplicated phase methods don't have to be - reimplemented per backend. - """ - - @abc.abstractmethod - async def run(self, spec: EnvRunSpec, role: EnvRole) -> EnvCommandResult: ... - - async def run_agent(self, spec: EnvRunSpec) -> EnvCommandResult: - return await self.run(spec, "agent") - - async def run_verifier(self, spec: EnvRunSpec) -> EnvCommandResult: - return await self.run(spec, "verifier") - - async def close(self) -> None: - return None - - -def _docker_run(image: str, spec: EnvRunSpec) -> EnvCommandResult: - """Run ``spec.command`` in a one-shot ``docker run --rm`` container. - - Shells out to the ``docker`` CLI (stdlib ``subprocess`` only), so no - ``agent-runtimes`` extra is needed — just a ``docker`` binary at call time. - """ - cmd = ["docker", "run", "--rm"] - if spec.workdir: - cmd += ["-w", spec.workdir] - for key, value in spec.env.items(): - cmd += ["-e", f"{key}={value}"] - for host_path, container_path in spec.mounts: - cmd += ["-v", f"{host_path}:{container_path}"] - cmd += spec.extra_args + os.environ.get("DOCKER_EXTRA_ARGS", "").split() - cmd += [image, *spec.command] - - print(f"[agent-eval-runtime] $ {_redact_for_logging(cmd)}") - try: - result = subprocess.run(cmd, check=False, text=True, timeout=spec.timeout) - except subprocess.TimeoutExpired: - return EnvCommandResult(exit_code=124, timed_out=True) - return EnvCommandResult(exit_code=result.returncode) - - -class DockerEnvironmentHandle(AbstractEnvironmentHandle): - """Docker-backed environment handle bound to one task image.""" - - def __init__(self, image: str) -> None: - self.image = image - - async def run(self, spec: EnvRunSpec, role: EnvRole = "agent") -> EnvCommandResult: - del role # Docker runs both roles identically against the same image. - return await asyncio.to_thread(_docker_run, self.image, spec) - - -class DockerEnvironmentProvider: - """Default provider that maps each task to its built Docker image.""" - - def __init__(self, *, image_tag_fn: Callable[[str], str] = default_image_tag) -> None: - self._image_tag_fn = image_tag_fn - - async def prepare( - self, - task: AgentEvalTask, - config: AgentEvalRunConfig | None = None, - ) -> DockerEnvironmentHandle: - del config - return DockerEnvironmentHandle(self._image_tag_fn(task.id)) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/_common.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/_common.py deleted file mode 100644 index 0d455fc9cf..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/_common.py +++ /dev/null @@ -1,158 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Shared helpers for the host and containerized NeMo Fabric agent-eval runtimes. - -:class:`~nemo_platform.beta.evaluator.agent_eval.runtimes.fabric.runtime.FabricAgentRuntime` (host) and -:class:`~nemo_platform.beta.evaluator.agent_eval.runtimes.fabric.container_runtime.FabricContainerRuntime` -(sandbox) map a Fabric ``RunResult`` to the *same* trial/evidence contract, so the pieces they share -live here — one definition, so the two runtimes cannot drift apart. - -Trajectory capture is built from ``nemo_relay``'s own typed config objects (a hard dependency), so -Relay owns its schema: a breaking Relay change fails construction here rather than silently producing -a malformed profile. -""" - -from __future__ import annotations - -import json -from collections.abc import Mapping -from pathlib import Path -from typing import Any - -from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalTask -from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus -from nemo_platform.beta.evaluator.values.evidence import CandidateEvidence, EvidenceDescriptor - -# Trajectory profile identity + the file-exporter output names we choose (Relay accepts these as -# inputs). Shared so both runtimes select/emit the trajectory under identical names. -TRAJECTORY_PROFILE_NAME = "eval_trajectory" -ATIF_FILENAME_TEMPLATE = "trajectory-{session_id}.atif.json" -ATOF_FILENAME = "events.atof.jsonl" -#: ATIF ``agent.version``. Both runtimes report the agent *framework* here so a consumer can group -#: host and container traces together; ``agent.name`` is what distinguishes them. Not a real version -#: yet — reporting the resolved nemo-fabric version would be the better answer. -FABRIC_AGENT_VERSION = "fabric" -# Fabric telemetry-profile selectors (Relay file exporter, no OTLP endpoint). -TELEMETRY_PROVIDER = "relay" -TELEMETRY_MODE = "sdk" - - -def safe_path_name(value: str) -> str: - """Filesystem-safe rendering of an arbitrary id (alnum/``._-`` kept, else ``-``; trimmed to 120).""" - return "".join(char if char.isalnum() or char in "._-" else "-" for char in value).strip(".-")[:120] - - -def task_subdir_name(index: int, task_id: str) -> str: - """Deterministic per-task evidence subdir name (``000000-``) shared by both runtimes.""" - safe = safe_path_name(task_id) - return f"{index:06d}-{safe}" if safe else f"task-{index:06d}" - - -def extract_output_text(output: object) -> str | None: - """Pull the user-visible message out of a Fabric output value (already unwrapped from the result). - - Harness outputs vary; adapters commonly nest the final message under ``response`` (the codex-cli - adapter does). Prefer a string ``response``/``output_text``/``text``/``message``, else stringify. - """ - if output is None: - return None - if isinstance(output, str): - return output - if isinstance(output, Mapping): - for key in ("response", "output_text", "text", "message"): - value = output.get(key) - if isinstance(value, str): - return value - return json.dumps(output, default=str) - - -def build_failed_trial( - task: AgentEvalTask, - evidence_dir: Path, - error: Exception | Mapping[str, Any], - *, - runtime_name: str, - trial_id_suffix: str, - extra_metadata: Mapping[str, Any] | None = None, -) -> AgentEvalTrial: - """Persist ``error.json`` and build a FAILED trial with the standard error evidence + metadata. - - ``error`` is either a raised exception or a Fabric error mapping (``stage``/``code``/``message``). - """ - if isinstance(error, Mapping): - error_type = str(error.get("code") or error.get("stage") or "FabricError") - error_message = str(error.get("message") or error) - else: - error_type = error.__class__.__name__ - error_message = str(error) - error_path = evidence_dir / "error.json" - error_path.write_text(json.dumps({"error_type": error_type, "error": error_message}) + "\n", encoding="utf-8") - return AgentEvalTrial( - id=f"{task.id}:{trial_id_suffix}", - task_id=task.id, - status=AgentEvalTrialStatus.FAILED, - output=None, - evidence=CandidateEvidence( - descriptors={"error": EvidenceDescriptor(kind="error", format="json", ref=str(error_path))}, - metadata={"runtime": runtime_name}, - ), - metadata={ - **(dict(extra_metadata) if extra_metadata else {}), - "runtime": runtime_name, - "error_type": error_type, - "error": error_message, - # A failed trial did not complete its agent phase; stamp it explicitly (matching the host - # Fabric/Codex runtimes) so AgentPhaseSuccessMetric scores it False rather than by omission. - "agent_ok": False, - }, - ) - - -def trajectory_telemetry(*, relay_dir: str, agent_name: str, agent_version: str) -> dict[str, Any]: - """The ``telemetry`` block of a Fabric trajectory profile: Relay's ATIF/ATOF file exporter (mode=sdk). - - Built from ``nemo_relay``'s own typed config so Relay owns its schema — no hand-maintained dict to - silently drift when Relay changes it. Callers wrap this in a profile with their own name + - ``runtime``/``environment`` blocks; ``relay_dir`` is where the ``trajectory-*.atif.json`` lands. - - ``nemo_relay`` is imported here rather than at module scope: it is a native extension costing - ~120ms to load, and this module is reachable from the evaluator plugin's job imports, so an - eager import would charge every consumer for trajectory capture they may never use. - """ - from nemo_relay.observability import ( - AtifConfig, - AtofConfig, - AtofFileSinkConfig, - ComponentSpec, - ObservabilityConfig, - ) - - observability = ComponentSpec( - config=ObservabilityConfig( - atif=AtifConfig( - enabled=True, - output_directory=relay_dir, - filename_template=ATIF_FILENAME_TEMPLATE, - agent_name=agent_name, - agent_version=agent_version, - ), - atof=AtofConfig( - enabled=True, - sinks=[ - AtofFileSinkConfig( - output_directory=relay_dir, - filename=ATOF_FILENAME, - mode="overwrite", - ) - ], - ), - ) - ) - return { - "enabled": True, - "provider": TELEMETRY_PROVIDER, - "mode": TELEMETRY_MODE, - "output_dir": relay_dir, - "config": {"version": 1, "components": [observability.to_dict()]}, - } diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/container_runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/container_runtime.py deleted file mode 100644 index 882919befc..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/container_runtime.py +++ /dev/null @@ -1,602 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Containerized NeMo Fabric agent-eval runtime. - -``FabricContainerRuntime`` is the sandboxed sibling of -:class:`~nemo_platform.beta.evaluator.agent_eval.runtimes.fabric.runtime.FabricAgentRuntime`: instead of -running the Fabric harness on the host filesystem, it runs it **inside a sandbox** (Docker now, -Kubernetes/agent-sandbox later) through the provider-neutral -:class:`~nemo_platform.beta.evaluator.agent_eval.runtimes.sandbox.api.AsyncSandbox` seam. - -Per task it: - -1. seeds ``/in`` with the composed Fabric agent config and framed input, plus the task's workspace - seed files; -2. execs Fabric's own CLI (``fabric run``), which writes a normalized ``RunResult`` to stdout and the - workspace + Relay ATIF trajectory under a fixed ``/out`` layout; -3. downloads ``/out`` across the boundary into the durable per-task evidence dir; and -4. maps it into the shared :class:`CandidateEvidence` contract the eval metrics consume — ``result`` - (json), ``trace`` (ATIF), plus ``workspace`` (filesystem) and ``logs`` — so the workspace-file, - held-out ``run_verifier``, and trajectory metrics score container trials with no metric changes. - (``FabricAgentRuntime`` surfaces ``workspace``/``logs`` only when Fabric promotes them as artifacts; - the container always captures them from the ``/out`` tree, so its evidence is a superset.) - -Relay writes ATIF **inside the image** (no host gateway), which removes the bare-``python3`` / -``tomli_w`` adapter-interpreter problem the host runtime has to work around. -""" - -from __future__ import annotations - -import asyncio -import copy -import json -import logging -import shlex -import shutil -import tempfile -from collections.abc import Mapping, Sequence -from pathlib import Path, PurePosixPath -from typing import TYPE_CHECKING, Any, cast - -from nemo_platform.beta.evaluator.agent_eval.runtimes.fabric import _common -from nemo_platform.beta.evaluator.agent_eval.runtimes.fabric.image import ensure_fabric_image -from nemo_platform.beta.evaluator.agent_eval.runtimes.fabric.skills import ( - CODEX_SKILLS_DIR, - SKILL_MODE_CODEX_SKILLS_DIR, - AgentSkill, - SkillInjectionError, - SkillMode, - SkillProvenance, - SkillSet, - resolve_skill_mode, - stage_skills_seed, -) -from nemo_platform.beta.evaluator.agent_eval.runtimes.sandbox.api import AsyncSandbox -from nemo_platform.beta.evaluator.agent_eval.runtimes.sandbox.base import SandboxExecResult, SandboxProvider, SandboxSpec -from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask -from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput, RunnerInfo -from nemo_platform.beta.evaluator.agent_eval.workspace_seeds import SEED_FILES_INPUT_KEY, seed_workspace -from nemo_platform.beta.evaluator.resolver_protocols import SecretResolver -from nemo_platform.beta.evaluator.resolvers import LocalSecretResolver -from nemo_platform.beta.evaluator.values.common import SecretRef -from nemo_platform.beta.evaluator.values.evidence import ( - EVIDENCE_FORMAT_ATIF, - EVIDENCE_LOGS, - EVIDENCE_TRACE, - CandidateEvidence, - EvidenceDescriptor, -) -from pydantic import JsonValue - -logger = logging.getLogger(__name__) - -if TYPE_CHECKING: - # nemo_fabric is an optional native dep (see FabricAgentRuntime); imported for typing only. Configs - # are consumed structurally via ``to_mapping()`` at runtime, so this module stays importable without it. - from nemo_fabric import FabricConfig # ty: ignore[unresolved-import] - -# Default per-task exec budget. Timeout is really task-specific (see AALGO-323 to move it onto -# AgentEvalTask); until then it is an internal default rather than a runtime-construction knob. -DEFAULT_FABRIC_TIMEOUT_S = 600 -_RUNTIME_NAME = "fabric_container" -_MISSING_FABRIC_MSG = ( - "FabricContainerRuntime skill injection requires the `nemo-fabric` package (native NeMo Fabric SDK) " - "on the host to resolve how a skill reaches the selected adapter; the container otherwise runs Fabric " - "only inside the sandbox." -) - -# Fixed in-container layout. The runtime seeds ``/in`` (agent config, input), execs Fabric's -# CLI, and reads the produced ``/out`` subtree back across the boundary. -_IN_DIR = "/in" -_OUT_DIR = "/out" -_WORKSPACE_DIR = f"{_OUT_DIR}/workspace" -_RELAY_DIR = f"{_OUT_DIR}/relay" -_ARTIFACTS_DIR = f"{_OUT_DIR}/artifacts" -_LOGS_DIR = f"{_OUT_DIR}/logs" -_RESULT_PATH = f"{_OUT_DIR}/fabric_result.json" -_FABRIC_STDERR = f"{_LOGS_DIR}/fabric-stderr.txt" -_AGENT_PATH = f"{_IN_DIR}/agent.yaml" -_INPUT_PATH = f"{_IN_DIR}/input.txt" -# In-sandbox root for a natively-injected skill bundle. It lives under ``/in`` (not ``/out``), so it is -# never part of the downloaded ``/out`` evidence — only codex-mode skills, which must sit in the workspace -# for the harness to self-discover them, need post-download cleanup. -_SKILLS_DIR = f"{_IN_DIR}/skills" -# Sentinel skill path attached only to probe Fabric's capability planner for the selected adapter's skills -# routing (mirrors the host runtime). Never staged and need not exist on disk. -_SKILL_PROBE_PATH = "nemo-eval-skill-capability-probe" - - -class FabricContainerRuntime: - """AgentTaskRunner that generates trials by running Fabric tasks inside a sandbox.""" - - def __init__( - self, - config: FabricConfig | Mapping[str, Any], - *, - provider: SandboxProvider, - secrets: Mapping[str, SecretRef] = {}, - image: str | None = None, - skills: Sequence[AgentSkill] | None = None, - ) -> None: - # The Fabric agent is fully described by its ``FabricConfig`` (harness + model + runtime); it is - # consumed structurally as a mapping to cross the sandbox boundary as JSON. - self._config = _to_mapping(config) - self._provider = provider - # ``secrets`` maps the env-var name a Fabric harness reads its credential from (declared by the - # adapter's ``requirements.env``) to a SecretRef. The runner only *declares* them; the resolver - # is owned by the orchestrator (see ``resolve_secrets``), mirroring ``MetricWithSecrets``. - self._secrets = dict(secrets) - self._resolved_env: dict[str, str] = {} - self._secrets_resolved = False - # Optional prebuilt image: the trial runs inside it, so it must contain the Fabric CLI + adapter. - # None -> stock harness-agnostic image built on first run. - self._image: str | None = image - # Optional agent skills injected per task (A/B: baseline vs. treated via ``with_skills``). How they - # reach the harness is resolved once per run (the adapter is constant across the taskset) in - # ``run_tasks``; only touched when a skill is set, so the no-skill path stays dependency-free. Names - # must be unique — each stages to its own ``/`` bundle, so a repeat would collide. - self._skill_set = SkillSet(tuple(skills or ())) - - def with_skills(self, skills: Sequence[AgentSkill]) -> FabricContainerRuntime: - """Return a copy of this runtime with ``skills`` *added* to its skill set; ``self`` is not modified. - - Mirrors :meth:`FabricAgentRuntime.with_skills`: additive and chainable - (``rt.with_skills([a]).with_skills([b])`` injects both), so an A/B eval derives a treated runtime - from a skill-free baseline (``baseline.with_skills(the_skills)``) and the arms differ in exactly the - injected skills. Names must be unique across the combined set (colliding ``/`` bundles), so - re-adding a present skill raises. A shallow copy suffices — the shared fields are immutable - config/paths/provider. (``run_tasks`` disposes the injected provider on completion, so an A/B run - over two arms should give each arm its own provider.) - """ - clone = copy.copy(self) - clone._skill_set = self._skill_set.with_skills(skills) - return clone - - def with_skill(self, skill: AgentSkill) -> FabricContainerRuntime: - """Return a copy of this runtime with ``skill`` *added*; ``self`` is not modified. - - Thin wrapper over :meth:`with_skills` for the single-skill case; equally chainable - (``rt.with_skill(a).with_skill(b)`` injects both). - """ - return self.with_skills([skill]) - - async def resolve_secrets(self, secret_resolver: SecretResolver) -> None: - """Resolve declared ``SecretRef``\\ s to values, keyed by the env var each harness reads. - - Mirrors ``MetricWithSecrets.resolve_secrets``: the resolver is owned by the orchestrator (the - AgentEvaluator / execution backend), not the runner. Call before :meth:`run_tasks`; a standalone - ``run_tasks`` falls back to local env resolution when this was not called. - """ - env: dict[str, str] = {} - for env_var, secret_ref in self._secrets.items(): - value = await secret_resolver.resolve_secret(secret_ref) - if value is None: - raise ValueError(f"could not resolve secret {secret_ref.root!r} for env var {env_var!r}") - env[env_var] = value - self._resolved_env = env - self._secrets_resolved = True - - def runner_info(self) -> RunnerInfo: - """Identify this runner and the Fabric container settings that shape its results. - - Records the provider only — never ``self._secrets``, which is persisted nowhere. - """ - return RunnerInfo( - name="fabric_container", - kind="runner", - config={ - "provider": self._provider.name, - "image": self._image, - "adapter_id": self._adapter_id(), - "skills": [skill.name for skill in self._skill_set.skills], - }, - ) - - async def run_tasks( - self, - tasks: Sequence[AgentEvalTask], - config: AgentEvalRunConfig | None = None, - ) -> Sequence[AgentEvalTrial]: - resolved_config = config or AgentEvalRunConfig() - semaphore = asyncio.Semaphore(resolved_config.parallelism) - - async def run_one(index: int, task: AgentEvalTask, skill_mode: SkillMode | None) -> AgentEvalTrial: - async with semaphore: - logger.info("running task", extra={"index": index + 1, "task_id": task.id}) - result = await self._run_task(index, task, resolved_config, skill_mode) - logger.info("task completed", extra={"index": index + 1, "task_id": task.id}) - return result - - try: - # Provision the harness-agnostic Fabric image once, build-if-missing (a first build compiles - # nemo-fabric — minutes); keep the blocking build off the shared event loop. Inside the guard - # so the provider is disposed even if provisioning or secret resolution raises. - if self._image is None: - self._image = await asyncio.to_thread(ensure_fabric_image) - if self._secrets and not self._secrets_resolved: - # No orchestrator resolved our secrets (standalone run) — fall back to local env resolution. - await self.resolve_secrets(LocalSecretResolver()) - # Resolve once (the adapter is constant across the taskset) how a skill reaches this harness, by - # probing Fabric's capability planner — the same authoritative routing the host runtime uses. - # Fail fast rather than silently run a skill-free trial mislabeled "with skill". Blocking pyo3 - # planning, so keep it off the shared event loop; only reached when a skill is set. - skill_mode = await asyncio.to_thread(self._resolve_skill_mode) if self._skill_set.skills else None - if self._skill_set.skills and skill_mode is None: - raise RuntimeError( - f"FabricContainerRuntime received one or more skills but adapter {self._adapter_id()!r} " - "has no known skill-injection strategy: Fabric does not route skills to it natively and " - "it is not a codex harness. Use a skills-native or codex harness, or drop the skills." - ) - return await asyncio.gather(*(run_one(index, task, skill_mode) for index, task in enumerate(tasks))) - finally: - # Each sandbox tears itself down; the provider is shared across the batch, so its - # process-wide resources are disposed once here, when the batch completes. - await self._provider.aclose() - - async def _run_task( - self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig, skill_mode: SkillMode | None - ) -> AgentEvalTrial: - evidence_dir = self._evidence_dir(index, task, config) - out_dir = evidence_dir / "out" - evidence_dir.mkdir(parents=True, exist_ok=True) - - # The whole per-task flow — framing input, seeding, exec, download, and parsing the result — is - # guarded so any failure (bad seed, sandbox crash, unreadable result) fails only this task's - # trial rather than aborting the gathered batch. - skill_provenances: list[SkillProvenance] = [] - try: - seed_files, skill_provenances = self._seed_files(task, skill_mode) - spec = SandboxSpec( - image=self._image, workdir=_WORKSPACE_DIR, env=dict(self._resolved_env), files=seed_files - ) - async with AsyncSandbox(self._provider, spec) as sandbox: - await sandbox.start() - await self._seed_workspace(sandbox, task) - result = await sandbox.exec(self._fabric_command(), timeout_s=DEFAULT_FABRIC_TIMEOUT_S) - await sandbox.download_dir(_OUT_DIR, out_dir) - # Codex self-injection seeds each bundle inside the workspace so the harness discovers it during - # the run; drop them from the downloaded evidence before the workspace is exposed (else the - # injected files read as agent output to workspace-reading metrics). Native staging lives under - # /in, which is never downloaded, so it never pollutes the evidence. - if skill_mode == SKILL_MODE_CODEX_SKILLS_DIR: - for provenance in skill_provenances: - await asyncio.to_thread(_remove_injected_bundle, out_dir / "workspace", provenance["location"]) - return self._to_trial(task, out_dir, evidence_dir, result, skill_provenances=skill_provenances) - except Exception as exc: # noqa: BLE001 - a task failure must not abort the whole run - # Stamp runtime + image + skills even on failures before _to_trial (startup/seeding/download). - return self._failed_trial( - task, evidence_dir, exc, extra_metadata={**self._base_metadata(), **_skill_metadata(skill_provenances)} - ) - - def _resolve_skill_mode(self) -> SkillMode | None: - """Ask Fabric how a skill would reach the selected harness, or ``None`` if it can't. - - Mirrors :meth:`FabricAgentRuntime._resolve_skill_mode`: plan a copy of the config with a sentinel - skill path attached (it need not exist on disk) and read how the adapter routes skills from the - capability plan. Querying the authoritative planner at runtime means any adapter that declares - native skills support — ours or an end-user's — is picked up without a hardcoded list. ``nemo_fabric`` - is imported lazily on the host (only when a skill is set), so the no-skill path never needs it. - """ - try: - from nemo_fabric import Fabric, FabricConfig # ty: ignore[unresolved-import] - except ImportError as exc: - raise RuntimeError(_MISSING_FABRIC_MSG) from exc - probe_config = FabricConfig.from_mapping(self._config) - probe_config.add_skill_path(_SKILL_PROBE_PATH) - plan = Fabric().plan(probe_config) - return resolve_skill_mode(capability_plan=plan.capability_plan, harness=plan.adapter.harness) - - def _adapter_id(self) -> str: - """The harness adapter id declared by the config mapping (for provenance + error messages).""" - harness = self._config.get("harness") - adapter_id = harness.get("adapter_id") if isinstance(harness, Mapping) else None - return str(adapter_id) if adapter_id is not None else "" - - def _fabric_command(self) -> str: - """The ``fabric run`` invocation: pre-create the /out dirs Fabric chdirs into, run, capture stdout.""" - run = f"fabric run {shlex.quote(_AGENT_PATH)} --input-file {shlex.quote(_INPUT_PATH)}" - return ( - f"mkdir -p {_WORKSPACE_DIR} {_RELAY_DIR} {_ARTIFACTS_DIR} {_LOGS_DIR} && " - f"{run} > {shlex.quote(_RESULT_PATH)} 2> {shlex.quote(_FABRIC_STDERR)}" - ) - - def _seed_files( - self, task: AgentEvalTask, skill_mode: SkillMode | None - ) -> tuple[dict[str, str], list[SkillProvenance]]: - """Return (files to seed into the sandbox, skill provenances). - - The agent config is written as JSON, which the Fabric CLI parses as YAML. Fabric 0.1.0rc2 removed - profile overlays (``--profile`` and the ``profiles`` config key are both gone), so everything — - the runtime's in-container settings and any natively-injected skill paths — is composed into the - single agent config here. When skills are injected each bundle is also rendered into the seed set - at the harness's in-sandbox discovery path (native: ``/in/skills/``; codex: - ``/.agents/skills/``). - """ - skill_paths: list[str] = [] - provenances: list[SkillProvenance] = [] - files: dict[str, str] = {_INPUT_PATH: task.agent_prompt()} - if self._skill_set.skills and skill_mode is not None: - if skill_mode == SKILL_MODE_CODEX_SKILLS_DIR: - _check_codex_skill_collision(self._skill_set.skills, task.inputs.get(SEED_FILES_INPUT_KEY) or {}) - seed = stage_skills_seed( - skills=self._skill_set.skills, - adapter_id=self._adapter_id(), - mode=skill_mode, - workspace_dir=_WORKSPACE_DIR, - skills_dir=_SKILLS_DIR, - ) - files.update(seed.files) - skill_paths = seed.skill_paths - provenances = seed.provenances - files[_AGENT_PATH] = json.dumps(self._composed_config(skill_paths)) - return files, provenances - - def _composed_config(self, skill_paths: Sequence[str] = ()) -> dict[str, Any]: - """The supplied agent config with the runtime's in-container settings merged on last. - - Mirrors the host runtime's ``_compose_config``: the workspace, artifact roots, trajectory - telemetry, and any natively-injected skill paths are evaluator-owned, so they are applied over - whatever the caller's config declared. Stays plain dicts rather than round-tripping through the - host's ``FabricConfig`` — the sandbox may run a different Fabric build, so the config is only - required to survive JSON transport, not to validate against the host's schema. - - Injected skill paths are APPENDED to the config's own ``skills.paths`` — mirroring - ``FabricConfig.add_skill_path`` — so skills the caller preconfigured survive injection and the - treated A/B arm differs from the baseline by exactly the injected skills. - """ - config = dict(self._config) - - # Each section is spread over the caller's, so sibling keys survive — pinning - # ``runtime.artifacts`` must not drop a configured ``runtime.transport``. - config["runtime"] = {**_section(config, "runtime"), "artifacts": _ARTIFACTS_DIR} - # ``provider: local`` is required by the native planner in the container (it does not inject the - # Python default), and the workspace pins the harness cwd to the retrievable /out subtree. - config["environment"] = { - **_section(config, "environment"), - "provider": "local", - "workspace": _WORKSPACE_DIR, - "artifacts": _ARTIFACTS_DIR, - } - # Relay ATIF/ATOF file exporter (sdk mode), built from nemo_relay's typed config via the shared - # helper so it stays a single source of truth with the host runtime. Replaced wholesale. - # ``agent_name`` distinguishes this runtime from the host one; ``agent_version`` records the - # agent framework and so matches the host's value, letting an ATIF consumer group both - # runtimes' traces. (Neither is a real version yet — see _common.trajectory_telemetry.) - config["telemetry"] = _common.trajectory_telemetry( - relay_dir=_RELAY_DIR, agent_name=_RUNTIME_NAME, agent_version=_common.FABRIC_AGENT_VERSION - ) - - declared_paths = _section(config, "skills").get("paths") or [] - merged_paths = list(dict.fromkeys([*(str(path) for path in declared_paths), *skill_paths])) - if merged_paths: - config["skills"] = {**_section(config, "skills"), "paths": merged_paths} - return config - - async def _seed_workspace(self, sandbox: AsyncSandbox, task: AgentEvalTask) -> None: - seeds = task.inputs.get(SEED_FILES_INPUT_KEY) - if not seeds: - return - # Transient host-side staging (a tmpdir, not part of the evidence bundle): seed with the SDK - # handlers, then upload across the boundary. seed_workspace is synchronous and a handler may do - # blocking I/O (e.g. a fileset download), so run it off the event loop shared by concurrent tasks. - with tempfile.TemporaryDirectory(prefix="nemo-fabric-seed-") as staging_dir: - staging = Path(staging_dir) - await asyncio.to_thread(seed_workspace, staging, seeds) - await sandbox.upload_dir(staging, _WORKSPACE_DIR) - - def _base_metadata(self) -> dict[str, object]: - """Metadata stamped on every trial from this runtime (success or failure), incl. the resolved image.""" - return {"runtime": _RUNTIME_NAME, "image": self._image, "sandbox_provider": self._provider.name} - - def _to_trial( - self, - task: AgentEvalTask, - out_dir: Path, - evidence_dir: Path, - result: SandboxExecResult, - *, - skill_provenances: list[SkillProvenance] | None = None, - ) -> AgentEvalTrial: - # Skill provenance (name + content hash + injection mode) rides on every trial for the A/B diff: - # a ``skills`` list plus the historical lone ``skill`` field, matching the host FabricAgentRuntime. - base_metadata = {**self._base_metadata(), **_skill_metadata(skill_provenances or [])} - - # Gate on the exec outcome first: a timed-out or non-zero ``fabric run`` is untrustworthy even - # when a stale/partial fabric_result.json is left behind (the shell ``>`` redirect truncates the - # file regardless), so never grade such a run off that file. - if result.error_type or result.return_code != 0: - stderr = _read_text(out_dir / "logs" / "fabric-stderr.txt") or (result.stderr or "") - detail = stderr.strip() or result.error_type or f"exit code {result.return_code}" - return self._failed_trial( - task, evidence_dir, RuntimeError(f"fabric run failed: {detail}"), extra_metadata=base_metadata - ) - - result_path = out_dir / "fabric_result.json" - result_payload = _read_json(result_path) - # `fabric run` writes a normalized RunResult object (a failed harness run still produces one, with - # status != "succeeded"). A missing, non-object, or unreadable payload means no usable result. - if not isinstance(result_payload, Mapping): - stderr = _read_text(out_dir / "logs" / "fabric-stderr.txt") or (result.stderr or "") - return self._failed_trial( - task, - evidence_dir, - RuntimeError(f"fabric run produced no usable result: {stderr.strip()}"), - extra_metadata=base_metadata, - ) - - status = str(result_payload.get("status")) - if status != "succeeded": - return self._failed_trial(task, evidence_dir, _result_error(result_payload), extra_metadata=base_metadata) - - return AgentEvalTrial( - id=f"{task.id}:fabric_container", - task_id=task.id, - status=AgentEvalTrialStatus.COMPLETED, - output=AgentOutput( - # ``response`` is the RunResult *output* payload (matching the host FabricAgentRuntime), - # not the whole normalized envelope, so metrics reading ``sample.response`` see one shape. - output_text=_common.extract_output_text(result_payload.get("output")), - response=cast(JsonValue, result_payload.get("output")), - metadata={**base_metadata, "evidence_dir": str(evidence_dir)}, - ), - evidence=self._evidence(out_dir, result_path), - metadata={**base_metadata, "generated": True, "agent_ok": True}, - ) - - def _evidence(self, out_dir: Path, result_path: Path) -> CandidateEvidence: - descriptors: dict[str, EvidenceDescriptor] = { - "result": EvidenceDescriptor(kind="json", format="json", ref=str(result_path)), - } - workspace_dir = out_dir / "workspace" - if workspace_dir.is_dir(): - descriptors["workspace"] = EvidenceDescriptor(kind="filesystem", ref=str(workspace_dir)) - logs_dir = out_dir / "logs" - if logs_dir.is_dir(): - descriptors[EVIDENCE_LOGS] = EvidenceDescriptor(kind="logs", ref=str(logs_dir)) - atif = _find_atif(out_dir / "relay") - if atif is not None: - descriptors[EVIDENCE_TRACE] = EvidenceDescriptor( - kind=EVIDENCE_TRACE, format=EVIDENCE_FORMAT_ATIF, ref=str(atif) - ) - return CandidateEvidence( - descriptors=descriptors, - metadata={"runtime": _RUNTIME_NAME, "sandbox_provider": self._provider.name, "image": self._image}, - ) - - def _failed_trial( - self, - task: AgentEvalTask, - evidence_dir: Path, - error: Exception | Mapping[str, object], - *, - extra_metadata: Mapping[str, object] | None = None, - ) -> AgentEvalTrial: - # Bind this runtime's name + trial-id suffix to the shared FAILED-trial builder. - return _common.build_failed_trial( - task, - evidence_dir, - error, - runtime_name=_RUNTIME_NAME, - trial_id_suffix=_RUNTIME_NAME, - extra_metadata=extra_metadata, - ) - - def _evidence_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> Path: - # Evidence lands under the run's output dir (like every other runtime); the container's own - # working state lives at /out inside the sandbox and is downloaded here. - root = (config.work_dir or Path.cwd()) / "evidence" / "fabric_container" - return root / _common.task_subdir_name(index, task.id) - - -def _to_mapping(config: FabricConfig | Mapping[str, Any]) -> dict[str, Any]: - """Normalize a typed Fabric config or a plain mapping to a plain dict for JSON transport.""" - # A typed Fabric config exposes ``to_mapping()``; a plain mapping is used as-is. Both are - # str-keyed at runtime, but the getattr + optional (unresolved) ``FabricConfig`` type defeat static - # narrowing, so cast the known-good source before building the dict. - to_mapping = getattr(config, "to_mapping", None) - source = to_mapping() if callable(to_mapping) else config - return dict(cast(Mapping[str, Any], source)) - - -def _skill_metadata(provenances: list[SkillProvenance]) -> dict[str, object]: - """Trial-metadata fields describing the injected skill set (the A/B provenance). - - ``skills`` is the full list of injected-skill provenances (empty = baseline). ``skill`` keeps the - historical single-provenance field — the lone provenance for a one-skill run, else ``None`` — so - single-skill consumers (e.g. ``SkillUsedMetric``) and existing trials/tests keep working unchanged. - Mirrors ``FabricAgentRuntime._skill_metadata``. - """ - return {"skill": provenances[0] if len(provenances) == 1 else None, "skills": provenances} - - -def _check_codex_skill_collision(skills: Sequence[AgentSkill], task_files: Mapping[str, object]) -> None: - """Raise if a task seed file targets the same bundle dir as a runtime-injected codex skill. - - ``.agents/skills/`` holds skills from two independent, equally valid sources: the runtime - ``skills`` parameter (the A/B knob — staged into the workspace before the sandbox starts) and - the task's own ``files`` inputs (skills the task definition always ships — uploaded after it - starts). Tasks are free to seed their own skills there; only writing the *same* - ``.agents/skills//`` from both sources is a conflict, since the task upload lands second - and would overwrite the injected bundle, leaving the stamped provenance hash describing content - the agent never saw. Fail that case rather than emit a silently mislabeled A/B trial. - """ - for skill in skills: - injected_bundle = PurePosixPath(CODEX_SKILLS_DIR) / skill.name - for rel_path in task_files: - seed = PurePosixPath(rel_path) - if seed == injected_bundle or injected_bundle in seed.parents: - raise SkillInjectionError( - f"task seed file {str(rel_path)!r} writes into {str(injected_bundle)!r}, which is " - f"also injected as the runtime skill {skill.name!r}; the task upload would overwrite " - "the injected bundle. Inject this skill via the runtime ``skills`` parameter or ship " - "it in the task's files, not both" - ) - - -def _remove_injected_bundle(workspace_dir: Path, location: str) -> None: - """Remove the Codex-injected skill subtree from a downloaded ``workspace`` dir and prune emptied parents. - - ``location`` is workspace-relative (``.agents/skills/``). Best-effort and mirrors the host - runtime's cleanup: the skill was already captured in the run's trajectory, so SkillUsedMetric (which - reads the trace, not the workspace) is unaffected, and any filesystem error here must not fail an - otherwise-successful trial. - """ - if not workspace_dir.is_dir(): - return - workspace_root = workspace_dir.resolve() - injected = (workspace_dir / location).resolve() - # Guard against a location escaping the workspace (defensive; provenance is evaluator-authored). - if workspace_root not in injected.parents or not injected.exists(): - return - shutil.rmtree(injected, ignore_errors=True) - # Prune now-empty reserved parents (``.agents/skills``, ``.agents``) but never the workspace itself. - parent = injected.parent - while parent != workspace_root and parent.is_dir(): - try: - parent.rmdir() # only succeeds while empty - except OSError: - break - parent = parent.parent - - -def _section(config: Mapping[str, Any], name: str) -> dict[str, Any]: - """A top-level config section as a plain dict — ``{}`` when absent or not a mapping.""" - value = config.get(name) - return dict(value) if isinstance(value, Mapping) else {} - - -def _find_atif(relay_dir: Path) -> Path | None: - # Relay nests the trajectory under a per-run subdir (relay/runtime-/trajectory-*.atif.json), - # so search recursively rather than only relay's direct children. - if not relay_dir.is_dir(): - return None - matches = sorted(relay_dir.rglob("trajectory-*.atif.json")) - return matches[0] if matches else None - - -def _read_json(path: Path) -> JsonValue | None: - if not path.is_file(): - return None - # A truncated/binary/unreadable result (e.g. a crashed CLI that left partial or non-UTF-8 bytes) - # is treated as "no usable result" rather than propagating and aborting the batch. - try: - return json.loads(path.read_text(encoding="utf-8")) - except (json.JSONDecodeError, UnicodeDecodeError, OSError): - return None - - -def _read_text(path: Path) -> str: - try: - return path.read_text(encoding="utf-8") if path.is_file() else "" - except (UnicodeDecodeError, OSError): - return "" - - -def _result_error(payload: object) -> Mapping[str, object]: - if not isinstance(payload, Mapping): - return {"code": "FabricError", "message": "Fabric run did not produce a result"} - error = payload.get("error") - if isinstance(error, Mapping): - return {"stage": error.get("stage"), "code": error.get("code"), "message": error.get("message")} - return {"code": payload.get("status"), "message": "Fabric run did not succeed"} diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/hook_loading.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/hook_loading.py deleted file mode 100644 index 3c3d893dc1..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/hook_loading.py +++ /dev/null @@ -1,141 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Load :class:`FabricTaskRunHook` implementations from string references. - -Authors register hooks without baking agent-specific code into the platform. -YAML may point at: - -* ``ref`` — ``module.path:Attr`` (importable object) -* ``path`` + ``attr`` — Python file on disk (no package install required) -* ``entry_point`` / ``type`` — name under ``nemo.fabric.task_hooks`` - -Remaining mapping keys are forwarded as constructor kwargs. -""" - -import importlib -import importlib.metadata -import importlib.util -import sys -from collections.abc import Mapping -from pathlib import Path -from typing import Any - -from nemo_platform.beta.evaluator.agent_eval.runtimes.fabric.hooks import FabricTaskRunHook - -FABRIC_TASK_HOOKS_GROUP = "nemo.fabric.task_hooks" - -_RESERVED = frozenset({"ref", "path", "attr", "entry_point", "type"}) - - -class FabricTaskHookLoadError(RuntimeError): - """Raised when a Fabric task-hook reference cannot be resolved or constructed.""" - - -def load_fabric_task_hook(spec: Mapping[str, Any] | None) -> FabricTaskRunHook | None: - """Construct a task hook from a mapping, or return ``None`` when ``spec`` is unset.""" - if spec is None: - return None - if not isinstance(spec, Mapping): - raise FabricTaskHookLoadError("run_hook spec must be a mapping when set.") - - ref = _optional_str(spec.get("ref")) - path = _optional_str(spec.get("path")) - attr = _optional_str(spec.get("attr")) - entry_point = _optional_str(spec.get("entry_point")) or _optional_str(spec.get("type")) - - modes = [bool(ref), bool(path), bool(entry_point)] - if sum(modes) == 0: - raise FabricTaskHookLoadError( - "run_hook requires one of: ref (module:attr), path+attr (file), or entry_point/type (nemo.fabric.task_hooks)." - ) - if sum(modes) > 1: - raise FabricTaskHookLoadError("run_hook accepts only one of: ref, path, or entry_point/type.") - - if path and not attr: - raise FabricTaskHookLoadError("run_hook.path requires run_hook.attr (class or factory name).") - - if ref: - target = _load_from_ref(ref) - elif path: - target = _load_from_path(Path(path).expanduser(), attr=attr or "") - else: - target = _load_from_entry_point(entry_point or "") - - kwargs = {key: value for key, value in spec.items() if key not in _RESERVED} - return _construct_hook(target, kwargs) - - -def _construct_hook(target: Any, kwargs: dict[str, Any]) -> FabricTaskRunHook: - if callable(target) and not isinstance(target, type): - # Module-level factory function. - hook = target(**kwargs) if kwargs else target() - elif isinstance(target, type): - hook = target(**kwargs) if kwargs else target() - else: - if kwargs: - raise FabricTaskHookLoadError("run_hook target is already an instance; constructor kwargs are not allowed.") - hook = target - - for method in ("prepare", "after_success", "cleanup"): - if not callable(getattr(hook, method, None)): - raise FabricTaskHookLoadError(f"run_hook object missing required method {method!r}.") - return hook # type: ignore[return-value] - - -def _load_from_ref(ref: str) -> Any: - module_name, _, attr_path = ref.partition(":") - if not module_name or not attr_path: - raise FabricTaskHookLoadError(f"run_hook.ref must look like 'module.path:Attr', got {ref!r}.") - try: - module = importlib.import_module(module_name) - except ImportError as exc: - raise FabricTaskHookLoadError(f"Could not import run_hook.ref module {module_name!r}.") from exc - return _resolve_attr(module, attr_path, label=f"run_hook.ref {ref!r}") - - -def _load_from_path(path: Path, attr: str) -> Any: - resolved = path.resolve() - if not resolved.is_file(): - raise FabricTaskHookLoadError(f"run_hook.path does not exist: {resolved}") - module_name = f"_nemo_fabric_task_hook_{resolved.stem}_{abs(hash(str(resolved)))}" - spec = importlib.util.spec_from_file_location(module_name, resolved) - if spec is None or spec.loader is None: - raise FabricTaskHookLoadError(f"Could not load run_hook.path: {resolved}") - module = importlib.util.module_from_spec(spec) - sys.modules[module_name] = module - try: - spec.loader.exec_module(module) - except Exception as exc: - sys.modules.pop(module_name, None) - raise FabricTaskHookLoadError(f"Failed executing run_hook.path {resolved}: {exc}") from exc - return _resolve_attr(module, attr, label=f"run_hook.path attr {attr!r}") - - -def _load_from_entry_point(name: str) -> Any: - matches = [ep for ep in importlib.metadata.entry_points(group=FABRIC_TASK_HOOKS_GROUP) if ep.name == name] - if not matches: - raise FabricTaskHookLoadError( - f"No entry point {name!r} in group {FABRIC_TASK_HOOKS_GROUP!r}. " - "Authors register hooks via packaging entry points, or use run_hook.ref / run_hook.path." - ) - try: - return matches[0].load() - except Exception as exc: - raise FabricTaskHookLoadError(f"Failed to load entry point {name!r} from {FABRIC_TASK_HOOKS_GROUP!r}.") from exc - - -def _resolve_attr(module: Any, attr_path: str, label: str) -> Any: - current = module - for part in attr_path.split("."): - if not hasattr(current, part): - raise FabricTaskHookLoadError(f"{label} not found.") - current = getattr(current, part) - return current - - -def _optional_str(value: Any) -> str | None: - if value is None: - return None - text = str(value).strip() - return text or None diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/hooks.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/hooks.py deleted file mode 100644 index debda3c94a..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/hooks.py +++ /dev/null @@ -1,54 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Per-task lifecycle hooks for :class:`FabricAgentRuntime`. - -Fabric already accepts a complete typed config per ``Fabric.run``. These hooks -exist so callers (e.g. optimize trials) can wrap each task with agent-specific -ephemeral state — run-scoped MCP bindings, credential handoffs — without -baking that logic into the runtime or into Fabric itself. -""" - -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any, Protocol - -from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalTask - - -@dataclass -class FabricTaskRunSession: - """Mutable bag owned by a hook for one task invocation.""" - - state: dict[str, Any] = field(default_factory=dict) - - -class FabricTaskRunHook(Protocol): - """Optional prepare / after-success / cleanup around one Fabric task run.""" - - def prepare( - self, - config: Any, - task: AgentEvalTask, - evidence_dir: Path, - workspace_dir: Path, - session: FabricTaskRunSession, - ) -> Any: - """Return the config that should be passed to ``Fabric.run`` for this task. - - ``config`` is a composed ``nemo_fabric.FabricConfig`` (typed when Fabric is installed). - """ - - def after_success( - self, - task: AgentEvalTask, - result: Any, - session: FabricTaskRunSession, - ) -> dict[str, Any] | None: - """Optional extras merged into trial ``output.metadata`` / ``metadata`` on success. - - ``result`` is a Fabric ``RunResult``. Raise to fail the trial (e.g. analyzer audit failed). - """ - - def cleanup(self, session: FabricTaskRunSession) -> None: - """Always invoked in ``finally`` after the task attempt (success or failure).""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/hooks_mcp_binding.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/hooks_mcp_binding.py deleted file mode 100644 index 0c3ef853c7..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/hooks_mcp_binding.py +++ /dev/null @@ -1,440 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Platform Fabric task hook for per-task MCP bindings (path-first). - -**Static MCP (no hook):** declare ``mcp.servers`` (transport, url, exposure, env, args) in the -optimize / Fabric YAML. That is the hero path for fixed stdio MCP servers. - -**Bound MCP (this hook):** use when the agent needs run-scoped MCP state (private input -binding, audit/verify, optional credential handoff). Configure via:: - - eval: - run_hook: - type: mcp_run_binding - agent_src: ${AGENT_SRC} # path-first: checkout .../src on sys.path - bindings: - - server: my-mcp # must match mcp.servers key - binding: my_pkg.audit:RunBinding - executable: ${AGENT_MCP_BIN} # MCP process from agent's own venv - config_paths: [settings.yaml] - handoff: # optional; at most one per binding - env: NVIDIA_API_KEY - ref: my_pkg.handoff:CredentialHandoff - -``mcp.servers`` still owns transport / placeholder url / exposure / env / args. This hook -only rebinds ``url`` to ``binding.mcp_command`` after ``Binding.create``, preserving -top-level ``env`` and ``args``. - -**Agent protocol (duck-typed, in the agent checkout):** - -* ``Binding.create(prompt, parent, **kwargs) -> binding`` -* ``binding.mcp_command`` — path/URL for this task -* ``binding.verify()`` or ``verify_exactly_once()`` — fail the trial on audit breach -* ``binding.cleanup()`` -* Optional handoff: ``Handoff.start(credential, timeout_seconds=...)`` with - ``.socket_path`` / ``.token`` / ``.close()`` - -Path isolation: do **not** pip-install the agent into the platform venv. Point -``agent_src`` at the checkout and ``executable`` at the agent-owned MCP binary. -Binding/handoff modules load into the platform process — keep them lightly dependent; -heavy runtime stays behind the MCP stdio boundary. -""" - -from __future__ import annotations - -import importlib -import importlib.util -import inspect -import json -import logging -import os -import sys -from collections.abc import Mapping, Sequence -from pathlib import Path -from typing import Any - -logger = logging.getLogger(__name__) - - -class McpRunBindingHookError(RuntimeError): - """Raised when MCP run-binding configuration or lifecycle fails.""" - - -def _load_ref(ref: str) -> Any: - """Load ``module.path:Attr`` or ``/abs/or/rel/file.py:Attr``.""" - module_name, _, attr = ref.partition(":") - if not module_name or not attr: - raise McpRunBindingHookError(f"ref must look like 'module.path:Attr' or 'file.py:Attr', got {ref!r}") - - path = Path(module_name).expanduser() - if path.suffix == ".py" or path.is_file(): - resolved = path.resolve() - if not resolved.is_file(): - raise McpRunBindingHookError(f"ref file does not exist: {resolved}") - mod_name = f"_mcp_run_binding_{resolved.stem}_{abs(hash(str(resolved)))}" - spec = importlib.util.spec_from_file_location(mod_name, resolved) - if spec is None or spec.loader is None: - raise McpRunBindingHookError(f"could not load ref file: {resolved}") - module = importlib.util.module_from_spec(spec) - sys.modules[mod_name] = module - spec.loader.exec_module(module) - else: - module = importlib.import_module(module_name) - - current: Any = module - for part in attr.split("."): - current = getattr(current, part) - return current - - -def _resolve_target(value: Any) -> Any: - """Resolve a string ref or pass through an already-imported class/callable.""" - if isinstance(value, str): - return _load_ref(value.strip()) - if value is None: - raise McpRunBindingHookError("binding/handoff ref is required") - return value - - -def _prepend_sys_path(path: str | Path) -> None: - resolved = str(Path(path).expanduser().resolve()) - if resolved not in sys.path: - sys.path.insert(0, resolved) - - -def _as_path_list(value: Any) -> list[Path]: - if value is None: - return [] - if isinstance(value, (str, Path)): - items: Sequence[Any] = [value] - elif isinstance(value, Sequence): - items = value - else: - raise McpRunBindingHookError(f"config_paths must be a path or list of paths, got {type(value)!r}") - paths: list[Path] = [] - for item in items: - path = Path(item).expanduser() - if not path.is_file(): - raise McpRunBindingHookError(f"config path does not exist: {path}") - paths.append(path.resolve()) - return paths - - -def _filter_kwargs(fn: Any, kwargs: dict[str, Any]) -> dict[str, Any]: - try: - params = inspect.signature(fn).parameters - except (TypeError, ValueError): - return kwargs - if any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values()): - return kwargs - return {key: value for key, value in kwargs.items() if key in params} - - -def _as_str_list(value: Any) -> list[str]: - if value is None: - return [] - if isinstance(value, str): - raise McpRunBindingHookError("MCP server args must be a list of strings, not a string") - if isinstance(value, Sequence): - return [str(item) for item in value] - raise McpRunBindingHookError(f"MCP server args must be a sequence of strings, got {type(value)!r}") - - -def _as_str_map(value: Any) -> dict[str, str]: - if value is None: - return {} - if not isinstance(value, Mapping): - raise McpRunBindingHookError(f"MCP server env must be a mapping, got {type(value)!r}") - return {str(key): str(item) for key, item in value.items()} - - -def _server_snapshot(config: Any, name: str) -> dict[str, Any]: - """Return preserved ``add_mcp_server`` kwargs for an existing MCP server. - - Fabric now owns ``env`` / ``args`` as top-level MCP server fields (not - ``extra_fields``). Legacy snapshots that still stash them under - ``extra_fields`` are lifted to top-level kwargs. - """ - mcp = getattr(config, "mcp", None) - servers = getattr(mcp, "servers", None) or {} - server = servers.get(name) if isinstance(servers, Mapping) else None - if server is None: - return {"transport": "stdio", "exposure": "harness_native"} - - transport = str(getattr(server, "transport", None) or "stdio") - exposure = str(getattr(server, "exposure", None) or "harness_native") - - extra: dict[str, Any] = {} - extra_fields = getattr(server, "extra_fields", None) - if isinstance(extra_fields, Mapping): - extra = dict(extra_fields) - elif callable(extra_fields): - extra = dict(extra_fields()) - elif hasattr(server, "model_extra") and isinstance(server.model_extra, Mapping): - extra = dict(server.model_extra) - - args = _as_str_list(getattr(server, "args", None)) - if not args and "args" in extra: - args = _as_str_list(extra.pop("args")) - - env = _as_str_map(getattr(server, "env", None)) - if not env and "env" in extra: - env = _as_str_map(extra.pop("env")) - - snapshot: dict[str, Any] = {"transport": transport, "exposure": exposure} - if args: - snapshot["args"] = args - if env: - snapshot["env"] = env - if extra: - snapshot["extra_fields"] = extra - return snapshot - - -def _verify_binding(binding: Any) -> Any: - verify = getattr(binding, "verify", None) - if callable(verify): - return verify() - verify_once = getattr(binding, "verify_exactly_once", None) - if callable(verify_once): - try: - return verify_once() - except Exception as exc: - # Agents sometimes re-call the tool after a successful analysis. Prefer the - # audited analysis over failing the whole optimize sample when one exists. - fallback = _audit_from_binding_path(binding) - if fallback is not None and _result_payload(fallback) is not None: - logger.warning( - "MCP binding exactly-once verify failed (%s); using audit analysis anyway", - exc, - ) - return fallback - raise McpRunBindingHookError(str(exc)) from exc - raise McpRunBindingHookError("binding has neither verify() nor verify_exactly_once()") - - -def _audit_from_binding_path(binding: Any) -> Any | None: - """Best-effort read of ``binding.audit_path`` when strict verify fails.""" - path = getattr(binding, "audit_path", None) - if path is None: - return None - audit_path = Path(path) - if not audit_path.is_file(): - return None - try: - payload = json.loads(audit_path.read_text(encoding="utf-8")) - except (OSError, ValueError): - return None - if not isinstance(payload, Mapping): - return None - - class _AuditShim: - def __init__(self, data: Mapping[str, Any]) -> None: - self._data = dict(data) - self.analysis = data.get("analysis") - self.result = data.get("result") - - def public_mapping(self) -> dict[str, Any]: - return {key: self._data[key] for key in ("run_id", "input_sha256", "invocation_count") if key in self._data} - - return _AuditShim(payload) - - -def _audit_mapping(audit: Any) -> dict[str, Any] | None: - public = getattr(audit, "public_mapping", None) - if callable(public): - mapping = public() - return dict(mapping) if isinstance(mapping, Mapping) else {"value": mapping} - if isinstance(audit, Mapping): - return dict(audit) - return None - - -def _result_payload(audit: Any) -> Any: - for attr in ("analysis", "result"): - value = getattr(audit, attr, None) - if value is None: - continue - dump = getattr(value, "model_dump", None) - if callable(dump): - return dump(mode="json") - return value - return None - - -class McpRunBindingHook: - """Ordered per-task MCP binding lifecycle around ``Fabric.run``.""" - - def __init__( - self, - bindings: Sequence[Mapping[str, Any]] | None = None, - *, - agent_src: str | Path | None = None, - pythonpath: str | Path | None = None, - binding_parent: str | Path | None = None, - ) -> None: - src = agent_src if agent_src is not None else pythonpath - if src is not None: - _prepend_sys_path(src) - - if not bindings: - raise McpRunBindingHookError("mcp_run_binding requires a non-empty bindings list") - - self._binding_parent = Path(binding_parent).expanduser() if binding_parent else None - self._entries: list[dict[str, Any]] = [] - for index, raw in enumerate(bindings): - if not isinstance(raw, Mapping): - raise McpRunBindingHookError(f"bindings[{index}] must be a mapping") - server = str(raw.get("server") or "").strip() - if not server or raw.get("binding") is None: - raise McpRunBindingHookError(f"bindings[{index}] requires server and binding") - - handoff_raw = raw.get("handoff") - handoff_env: str | None = None - handoff_cls: Any | None = None - if handoff_raw is not None: - if not isinstance(handoff_raw, Mapping): - raise McpRunBindingHookError(f"bindings[{index}].handoff must be a mapping") - handoff_env = str(handoff_raw.get("env") or "").strip() or None - handoff_ref = handoff_raw.get("ref") - if not handoff_env or handoff_ref is None: - raise McpRunBindingHookError(f"bindings[{index}].handoff requires env and ref") - try: - handoff_cls = _resolve_target(handoff_ref) - except Exception as exc: - raise McpRunBindingHookError( - f"Could not resolve bindings[{index}].handoff.ref={handoff_ref!r}" - ) from exc - - binding_raw = raw.get("binding") - try: - binding_cls = _resolve_target(binding_raw) - except Exception as exc: - raise McpRunBindingHookError( - f"Could not resolve bindings[{index}].binding={binding_raw!r}. " - "Set agent_src to the agent checkout .../src (path-first; do not install " - "the agent into the platform venv)." - ) from exc - - executable_raw = raw.get("executable") - executable = Path(executable_raw).expanduser() if executable_raw else None - if executable is not None and not executable.is_file(): - raise McpRunBindingHookError(f"bindings[{index}].executable does not exist: {executable}") - - config_paths = _as_path_list(raw.get("config_paths") or raw.get("config_path")) - - self._entries.append( - { - "server": server, - "binding_cls": binding_cls, - "handoff_cls": handoff_cls, - "handoff_env": handoff_env, - "executable": executable.resolve() if executable is not None else None, - "config_paths": config_paths, - } - ) - - def prepare(self, config: Any, task: Any, evidence_dir: Path, workspace_dir: Path, session: Any) -> Any: - del workspace_dir - if not hasattr(config, "add_mcp_server"): - raise McpRunBindingHookError("Fabric config does not expose add_mcp_server; cannot rebind MCP.") - - prompt = task.agent_prompt() - parent = self._binding_parent or (evidence_dir / "mcp-bindings") - parent.mkdir(parents=True, exist_ok=True) - - started: list[dict[str, Any]] = [] - session.state["mcp_bindings"] = started - - try: - for entry in self._entries: - handoff = None - handoff_cls = entry["handoff_cls"] - handoff_env = entry["handoff_env"] - if handoff_cls is not None and handoff_env: - credential = os.environ.get(handoff_env) - if credential: - handoff = handoff_cls.start(credential, timeout_seconds=60.0) - - create_kwargs: dict[str, Any] = { - "credential_socket": handoff.socket_path if handoff is not None else None, - "credential_token": handoff.token if handoff is not None else None, - } - if entry["executable"] is not None: - create_kwargs["executable"] = entry["executable"] - config_paths: list[Path] = entry["config_paths"] - if config_paths: - create_kwargs["config_paths"] = config_paths - create_kwargs["config_path"] = config_paths[0] - - try: - binding = entry["binding_cls"].create( - prompt, - parent, - **_filter_kwargs(entry["binding_cls"].create, create_kwargs), - ) - except Exception: - if handoff is not None: - handoff.close() - raise - - # Register before rebinding so prepare failures can still cleanup. - started.append({"server": entry["server"], "binding": binding, "handoff": handoff}) - preserved = _server_snapshot(config, entry["server"]) - config = config.add_mcp_server( - entry["server"], - url=str(binding.mcp_command), - **preserved, - ) - except Exception: - self.cleanup(session) - raise - - return config - - def after_success(self, task: Any, result: Any, session: Any) -> dict[str, Any] | None: - del task, result - started = session.state.get("mcp_bindings") or [] - if not started: - raise McpRunBindingHookError("mcp bindings missing after Fabric.run") - - mcp_bindings: dict[str, Any] = {} - first_result: Any = None - for item in started: - server = item["server"] - binding = item["binding"] - audit = _verify_binding(binding) - entry_extras: dict[str, Any] = {} - mapping = _audit_mapping(audit) - if mapping is not None: - entry_extras["audit"] = mapping - payload = _result_payload(audit) - if payload is not None: - entry_extras["result"] = payload - if first_result is None: - first_result = payload - mcp_bindings[server] = entry_extras - - extras: dict[str, Any] = {"mcp_bindings": mcp_bindings} - # Deprecated alias for one release — FabricAgentRuntime historically read this key. - if first_result is not None: - extras["analyzer_analysis"] = first_result - return extras - - def cleanup(self, session: Any) -> None: - started: list[dict[str, Any]] = list(session.state.pop("mcp_bindings", []) or []) - for item in reversed(started): - binding = item.get("binding") - handoff = item.get("handoff") - server = item.get("server") - try: - if binding is not None: - binding.cleanup() - except Exception: - logger.exception("Failed to cleanup MCP binding for %s", server) - try: - if handoff is not None: - handoff.close() - except Exception: - logger.exception("Failed to close MCP handoff for %s", server) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/image.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/image.py deleted file mode 100644 index 6d218c8589..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/image.py +++ /dev/null @@ -1,173 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Build-if-missing provisioning for the Fabric sandbox image. - -``FabricContainerRuntime`` needs a container image with Fabric + its harness runtimes. Rather than -make callers hand-write a Dockerfile, the SDK owns the recipe (:mod:`sandbox.Dockerfile`, a -multi-stage build) and provisions the image opaquely: :func:`ensure_fabric_image` returns a usable -image tag, building it only when it isn't already present locally. This mirrors the -``ensure_task_image`` build-if-missing pattern (``docker image inspect`` → ``docker build``). - -The tag is content-addressed on the recipe + selected extras, so a recipe change produces a new tag -(cache-bust) and an unchanged recipe reuses the cached image. The Fabric source is private/native -(no public wheel), so the build needs a local NeMo-Fabric checkout — resolved from ``fabric_repo`` / -``$NEMO_FABRIC_REPO`` / ``~/workspace/NeMo-Fabric``. Only the maturin build inputs are staged into the -context (not the whole repo), and the multi-stage build keeps the source and Rust toolchain out of the -final image. - -This is the local-Docker provisioning path. The intended evolution is a remote image registry as a -cache: :func:`ensure_fabric_image` keeps the same "return a usable tag" contract, its body swapping -local build for a registry pull (build-and-push on miss). - -DEPENDENCY (as of July 2026): the multi-stage image installs the ``nemo-fabric`` wheel and discards -the source, so Fabric must be able to resolve built-in adapters *from the installed distribution*. -That only works on NeMo-Fabric's ``installed-adapter-discovery`` branch (which bundles the adapters -under ``python/src/nemo_fabric/adapters`` and adds ``AdapterDescriptorSource::Installed``). On today's -``main`` the wheel ships no adapter descriptors, so a wheel-only image cannot resolve e.g. -``nvidia.fabric.hermes``. Once that lands on ``main``, switch to installing the top-level -``adapters/*`` packages explicitly here instead of relying on the branch's packaging. -""" - -from __future__ import annotations - -import hashlib -import logging -import os -import shutil -import subprocess -import tempfile -from pathlib import Path - -logger = logging.getLogger(__name__) - -# ``localhost/`` prefix so Docker treats it as an explicit local registry and does NOT qualify the tag -# to ``docker.io/…`` — this image is built locally and never pushed to Docker Hub. -DEFAULT_FABRIC_IMAGE_REPO = "localhost/nemo-evaluator/fabric-sandbox" -FABRIC_REPO_ENV = "NEMO_FABRIC_REPO" -_DEFAULT_FABRIC_REPO = Path.home() / "workspace" / "NeMo-Fabric" -_DOCKERFILE = Path(__file__).with_name("sandbox.Dockerfile") - -# Bound the docker subprocess calls so an unresponsive daemon fails fast instead of hanging the runtime. -# ``inspect`` is near-instant; the build compiles nemo-fabric (minutes), so it gets a generous ceiling. -_INSPECT_TIMEOUT_S = 30 -_BUILD_TIMEOUT_S = 3600 - -#: Harness runtime deps baked into the (single, harness-agnostic) Fabric image. The native -#: ``nemo-fabric`` build plus *all* built-in adapter descriptors are always present, so the CLI can -#: resolve any built-in harness; these extras add the per-harness *runtime* deps (``hermes`` → -#: ``hermes-agent``; ``relay`` → the ATIF exporter). Codex additionally needs node + the codex CLI + -#: the nemo-relay gateway binary and is not provisioned yet (see AALGO-321); append it here when ready. -_EXTRAS: tuple[str, ...] = ("hermes", "relay") - -#: Paths under the NeMo-Fabric checkout that the maturin build actually needs. Staging only these -#: (rather than the whole repo) keeps the build context small; the multi-stage build keeps them out -#: of the final image entirely. -_BUILD_SOURCE_PATHS = ("Cargo.toml", "Cargo.lock", "pyproject.toml", "README.md", "crates", "python") - - -class FabricImageError(RuntimeError): - """Raised when the Fabric sandbox image cannot be provisioned.""" - - -def _extras_arg() -> str: - return ",".join(_EXTRAS) - - -def fabric_image_tag(*, repo: str = DEFAULT_FABRIC_IMAGE_REPO) -> str: - """Content-addressed tag for the harness-agnostic Fabric image: ``:``. - - Not keyed by harness: one Fabric install + the bundled adapters runs any built-in harness, so the - image is the same regardless of which harness a task's config selects. - """ - recipe = _DOCKERFILE.read_bytes() + _extras_arg().encode("utf-8") - return f"{repo}:{hashlib.sha256(recipe).hexdigest()[:12]}" - - -def image_exists(tag: str, *, docker_bin: str = "docker") -> bool: - """Whether an image tag is present in the local Docker image store. - - Raises :class:`FabricImageError` when the Docker daemon is unreachable, so a stopped/misconfigured - daemon surfaces as a clear error instead of masquerading as "image absent" and triggering a build - that then also fails confusingly. - """ - try: - result = subprocess.run( - [docker_bin, "image", "inspect", tag], capture_output=True, check=False, timeout=_INSPECT_TIMEOUT_S - ) - except subprocess.TimeoutExpired as exc: - raise FabricImageError( - f"`docker image inspect` timed out after {_INSPECT_TIMEOUT_S}s (daemon unresponsive?)" - ) from exc - if result.returncode == 0: - return True - stderr = result.stderr.decode("utf-8", errors="replace") - if "cannot connect to the docker daemon" in stderr.lower(): - raise FabricImageError(f"cannot reach the Docker daemon (is it running?): {stderr.strip()}") - logger.debug("Fabric image %s not present in local store", tag) - return False - - -def _resolve_fabric_repo(fabric_repo: str | Path | None) -> Path: - default = Path(os.environ.get(FABRIC_REPO_ENV, _DEFAULT_FABRIC_REPO)) - repo = (Path(fabric_repo) if fabric_repo is not None else default).expanduser() - if not (repo / "pyproject.toml").is_file(): - raise FabricImageError( - f"NeMo-Fabric source not found at {repo}. The Fabric image is built from source " - f"(no public wheel); set {FABRIC_REPO_ENV} or pass fabric_repo to point at a checkout." - ) - return repo - - -def _stage_source(repo: Path, dest: Path) -> None: - """Copy only the maturin build inputs from ``repo`` into ``dest`` (not the whole checkout).""" - dest.mkdir(parents=True, exist_ok=True) - for name in _BUILD_SOURCE_PATHS: - src = repo / name - if src.is_dir(): - shutil.copytree(src, dest / name, ignore=shutil.ignore_patterns("target", "__pycache__", "*.whl")) - elif src.is_file(): - shutil.copy2(src, dest / name) - else: - raise FabricImageError(f"expected Fabric build input {name!r} not found under {repo}") - - -def ensure_fabric_image( - *, - fabric_repo: str | Path | None = None, - docker_bin: str = "docker", - force_build: bool = False, -) -> str: - """Return a usable Fabric image tag, building it only if not already present. - - One harness-agnostic image serves every built-in harness. Idempotent and content-addressed: an - unchanged recipe reuses the cached image; a changed recipe yields a new tag. Builds from a staged - copy of the local NeMo-Fabric source (build inputs only). - """ - tag = fabric_image_tag() - if not force_build and image_exists(tag, docker_bin=docker_bin): - logger.debug("Fabric image %s already present; skipping build.", tag) - return tag - - repo = _resolve_fabric_repo(fabric_repo) - logger.info( - "Building Fabric image (first build compiles nemo-fabric; this can take minutes)...", - extra=dict(tag=tag, repo=repo), - ) - with tempfile.TemporaryDirectory(prefix="nemo-fabric-image-") as ctx_dir: - ctx = Path(ctx_dir) - _stage_source(repo, ctx / "nemo-fabric") - shutil.copy2(_DOCKERFILE, ctx / "Dockerfile") - try: - subprocess.run( - [docker_bin, "build", "--build-arg", f"EXTRAS={_extras_arg()}", "-t", tag, str(ctx)], - check=True, - env={**os.environ, "DOCKER_BUILDKIT": "1"}, - timeout=_BUILD_TIMEOUT_S, - ) - except subprocess.TimeoutExpired as exc: - raise FabricImageError(f"docker build timed out after {_BUILD_TIMEOUT_S}s for {tag}") from exc - except subprocess.CalledProcessError as exc: - raise FabricImageError(f"docker build failed for {tag}: {exc}") from exc - logger.info("Built Fabric image %s.", tag, extra=dict(tag=tag)) - return tag diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py deleted file mode 100644 index 712128bd4e..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py +++ /dev/null @@ -1,724 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""NeMo Fabric-backed agent-eval runtime. - -``FabricAgentRuntime`` drives an agent harness (Codex, Hermes, ...) through the -NeMo Fabric Python SDK and adapts each normalized Fabric ``RunResult`` into an -:class:`AgentEvalTrial`. The harness is chosen by the supplied Fabric config's -``harness.adapter_id`` (never inferred from a model); an optional ``model`` slug -is applied as the config's default model, mirroring Fabric's own Harbor integration. - -Per-task settings (workspace, model, trajectory capture) are composed directly onto -a copy of the supplied config via the SDK's config helpers (``model_copy`` + -``enable_relay`` + ``environment``). Fabric removed profile overlays in 0.1.0rc2 — -``FabricConfig`` rejects a ``profiles`` key and ``Fabric.run`` takes no ``profiles`` -argument — so a run is described by exactly one complete typed config, and the -evaluator-owned per-task settings are authoritative simply by being applied last. - -Every task runs in its own fresh workspace: the runtime seeds it from -``inputs['files']`` (a no-op when there are none), runs the harness in it (via -``environment.workspace``), and exposes its final file tree as ``workspace`` -filesystem evidence, so workspace-reading metrics score a Fabric trial alongside -the ATIF trajectory. Any ``environment.workspace`` set in the supplied config is -overridden per task. - -``nemo_fabric`` is an optional native dependency: its types are imported for -annotations under ``TYPE_CHECKING`` and the package is loaded lazily at runtime, -so this module stays importable without it. -""" - -from __future__ import annotations - -import asyncio -import copy -import json -import logging -import shutil -from collections.abc import Mapping, Sequence -from datetime import UTC, datetime -from pathlib import Path -from typing import TYPE_CHECKING, Any -from uuid import uuid4 - -from nemo_platform.beta.evaluator.agent_eval.runtimes.fabric import _common -from nemo_platform.beta.evaluator.agent_eval.runtimes.fabric.hooks import FabricTaskRunHook, FabricTaskRunSession -from nemo_platform.beta.evaluator.agent_eval.runtimes.fabric.skills import ( - SKILL_MODE_CODEX_SKILLS_DIR, - AgentSkill, - SkillMode, - SkillProvenance, - SkillSet, - install_skills, - resolve_skill_mode, -) -from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask -from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput, RunnerInfo -from nemo_platform.beta.evaluator.agent_eval.workspace_seeds import SEED_FILES_INPUT_KEY, seed_workspace -from nemo_platform.beta.evaluator.values.evidence import ( - EVIDENCE_FORMAT_ATIF, - EVIDENCE_TRACE, - CandidateEvidence, - EvidenceDescriptor, -) -from pydantic import JsonValue - -if TYPE_CHECKING: - # Annotations use nemo_fabric's real types (single source of truth). nemo_fabric is an optional - # native package not yet in our locked dependency set, so it is imported for typing only and - # loaded lazily at runtime (see ``run_tasks``). Drop the ty:ignore once nemo-fabric is a - # resolvable dependency and the checker can see it. - from nemo_fabric import ( # ty: ignore[unresolved-import] - Fabric, - FabricConfig, - RelayObservabilityConfig, - RunOutput, - RunResult, - ) - -DEFAULT_FABRIC_TIMEOUT_S = 600 -_RUNTIME_NAME = "fabric" -_MISSING_FABRIC_MSG = "FabricAgentRuntime requires the `nemo-fabric` package (native NeMo Fabric SDK)." -_MISSING_RELAY_MSG = ( - "FabricAgentRuntime trajectory capture requires the `nemo-relay` package " - "(install `nemo-fabric[relay]`), or set capture_trajectory=False." -) - -logger = logging.getLogger(__name__) - -# Evidence-dir layout for trajectory capture. These subdir names are our own local layout — we create -# them and hand them to Fabric/Relay, so they are not derived from either library. -_RELAY_SUBDIR = "relay" -_ARTIFACTS_SUBDIR = "artifacts" -# Per-task workspace: where seed files are staged and where the harness reads/writes. We -# create it, point Fabric's ``environment.workspace`` at it, and expose it as ``workspace`` evidence. -_WORKSPACE_SUBDIR = "workspace" -# Per-task skill staging dir (native injection): the skill's files are resolved here and the staged -# root is added to the task config's ``skills.paths``. For codex self-injection the skill lands in the -# workspace instead (no path added). -_SKILL_SUBDIR = "skill" -# Sentinel skill path attached only to probe Fabric's capability planner for the selected adapter's -# skills routing (see ``_resolve_skill_mode``). Never staged and need not exist on disk — the planner -# just reports how it would route a skill for this adapter. -_SKILL_PROBE_PATH = "nemo-eval-skill-capability-probe" -# Evidence key + descriptor kind for the staged workspace, consumed by the -# workspace-reading metrics. -_WORKSPACE_EVIDENCE_KEY = "workspace" -_WORKSPACE_EVIDENCE_KIND = "filesystem" -# File-exporter output names we choose for the Relay ATIF/ATOF trajectory (Relay accepts these as inputs). -_ATIF_FILENAME_TEMPLATE = "trajectory-{session_id}.atif.json" -_ATOF_FILENAME = "events.atof.jsonl" -# ``kind`` Fabric stamps on the promoted Relay ATIF artifact; used to surface it as trace evidence. -_ATIF_ARTIFACT_KIND = "atif" - - -class FabricAgentRuntime: - """AgentTaskRunner that generates trials by running tasks through NeMo Fabric. - - The harness is selected entirely by ``config["harness"]["adapter_id"]``. Across harnesses the - config shape differs mainly in that ``adapter_id``, ``runtime.transport``, and any harness-specific - ``harness.settings`` — e.g. Codex runs as a subprocess (``transport="cli"``) while the Hermes SDK - harness runs in-library (``transport="library"``). See - ``examples/fabric_harness_runtimes.py`` for full Codex-CLI and Hermes-SDK config examples. - """ - - def __init__( - self, - *, - config: Mapping[str, Any], - model: str | None = None, - base_dir: str | Path | None = None, - work_root: str | Path | None = None, - timeout_s: int = DEFAULT_FABRIC_TIMEOUT_S, - capture_trajectory: bool = True, - trajectory_extra: Mapping[str, Any] | None = None, - runtime_name: str = _RUNTIME_NAME, - skills: Sequence[AgentSkill] | None = None, - task_hook: FabricTaskRunHook | None = None, - ) -> None: - self._config = config - self._model = model - self._base_dir = Path(base_dir).expanduser() if base_dir is not None else None - self._work_root = Path(work_root).expanduser() if work_root is not None else None - self._timeout_s = timeout_s - self._capture_trajectory = capture_trajectory - self._trajectory_extra = dict(trajectory_extra) if trajectory_extra else None - self._runtime_name = runtime_name - self._skill_set = SkillSet(tuple(skills or ())) - self._task_hook = task_hook - - def with_skills(self, skills: Sequence[AgentSkill]) -> FabricAgentRuntime: - """Return a copy of this runtime with ``skills`` *added* to its skill set; ``self`` is not modified. - - Additive and chainable: ``rt.with_skills([a]).with_skills([b])`` injects both a and b. Lets an A/B - eval derive a treated runtime from a skill-free baseline (``baseline.with_skills(the_skills)``) so - the two arms differ in exactly the injected skills. Skill names must be unique across the combined - set — two bundles claiming the same ``/`` would collide — so re-adding a skill already - present raises. A shallow copy suffices — the shared fields are immutable config/paths. - """ - clone = copy.copy(self) - clone._skill_set = self._skill_set.with_skills(skills) - return clone - - def with_skill(self, skill: AgentSkill) -> FabricAgentRuntime: - """Return a copy of this runtime with ``skill`` *added*; ``self`` is not modified. - - Thin wrapper over :meth:`with_skills` for the common single-skill case; equally chainable - (``rt.with_skill(a).with_skill(b)`` injects both). - """ - return self.with_skills([skill]) - - def _adapter_id(self) -> str: - """Harness adapter selected by the Fabric config (empty when unset).""" - harness = self._config.get("harness") if isinstance(self._config, Mapping) else None - adapter_id = harness.get("adapter_id") if isinstance(harness, Mapping) else None - return str(adapter_id) if adapter_id is not None else "" - - def _effective_model(self) -> str | None: - """The model a run will actually use, mirroring :meth:`_compose_config`'s precedence. - - ``_compose_config`` only overwrites the config's default model when ``self._model`` is set, so - a model supplied purely through ``config`` is what runs. Reporting ``self._model`` alone would - record ``None`` for those runs, giving two runs with *different* models identical provenance — - the one thing this metadata exists to prevent. - """ - if self._model: - return self._model - models = self._config.get("models") if isinstance(self._config, Mapping) else None - default = models.get("default") if isinstance(models, Mapping) else None - model = default.get("model") if isinstance(default, Mapping) else getattr(default, "model", None) - return str(model) if model is not None else None - - def runner_info(self) -> RunnerInfo: - """Identify this runner and the Fabric settings that shape its results.""" - return RunnerInfo( - name=self._runtime_name, - kind="runner", - config={ - "model": self._effective_model(), - "timeout_s": self._timeout_s, - "adapter_id": self._adapter_id(), - "skills": [skill.name for skill in self._skill_set.skills], - # Off means no relay/ATIF exporter, so the run captures no trajectory evidence — a - # metric that scores trajectories sees something different. - "capture_trajectory": self._capture_trajectory, - }, - ) - - async def run_tasks( - self, - tasks: Sequence[AgentEvalTask], - config: AgentEvalRunConfig | None = None, - ) -> Sequence[AgentEvalTrial]: - try: - # nemo_fabric ships a native (pyo3) core and is an optional dependency, so it is imported - # lazily here rather than at module load. - from nemo_fabric import Fabric, FabricConfig # ty: ignore[unresolved-import] - except ImportError as exc: - raise RuntimeError(_MISSING_FABRIC_MSG) from exc - - resolved_config = config or AgentEvalRunConfig() - # Assign a run id once per run so two runs (e.g. an A/B baseline vs. skilled variant) written - # under the same work_root/output_dir land in distinct, non-colliding evidence trees. Callers - # that set run_id keep their identifier. - if resolved_config.run_id is None: - resolved_config = resolved_config.model_copy(update={"run_id": _new_run_id()}) - agent_config = FabricConfig.from_mapping(self._config) - # Fail fast (once) if trajectory capture is requested but the nemo-relay gateway isn't - # importable, rather than failing every task the same way inside the per-task guard. - if self._capture_trajectory: - try: - import nemo_relay.observability # noqa: F401 # ty: ignore[unresolved-import] - except ImportError as exc: - raise RuntimeError(_MISSING_RELAY_MSG) from exc - # ``Fabric`` (formerly ``FabricClient``) is a lightweight, reusable facade — not a lifecycle - # context manager — so it is created once and reused across tasks with no cleanup. - client = Fabric() - - # Resolve once how a skill would reach this harness (the adapter is constant across tasks) by - # asking Fabric's own capability planner, so any adapter that declares native skills support — ours - # or an end-user's — is picked up automatically instead of via a hardcoded allow-list. Fail fast - # rather than silently run a skill-free trial mislabeled as "with skill", which would corrupt an - # A/B comparison. Only touched when a skill is set, so the no-skill path is unaffected. - skill_mode: SkillMode | None = None - if self._skill_set.skills: - skill_mode = self._resolve_skill_mode(client, agent_config) - if skill_mode is None: - adapter_id = agent_config.harness.adapter_id - raise RuntimeError( - f"FabricAgentRuntime received one or more skills but adapter {adapter_id!r} has no known " - "skill-injection strategy: Fabric does not route skills to it natively and it is not a " - "codex harness. Use a skills-native or codex harness, or drop the skills." - ) - - semaphore = asyncio.Semaphore(resolved_config.parallelism) - - async def run_one(index: int, task: AgentEvalTask) -> AgentEvalTrial: - async with semaphore: - return await self._run_task(client, agent_config, index, task, resolved_config, skill_mode) - - return await asyncio.gather(*(run_one(index, task) for index, task in enumerate(tasks))) - - def _resolve_skill_mode(self, client: Fabric, agent_config: FabricConfig) -> SkillMode | None: - """Ask Fabric how a skill would reach the selected harness, or ``None`` if it can't. - - Probes Fabric's capability planner: plan a copy of the config with a sentinel skill path attached - (it need not exist on disk) and read how the adapter routes skills. Querying the authoritative - source at runtime means adapters that declare native skills support — ours or an end-user's — are - detected without a hardcoded list. See :func:`~...skills.resolve_skill_mode`. - """ - probe_config = agent_config.model_copy(deep=True) - probe_config.add_skill_path(_SKILL_PROBE_PATH) - plan = client.plan(probe_config, base_dir=self._base_dir) - return resolve_skill_mode(capability_plan=plan.capability_plan, harness=plan.adapter.harness) - - async def _run_task( - self, - client: Fabric, - agent_config: FabricConfig, - index: int, - task: AgentEvalTask, - config: AgentEvalRunConfig, - skill_mode: SkillMode | None, - ) -> AgentEvalTrial: - # nemo_fabric is already imported+validated in ``run_tasks``; this is a cached sys.modules - # lookup, not a re-load, so the types are used where they're constructed instead of threaded down. - from nemo_fabric import RunRequest # ty: ignore[unresolved-import] - - evidence_dir = self._evidence_dir(index, task, config) - evidence_dir.mkdir(parents=True, exist_ok=True) - - # Every task runs in its own fresh workspace: seed any ``inputs['files']`` into it (a no-op when - # there are none), point the harness at it, and expose it as ``workspace`` filesystem evidence — - # a uniform per-task dir that maps cleanly onto a per-task container volume later. Seeding runs - # inside the guarded block so a bad seed (a path escaping the workspace, an unresolvable fileset) - # fails just this task, not the whole run; it is synchronous and may block (a fileset handler - # downloads), so it is offloaded off the shared event loop. - workspace_dir = evidence_dir / _WORKSPACE_SUBDIR - workspace_dir.mkdir(parents=True, exist_ok=True) - skill_provenances: list[SkillProvenance] = [] - hook_session = FabricTaskRunSession() - hook_extras: dict[str, Any] | None = None - try: - # Stage seed files into the workspace for their on-disk side effect; the prompt is the task - # instruction only, so the returned paths are unused. - await asyncio.to_thread(seed_workspace, workspace_dir, task.inputs.get(SEED_FILES_INPUT_KEY)) - - # Inject the skill set (if any) for this task. A native harness gets each staged bundle added - # to the config's ``skills.paths``; codex self-injection stages each bundle into the - # workspace and adds no path. One provenance per skill is stamped on the trial for the A/B - # diff. Blocking file I/O, off the event loop. - skill_paths: list[str] = [] - if self._skill_set.skills and skill_mode is not None: - installation = await asyncio.to_thread( - install_skills, - skills=self._skill_set.skills, - adapter_id=agent_config.harness.adapter_id, - mode=skill_mode, - workspace_dir=workspace_dir, - skill_stage_dir=(evidence_dir / _SKILL_SUBDIR).resolve(), - ) - skill_provenances = installation.provenances - skill_paths = installation.skill_paths - - # Everything the run needs lives in one typed config: Fabric no longer layers profile - # overlays, so the per-task workspace/model/trajectory settings are composed on last and are - # authoritative by construction. ``add_skill_path`` appends, so config-declared skills survive. - task_config = self._compose_config(agent_config, evidence_dir, workspace_dir, task=task) - for skill_path in skill_paths: - task_config.add_skill_path(skill_path) - - if self._task_hook is not None: - task_config = self._task_hook.prepare( - config=task_config, - task=task, - evidence_dir=evidence_dir, - workspace_dir=workspace_dir, - session=hook_session, - ) - - result = await asyncio.wait_for( - # ``Fabric.run`` folds the per-invocation input + request id into a ``RunRequest``. - client.run( - task_config, - base_dir=self._base_dir, - request=RunRequest(input=task.agent_prompt(), request_id=task.id), - ), - timeout=self._timeout_s, - ) - # Always try to harvest MCP binding results. Hermes often ends with - # ``completed=false`` / empty finals after a successful tool call; the binding - # audit is still the authoritative analyzer output for scoring. - if self._task_hook is not None: - try: - hook_extras = self._task_hook.after_success(task=task, result=result, session=hook_session) - except Exception as exc: # noqa: BLE001 - binding harvest must not abort the batch - logger.warning("Fabric task hook after_success failed: %s", exc) - if result.status == "succeeded": - raise - hook_extras = None - except TimeoutError as exc: - return self._failed_trial(task, evidence_dir, exc, extra_metadata=self._skill_metadata(skill_provenances)) - except Exception as exc: # noqa: BLE001 - a task failure must not abort the whole run - return self._failed_trial(task, evidence_dir, exc, extra_metadata=self._skill_metadata(skill_provenances)) - finally: - if self._task_hook is not None: - try: - self._task_hook.cleanup(session=hook_session) - except Exception: # noqa: BLE001 - hook cleanup must not mask the trial outcome - pass - # Codex self-injection staged each bundle *inside* the workspace so the harness could discover - # it. Remove them once the run is over (it is already captured in the trajectory) so the injected - # files don't linger in the durable workspace and, on any path that exposes it as filesystem - # evidence, read as agent output and skew workspace-reading metrics. In ``finally`` so a - # timed-out or errored run cleans up too, not just the success path. Best-effort per - # ``_remove_injected_bundle``; a no-op for native mode and when nothing was staged. - if skill_mode == SKILL_MODE_CODEX_SKILLS_DIR: - for provenance in skill_provenances: - await asyncio.to_thread(_remove_injected_bundle, workspace_dir, provenance["location"]) - - return self._to_trial( - task, - result, - evidence_dir, - workspace_dir, - skill_provenances=skill_provenances, - hook_extras=hook_extras, - ) - - @staticmethod - def _skill_metadata(provenances: list[SkillProvenance]) -> dict[str, Any]: - """Trial-metadata fields describing the injected skill set (the A/B provenance). - - ``skills`` is the full list of injected-skill provenances (empty = baseline). ``skill`` keeps the - historical single-provenance field — the lone provenance for a one-skill run, else ``None`` — so - single-skill consumers (e.g. ``SkillUsedMetric``) and existing trials keep working unchanged. - """ - return {"skill": provenances[0] if len(provenances) == 1 else None, "skills": provenances} - - def _to_trial( - self, - task: AgentEvalTask, - result: RunResult, - evidence_dir: Path, - workspace_dir: Path, - skill_provenances: list[SkillProvenance] | None = None, - hook_extras: Mapping[str, Any] | None = None, - ) -> AgentEvalTrial: - # Persist the full normalized Fabric result so graders (and debugging) can see the raw - # envelope, and expose it as an evidence descriptor. - result_path = evidence_dir / "fabric_result.json" - result_path.write_text(json.dumps(result.to_mapping(), indent=2, default=str), encoding="utf-8") - - extras = dict(hook_extras) if hook_extras else {} - base_metadata: dict[str, Any] = { - "runtime": self._runtime_name, - "harness": result.harness, - "adapter_id": result.adapter_id, - "adapter_kind": result.adapter_kind, - "invocation_id": result.invocation_id, - "agent_model": self._model, - # Skill provenance (name + content hash + injection mode) for the A/B diff. - **self._skill_metadata(skill_provenances or []), - **extras, - } - - if result.status != "succeeded": - # Hermes may report a non-success final message after a successful MCP tool - # call. Prefer the binding audit result over a hard fail when present. - binding_result = _first_mcp_binding_result(extras) - analysis = binding_result if binding_result is not None else extras.get("analyzer_analysis") - if analysis is not None: - base_metadata = { - **base_metadata, - "fabric_status": result.status, - "recovered_from_mcp_binding": True, - } - return AgentEvalTrial( - id=f"{task.id}:fabric", - task_id=task.id, - status=AgentEvalTrialStatus.COMPLETED, - output=AgentOutput( - output_text=json.dumps(analysis, default=str), - response=_normalize_output(result.output), - metadata={**base_metadata, "evidence_dir": str(evidence_dir)}, - ), - evidence=self._evidence(result, result_path, workspace_dir), - metadata={**base_metadata, "generated": True, "agent_ok": True}, - ) - return self._failed_trial(task, evidence_dir, _result_error(result), extra_metadata=base_metadata) - - # Fabric wraps the output in a ``RunOutput`` mapping (RunOutput response contract, #52), - # which is not itself a JSON value; normalize it to a plain mapping so it round-trips through the - # trial's ``JsonValue``-typed response. - output = _normalize_output(result.output) - # Author / mcp_run_binding hooks may attach a structured result. Prefer that when the - # harness returns an empty final message after a successful tool call. - output_text = _extract_output_text(output) - if not output_text or not str(output_text).strip(): - binding_result = _first_mcp_binding_result(extras) - analysis = binding_result if binding_result is not None else extras.get("analyzer_analysis") - if analysis is not None: - output_text = json.dumps(analysis, default=str) - return AgentEvalTrial( - id=f"{task.id}:fabric", - task_id=task.id, - status=AgentEvalTrialStatus.COMPLETED, - output=AgentOutput( - output_text=output_text, - response=output, - metadata={**base_metadata, "evidence_dir": str(evidence_dir)}, - ), - evidence=self._evidence(result, result_path, workspace_dir), - # AgentPhaseSuccessMetric reads agent_ok to score whether the agent phase finished cleanly - # (an explicit bool, not just trial status). - metadata={**base_metadata, "generated": True, "agent_ok": True}, - ) - - def _evidence(self, result: RunResult, result_path: Path, workspace_dir: Path) -> CandidateEvidence: - # The workspace is a host directory the harness ran in, so its final file tree is available on - # disk — expose it as filesystem evidence so workspace-reading metrics can score a Fabric trial. - descriptors: dict[str, EvidenceDescriptor] = { - "result": EvidenceDescriptor(kind="json", format="json", ref=str(result_path)), - _WORKSPACE_EVIDENCE_KEY: EvidenceDescriptor(kind=_WORKSPACE_EVIDENCE_KIND, ref=str(workspace_dir)), - } - for artifact in result.artifacts.artifacts: - descriptors[artifact.name] = EvidenceDescriptor( - kind=artifact.kind or "file", - ref=str(artifact.path), - metadata={"media_type": artifact.media_type}, - ) - # Surface the Relay ATIF trajectory under the standard trace evidence key so graders - # that consume a normalized trajectory find it. - if artifact.kind == _ATIF_ARTIFACT_KIND: - descriptors[EVIDENCE_TRACE] = EvidenceDescriptor( - kind=EVIDENCE_TRACE, - format=EVIDENCE_FORMAT_ATIF, - ref=str(artifact.path), - ) - return CandidateEvidence( - descriptors=descriptors, - metadata={ - "runtime": self._runtime_name, - "harness": result.harness, - "telemetry": [ - {"provider": ref.provider, "kind": ref.kind, "uri": ref.uri, "trace_id": ref.trace_id} - for ref in result.telemetry - ], - "events": [{"kind": event.kind, "message": event.message} for event in result.events], - }, - ) - - def _failed_trial( - self, - task: AgentEvalTask, - evidence_dir: Path, - error: Exception | Mapping[str, Any], - extra_metadata: Mapping[str, Any] | None = None, - ) -> AgentEvalTrial: - if isinstance(error, Mapping): - error_type = str(error.get("code") or error.get("stage") or "FabricError") - error_message = str(error.get("message") or error) - else: - error_type = error.__class__.__name__ - error_message = str(error) - error_path = evidence_dir / "error.json" - error_path.write_text(json.dumps({"error_type": error_type, "error": error_message}) + "\n", encoding="utf-8") - return AgentEvalTrial( - id=f"{task.id}:fabric", - task_id=task.id, - status=AgentEvalTrialStatus.FAILED, - output=None, - evidence=CandidateEvidence( - descriptors={"error": EvidenceDescriptor(kind="error", format="json", ref=str(error_path))}, - metadata={"runtime": self._runtime_name}, - ), - metadata={ - **(dict(extra_metadata) if extra_metadata else {}), - "runtime": self._runtime_name, - "agent_ok": False, - "error_type": error_type, - "error": error_message, - }, - ) - - def _compose_config( - self, - agent_config: FabricConfig, - evidence_dir: Path, - workspace_dir: Path, - task: AgentEvalTask, - ) -> FabricConfig: - # nemo_fabric is already imported+validated in ``run_tasks``; this is a cached sys.modules - # lookup, not a re-load, so the type is used where it's constructed instead of threaded down. - from nemo_fabric import EnvironmentConfig, ModelConfig # ty: ignore[unresolved-import] - - # Copy the base config and apply this task's workspace, model, and trajectory settings directly - # onto it. These land last, so they override anything the supplied config declared. - cfg = agent_config.model_copy(deep=True) - - # Point the harness at this task's staged workspace (the codex-cli adapter resolves its cwd from - # it). ``provider="local"`` is required by the native planner. Any config-supplied - # environment.workspace is overridden per task. - environment = cfg.environment or EnvironmentConfig(provider="local") - environment.provider = environment.provider or "local" - environment.workspace = str(workspace_dir.resolve()) - cfg.environment = environment - - # Apply the model as the config's default (mirrors nemo_fabric.integrations.harbor). - if self._model: - provider = self._model.split("/", maxsplit=1)[0] if "/" in self._model else "openai" - cfg.models["default"] = ModelConfig(provider=provider, model=self._model) - - if self._capture_trajectory: - # Enable Relay's ATIF/ATOF file exporter under this task's durable evidence dir, and pin the - # Fabric artifact root so the promoted ``trajectory-*.atif.json`` persists. Requires the - # ``nemo-relay`` gateway on PATH in the runtime. Stamp the task id (and any caller - # ``trajectory_extra``) onto ATIF ``extra`` so optimizer trials can join traces to rows. - relay_dir = evidence_dir / _RELAY_SUBDIR - artifacts_dir = evidence_dir / _ARTIFACTS_SUBDIR - relay_dir.mkdir(parents=True, exist_ok=True) - artifacts_dir.mkdir(parents=True, exist_ok=True) - row_extra = {"nemo.optimizer.row_id": task.id} if task.id else None - cfg.enable_relay( - output_dir=str(relay_dir), - observability=self._relay_config(relay_dir, extra=row_extra), - ) - cfg.runtime.artifacts = str(artifacts_dir) - cfg.environment.artifacts = str(artifacts_dir) - - return cfg - - def _relay_config( - self, - relay_dir: Path, - extra: Mapping[str, Any] | None = None, - ) -> RelayObservabilityConfig: - # The ATIF/ATOF observability config is built from Fabric's own typed relay-config objects so - # Fabric owns the schema (no hand-maintained dict that silently drifts when Fabric changes it), - # mirroring nemo_fabric's own Harbor integration. It is handed straight to ``enable_relay`` via - # its ``observability=`` parameter — the SDK only configures ATIF/ATOF observability, so it needs - # neither a generic ``components`` list nor the legacy component-wrapped shape. nemo_fabric is - # already imported+validated in ``run_tasks``, so this is a cached sys.modules lookup. - from nemo_fabric import ( # ty: ignore[unresolved-import] - RelayAtifConfig, - RelayAtofConfig, - RelayAtofFileSinkConfig, - RelayObservabilityConfig, - ) - - relay_dir_str = str(relay_dir) - atif_extra: dict[str, Any] | None = None - if self._trajectory_extra or extra: - atif_extra = {**(self._trajectory_extra or {}), **(dict(extra) if extra else {})} - return RelayObservabilityConfig( - atif=RelayAtifConfig( - enabled=True, - output_directory=relay_dir_str, - filename_template=_ATIF_FILENAME_TEMPLATE, - agent_name=self._runtime_name, - agent_version=_common.FABRIC_AGENT_VERSION, - extra=atif_extra, - ), - atof=RelayAtofConfig( - enabled=True, - sinks=[ - RelayAtofFileSinkConfig( - output_directory=relay_dir_str, - filename=_ATOF_FILENAME, - mode="overwrite", - ) - ], - ), - ) - - def _evidence_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> Path: - root = self._work_root - if root is None: - root = (config.work_dir or Path.cwd()) / "evidence" / "fabric" - # The run id isolates this run's evidence from other runs sharing the same root (A/B baseline - # vs. skilled); run_tasks always populates it, so the fallback only guards a direct call. - run_id = config.run_id or _new_run_id() - safe_task_id = _safe_path_name(task.id) - task_dir = f"{index:06d}-{safe_task_id}" if safe_task_id else f"task-{index:06d}" - return Path(root) / _safe_path_name(run_id) / task_dir - - -def _remove_injected_bundle(workspace_dir: Path, location: str) -> None: - """Remove the Codex-injected skill subtree from ``workspace_dir`` and prune emptied parents. - - ``location`` is workspace-relative (``.agents/skills/``). Best-effort: the skill was already - captured in the run's trajectory, so SkillUsedMetric (which reads the trace, not the workspace) is - unaffected, and any filesystem error here must not fail an otherwise-successful trial. - """ - workspace_root = workspace_dir.resolve() - injected = (workspace_dir / location).resolve() - # Guard against a location escaping the workspace (defensive; provenance is evaluator-authored). - if workspace_root not in injected.parents or not injected.exists(): - return - shutil.rmtree(injected, ignore_errors=True) - # Prune now-empty reserved parents (``.agents/skills``, ``.agents``) but never the workspace itself. - parent = injected.parent - while parent != workspace_root and parent.is_dir(): - try: - parent.rmdir() # only succeeds while empty - except OSError: - break - parent = parent.parent - - -def _normalize_output(output: RunOutput | JsonValue) -> JsonValue: - """Unwrap a Fabric ``RunResult.output`` into the plain JSON value the trial response stores. - - Newer Fabric wraps output in a ``RunOutput`` (the RunOutput response contract), which is a - ``Mapping``; copy it into a plain dict (equivalent to its ``to_mapping()``). Raw/older JSON outputs - are already JSON values and pass through unchanged. - """ - if isinstance(output, Mapping): - return dict(output) - return output - - -def _first_mcp_binding_result(extras: Mapping[str, Any]) -> Any | None: - """Return the first ``mcp_bindings..result`` payload, if any.""" - bindings = extras.get("mcp_bindings") - if not isinstance(bindings, Mapping): - return None - for entry in bindings.values(): - if isinstance(entry, Mapping) and "result" in entry: - return entry.get("result") - return None - - -def _extract_output_text(output: object) -> str | None: - """Pull the user-visible message out of a Fabric ``RunResult.output`` (JSON-shaped). - - Harness outputs vary; adapters commonly nest the final message under ``response`` (the codex-cli - adapter does). Prefer a string ``response``/``output_text``, else stringify the whole value. - """ - if output is None: - return None - if isinstance(output, str): - return output - if isinstance(output, Mapping): - for key in ("response", "output_text", "text", "message"): - value = output.get(key) - if isinstance(value, str): - return value - return json.dumps(output, default=str) - - -def _result_error(result: RunResult) -> Mapping[str, Any]: - error = result.error - if error is None: - return {"code": result.status, "message": "Fabric run did not succeed"} - return {"stage": error.stage, "code": error.code, "message": error.message} - - -def _safe_path_name(value: str) -> str: - return "".join(char if char.isalnum() or char in "._-" else "-" for char in value).strip(".-")[:120] - - -def _new_run_id() -> str: - timestamp = datetime.now(UTC).strftime("%Y%m%d%H%M%S%f") - return f"fabric-{timestamp}-{uuid4().hex[:8]}" diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/sandbox.Dockerfile b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/sandbox.Dockerfile deleted file mode 100644 index 8b393d2059..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/sandbox.Dockerfile +++ /dev/null @@ -1,51 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# -# Multi-stage image for FabricContainerRuntime. The builder compiles nemo-fabric (maturin/Rust) into -# an isolated venv AND builds Fabric's own `fabric` CLI (the runtime execs `fabric run` to kick off -# the harness). The final stage copies only the venv + the CLI binary + the built-in adapters — no -# source tree and no Rust toolchain. Harness extras are selected via the EXTRAS build arg. -ARG PYTHON_VERSION=3.12 - -FROM python:${PYTHON_VERSION}-slim-bookworm AS builder -ARG EXTRAS=hermes,relay -RUN apt-get update \ - && apt-get install -y --no-install-recommends build-essential curl git pkg-config libssl-dev \ - && rm -rf /var/lib/apt/lists/* -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs -o /tmp/rustup-init.sh \ - && sh /tmp/rustup-init.sh -y --profile minimal \ - && rm -f /tmp/rustup-init.sh -ENV PATH=/root/.cargo/bin:$PATH -RUN python -m venv /opt/venv -ENV PATH=/opt/venv/bin:$PATH -# Only the maturin build inputs are in the context (see image._stage_source): the native nemo-fabric -# extension is compiled here and the harness/relay wheels are pulled from PyPI — into the venv only. -COPY nemo-fabric /src -RUN pip install --no-cache-dir "/src[${EXTRAS}]" -# Fabric's own CLI (Rust). The runtime execs `fabric run --profile … --input-file …`, -# which prints a normalized RunResult to stdout — so no in-image Python driver is needed. -RUN cargo build --release --manifest-path /src/Cargo.toml -p fabric-cli - -FROM python:${PYTHON_VERSION}-slim-bookworm AS runtime -COPY --from=builder /opt/venv /opt/venv -COPY --from=builder /src/target/release/fabric /usr/local/bin/fabric -# The CLI binary resolves built-in adapters from its compile-time repository path -# (CARGO_MANIFEST_DIR/../../python/src/nemo_fabric/adapters); ship just that dir to the baked path so a -# wheel-only image can resolve them. Depends on NeMo-Fabric's installed-adapter-discovery layout -# (see image.py); swap to installing the top-level adapters/* packages once that lands on main. -COPY --from=builder /src/python/src/nemo_fabric/adapters /src/python/src/nemo_fabric/adapters -# The CLI's baked path is the literal `/../../python/src/nemo_fabric/adapters`; the -# kernel needs `/src/crates/fabric-core` to exist to walk the `..`, even though nothing lives there. -RUN mkdir -p /src/crates/fabric-core -ENV PATH=/opt/venv/bin:$PATH -RUN python -c "from nemo_fabric import FabricClient" && fabric version -# Run agent-generated code as a non-root user: this sandbox execs `fabric run` over untrusted, -# agent-produced content, so dropping root narrows the blast radius of a container escape. Pre-create -# and own the fixed /in (seeded inputs) and /out (workspace + results) trees, since a non-root process -# cannot mkdir under / at exec time and the runtime creates /out/{workspace,relay,artifacts,logs} then. -RUN useradd --create-home --uid 1000 sandbox \ - && mkdir -p /in /out \ - && chown -R sandbox:sandbox /in /out -WORKDIR /out -USER sandbox diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/skills.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/skills.py deleted file mode 100644 index 4e1a912315..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/skills.py +++ /dev/null @@ -1,497 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Agent-skill injection for the Fabric agent-eval runtimes (PROTOTYPE). - -An *agent skill* is a directory following the `agentskills.io `_ -spec: a folder named ``/`` containing a required ``SKILL.md`` (YAML frontmatter with ``name`` + -``description``, then instructions) plus optional ``scripts/`` / ``references/`` / ``assets/``. We make -that bundle available to the harness before it runs a task so an A/B eval can score the same taskset -with and without the skill. The skill is a runtime-level knob: build one runtime with ``skill=None`` -and one with ``skill=`` over the same tasks, then diff the scores. - -An :class:`AgentSkill` points at a local skill directory; staging is an OS-level ``copytree`` (file -contents never pass through Python memory). The plugin resolves a platform fileset to a local -directory and constructs an ``AgentSkill`` from it — the SDK has no fileset concept of its own. - -How the skill reaches the harness depends on the selected Fabric adapter, and which mode applies is -decided by *querying Fabric's own capability planner at runtime* (:func:`resolve_skill_mode` over a -``RunPlan.capability_plan``), not a hardcoded adapter list — so it tracks whatever the installed -adapters declare, including end-user adapters we don't ship: - -* **Native** (:data:`SKILL_MODE_NATIVE`): the adapter advertises ``accepts: ["skills", ...]``, so - Fabric's planner routes skills to ``harness_native``. We stage the bundle into an isolated - ``/`` dir and add it to the config's ``skills.paths``; the adapter loads it (Hermes → harness - ``skills.external_dirs``). As of nemo-fabric 0.1.0rc3 the hermes, claude AND **codex** adapters all - declare ``skills``, so this is the path every harness we ship currently takes. -* **Codex skills dir** (:data:`SKILL_MODE_CODEX_SKILLS_DIR`): a fallback for a codex-harness adapter - that does *not* accept the native skills config. The Codex CLI itself discovers agentskills bundles - from ``.agents/skills/`` in its working directory, so we place the bundle at - ``/.agents/skills//`` and let Codex find it — same discoverable-skill semantics as - native (cross-harness A/B stays apples-to-apples), no Fabric adapter change needed. - NOTE: the shipped codex adapter accepts ``skills`` today, so this branch is currently unreachable in - production and is exercised only by the fake-backed tests. It is kept for adapters (ours or an - end-user's) that route skills ``unsupported`` on a codex harness. - -If an adapter neither routes skills natively nor is a Codex harness, :func:`resolve_skill_mode` returns -``None`` and the runtime fails fast rather than silently running a skill-free trial. -""" - -from __future__ import annotations - -import hashlib -import re -import shutil -from collections.abc import Mapping, Sequence -from dataclasses import dataclass -from pathlib import Path -from typing import Literal, TypedDict - -from pydantic import BaseModel, ConfigDict, Field, field_validator - -#: Required entry document of an agentskills bundle. -PRIMARY_SKILL_DOC = "SKILL.md" -#: Directory Codex scans (relative to its working dir) for agentskills bundles. -CODEX_SKILLS_DIR = ".agents/skills" - -#: How an injected skill reaches the selected harness (resolved from Fabric's capability plan). The two -#: runtimes thread this value from :func:`resolve_skill_mode` down to :func:`install_skill` / -#: :func:`stage_skills_seed`, so a mistyped mode is a type error rather than a silent no-op. -SkillMode = Literal["native", "codex_skills_dir"] - -#: Skill reaches the harness via the native Fabric ``skills`` config (adapter accepts it). -SKILL_MODE_NATIVE: SkillMode = "native" -#: Skill is placed under ``/.agents/skills//`` for Codex to discover. -SKILL_MODE_CODEX_SKILLS_DIR: SkillMode = "codex_skills_dir" - -# agentskills.io name rule: 1-64 chars, lowercase alphanumeric + single interior hyphens. -_SKILL_NAME_RE = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$") -_MAX_NAME_LEN = 64 - -# Fabric capability-planner vocabulary (``RunPlan.capability_plan['routes']`` entries). A ``skills`` -# route with target ``harness_native`` means the selected adapter declared native skills support; the -# runtime plans a probe skill path and reads these to decide the injection mode (see resolve_skill_mode). -_SKILLS_ROUTE_KIND = "skills" -_SKILLS_TARGET_NATIVE = "harness_native" -# Fabric harness name of the Codex CLI adapter, which self-discovers ``.agents/skills/`` rather than -# accepting the native ``skills`` config. -_CODEX_HARNESS = "codex" - - -class SkillInjectionError(ValueError): - """A skill could not be resolved, staged, or wired into the selected harness. - - Subclasses ``ValueError`` so the runtime's per-task error handling still catches it and fails - only that task. - """ - - -class AgentSkill(BaseModel): - """An agentskills.io bundle (a local directory) to make available to the agent before a task. - - ``name`` must satisfy the agentskills naming rule and is used as the staged bundle's directory name - (spec: the name matches the directory name). ``directory`` is the local skill directory, which must - contain a top-level ``SKILL.md``. - """ - - model_config = ConfigDict(extra="forbid") - - name: str = Field(description="agentskills skill name; also the bundle directory name and provenance id.") - directory: Path = Field(description="Local agentskills bundle directory (a SKILL.md at its root).") - - @field_validator("name") - @classmethod - def _valid_name(cls, value: str) -> str: - if len(value) > _MAX_NAME_LEN or not _SKILL_NAME_RE.match(value): - raise ValueError( - f"skill name {value!r} must be 1-{_MAX_NAME_LEN} chars, lowercase alphanumeric with " - "single interior hyphens (agentskills.io naming rule)" - ) - return value - - @classmethod - def from_directory(cls, directory: str | Path, *, name: str | None = None) -> AgentSkill: - """Build a skill from an on-disk agentskills bundle. ``name`` defaults to the directory basename.""" - root = Path(directory).expanduser().resolve() - if not (root / PRIMARY_SKILL_DOC).is_file(): - raise SkillInjectionError(f"skill directory {str(directory)!r} has no {PRIMARY_SKILL_DOC}") - return cls(name=name or root.name, directory=root) - - -class SkillProvenance(TypedDict): - """Which skill was injected into a trial and how; stamped into trial metadata for the A/B diff. - - A plain (JSON-serializable) dict so it drops straight into trial metadata. ``None`` in that slot - means the baseline (no skill). - """ - - name: str #: The skill's agentskills name. - hash: str #: sha256 over the staged bundle — attributes a score delta to an exact skill version. - mode: SkillMode #: How it was injected (:data:`SKILL_MODE_NATIVE` / :data:`SKILL_MODE_CODEX_SKILLS_DIR`). - adapter_id: str #: The harness adapter the skill was wired into. - location: str #: Where the bundle was staged (absolute for native, workspace-relative for codex). - - -@dataclass -class SkillInstallation: - """Result of installing a skill for one task. - - ``skill_paths`` are staged bundle roots the runtime hands to ``FabricConfig.add_skill_path`` (the - native branch emits one; the Codex branch emits none because placement in the workspace is the - delivery mechanism). ``provenance`` is stamped into trial metadata so the A/B comparison is - auditable. - """ - - skill_paths: list[str] - provenance: SkillProvenance - - -def native_skills_route(capability_plan: Mapping[str, object]) -> bool: - """Whether Fabric's capability planner routed skills to the harness natively. - - ``capability_plan`` is the ``RunPlan.capability_plan`` mapping from ``Fabric.plan(...)`` planned with - a skill path attached; its ``routes`` record each capability decision. A ``skills`` route with target - ``harness_native`` means the selected adapter declares ``accepts: ["skills", ...]`` and Fabric hands - the bundle to the harness itself. Any other outcome (``unsupported``, or no skills route) is False. - """ - routes = capability_plan.get("routes") - if not isinstance(routes, list): - return False - return any( - isinstance(route, Mapping) - and route.get("kind") == _SKILLS_ROUTE_KIND - and route.get("target") == _SKILLS_TARGET_NATIVE - for route in routes - ) - - -def resolve_skill_mode(*, capability_plan: Mapping[str, object], harness: str) -> SkillMode | None: - """Resolve how a skill would reach the selected harness, or ``None`` if it can't. - - Driven by Fabric's own capability routing (queried at runtime via ``Fabric.plan``) rather than a - hardcoded adapter list, so it tracks whatever the installed adapters declare — including end-user - adapters we don't ship: - - * skills route natively (:func:`native_skills_route`) -> :data:`SKILL_MODE_NATIVE`; - * else a Codex harness (self-discovers ``.agents/skills/``) -> :data:`SKILL_MODE_CODEX_SKILLS_DIR`; - * else ``None`` -> the runtime fails fast rather than run a skill-free trial labeled "with skill". - """ - if native_skills_route(capability_plan): - return SKILL_MODE_NATIVE - if harness.strip().lower() == _CODEX_HARNESS: - return SKILL_MODE_CODEX_SKILLS_DIR - return None - - -def install_skill( - *, - skill: AgentSkill, - adapter_id: str, - mode: SkillMode, - workspace_dir: Path, - skill_stage_dir: Path, -) -> SkillInstallation: - """Stage ``skill`` as a ``/`` bundle and wire it into the harness per ``mode``. - - Blocking file I/O — call via ``asyncio.to_thread`` from the async runtime. The bundle is always - namespaced under ``/`` so it never collides with task-seeded workspace-root files; the content - hash is computed over the staged bytes so provenance tracks the actual skill content. - - The native branch returns the staged root for ``FabricConfig.add_skill_path``, which appends to - whatever the base config already declares. Any preconfigured skills therefore survive injection - without this function having to re-list them. - """ - if mode == SKILL_MODE_NATIVE: - skill_root = skill_stage_dir / skill.name - _stage_bundle(skill.directory, skill_root, reserved=False) - return SkillInstallation( - skill_paths=[str(skill_root)], - provenance=_provenance(skill, _hash_directory(skill_root), mode, adapter_id, str(skill_root)), - ) - - if mode == SKILL_MODE_CODEX_SKILLS_DIR: - skill_root = workspace_dir / CODEX_SKILLS_DIR / skill.name - _stage_bundle(skill.directory, skill_root, reserved=True) - location = (Path(CODEX_SKILLS_DIR) / skill.name).as_posix() - return SkillInstallation( - skill_paths=[], - provenance=_provenance(skill, _hash_directory(skill_root), mode, adapter_id, location), - ) - - raise SkillInjectionError(f"unknown skill injection mode {mode!r} for adapter {adapter_id!r}") - - -@dataclass -class SkillsInstallation: - """Result of installing several skills for one task (see :func:`install_skills`). - - ``skill_paths`` is every staged native bundle root, in the given order, for the runtime to feed to - ``FabricConfig.add_skill_path``; the Codex branch emits none because workspace placement is the - delivery mechanism. ``provenances`` is one entry per skill, in the given order, stamped into trial - metadata so a multi-skill A/B comparison is auditable. - """ - - skill_paths: list[str] - provenances: list[SkillProvenance] - - -def require_unique_skill_names(skills: Sequence[AgentSkill]) -> None: - """Raise if two skills share a name — their ``/`` bundles would collide when staged. - - Each skill stages into its own ``/`` directory (native stage dir or ``.agents/skills/``), so a - repeated name would clobber (or fail to stage over) an earlier bundle. Checked up front so a - misconfigured runtime fails before any task runs, not mid-stage on the second collision. - """ - seen: set[str] = set() - duplicates: list[str] = [] - for skill in skills: - if skill.name in seen and skill.name not in duplicates: - duplicates.append(skill.name) - seen.add(skill.name) - if duplicates: - raise SkillInjectionError( - f"duplicate skill name(s) {duplicates}: each skill stages to its own '/' bundle, so " - "skill names must be unique within one runtime" - ) - - -@dataclass(frozen=True) -class SkillSet: - """Immutable, name-validated collection of :class:`AgentSkill`\\s shared by both Fabric runtimes. - - Centralizes the uniqueness check and clone-on-mutation pattern that - :class:`~...FabricAgentRuntime` and :class:`~...FabricContainerRuntime` would otherwise - duplicate: construction validates that skill names are unique; :meth:`with_skills` and - :meth:`with_skill` each return a new ``SkillSet`` without modifying ``self``. - """ - - skills: tuple[AgentSkill, ...] = () - - def __post_init__(self) -> None: - require_unique_skill_names(self.skills) - - def with_skills(self, skills: Sequence[AgentSkill]) -> SkillSet: - """Return a new ``SkillSet`` with ``skills`` appended; ``self`` is not modified.""" - return SkillSet((*self.skills, *skills)) - - def with_skill(self, skill: AgentSkill) -> SkillSet: - """Return a new ``SkillSet`` with ``skill`` appended; ``self`` is not modified.""" - return self.with_skills([skill]) - - -def install_skills( - *, - skills: Sequence[AgentSkill], - adapter_id: str, - mode: SkillMode, - workspace_dir: Path, - skill_stage_dir: Path, -) -> SkillsInstallation: - """Stage every skill in ``skills`` for one task and wire them all into the harness per ``mode``. - - Loops :func:`install_skill` — each skill stages into its own namespaced ``/`` bundle — and - collects the staged roots for the native mode. ``FabricConfig.add_skill_path`` appends and - de-duplicates, so every injected skill lands alongside whatever the base config already declared, - with no re-listing. Skill names must be unique (their ``/`` bundles would otherwise collide). - Blocking file I/O — call via ``asyncio.to_thread`` from the async runtime. - - Installation is all-or-nothing: if any skill fails to stage, the bundles already staged in this call - are rolled back before the error propagates, so a partial skill set never lingers on disk (the caller - raises before it ever sees provenances, so it cannot clean up itself). Only bundles this call staged - are removed, so a reserved-path collision can never delete a pre-existing task-seeded file. - """ - require_unique_skill_names(skills) - provenances: list[SkillProvenance] = [] - staged_roots: list[Path] = [] - try: - for skill in skills: - # Register the target BEFORE staging: install_skill can raise after it has already written - # files (a copytree failing partway, an unreadable file while hashing), and a root recorded - # only on success would leave that partial bundle behind. A target that already exists is - # never registered — in codex mode that is a task-seeded file install_skill refuses to - # clobber, and rolling it back would delete task input this call did not create. - stage_root = _skill_stage_root(skill, mode, workspace_dir, skill_stage_dir) - if not stage_root.exists(): - staged_roots.append(stage_root) - provenance = install_skill( - skill=skill, - adapter_id=adapter_id, - mode=mode, - workspace_dir=workspace_dir, - skill_stage_dir=skill_stage_dir, - ).provenance - provenances.append(provenance) - except Exception: - for root in staged_roots: - shutil.rmtree(root, ignore_errors=True) - raise - - skill_paths: list[str] = [] - if mode == SKILL_MODE_NATIVE: - # Each staged bundle root, order-preserved and de-duplicated (a native provenance's - # ``location`` is its absolute staged skill root). - skill_paths = list(dict.fromkeys(prov["location"] for prov in provenances)) - return SkillsInstallation(skill_paths=skill_paths, provenances=provenances) - - -def _skill_stage_root(skill: AgentSkill, mode: SkillMode, workspace_dir: Path, skill_stage_dir: Path) -> Path: - """Absolute on-disk root :func:`install_skill` would stage ``skill`` into, computed before staging. - - Mirrors install_skill's per-mode placement so :func:`install_skills` can register a rollback target - up front (an unknown mode raises there, not here; the returned path is simply never created, and - rolling back a path that does not exist is a no-op).""" - if mode == SKILL_MODE_NATIVE: - return skill_stage_dir / skill.name - return workspace_dir / CODEX_SKILLS_DIR / skill.name - - -def _render_skill_seed( - *, skill: AgentSkill, adapter_id: str, mode: SkillMode, workspace_dir: str, skills_dir: str -) -> tuple[dict[str, str], SkillProvenance]: - """Render one skill bundle into an in-sandbox ``{path: text}`` seed map + its provenance. - - The per-skill core of :func:`stage_skills_seed` (the containerized counterpart of :func:`install_skill`, - which ``copytree``\\ s onto host disk): the container has no host workspace, so the bundle is read into - memory as UTF-8 text and keyed at the harness's in-sandbox discovery path — native: ``/ - /``; codex: ``/.agents/skills//``. The content hash is over the source bundle - (matching :func:`install_skill`). The caller merges these into one seed set and, for native mode, a - single ``skills`` overlay — so no per-skill overlay is emitted here. - """ - bundle = _read_text_bundle(skill.directory) - skill_hash = _hash_directory(skill.directory) - if mode == SKILL_MODE_NATIVE: - skill_root = f"{skills_dir.rstrip('/')}/{skill.name}" - files = {f"{skill_root}/{rel}": text for rel, text in bundle.items()} - return files, _provenance(skill, skill_hash, mode, adapter_id, skill_root) - - if mode == SKILL_MODE_CODEX_SKILLS_DIR: - skill_root = f"{workspace_dir.rstrip('/')}/{CODEX_SKILLS_DIR}/{skill.name}" - files = {f"{skill_root}/{rel}": text for rel, text in bundle.items()} - location = f"{CODEX_SKILLS_DIR}/{skill.name}" - return files, _provenance(skill, skill_hash, mode, adapter_id, location) - - raise SkillInjectionError(f"unknown skill injection mode {mode!r} for adapter {adapter_id!r}") - - -def _read_text_bundle(directory: Path) -> dict[str, str]: - """Read an agentskills bundle into a ``{posix_relpath: text}`` map (requires a top-level ``SKILL.md``). - - Every file is decoded as UTF-8: the containerized seed set (``SandboxSpec.files``) is text-only, so a - binary file (e.g. an image under ``assets/``) raises here rather than silently corrupting the staged - bundle — the host :func:`install_skill` path (OS-level ``copytree``) handles binary bundles instead. - """ - src = directory.expanduser() - if not (src / PRIMARY_SKILL_DOC).is_file(): - raise SkillInjectionError(f"skill directory {str(directory)!r} has no {PRIMARY_SKILL_DOC}") - bundle: dict[str, str] = {} - for path in sorted(candidate for candidate in src.rglob("*") if candidate.is_file()): - rel = path.relative_to(src).as_posix() - try: - bundle[rel] = path.read_text(encoding="utf-8") - except UnicodeDecodeError as exc: - raise SkillInjectionError( - f"skill file {rel!r} is not UTF-8 text; containerized skill injection (via the sandbox " - "seed set) supports text bundles only" - ) from exc - return bundle - - -@dataclass -class SkillsSeed: - """Result of rendering several skills into one sandbox seed set (see :func:`stage_skills_seed`). - - The plural, containerized sibling of :class:`SkillsInstallation`: - - * ``files`` — the merged ``{absolute_in_sandbox_path: text}`` seed map for every staged bundle. - * ``skill_paths`` — every staged native bundle root, in order, for the runtime to merge into the - composed config's ``skills.paths``; the codex branch emits none. - * ``provenances`` — one entry per skill, in the given order, for the multi-skill A/B trial metadata. - """ - - files: dict[str, str] - skill_paths: list[str] - provenances: list[SkillProvenance] - - -def stage_skills_seed( - *, - skills: Sequence[AgentSkill], - adapter_id: str, - mode: SkillMode, - workspace_dir: str, - skills_dir: str, -) -> SkillsSeed: - """Render every skill in ``skills`` into one sandbox seed set for the container runtime. - - The plural, containerized sibling of :func:`install_skills`: renders each bundle (via - :func:`_render_skill_seed`) under its own ``/`` at the harness's in-sandbox discovery path and - collects the native in-sandbox roots. The caller merges those into the composed config's - ``skills.paths`` alongside whatever it already declared, so nothing has to be re-listed here. Skill - names must be unique — their ``/`` bundles would otherwise collide. No on-disk rollback is - needed (unlike :func:`install_skills`): the seed set is an in-memory map, so a failure to render any - skill just discards the accumulated map and raises, leaving nothing staged. - """ - require_unique_skill_names(skills) - files: dict[str, str] = {} - provenances: list[SkillProvenance] = [] - for skill in skills: - rendered, provenance = _render_skill_seed( - skill=skill, adapter_id=adapter_id, mode=mode, workspace_dir=workspace_dir, skills_dir=skills_dir - ) - files.update(rendered) - provenances.append(provenance) - - skill_paths: list[str] = [] - if mode == SKILL_MODE_NATIVE: - # Each staged bundle (a native provenance's ``location`` is its absolute in-sandbox skill - # root), order-preserved and de-duplicated. - skill_paths = list(dict.fromkeys(prov["location"] for prov in provenances)) - return SkillsSeed(files=files, skill_paths=skill_paths, provenances=provenances) - - -def _stage_bundle(directory: Path, skill_root: Path, *, reserved: bool) -> None: - """Stage the skill ``directory`` as an *exact* copy at ``skill_root`` (the ``/`` bundle dir). - - The staged bundle must reflect exactly the supplied directory, so provenance and behaviour track the - real content. ``reserved`` picks the collision policy for the destination: - - * ``reserved=False`` — the evaluator-owned native stage dir: recreate it, so a reused run id can't - leave a file that was since removed from the source bundle surviving in the stage. - * ``reserved=True`` — the Codex workspace path (``.agents/skills/``): refuse to clobber - pre-existing content there, since it can only be a task-seeded file colliding with the reserved - skill path. - """ - src = directory.expanduser() - if not (src / PRIMARY_SKILL_DOC).is_file(): - raise SkillInjectionError(f"skill directory {str(directory)!r} has no {PRIMARY_SKILL_DOC}") - if skill_root.exists(): - if reserved: - raise SkillInjectionError( - f"cannot stage skill into reserved path {str(skill_root)!r}: it already exists " - "(a task-seeded file collides with the injected skill bundle)" - ) - shutil.rmtree(skill_root) # evaluator-owned: recreate so the stage is an exact copy - skill_root.parent.mkdir(parents=True, exist_ok=True) - # OS-level copy — file contents never pass through Python memory. - shutil.copytree(src, skill_root) - - -def _provenance(skill: AgentSkill, skill_hash: str, mode: SkillMode, adapter_id: str, location: str) -> SkillProvenance: - return { - "name": skill.name, - "hash": skill_hash, - "mode": mode, - "adapter_id": adapter_id, - "location": location, - } - - -def _hash_directory(directory: Path) -> str: - """Stable sha256 over a directory's file tree (sorted relpath + contents).""" - digest = hashlib.sha256() - for path in sorted(path for path in directory.rglob("*") if path.is_file()): - digest.update(path.relative_to(directory).as_posix().encode("utf-8")) - digest.update(b"\0") - digest.update(path.read_bytes()) - digest.update(b"\0") - return digest.hexdigest() diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/harbor_runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/harbor_runtime.py deleted file mode 100644 index e96b8132d4..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/harbor_runtime.py +++ /dev/null @@ -1,1413 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Harbor-backed :class:`AgentTaskRunner` for the agent-eval pipeline. - -Harbor already runs trials in (Docker) environments, retries them, and writes a -documented results tree: one ``__/result.json`` per trial under the -job directory. This runtime adapts that tree into SDK :class:`AgentEvalTrial` -objects so an :class:`AgentEvaluator` can score and report Harbor runs through the -same seam as any other runtime. - -Two ways to drive it: - -* **Native** — pass a :class:`HarborRuntimeConfig` and a dataset directory; the - runtime builds Harbor's ``JobConfig`` and runs it itself. The one-call - :func:`run_harbor_eval` loads the tasks, runs, and scores, so caller code is a - couple of lines. ``harbor`` is imported lazily inside ``run_tasks`` (it is an - optional extra), so importing this module never requires Harbor. Custom - ``import_path`` agents are supported too: set ``agent_import_path`` and, for a - loose ``harbor_wrapper.py``, also ``agent_dir`` — the runtime then injects that - directory into ``sys.modules`` for the duration of the run and tears it down - after (see :func:`scoped_harbor_agent_import`). When ``agent_dir`` is omitted - (the module is already importable) the path is handed to Harbor's importer - unchanged, so nothing is imposed on how the agent is packaged. -* **Injected / offline** — pass a ``job_dir`` (and optionally a ``run_job`` - callback) to adapt an already-completed job dir or to run a caller-built job. - -Trial *adaptation* only ever reads Harbor's on-disk ``result.json`` files, so -that half stays dependency-free regardless of how the job was produced. -""" - -from __future__ import annotations - -import contextlib -import hashlib -import importlib.machinery -import json -import logging -import os -import re -import shutil -import sys -import threading -import tomllib -from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence -from datetime import datetime, timezone -from pathlib import Path -from types import ModuleType -from typing import Any - -from nemo_platform.beta.evaluator.agent_eval.results import AgentEvalResult -from nemo_platform.beta.evaluator.agent_eval.scores import AgentEvalScoreStatus -from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask, AgentEvalTaskset -from nemo_platform.beta.evaluator.agent_eval.trials import ( - AgentEvalTrial, - AgentEvalTrialStatus, - AgentOutput, - RunnerInfo, - standard_evidence_descriptors, -) -from nemo_platform.beta.evaluator.metrics.protocol import Metric, MetricInput, MetricOutput, MetricOutputSpec, MetricResult -from nemo_platform.beta.evaluator.values.evidence import CandidateEvidence -from pydantic import BaseModel, ConfigDict, Field, model_validator - -logger = logging.getLogger(__name__) - -# Default reward key inside Harbor's ``verifier_result.rewards`` mapping. -DEFAULT_REWARD_KEY = "reward" -# Filename that marks a directory as a Harbor task, and the template dir to skip. -_TASK_CONFIG_FILENAME = "task.toml" -_TASK_TEMPLATE_DIRNAME = "task_template" -# Synthetic sys.modules root a custom ``import_path`` agent package is injected under. -_AGENT_IMPORT_ROOT = "_nemo_evaluator_harbor_agents" -# Guards the sys.modules mutation while injecting/removing scoped agent packages. -_IMPORT_LOCK = threading.Lock() -# Open scopes per content-addressed agent package. Identical agent contents share a -# package name, so teardown must wait for the last scope rather than the first. -_AGENT_PACKAGE_REFCOUNTS: dict[str, int] = {} -# Characters of the agent-content digest used to disambiguate the package name. Long -# enough that distinct agents don't collide; short enough to keep import paths (and -# Harbor's persisted JobConfig) readable. -_IMPORT_DIGEST_CHARS = 12 -# Records which inputs produced a job dir, so a rerun can tell a reusable cache from -# a stale one. A file, not a directory: Harbor rmtree's stray directories in a job dir. -CACHE_STAMP_FILENAME = ".nemo-eval-harbor-cache.json" -# Public so downstreams can assert the SDK is new enough to own cache staleness. -CACHE_STAMP_VERSION = 1 -# Excluded from the cache fingerprint — see :func:`_cache_stamp` for why each one. -_CACHE_IRRELEVANT_OPTIONS = frozenset( - {"jobs_dir", "job_name", "force_rerun", "quiet", "n_concurrent_trials", "agent_dir", "reward_key"} -) -# Where Harbor persists the JobConfig it will compare a resume against. -_HARBOR_JOB_CONFIG_FILENAME = "config.json" -# Fields Harbor's own JobConfig equality ignores, so they can never be why it refused -# to resume. Pinned against Harbor upstream by the drift-guard test. -_HARBOR_EQ_IGNORED_FIELDS = frozenset({"job_name", "debug"}) -# How Harbor says "this job dir cannot be resumed": one from the JobConfig comparison -# in `Job.create`, one from the lock.json check early in `Job.run`. Matching on the -# message is deliberate coupling, and it fails in the safe direction — an -# unrecognized FileExistsError propagates untouched rather than costing a job dir, so -# a Harbor reword degrades to a loud crash, never to a silent deletion. -_HARBOR_RESUME_REFUSALS = ("resumed with a different config", "does not match the resolved job lock") -# Cap on each value rendered into the "what differed" log line: enough for a scalar -# like `n_concurrent_trials`, bounded for a whole nested `agents` list. -_DRIFT_VALUE_CHARS = 80 -# Markdown instruction files may carry repository license comments. Those are file metadata, not -# agent-facing task instructions. -_SPDX_HTML_COMMENT_RE = re.compile(r"\s*") -# Derived/VCS noise skipped when digesting a directory. Deliberately NOT skipped: -# `node_modules` and other vendored dependency trees, which ship with the agent and -# change what it does. `.venv`/`.uv` stay skipped because they are environment, not -# deliverable — the Harbor wrapper does not upload them into the task container. -_DIGEST_SKIP_DIRS = frozenset({".git", "__pycache__", ".venv", ".uv", ".mypy_cache", ".pytest_cache"}) -_DIGEST_CHUNK_BYTES = 1 << 20 - -RunJob = Callable[[], Awaitable[None]] - - -class HarborRuntimeConfig(BaseModel): - """Declarative config for running a Harbor job natively through the SDK. - - Holds only plain/pydantic fields so importing this module never needs Harbor; - the fields are mapped onto Harbor's ``JobConfig`` lazily at run time. - """ - - model_config = ConfigDict(extra="forbid") - - jobs_dir: Path = Field(description="Parent directory Harbor writes the ``/`` results tree into.") - job_name: str | None = Field(default=None, description="Harbor job name; a timestamp is generated when omitted.") - agent_name: str | None = Field( - default="oracle", - description="Built-in Harbor agent to run (e.g. 'oracle'). Ignored when ``agent_import_path`` is set.", - ) - agent_import_path: str | None = Field( - default=None, - description="Custom Harbor agent import path (e.g. 'harbor_wrapper:WrappedAgent'); overrides ``agent_name``.", - ) - agent_dir: Path | None = Field( - default=None, - description=( - "Directory holding the module named by ``agent_import_path``. Set it for a loose " - "wrapper file (the SDK makes it importable); leave it unset when the module is " - "already importable (installed package), and Harbor imports it directly." - ), - ) - agent_model_name: str | None = Field(default=None, description="Optional model slug passed to the Harbor agent.") - n_attempts: int = Field(default=1, ge=1, description="Number of attempts Harbor runs per task.") - n_concurrent_trials: int = Field(default=4, ge=1, description="Maximum concurrent Harbor trials.") - quiet: bool = Field(default=True, description="Suppress Harbor's trial progress displays.") - force_rerun: bool = Field(default=False, description="Delete an existing job dir before running.") - artifacts: list[str] = Field(default_factory=list, description="Harbor artifact sources to collect per trial.") - trace_dir: str | None = Field( - default=None, - description="Container path of agent traces to collect as the 'traces' artifact (e.g. '/app/traces').", - ) - max_retries: int = Field(default=0, ge=0, description="Harbor per-trial retry attempts on transient failures.") - timeout_multiplier: float | None = Field(default=None, description="Global Harbor timeout multiplier.") - agent_timeout_multiplier: float | None = Field(default=None, description="Agent-phase timeout multiplier.") - verifier_timeout_multiplier: float | None = Field(default=None, description="Verifier-phase timeout multiplier.") - agent_setup_timeout_multiplier: float | None = Field(default=None, description="Agent-setup timeout multiplier.") - environment_build_timeout_multiplier: float | None = Field( - default=None, description="Environment-build timeout multiplier." - ) - reward_key: str = Field(default=DEFAULT_REWARD_KEY, description="Key read from Harbor's rewards mapping.") - - @model_validator(mode="after") - def _agent_dir_needs_import_path(self) -> HarborRuntimeConfig: - if self.agent_dir is not None and self.agent_import_path is None: - raise ValueError("agent_dir only applies to a custom agent_import_path") - return self - - -class HarborRewardMetric: - """Score the verifier reward Harbor stamped onto trial metadata. - - Reads ``reward`` from the candidate metadata (populated by - :func:`build_trials_from_job_dir`); a trial with no verifier reward scores - ``0.0``. This is the Harbor analogue of the example ``VerifierRewardMetric`` - — a reward-off-metadata scorer. - """ - - def __init__(self, *, output_name: str = "reward", metric_type: str = "harbor_reward") -> None: - self._output_name = output_name - self._metric_type = metric_type - - @property - def type(self) -> str: - return self._metric_type - - def output_spec(self) -> list[MetricOutputSpec]: - return [MetricOutputSpec.continuous_score(self._output_name)] - - async def compute_scores(self, input: MetricInput) -> MetricResult: - reward = input.candidate.metadata.get("reward") - value = float(reward) if reward is not None else 0.0 - return MetricResult(outputs=[MetricOutput(name=self._output_name, value=value)]) - - -def _effective_harbor_agent(config: HarborRuntimeConfig | None) -> str | None: - """The agent a run will actually use, mirroring ``run_job``'s resolution order. - - ``agent_import_path`` wins when set; otherwise the built-in ``agent_name``, which itself falls back - to Harbor's ``oracle`` default. Recording the resolved value keeps two runs with different custom - agents distinguishable in provenance. - """ - if config is None: - return None - return config.agent_import_path or config.agent_name or "oracle" - - -class HarborAgentTaskRunner: - """An :class:`AgentTaskRunner` that runs a Harbor job, then adapts its results. - - Two construction modes: - - * **Native** — pass ``config`` (a :class:`HarborRuntimeConfig`); the runtime - builds and runs Harbor's ``JobConfig`` itself (Harbor is imported lazily). - The dataset directory is taken from the tasks handed to :meth:`run_tasks` - (each carries ``metadata['harbor_dataset_path']`` from - :func:`discover_harbor_tasks`), or from an explicit ``dataset_path`` - override, so it isn't repeated. ``task_names`` optionally restricts the run - to a subset of tasks, and the ``config``'s ``job_dir`` doubles as a cache: - an existing run whose results already cover every requested task (with - ``n_attempts`` completed, non-errored trials each) is re-adapted instead of - re-run (unless ``force_rerun`` is set). Caching only takes effect when a - stable ``job_name`` is set on the config — the default timestamped - ``job_name`` writes a fresh dir per run and never hits the cache. - * **Injected / offline** — pass ``job_dir`` (and optionally a ``run_job`` - callback); ``run_job`` is awaited before the job dir is read, and - ``run_job=None`` simply adapts an already-completed job dir. - - ``job_dir`` is the directory Harbor writes its per-trial - ``__/result.json`` files into. - """ - - def __init__( - self, - *, - config: HarborRuntimeConfig | None = None, - dataset_path: str | Path | None = None, - task_names: Sequence[str] | None = None, - job_dir: str | Path | None = None, - run_job: RunJob | None = None, - reward_key: str = DEFAULT_REWARD_KEY, - ) -> None: - if config is None and job_dir is None: - raise ValueError("provide either a HarborRuntimeConfig or an explicit job_dir") - self._config = config - self._dataset_path = Path(dataset_path) if dataset_path is not None else None - self._task_names = task_names - self._job_dir = Path(job_dir) if job_dir is not None else None - self._run_job = run_job - self._reward_key = config.reward_key if config is not None else reward_key - - def runner_info(self) -> RunnerInfo: - """Identify this runner and the Harbor settings that shape its results. - - Records the *effective* agent, mirroring how ``run_job`` resolves it: ``agent_import_path`` - overrides ``agent_name`` (which itself defaults to ``oracle``). Reporting the configured - ``agent_name`` alone would give two runs using different custom agents identical provenance. - """ - config = self._config - return RunnerInfo( - name="harbor", - kind="runner", - config={ - "agent_name": config.agent_name if config is not None else None, - "agent_import_path": config.agent_import_path if config is not None else None, - "agent_model_name": config.agent_model_name if config is not None else None, - "effective_agent": _effective_harbor_agent(config), - "n_attempts": config.n_attempts if config is not None else None, - # Native mode resolves the concrete job directory inside run_tasks (the name defaults - # to a timestamp), so record the configured location rather than a not-yet-known path. - "job_dir": str(self._job_dir) if self._job_dir is not None else None, - "jobs_dir": str(config.jobs_dir) if config is not None else None, - "job_name": config.job_name if config is not None else None, - "reward_key": self._reward_key, - }, - ) - - async def run_tasks( - self, - tasks: Sequence[AgentEvalTask], - config: AgentEvalRunConfig | None = None, - ) -> list[AgentEvalTrial]: - """Run the Harbor job when needed, then return one trial per Harbor trial. - - In native mode the dataset directory is recovered from the tasks (each - carries ``metadata['harbor_dataset_path']`` from - :func:`discover_harbor_tasks`) unless a ``dataset_path`` override was given, - so callers don't repeat it. - - ``job_dir`` doubles as a cache. Results are served straight off it, without - importing Harbor at all, only when **both** hold: every requested task already - has ``n_attempts`` completed, non-errored results there, *and* the directory - carries a cache stamp matching this run's inputs (agent contents, task - contents, result-affecting options). - - Otherwise Harbor runs, and what happens to the directory depends on *which* - check failed. A **stamp mismatch** discards it first: those results came from - different inputs, so there is nothing safe to resume onto. A directory that - merely lacks **coverage** — stamp matches, but not enough completed results — - is handed to Harbor intact so its per-trial resume keeps the finished trials - and runs only what is missing. Harbor may still refuse a directory on its own - (stricter) terms; :func:`_build_native_job` then discards it and re-runs. - - The cache only engages when the config pins a stable ``job_name``; with the - default timestamped name no fingerprint is computed at all. - - Assumes a **single writer per job directory**. Neither this runtime nor - Harbor locks it, so two processes sharing a pinned ``job_name`` on a shared - volume will race. - """ - if self._config is not None: - dataset_path = self._dataset_path or _dataset_path_from_tasks(tasks) - job_name, job_dir = _resolve_job_dir(self._config) - - # Only fingerprint when the answer can depend on it: an unpinned job name - # can never hit, and force_rerun/a missing dir already decided. This keeps - # the digest I/O off every run of callers that don't pin a job name. - stamp: dict[str, Any] | None = None - if self._config.job_name is None or self._config.force_rerun or not job_dir.is_dir(): - stale = True - else: - stamp = _cache_stamp(self._config, dataset_path, tasks) - stale = _cache_is_stale(job_dir, stamp) - - if stale or not _all_tasks_cached(job_dir, tasks, n_attempts=self._config.n_attempts): - job_dir, run_job = _build_native_job( - self._config, - dataset_path, - self._task_names, - job_name=job_name, - # Discard only when the inputs changed. Otherwise leave it off so - # Harbor resumes per trial and keeps completed work — including - # with `agent_dir` set, now that the scoped import path is - # content-addressed rather than a fresh uuid per run and Harbor's - # JobConfig comparison can therefore match (AALGO-430). - force_rerun=(self._config.force_rerun or stale), - ) - # Fingerprint the inputs *before* running and confirm they are - # unchanged afterwards. Stamping only the post-run state would label - # results produced from the old sources with the new fingerprint, so - # a later run would happily serve them. Covers what Harbor actually - # ran: with no task_names filter that is the whole dataset, and - # recording only the requested subset would make the next full-set - # run look stale and re-run a complete job dir. - coverage = _stamp_coverage(dataset_path, tasks, self._task_names) - before = _cache_stamp(self._config, dataset_path, coverage) - await run_job() - if self._config.job_name is not None: - after = _cache_stamp(self._config, dataset_path, coverage) - if after == before: - _write_cache_stamp(job_dir, after) - else: - # Leaving it unstamped re-runs next time, which is the safe - # direction: we cannot say which inputs produced these results. - logger.warning( - "Agent or task contents changed while Harbor job %s was running; leaving it unstamped " - "so the next run re-executes rather than trusting these results.", - job_dir, - ) - return build_trials_from_job_dir(job_dir, tasks, reward_key=self._reward_key) - - if self._job_dir is None: # unreachable: __init__ guarantees config or job_dir - raise ValueError("no job_dir configured") - if self._run_job is not None: - await self._run_job() - return build_trials_from_job_dir(self._job_dir, tasks, reward_key=self._reward_key) - - -def _dataset_path_from_tasks(tasks: Sequence[AgentEvalTask]) -> Path: - """Recover the Harbor dataset dir stamped on tasks by :func:`discover_harbor_tasks`.""" - for task in tasks: - stamped = task.metadata.get("harbor_dataset_path") - if isinstance(stamped, str) and stamped: - return Path(stamped) - raise ValueError( - "native Harbor run needs a dataset path: pass dataset_path, or build tasks with " - "discover_harbor_tasks/HarborTasksetLoader (which stamp metadata['harbor_dataset_path'])" - ) - - -def _all_tasks_cached(job_dir: Path, tasks: Sequence[AgentEvalTask], *, n_attempts: int) -> bool: - """Return True when every requested task already has ``n_attempts`` completed results. - - Lets ``job_dir`` act as a cache so a native run whose results are all present - is re-adapted instead of re-run. The cache is **success-aware**: only trials - that finished without an ``exception_info`` count, and a task must have at - least ``n_attempts`` of them, so an interrupted, errored, or under-sampled run - is re-run rather than silently served from a partial cache. Caching only takes - effect when a stable ``job_name`` is set on the config; with the default - timestamped ``job_name`` every run writes a fresh dir and never hits the cache. - """ - if not job_dir.is_dir(): - return False - counts: dict[str, int] = {} - for result_path in job_dir.glob("*/result.json"): - try: - data = json.loads(result_path.read_text()) - except (json.JSONDecodeError, OSError): - continue - if data.get("exception_info") is not None: - continue - name = data.get("task_name") - if isinstance(name, str): - counts[name] = counts.get(name, 0) + 1 - return all(counts.get(task.id, 0) >= n_attempts for task in tasks) - - -def _feed(digest: "hashlib._Hash", label: bytes, payload: bytes) -> None: - """Append a length-framed field to ``digest``. - - Framing matters: concatenating ``name \0 content \0`` is ambiguous because file - *contents* may contain NUL, so two different trees can produce an identical byte - stream. Prefixing every variable-length field with its length makes the encoding - injective, which is what stops a collision from being read as "unchanged". - """ - digest.update(label) - digest.update(len(payload).to_bytes(8, "big")) - digest.update(payload) - - -def _safe_resolve(path: Path) -> Path: - """``Path.resolve()`` that degrades instead of raising. - - The digest is a best-effort guard, not a reason to fail a run that would - otherwise succeed, so fall back to the unresolved absolute path. - - ``RuntimeError`` is caught alongside ``OSError`` and is the case that actually - fires: on CPython 3.12 — the floor this package targets — a **symlink loop** - surfaces as ``RuntimeError("Symlink loop from ...")``, because ``resolve()`` - translates ``ELOOP`` before re-raising. It is not an ``OSError``, so catching - only that would let a loop under a task directory kill the run. A loop raises - deterministically, not as a race. ``OSError`` covers the narrower case of a - symlink that disappears mid-walk, since ``resolve()`` calls ``os.readlink``. - - Both are 3.12/3.13 behaviours: 3.14 resolves a loop without raising at all. - """ - try: - return path.resolve() - except (OSError, RuntimeError): - return path.absolute() - - -def _is_executable(path: Path) -> bool: - """Whether the owner-execute bit is set, following symlinks. - - Only the execute bit, mirroring what git tracks: read/write bits vary with umask - and would evict the cache for nothing, but flipping +x on ``tests/test.sh`` or an - agent entrypoint genuinely changes what Harbor does. - """ - try: - return bool(path.stat().st_mode & 0o100) - except OSError: - return False - - -def _digest_directory(root: Path, *, exclude: frozenset[Path] = frozenset()) -> str: - """Content-hash a directory tree, skipping build/VCS noise and excluded roots. - - Contents rather than mtimes: callers routinely materialize the directory with - ``copytree`` (the optimizer does, per candidate), which rewrites every mtime and - would defeat the cache entirely. - - ``exclude`` takes *resolved* directories to skip wholesale. It exists because - ``jobs_dir`` is caller-chosen and may sit **under** the dataset or agent - directory; without excluding it the digest would hash the growing results tree - it is meant to validate, and would never stabilize. - - Symlinks are **followed**, not skipped. Skipping them silently defeats the whole - guard: a directory assembled out of links to shared sources would hash to the - empty digest, so edits behind those links would never invalidate the cache. The - link target is folded in alongside the contents, so re-pointing a link is a - change even when both targets happen to hold identical bytes. - - Unreadable or vanished files are folded in as a marker rather than raised: a - transient read failure must not kill a run that would otherwise have succeeded. - """ - digest = hashlib.sha256() - if not root.is_dir(): - return digest.hexdigest() - # Keep only exclusions strictly *inside* this tree. An excluded root that is an - # ancestor of (or equal to) `root` would otherwise match every entry and yield - # an empty digest — silently disabling invalidation for the whole directory, - # which is exactly the failure this function exists to prevent. jobs_dir being - # a parent of the agent/task dir is a legitimate layout, not a reason to stop - # hashing it. - root_resolved = _safe_resolve(root) - excluded = { - resolved - for resolved in (_safe_resolve(path) for path in exclude) - if resolved != root_resolved and resolved.is_relative_to(root_resolved) - } - # Resolved dirs already walked, so a symlink cycle terminates instead of hanging. - visited: set[Path] = set() - - def walk(directory: Path) -> None: - resolved_dir = _safe_resolve(directory) - if resolved_dir in visited: - return - visited.add(resolved_dir) - try: - entries = sorted(directory.iterdir()) - except OSError as exc: - logger.warning("Could not list %s while fingerprinting %s: %s", directory, root, exc) - _feed(digest, b"unlistable", b"") - return - for path in entries: - if path.name in _DIGEST_SKIP_DIRS: - continue - resolved = _safe_resolve(path) - if any(resolved == item or item in resolved.parents for item in excluded): - continue - - _feed(digest, b"name", path.relative_to(root).as_posix().encode("utf-8")) - _feed(digest, b"mode", b"x" if _is_executable(path) else b"-") - if path.is_symlink(): - # Record where the link points, so retargeting counts as a change. - try: - target = os.readlink(path).encode("utf-8") - except OSError as exc: - # The link vanished mid-walk. Same contract as an unreadable - # file: degrade to a marker rather than kill the run. - logger.warning("Could not read link %s while fingerprinting %s: %s", path, root, exc) - target = b"" - _feed(digest, b"symlink", target) - if path.is_dir(): - _feed(digest, b"dir", b"") - walk(path) - continue - if not path.is_file(): - # Broken link, socket, fifo: nothing to hash, but its presence counts. - _feed(digest, b"not-a-file", b"") - continue - content = hashlib.sha256() - try: - with path.open("rb") as handle: - # Streamed: Harbor datasets may ship large seeds or build contexts. - for chunk in iter(lambda: handle.read(_DIGEST_CHUNK_BYTES), b""): - content.update(chunk) - except OSError as exc: - logger.warning("Could not read %s while fingerprinting %s: %s", path, root, exc) - content.update(b"") - # The sub-digest is fixed width, so file bytes can never be confused with - # the framing around them. - _feed(digest, b"file", content.digest()) - - walk(root) - return digest.hexdigest() - - -def _task_dirs_for(dataset_path: Path, tasks: Sequence[AgentEvalTask]) -> dict[str, Path | None]: - """Resolve each task's on-disk directory, falling back to re-discovery. - - :func:`discover_harbor_tasks` stamps ``metadata['harbor_task_dir']``, but callers - may build :class:`AgentEvalTask` objects by hand (the Evaluator plugin builds - them from a job spec), so the metadata is not guaranteed. - - A task that cannot be resolved maps to ``None`` — the caller must treat that as - un-cacheable rather than silently omitting it from the fingerprint, which would - be a stale-cache hole. - """ - # `_safe_resolve`, not bare `resolve()`: this walk is a best-effort cache guard, so - # a symlink that vanishes mid-run must degrade to an unresolved absolute path - # rather than raise out of a job that would otherwise succeed. - dataset_root = _safe_resolve(dataset_path) - resolved: dict[str, Path | None] = {} - for task in tasks: - stamped = task.metadata.get("harbor_task_dir") - candidate = Path(stamped) if isinstance(stamped, str) and stamped else None - # The stamp records where a task was *discovered*, which is not necessarily - # where this run executes it: `dataset_path` can be overridden on the runner. - # Trusting a stale or foreign path would fingerprint one dataset while Harbor - # runs another, so anything missing or outside the active dataset is dropped - # and re-discovered below. - if candidate is not None: - candidate_resolved = _safe_resolve(candidate) - if not candidate.is_dir() or not candidate_resolved.is_relative_to(dataset_root): - logger.debug( - "Ignoring stamped harbor_task_dir %s for task %r: not a directory under the active dataset %s", - candidate, - task.id, - dataset_root, - ) - candidate = None - resolved[task.id] = candidate - if all(path is not None for path in resolved.values()): - return resolved - - try: - discovered = { - task.id: Path(str(task.metadata["harbor_task_dir"])) for task in discover_harbor_tasks(dataset_path) - } - except (OSError, ValueError) as exc: - # discover_harbor_tasks raises on ANY malformed task.toml in the dataset. - # Refusing the cache is the safe reading; failing the run is not, since this - # path previously never read those files. - logger.warning( - "Could not resolve Harbor task dirs under %s; treating the cache as stale: %s", dataset_path, exc - ) - return dict.fromkeys(resolved, None) - return {task_id: path or discovered.get(task_id) for task_id, path in resolved.items()} - - -def _stamp_coverage( - dataset_path: Path, - tasks: Sequence[AgentEvalTask], - task_names: Sequence[str] | None, -) -> Sequence[AgentEvalTask]: - """Tasks a written stamp must cover: everything Harbor was asked to run. - - ``task_names`` is the filter handed to Harbor's ``DatasetConfig``. When it is - ``None`` Harbor runs every task in the dataset, which can be a superset of the - tasks this call was asked to score — and a stamp that recorded only the smaller - set would report the larger one as stale on the next run. - """ - if task_names is not None: - return tasks - try: - discovered = discover_harbor_tasks(dataset_path) - except (OSError, ValueError): - # Same reasoning as _task_dirs_for: a malformed sibling task must not fail a - # run. Recording only the requested tasks just costs a re-run later. - return tasks - covered = {task.id: task for task in discovered} - covered.update({task.id: task for task in tasks}) - return list(covered.values()) - - -def _cache_stamp( - config: HarborRuntimeConfig, - dataset_path: Path, - tasks: Sequence[AgentEvalTask], -) -> dict[str, Any]: - """Fingerprint the inputs that decide whether a job dir can be reused. - - Covers the result-affecting options, the contents of ``agent_dir``, and the - contents of every task directory. Two gaps are deliberate and worth knowing - before trusting a hit: when ``agent_dir`` is ``None`` the agent is an already - importable module, so only its *import path* is fingerprinted and edits to that - installed package are invisible; and a task whose directory cannot be resolved - is recorded as ````, which always forces a re-run. - - Recorded per task rather than as one job-wide hash so that evaluating a - **subset** of a previously-cached job still hits: staleness is decided only over - the tasks actually requested. - - Excluded from the option hash: presentation and placement knobs (``quiet``, - ``n_concurrent_trials``, ``jobs_dir``, ``job_name``, ``force_rerun``), which - change nothing about the results; ``agent_dir``, an absolute path whose - *content* is hashed separately, so a relocated but identical agent still hits; - and ``reward_key``, which only selects which reward - :func:`build_trials_from_job_dir` reads back and must not cost a Docker re-run. - """ - options = config.model_dump(exclude=set(_CACHE_IRRELEVANT_OPTIONS), mode="json") - # `_safe_resolve` throughout, matching `_task_dirs_for`: fingerprinting is - # best-effort, so a symlink loop or a vanished link under any of these must - # degrade to an unresolved path rather than raise out of `run_tasks` and fail a - # run that would otherwise succeed. - excluded_roots = frozenset({_safe_resolve(config.jobs_dir.expanduser())}) - - agent_digest = "" - if config.agent_dir is not None: - agent_digest = _digest_directory(_safe_resolve(config.agent_dir.expanduser()), exclude=excluded_roots) - - task_digests: dict[str, str] = {} - for task_id, task_dir in sorted(_task_dirs_for(dataset_path, tasks).items()): - task_digests[task_id] = ( - "" if task_dir is None else _digest_directory(_safe_resolve(task_dir), exclude=excluded_roots) - ) - - return { - "version": CACHE_STAMP_VERSION, - "options": hashlib.sha256(json.dumps(options, sort_keys=True, default=str).encode("utf-8")).hexdigest(), - "agent": agent_digest, - "tasks": task_digests, - } - - -def _cache_is_stale(job_dir: Path, stamp: Mapping[str, Any]) -> bool: - """Return True when ``job_dir`` was not produced by the inputs in ``stamp``. - - A directory with no stamp is stale: it predates this check, or was written by - plain Harbor, and re-running is the safe reading. An ```` task - digest is likewise always stale — we could not prove the inputs match. A - directory that does not exist is stale too: there is nothing there to reuse, and - answering "not stale" would be an invitation to serve zero trials. - """ - if not job_dir.is_dir(): - return True - try: - stored = json.loads((job_dir / CACHE_STAMP_FILENAME).read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - stored = None - - reason: str | None = None - if not isinstance(stored, Mapping): - reason = "no usable cache stamp" - elif stored.get("version") != stamp["version"]: - reason = "cache stamp version changed" - elif stored.get("options") != stamp["options"]: - reason = "a result-affecting option changed" - elif stored.get("agent") != stamp["agent"]: - reason = "the agent directory changed" - else: - stored_tasks = stored.get("tasks") - stored_tasks = stored_tasks if isinstance(stored_tasks, Mapping) else {} - for task_id, digest in stamp["tasks"].items(): - if digest == "": - reason = f"task {task_id!r} could not be resolved on disk" - break - if stored_tasks.get(task_id) != digest: - reason = f"task {task_id!r} changed or was not part of the cached run" - break - - if reason is None: - return False - logger.info("Re-running Harbor job %s instead of serving it from cache: %s.", job_dir, reason) - return True - - -def _write_cache_stamp(job_dir: Path, stamp: Mapping[str, Any]) -> None: - """Record the inputs a completed job dir was produced from. - - Best-effort: a job dir that could not be stamped simply re-runs next time, which - is the safe direction. Written as a *file* deliberately — Harbor deletes any - stray *directory* in a job dir that lacks ``result.json``. - """ - try: - (job_dir / CACHE_STAMP_FILENAME).write_text( - json.dumps(stamp, indent=2, sort_keys=True) + "\n", encoding="utf-8" - ) - except OSError as exc: - logger.warning("Could not stamp Harbor job dir %s with its cache key: %s", job_dir, exc) - - -def _resolve_job_dir(config: HarborRuntimeConfig) -> tuple[str, Path]: - """Resolve ``(job_name, job_dir)`` without importing Harbor. - - Split out from :func:`_build_native_job` because the caller must know the job - directory *before* deciding whether to run: an unpinned ``job_name`` is a - timestamp with microsecond precision, so resolving it twice would yield two - different directories and the cache decision would be made about the wrong one. - """ - job_name = config.job_name or datetime.now(timezone.utc).strftime("%Y-%m-%d__%H-%M-%S__%f") - return job_name, config.jobs_dir / job_name - - -def _build_native_job( - config: HarborRuntimeConfig, - dataset_path: Path, - task_names: Sequence[str] | None, - *, - job_name: str | None = None, - force_rerun: bool | None = None, -) -> tuple[Path, RunJob]: - """Build a Harbor ``JobConfig`` from ``config`` and return ``(job_dir, run_job)``. - - Harbor is imported inside ``run_job`` (not at module load) because it is an - optional extra. The job name is resolved up front so ``job_dir`` is known - without importing Harbor. When ``agent_import_path`` is set, ``run_job`` - scopes the user's agent package into ``sys.modules`` for the run and removes - it afterwards (see :func:`scoped_harbor_agent_import`). - - Args: - job_name: Pre-resolved job name from :func:`_resolve_job_dir`. Pass it when - the caller already resolved the directory, so an unpinned name is not - re-generated into a different timestamp. - force_rerun: Overrides ``config.force_rerun`` for this build. Passed rather - than applied via ``model_copy`` so the caller's config is never mutated - and the job name stays fixed. - """ - resolved_name = job_name if job_name is not None else _resolve_job_dir(config)[0] - job_dir = config.jobs_dir / resolved_name - effective_force_rerun = config.force_rerun if force_rerun is None else force_rerun - - async def run_job() -> None: - try: - from harbor.job import DatasetConfig, Job, JobConfig # ty: ignore[unresolved-import] - from harbor.models.job.config import RetryConfig # ty: ignore[unresolved-import] - from harbor.models.trial.config import AgentConfig, ArtifactConfig # ty: ignore[unresolved-import] - except ModuleNotFoundError as exc: - raise ModuleNotFoundError( - "the native Harbor runtime needs `harbor`, which is not an SDK dependency " - '(it requires Python >=3.12). Install it separately: uv pip install "harbor>=0.16.1"' - ) from exc - - if effective_force_rerun and job_dir.exists(): - shutil.rmtree(job_dir) - - artifacts: list[str | ArtifactConfig] = list(config.artifacts) - if config.trace_dir is not None: - artifacts = [ArtifactConfig(source=config.trace_dir, destination="traces"), *artifacts] - - timeout_kwargs = { - key: value - for key, value in { - "timeout_multiplier": config.timeout_multiplier, - "agent_timeout_multiplier": config.agent_timeout_multiplier, - "verifier_timeout_multiplier": config.verifier_timeout_multiplier, - "agent_setup_timeout_multiplier": config.agent_setup_timeout_multiplier, - "environment_build_timeout_multiplier": config.environment_build_timeout_multiplier, - }.items() - if value is not None - } - - async def _create_and_run(agent: Any) -> None: - job_config = JobConfig( - job_name=resolved_name, - jobs_dir=config.jobs_dir, - n_attempts=config.n_attempts, - n_concurrent_trials=config.n_concurrent_trials, - quiet=config.quiet, - retry=RetryConfig(max_retries=config.max_retries), - artifacts=artifacts, - agents=[agent], - datasets=[DatasetConfig(path=dataset_path, task_names=list(task_names) if task_names else None)], - **timeout_kwargs, - ) - - async def _attempt() -> None: - job = await Job.create(job_config) - await job.run() - - try: - await _attempt() - except FileExistsError as exc: - # Harbor refuses to resume a job dir whose persisted `config.json` or - # `lock.json` differs from this run's — and it refuses by raising, not - # by re-running. Its comparison is deliberately stricter than the SDK - # cache stamp: `quiet`, `n_concurrent_trials` and the `task_names` - # filter all change the JobConfig without changing the results, so the - # stamp excludes them (a full cache hit must not pay for a concurrency - # tweak) while Harbor still rejects the directory. Honour the intent of - # the rerun rather than propagating a crash. - # - # Identify the refusal positively before deleting anything. Both of - # Harbor's refusals fire before any trial executes, so discarding costs - # only completed work — but that reasoning holds *only* for those two. - # An ordinary "file exists" raised from inside a trial, a hook, or an - # environment build must not be mistaken for drift and answered by - # destroying the directory. - if not (job_dir.exists() and _is_harbor_resume_refusal(exc, job_dir)): - raise - drift = _describe_job_config_drift(job_dir, job_config) - logger.warning( - "Harbor refused to resume job dir %s, so it is being discarded and re-run from scratch: %s%s", - job_dir, - exc, - f" Differing config: {drift}." if drift else "", - ) - shutil.rmtree(job_dir) - await _attempt() - - if config.agent_import_path is None: - await _create_and_run(AgentConfig(name=config.agent_name or "oracle", model_name=config.agent_model_name)) - elif config.agent_dir is not None: - # Loose wrapper file: make its directory importable for the run. The - # jobs_dir exclusion must match _cache_stamp's, or a jobs_dir nested under - # agent_dir would shift the package name as results accumulate. - agent_dir = config.agent_dir.expanduser().resolve() - excluded_roots = frozenset({config.jobs_dir.expanduser().resolve()}) - with scoped_harbor_agent_import( - agent_dir, config.agent_import_path, exclude=excluded_roots - ) as scoped_import: - await _create_and_run(AgentConfig(import_path=scoped_import, model_name=config.agent_model_name)) - else: - # Already-importable module (installed package): let Harbor import it directly. - await _create_and_run(AgentConfig(import_path=config.agent_import_path, model_name=config.agent_model_name)) - - return job_dir, run_job - - -def _is_harbor_resume_refusal(exc: FileExistsError, job_dir: Path) -> bool: - """Return True when ``exc`` is Harbor declining to resume ``job_dir``. - - Separates Harbor's refusal — the one case where deleting the directory is the - right answer — from an ordinary "file exists" surfacing from a trial, a hook or an - environment build, where deleting it would destroy completed work to no purpose. - - Two signals, both required. Harbor constructs its refusals with a bare message, so - ``errno`` is unset, while an OS-level ``EEXIST`` always carries one; and both - refusals name the job directory and end in a known phrase. - """ - if exc.errno is not None: - return False - message = str(exc) - return str(job_dir) in message and any(phrase in message for phrase in _HARBOR_RESUME_REFUSALS) - - -def _describe_job_config_drift(job_dir: Path, job_config: Any) -> str: - """Name the fields that differ between ``job_dir``'s persisted JobConfig and this run's. - - Harbor reports *that* an existing config differs, never *which* field, which - leaves the resulting discard looking arbitrary. This reproduces enough of its - comparison to say — turning "Harbor refused" into "n_concurrent_trials: 10 -> 4". - - Best-effort by construction. Returns ``""`` when the difference cannot be - located: no ``config.json``, unparseable, or a refusal that came from - ``lock.json`` instead, which has no JobConfig difference to report. Diagnostics - must never mask the failure they explain, so every error here is swallowed. - """ - try: - stored_text = (job_dir / _HARBOR_JOB_CONFIG_FILENAME).read_text(encoding="utf-8") - # Harbor persists with exclude_defaults=True, so the JSON omits every field - # left at its default and comparing it raw would report phantom differences. - # Round-tripping through the model refills them, which is what Harbor itself - # compares after re-validating the stored config. - stored = type(job_config).model_validate_json(stored_text).model_dump() - current = job_config.model_dump() - return ", ".join( - f"{field}: {_truncated_repr(stored.get(field))} -> {_truncated_repr(value)}" - for field, value in current.items() - if field not in _HARBOR_EQ_IGNORED_FIELDS and stored.get(field) != value - ) - except Exception: - return "" - - -def _truncated_repr(value: Any) -> str: - """Render ``value`` for a log line, bounded so a nested config can't flood it.""" - text = repr(value) - return text if len(text) <= _DRIFT_VALUE_CHARS else f"{text[:_DRIFT_VALUE_CHARS]}..." - - -@contextlib.contextmanager -def scoped_harbor_agent_import( - agent_dir: Path, import_path: str, *, exclude: frozenset[Path] = frozenset() -) -> Iterator[str]: - """Make ``agent_dir`` importable under a content-addressed package for the block. - - Args: - agent_dir: directory containing the module referenced by ``import_path``. - import_path: Harbor agent path, ``"module"`` or ``"module:attribute"``. - exclude: resolved directories to leave out of the content digest. Pass the - same set :func:`_cache_stamp` uses — in practice ``jobs_dir``, which is - caller-chosen and may sit *under* ``agent_dir``. Omitting it lets the - growing results tree feed the package name, so the import path would - change on every run and the resume this function exists to enable would - never happen. See :func:`_digest_directory`. - - Yields: - str: the rewritten import path Harbor should load (the module rooted under - the injected synthetic package, preserving any ``:attribute`` suffix). - - Raises: - ValueError: if ``import_path`` has no module component. - - **The package name is derived from the directory's contents, not a random - UUID, and that is load-bearing.** This string becomes ``AgentConfig.import_path`` - and therefore part of Harbor's ``JobConfig``, which Harbor compares field-by-field - when deciding whether an existing job directory may be resumed. A random suffix - made that comparison fail on every rerun, so Harbor raised ``FileExistsError`` - instead of resuming and its per-trial resume was unreachable for any caller that - sets ``agent_dir`` (AALGO-430). Content-addressing keeps distinct agents isolated - while letting an unchanged agent resume — and makes an *edited* agent invalidate - the job dir on Harbor's own terms. - - Identical contents therefore share a package name, so overlapping scopes are - refcounted: the injected ``sys.modules`` entries are removed when the last - scope exits, not the first (see :func:`_uninstall_agent_package`). The mutation - is guarded by a process-wide lock. ``sys.modules`` is per-process, so concurrent - *processes* were never at risk here. - - The name is ``_``, so it tracks the directory's *location* as - well as its contents — deliberately, because an opaque hash makes every traceback - and import error unreadable. That is a narrow, knowing divergence from the cache - stamp, which excludes ``agent_dir`` so a relocated but identical agent still hits - (see :func:`_cache_stamp`). Relocating an agent while pinning the same - ``job_name`` therefore leaves the stamp valid but changes this string, and Harbor - declines to resume; :func:`_build_native_job` absorbs that into a clean re-run. - The results stay correct — it costs one repeated job. Callers that rebuild agents - under changing directory names (the Experimentalist does) are unaffected, because - the agent name feeds their ``job_name`` too, so a rename lands in a different job - dir with nothing to resume. - - **That last sentence only holds if the caller derives its job name from the - *resolved* directory, as this function does.** Deriving it from the caller's - spelling instead lets the two disagree: a symlink keeps its own name while - resolving elsewhere, so flipping it at a fixed ``job_name`` would reuse one job - dir for two different agents, caught only by Harbor's refusal rather than by - design. The Experimentalist resolves first for exactly this reason - (``resolve_harbor_run_inputs``). - - Only ``agent_dir`` (not ``sys.path``) is made importable, so a loose wrapper - must be self-contained: a single module, or one that reaches siblings via - relative imports (``from .helper import ...``). A wrapper that does an absolute - ``import helper`` of a sibling file won't resolve — install it as a package and - use the ``agent_dir``-less path instead. - """ - module_name, sep, attribute = import_path.partition(":") - module_name = module_name.strip().lstrip(".") - if not module_name: - raise ValueError("import_path must be 'module' or 'module:attribute'") - # Hashed here rather than reused from the cache stamp: this must describe the tree - # as it is about to be imported, and the extra walk is noise next to Docker. - suffix = _digest_directory(agent_dir, exclude=exclude)[:_IMPORT_DIGEST_CHARS] - package = f"{_AGENT_IMPORT_ROOT}.{_safe_identifier(agent_dir.name)}_{suffix}" - with _IMPORT_LOCK: - _install_agent_package(package, agent_dir) - try: - scoped = f"{package}.{module_name}" - yield f"{scoped}:{attribute}" if sep else scoped - finally: - with _IMPORT_LOCK: - _uninstall_agent_package(package) - - -def _safe_identifier(value: str) -> str: - """Turn an arbitrary directory name into a valid Python identifier.""" - identifier = re.sub(r"\W+", "_", value).strip("_") - if not identifier: - return "agent" - return identifier if identifier[0].isalpha() or identifier[0] == "_" else f"_{identifier}" - - -def _install_agent_package(package: str, agent_dir: Path) -> None: - """Register ``package`` (and its parents) in ``sys.modules`` rooted at ``agent_dir``. - - Refcounted: package names are content-addressed, so two overlapping scopes on the - same agent directory legitimately share one. Callers must hold ``_IMPORT_LOCK``. - """ - parts = package.split(".") - for idx in range(1, len(parts) + 1): - name = ".".join(parts[:idx]) - if name not in sys.modules: - module = ModuleType(name) - module.__path__ = [] # namespace package; leaf __path__ is set below - module.__spec__ = importlib.machinery.ModuleSpec(name, loader=None, is_package=True) - sys.modules[name] = module - if idx > 1: - setattr(sys.modules[".".join(parts[: idx - 1])], parts[idx - 1], module) - installed = sys.modules[package] - if not installed.__path__: - installed.__path__ = [str(agent_dir)] - elif installed.__path__ != [str(agent_dir)]: - # Two directories sharing this package name share a content digest, so their - # trees are byte-identical and the path already installed is exactly as - # correct as this one — the excluded content (`.git`, `__pycache__`, the - # env dirs, `jobs_dir`) is not importable. Repointing would swap the - # directory out from under a scope that is still open, for no gain. - logger.debug( - "Agent package %s is already installed from %s; keeping it for the identical tree at %s", - package, - installed.__path__[0], - agent_dir, - ) - # Counted only once the injection it guards has succeeded. Incrementing first - # would strand the count above zero if any step above raised — the scope never - # opens, so nothing ever decrements it, and the package could never be torn down - # again for the life of the process. - _AGENT_PACKAGE_REFCOUNTS[package] = _AGENT_PACKAGE_REFCOUNTS.get(package, 0) + 1 - - -def _uninstall_agent_package(package: str) -> None: - """Remove ``package`` and its submodules from ``sys.modules`` on the last exit. - - Tearing down on the *first* exit would break a still-open scope sharing the same - content-addressed name, so the removal waits for the refcount to reach zero. - Callers must hold ``_IMPORT_LOCK``. - """ - remaining = _AGENT_PACKAGE_REFCOUNTS.get(package, 0) - 1 - if remaining > 0: - _AGENT_PACKAGE_REFCOUNTS[package] = remaining - return - _AGENT_PACKAGE_REFCOUNTS.pop(package, None) - - for name in [n for n in sys.modules if n == package or n.startswith(f"{package}.")]: - sys.modules.pop(name, None) - parent, _, child = package.rpartition(".") - parent_module = sys.modules.get(parent) - if parent_module is not None: - with contextlib.suppress(AttributeError): - delattr(parent_module, child) - - -def build_trials_from_job_dir( - job_dir: str | Path, - tasks: Sequence[AgentEvalTask], - *, - reward_key: str = DEFAULT_REWARD_KEY, -) -> list[AgentEvalTrial]: - """Adapt Harbor's per-trial ``result.json`` files into :class:`AgentEvalTrial` objects. - - Reads ``/__/result.json`` (the top-level aggregate - ``/result.json`` is skipped because it is not nested). Each Harbor - trial whose ``task_name`` matches a supplied task id becomes one trial, with - the verifier reward, exception type, and token/cost measurements stamped on - ``metadata`` and standard evidence descriptors pointing at the trial's - on-disk artifacts. - """ - job_path = Path(job_dir) - known_task_ids = {task.id for task in tasks} - trials: list[AgentEvalTrial] = [] - for result_path in sorted(job_path.glob("*/result.json")): - try: - data = json.loads(result_path.read_text()) - except (json.JSONDecodeError, OSError) as exc: - logger.warning("Skipping unreadable Harbor trial result %s: %s", result_path, exc) - continue - task_id = data.get("task_name") - if task_id not in known_task_ids: - # Trial for a task we weren't asked to score (e.g. a wider dataset run). - continue - trials.append(_trial_from_harbor_result(result_path.parent, data, reward_key=reward_key)) - - # Surface tasks that produced no trial loudly: a mis-pointed job_dir or a - # crashed run would otherwise silently score fewer tasks than requested. - missing = known_task_ids - {trial.task_id for trial in trials} - if missing: - logger.warning("No Harbor trial result found for %d requested task(s): %s", len(missing), sorted(missing)) - if not trials: - logger.warning( - "No Harbor trial results under %s matched the requested tasks; nothing will be scored.", job_path - ) - return trials - - -def _trial_from_harbor_result(trial_dir: Path, data: Mapping[str, Any], *, reward_key: str) -> AgentEvalTrial: - task_id = str(data["task_name"]) - trial_id = str(data.get("trial_name") or trial_dir.name) - rewards = _rewards_mapping(data) - reward = _primary_reward(rewards, reward_key) - exception_type = _exception_type(data.get("exception_info")) - - metadata: dict[str, Any] = { - "reward": reward, - "reward_details": dict(rewards), - "harbor_trial_dir": str(trial_dir), - } - if exception_type is not None: - metadata["exception_type"] = exception_type - metadata.update(_token_measurements(data.get("agent_result"))) - - # An errored trial (or one with no reward) stays PARTIAL so it is still scored - # as 0 and counted in the summary; FAILED would exclude it from scoring. - status = ( - AgentEvalTrialStatus.COMPLETED - if exception_type is None and reward is not None - else AgentEvalTrialStatus.PARTIAL - ) - - trace_path = trial_dir / "agent" / "trajectory.json" - descriptors = standard_evidence_descriptors( - logs_dir=trial_dir / "agent", - final_state_dir=trial_dir / "artifacts", - trace_path=trace_path if trace_path.exists() else None, - verifier_logs_dir=trial_dir / "verifier", - ) - - return AgentEvalTrial( - id=trial_id, - task_id=task_id, - status=status, - output=AgentOutput(metadata={"harbor_trial_dir": str(trial_dir)}), - evidence=CandidateEvidence(descriptors=descriptors), - metadata=metadata, - ) - - -def _rewards_mapping(data: Mapping[str, Any]) -> dict[str, float]: - verifier_result = data.get("verifier_result") - if not isinstance(verifier_result, Mapping): - return {} - rewards = verifier_result.get("rewards") - if not isinstance(rewards, Mapping): - return {} - out: dict[str, float] = {} - for key, value in rewards.items(): - try: - out[str(key)] = float(value) - except (TypeError, ValueError): - continue - return out - - -def _primary_reward(rewards: Mapping[str, float], reward_key: str) -> float | None: - """Return the single reward a trial is scored on. - - Returns the reward named by ``reward_key`` when the verifier emitted it. - Returns ``None`` otherwise (the trial is treated as having no reward, so it - stays PARTIAL rather than scoring a misleading 0.0): if the verifier emitted - rewards but none matches ``reward_key`` a warning is logged, since we do not - guess among the emitted rewards (point ``reward_key`` at the intended one, or - score the others with additional metrics over ``reward_details``). - """ - if reward_key in rewards: - return rewards[reward_key] - if rewards: - logger.warning( - "Harbor trial emitted rewards %s but none matches reward_key=%r; treating the trial as having no reward", - sorted(rewards), - reward_key, - ) - return None - - -def _exception_type(exception_info: Any) -> str | None: - if exception_info is None: - return None - if isinstance(exception_info, Mapping): - for key in ("exception_type", "type", "name", "class"): - value = exception_info.get(key) - if isinstance(value, str) and value: - return value - return "UnknownException" - return str(exception_info) - - -def _token_measurements(agent_result: Any) -> dict[str, int | float]: - """Map Harbor's ``agent_result`` token counts onto SDK ``TrialMeasurements`` keys.""" - if not isinstance(agent_result, Mapping): - return {} - mapping = { - "prompt_tokens": "n_input_tokens", - "completion_tokens": "n_output_tokens", - "cache_read_tokens": "n_cache_tokens", - } - out: dict[str, int | float] = {} - for sdk_key, harbor_key in mapping.items(): - value = agent_result.get(harbor_key) - if isinstance(value, int) and not isinstance(value, bool): - out[sdk_key] = value - cost = agent_result.get("cost_usd") - if isinstance(cost, (int, float)) and not isinstance(cost, bool): - out["cost_usd"] = float(cost) - return out - - -def _harbor_task_dirs(dataset_path: Path) -> list[Path]: - """Return the Harbor task folders under ``dataset_path`` (or itself if it is one).""" - if (dataset_path / _TASK_CONFIG_FILENAME).is_file(): - return [dataset_path] - return sorted( - path - for path in dataset_path.iterdir() - if path.is_dir() and path.name != _TASK_TEMPLATE_DIRNAME and (path / _TASK_CONFIG_FILENAME).is_file() - ) - - -def _strip_leading_spdx_html_comments(text: str) -> str: - """Remove leading SPDX HTML comments from Markdown prompt content.""" - position = 0 - while match := _SPDX_HTML_COMMENT_RE.match(text, position): - position = match.end() - return text[position:] - - -def discover_harbor_tasks(dataset_path: str | Path) -> list[AgentEvalTask]: - """Build one :class:`AgentEvalTask` per Harbor task folder in ``dataset_path``. - - Mirrors Harbor's own local-dataset discovery: every immediate subdirectory - with a ``task.toml`` is a task. The task id is read from ``[task] name`` so it - matches the ``task_name`` Harbor writes into each trial's ``result.json``, and - each task is scored by a :class:`HarborRewardMetric`. - - Raises: - ValueError: if a task's ``task.toml`` or ``instruction.md`` is malformed or - unreadable — the offending path is named. A discovered task is never - silently dropped, since that would quietly shrink eval coverage. - """ - dataset_path = Path(dataset_path) - tasks: list[AgentEvalTask] = [] - for task_dir in _harbor_task_dirs(dataset_path): - config_path = task_dir / _TASK_CONFIG_FILENAME - try: - config = tomllib.loads(config_path.read_text(encoding="utf-8")) - except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError) as exc: - raise ValueError(f"malformed Harbor task config at {config_path}: {exc}") from exc - task_name = config.get("task", {}).get("name", task_dir.name) - instruction_path = task_dir / "instruction.md" - try: - instruction = ( - _strip_leading_spdx_html_comments(instruction_path.read_text(encoding="utf-8")).strip() - if instruction_path.is_file() - else task_name - ) - except (OSError, UnicodeDecodeError) as exc: - raise ValueError(f"unreadable Harbor instruction at {instruction_path}: {exc}") from exc - tasks.append( - AgentEvalTask( - id=task_name, - # `intent` is human-facing metadata, never shown to the agent; the task name is the - # only human label Harbor's task.toml provides. The instruction the agent acts on - # lives in `inputs["instruction"]`. - intent=task_name, - inputs={"instruction": instruction}, - metrics=[HarborRewardMetric()], - metadata={"harbor_dataset_path": str(dataset_path), "harbor_task_dir": str(task_dir)}, - ) - ) - return tasks - - -class HarborTasksetLoader: - """Load a Harbor local-dataset directory as an :class:`AgentEvalTaskset`. - - Implements the :class:`AgentEvalTasksetLoader` protocol so "dataset dir in → - tasks out" is a single call. - """ - - def __init__(self, dataset_path: str | Path, *, name: str = "harbor") -> None: - self._dataset_path = Path(dataset_path) - self._name = name - - @property - def name(self) -> str: - return self._name - - def load( - self, - *, - source: str | Path | None = None, - limit: int | None = None, - evidence_dir: Path | None = None, - ) -> AgentEvalTaskset: - """Discover Harbor tasks under ``source`` (or the configured path) into a taskset.""" - dataset_path = Path(source) if source is not None else self._dataset_path - tasks = discover_harbor_tasks(dataset_path) - if limit is not None: - tasks = tasks[:limit] - return AgentEvalTaskset(tasks=tasks, metadata={"harbor_dataset_path": str(dataset_path)}) - - -async def run_harbor_eval( - config: HarborRuntimeConfig, - dataset_path: str | Path, - *, - task_names: Sequence[str] | None = None, - metrics: Sequence[Metric] | None = None, - run_config: AgentEvalRunConfig | None = None, -) -> AgentEvalResult: - """Run a Harbor dataset natively and score it — the minimal-plumbing entry point. - - Loads the taskset from ``dataset_path``, runs Harbor via ``config``, and scores - through :class:`AgentEvaluator`. Tasks are scored by :class:`HarborRewardMetric` - unless ``metrics`` overrides them. Returns the scored :class:`AgentEvalResult`. - """ - from nemo_platform.beta.evaluator.agent_eval.evaluator import AgentEvaluator - - dataset_path = Path(dataset_path) - tasks = HarborTasksetLoader(dataset_path).load().tasks - if task_names is not None: - wanted = set(task_names) - tasks = [task for task in tasks if task.id in wanted] - if metrics is not None: - tasks = [task.model_copy(update={"metrics": list(metrics)}) for task in tasks] - - runner = HarborAgentTaskRunner(config=config, task_names=task_names) - return await AgentEvaluator().run( - tasks=tasks, - target=runner, - config=run_config or AgentEvalRunConfig(), - ) - - -def reward_payload_from_result( - result: AgentEvalResult, - *, - reward_key: str = DEFAULT_REWARD_KEY, -) -> dict[str, Any]: - """Reconstruct the optimizer's legacy ``{reward, reward_details, exceptions}`` payload. - - Phase-1 adapter so consumers that still expect Harbor's aggregate shape can - read it off an :class:`AgentEvalResult`: - - * ``reward`` — mean of each metric output, keyed ``"."``. - * ``reward_details`` — ``{output: {value_str: [task_id, ...]}}`` grouped from - per-trial scores (Harbor's ``reward_stats`` analogue). - * ``exceptions`` — ``{exception_type: [task_id, ...]}`` from trial metadata - (Harbor's ``exception_stats`` analogue). - """ - reward = {score.name: score.mean for score in result.summary.scores.scores if score.mean is not None} - - reward_details: dict[str, dict[str, list[str]]] = {} - for score in result.scores: - if score.status == AgentEvalScoreStatus.FAILED: - continue - for output in score.outputs: - value = output.value - value_str = ( - str(float(value)) if isinstance(value, (int, float)) and not isinstance(value, bool) else str(value) - ) - reward_details.setdefault(output.name, {}).setdefault(value_str, []).append(score.task_id) - - exceptions: dict[str, list[str]] = {} - for trial in result.trials: - exc = trial.metadata.get("exception_type") - if isinstance(exc, str) and exc: - exceptions.setdefault(exc, []).append(trial.task_id) - - return { - "reward": reward, - "reward_details": reward_details, - "exceptions": exceptions, - } - - -__all__ = [ - "CACHE_STAMP_FILENAME", - "CACHE_STAMP_VERSION", - "DEFAULT_REWARD_KEY", - "HarborAgentTaskRunner", - "HarborRewardMetric", - "HarborRuntimeConfig", - "HarborTasksetLoader", - "build_trials_from_job_dir", - "discover_harbor_tasks", - "reward_payload_from_result", - "run_harbor_eval", - "scoped_harbor_agent_import", -] diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/api.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/api.py deleted file mode 100644 index 674e0fa19e..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/api.py +++ /dev/null @@ -1,127 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Provider-neutral sandbox facade. - -:class:`AsyncSandbox` is the object agent-eval runtimes use: it drives one sandbox's -``create → seed → exec → transfer → close`` lifecycle over a -:class:`~nemo_platform.beta.evaluator.agent_eval.runtimes.sandbox.base.SandboxProvider`, and writes -``spec.files`` in on ``start()``. It does **not** own the provider's lifetime: the provider is -typically shared across a batch of concurrent sandboxes, so ``stop()`` tears down only this -sandbox (``provider.close(handle)``). Disposing the provider's process-wide resources -(``provider.aclose()``) is the batch owner's job — the runtime that created the provider. -""" - -from __future__ import annotations - -from pathlib import Path - -from nemo_platform.beta.evaluator.agent_eval.runtimes.sandbox.base import ( - SandboxExecResult, - SandboxHandle, - SandboxProvider, - SandboxSpec, - SandboxStatus, -) - - -class AsyncSandbox: - """Async sandbox backed by a :class:`SandboxProvider`.""" - - def __init__(self, provider: SandboxProvider, spec: SandboxSpec | None = None) -> None: - self._provider = provider - self._spec = spec - self._handle: SandboxHandle | None = None - self._started = False - self._closed = False - - def _require_handle(self) -> SandboxHandle: - if self._handle is None or not self._started: - raise RuntimeError("Sandbox has not been started") - return self._handle - - async def start(self, spec: SandboxSpec | None = None) -> AsyncSandbox: - if self._closed: - raise RuntimeError("Sandbox has been stopped") - if self._started: - raise RuntimeError("Sandbox is already started") - resolved_spec = spec if spec is not None else self._spec - if resolved_spec is None: - raise ValueError("Sandbox.start() requires a SandboxSpec") - - handle = await self._provider.create(resolved_spec) - # Seed startup files after the sandbox is up; tear the sandbox down on any seed failure so a - # half-created sandbox never leaks. - try: - for target_path, contents in resolved_spec.files.items(): - await _write_file(self._provider, handle, target_path, contents) - except BaseException: - # Close just this half-created sandbox; the shared provider's lifetime is the owner's. - await self._provider.close(handle) - self._closed = True - raise - - self._spec = resolved_spec - self._handle = handle - self._started = True - return self - - async def exec( - self, - command: str, - *, - cwd: str | None = None, - env: dict[str, str] | None = None, - timeout_s: int | float | None = None, - stdin: bytes | None = None, - ) -> SandboxExecResult: - resolved_cwd = cwd if cwd is not None else (self._spec.workdir if self._spec is not None else None) - return await self._provider.exec( - self._require_handle(), command, cwd=resolved_cwd, env=env, timeout_s=timeout_s, stdin=stdin - ) - - async def upload_file(self, local_path: Path | str, remote_path: str) -> None: - await self._provider.upload_file(self._require_handle(), Path(local_path), remote_path) - - async def upload_dir(self, local_dir: Path | str, remote_dir: str) -> None: - await self._provider.upload_dir(self._require_handle(), Path(local_dir), remote_dir) - - async def download_file(self, remote_path: str, local_path: Path | str) -> None: - await self._provider.download_file(self._require_handle(), remote_path, Path(local_path)) - - async def download_dir(self, remote_dir: str, local_dir: Path | str) -> None: - await self._provider.download_dir(self._require_handle(), remote_dir, Path(local_dir)) - - async def status(self) -> SandboxStatus: - if self._handle is None: - return SandboxStatus.UNKNOWN - if self._closed: - return SandboxStatus.STOPPED - return await self._provider.status(self._handle) - - async def stop(self) -> None: - # Tears down only *this* sandbox. The provider is shared across sibling sandboxes, so its - # process-wide resources (``aclose``) are the owner's to dispose — closing them here would tear - # the provider down under still-running siblings when the first one exits. - if self._closed: - return - self._closed = True - if self._handle is not None and self._started: - self._started = False - await self._provider.close(self._handle) - - async def __aenter__(self) -> AsyncSandbox: - return self - - async def __aexit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None: - await self.stop() - - -async def _write_file(provider: SandboxProvider, handle: SandboxHandle, target_path: str, contents: str) -> None: - """Write one text seed file into the sandbox via a host temp file + upload.""" - import tempfile - - with tempfile.TemporaryDirectory(prefix="nemo-eval-sandbox-seed-") as tmp_dir: - source = Path(tmp_dir) / "seed" - source.write_text(contents, encoding="utf-8") - await provider.upload_file(handle, source, target_path) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/base.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/base.py deleted file mode 100644 index bc7db1790d..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/base.py +++ /dev/null @@ -1,193 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Provider-neutral sandbox seam for containerized agent-eval runtimes. - -A minimal, boundary-crossing sandbox contract: a runtime describes a sandbox with -:class:`SandboxSpec`, a :class:`SandboxProvider` creates it and runs commands, and -context/artifacts move across the boundary by *file transfer* (``upload_*`` / -``download_*``) rather than shared mounts. That transfer model is what lets the same -runtime run on a local Docker backend today and a remote Kubernetes backend later: -bind mounts do not cross the Kubernetes API boundary, but ``docker cp`` / ``kubectl cp`` -do. - -The shape (exec + programmatic file I/O + the ``error_type`` sentinel convention) is -deliberately modeled on NeMo Gym's ``nemo_gym.sandbox`` provider protocol so a Gym -backend could be adapted later, but it is intentionally scoped to what the agent-eval -evidence contract needs — we own it, so it carries no heavyweight dependency. -""" - -from __future__ import annotations - -from collections.abc import Mapping -from dataclasses import dataclass, field -from enum import Enum -from pathlib import Path -from typing import Any, Protocol - - -class SandboxStatus(str, Enum): - """Provider-neutral sandbox lifecycle status.""" - - STARTING = "starting" - RUNNING = "running" - STOPPED = "stopped" - ERROR = "error" - UNKNOWN = "unknown" - - -#: Sentinel ``return_code`` a provider uses when the *sandbox runtime* (not the user's -#: command) failed to run the command — e.g. a timeout or a dead container. Distinguishes -#: "the sandbox broke" from "the command exited non-zero". -SANDBOX_RUNTIME_RETURN_CODE = 125 - - -@dataclass(frozen=True) -class SandboxResources: - """Provider-neutral resource request (providers map or ignore fields they can't honor).""" - - cpu: float | None = None - memory_mib: int | None = None - disk_gib: int | None = None - gpu: int | None = None - gpu_type: str | None = None - - @classmethod - def from_mapping(cls, resources: Mapping[str, Any] | None) -> SandboxResources: - if resources is None: - return cls() - allowed = set(cls.__dataclass_fields__) - unknown = set(resources) - allowed - if unknown: - raise ValueError( - f"Unknown sandbox resource keys: {', '.join(sorted(unknown))}. " - f"Expected keys: {', '.join(sorted(allowed))}" - ) - return cls( - cpu=float(resources["cpu"]) if resources.get("cpu") is not None else None, - memory_mib=int(resources["memory_mib"]) if resources.get("memory_mib") is not None else None, - disk_gib=int(resources["disk_gib"]) if resources.get("disk_gib") is not None else None, - gpu=int(resources["gpu"]) if resources.get("gpu") is not None else None, - gpu_type=str(resources["gpu_type"]) if resources.get("gpu_type") is not None else None, - ) - - -@dataclass(frozen=True) -class SandboxSpec: - """A sandbox creation request. - - ``files`` are seed files written into the sandbox at ``start()`` as a - ``{absolute_container_path: text_contents}`` map; larger or binary payloads use - :meth:`SandboxProvider.upload_file` / ``upload_dir`` after start. ``provider_options`` - carries backend-specific knobs (e.g. the Docker network) the neutral spec doesn't model. - - ``resources`` is a typed :class:`SandboxResources`; build one from an untyped mapping with - :meth:`SandboxResources.from_mapping` at the edge rather than passing a raw mapping here. - """ - - image: str | None = None - workdir: str | None = None - ttl_s: int | float | None = None - env: dict[str, str] = field(default_factory=dict) - files: dict[str, str] = field(default_factory=dict) - metadata: dict[str, str] = field(default_factory=dict) - resources: SandboxResources = field(default_factory=SandboxResources) - provider_options: dict[str, Any] = field(default_factory=dict) - - -@dataclass -class SandboxHandle: - """Provider-neutral handle to a created sandbox. - - ``raw`` is provider-owned opaque state. Callers pass it back to the provider through - this handle rather than inspecting it — it is typed ``object`` so no consumer depends - on a provider's internal representation. - """ - - sandbox_id: str - provider_name: str - raw: object - - -@dataclass(frozen=True) -class SandboxExecResult: - """Provider-neutral process-execution result. - - ``return_code`` is the process exit code when the sandbox actually ran the command. - On a sandbox-runtime failure (timeout, dead sandbox) it is - :data:`SANDBOX_RUNTIME_RETURN_CODE` and ``error_type`` names the failure. - """ - - stdout: str | None - stderr: str | None - return_code: int - error_type: str | None = None - - @property - def ok(self) -> bool: - """Whether the command exited 0 and the sandbox runtime did not fail.""" - return self.return_code == 0 and self.error_type is None - - -class SandboxCreateError(RuntimeError): - """Raised when a provider cannot create a sandbox.""" - - -class SandboxProvider(Protocol): - """Runtime/infra provider contract used by the public sandbox facade. - - Concrete providers (Docker now, Kubernetes/agent-sandbox next) implement this - structurally. File transfer is programmatic so it crosses a remote API boundary, - not just a shared host filesystem. - """ - - name: str - - async def create(self, spec: SandboxSpec) -> SandboxHandle: - """Create a ready sandbox and return a provider-neutral handle. - - Providers must return only once the sandbox can run commands and transfer files, - raising :class:`SandboxCreateError` (or a subclass) if it cannot become ready. - """ - ... - - async def exec( - self, - handle: SandboxHandle, - command: str, - *, - cwd: str | None = None, - env: dict[str, str] | None = None, - timeout_s: int | float | None = None, - stdin: bytes | None = None, - ) -> SandboxExecResult: - """Run a shell command inside a sandbox; never raises for command failure.""" - ... - - async def upload_file(self, handle: SandboxHandle, source_path: Path, target_path: str) -> None: - """Upload one local file into a sandbox.""" - ... - - async def upload_dir(self, handle: SandboxHandle, source_dir: Path, target_dir: str) -> None: - """Upload a local directory tree into a sandbox.""" - ... - - async def download_file(self, handle: SandboxHandle, source_path: str, target_path: Path) -> None: - """Download one sandbox file to the local filesystem.""" - ... - - async def download_dir(self, handle: SandboxHandle, source_dir: str, target_dir: Path) -> None: - """Download a sandbox directory tree to the local filesystem.""" - ... - - async def status(self, handle: SandboxHandle) -> SandboxStatus: - """Return the current sandbox lifecycle status.""" - ... - - async def close(self, handle: SandboxHandle) -> None: - """End the sandbox lifecycle and release provider resources for it.""" - ... - - async def aclose(self) -> None: - """Close provider-scoped resources (SDK clients, pools).""" - ... diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/_compose_cli.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/_compose_cli.py deleted file mode 100644 index 4e804644f9..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/_compose_cli.py +++ /dev/null @@ -1,485 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# ruff: noqa: I001 - the vendored SDK mirror uses different import-order settings. - -"""Docker and Compose command execution for the Compose sandbox provider.""" - -from __future__ import annotations - -import asyncio -import os -import re -import signal -from collections.abc import Awaitable, Callable, Mapping, Sequence -from pathlib import Path -from typing import IO - -from ..base import SANDBOX_RUNTIME_RETURN_CODE -from ._compose_contracts import ComposeCleanupError, ComposeCommandResult -from ._compose_state import _ComposeCommandScope - -_CLEANUP_ATTEMPTS = 3 -_CLEANUP_RETRY_DELAY_SECONDS = 0.5 -_SECRET_ENV_FRAGMENT = re.compile(r"(?:TOKEN|KEY|PASSWORD|SECRET)", re.IGNORECASE) -_INLINE_SECRET = re.compile( - r"""(?ix) - (?P - ["']?(?:authorization|x-api-key|api[_-]?key|token|password)["']? - \s*[:=]\s* - ) - (?: - (?P["'])(?P.*?)(?P=quote) - | - (?P[^\r\n,}\]]+) - ) - """ -) - - -class _ComposeCli: - """Command gateway for one lifecycle-aware Compose provider.""" - - def __init__(self, scope: Callable[[], _ComposeCommandScope]) -> None: - """Bind command execution to a provider command-scope resolver. - - Args: - scope: Zero-argument function returning active lifecycle settings, or the - provider's current settings when no lifecycle is active. - """ - self._scope = scope - - async def run_compose( - self, - args: Sequence[str], - *, - environment: Mapping[str, str], - timeout: float, - stdin: bytes | None = None, - stream_output: IO[str] | None = None, - ) -> ComposeCommandResult: - """Run one project-scoped Docker Compose command. - - Args: - args: Compose subcommand and arguments, excluding configured global options. - environment: Complete subprocess environment. - timeout: Command deadline in seconds. - stdin: Optional bytes forwarded to standard input. - stream_output: Optional text sink for line-buffered, redacted progress output. - - Returns: - Captured command result with the fully rendered argument vector. - """ - scope = self._scope() - argv: tuple[str, ...] = ( - scope.docker_bin, - "compose", - "--ansi", - "never", - "--progress", - "plain", - "--project-directory", - str(scope.project_directory), - *(item for compose_file in scope.compose_files for item in ("--file", str(compose_file))), - "--project-name", - scope.project_name, - *(item for profile in scope.profiles for item in ("--profile", profile)), - *args, - ) - return await _run_command( - argv, - cwd=scope.project_directory, - environment=environment, - timeout=timeout, - stdin=stdin, - stream_output=stream_output, - ) - - async def run_docker( - self, - args: Sequence[str], - *, - environment: Mapping[str, str], - timeout: float, - ) -> ComposeCommandResult: - """Run one Docker CLI command outside the Compose subcommand. - - Args: - args: Docker subcommand and arguments. - environment: Complete subprocess environment. - timeout: Command deadline in seconds. - - Returns: - Captured command result. - """ - scope = self._scope() - return await _run_command( - (scope.docker_bin, *args), - cwd=scope.project_directory, - environment=environment, - timeout=timeout, - stdin=None, - stream_output=None, - ) - - async def retry_compose( - self, - args: Sequence[str], - *, - environment: Mapping[str, str], - timeout: float, - ) -> ComposeCommandResult: - """Run a Compose operation with the cleanup retry policy. - - Args: - args: Compose subcommand and arguments, excluding global project options. - environment: Environment forwarded to Compose. - timeout: Deadline applied independently to each attempt. - - Returns: - First successful result or the final failed result after all attempts. - """ - return await _retry_command( - lambda: self.run_compose( - args, - environment=environment, - timeout=timeout, - ) - ) - - @staticmethod - def failure_message( - prefix: str, - result: ComposeCommandResult, - environment: Mapping[str, str], - ) -> str: - """Build a redacted message from command output. - - Args: - prefix: Human-readable operation description. - result: Failed or timed-out command result. - environment: Command environment used to identify secrets. - - Returns: - Message containing the prefix, timeout state, and redacted output or return code. - """ - captured = "\n".join(stream.strip() for stream in (result.stdout, result.stderr) if stream.strip()) - details = _redact(captured, environment) - timeout = " (timed out)" if result.timed_out else "" - return f"{prefix}{timeout}: {details or f'exit {result.return_code}'}" - - -async def _run_command( - argv: tuple[str, ...], - *, - cwd: Path, - environment: Mapping[str, str], - timeout: float, - stdin: bytes | None, - stream_output: IO[str] | None = None, -) -> ComposeCommandResult: - """Run a subprocess with process-group cancellation and timeout handling. - - Args: - argv: Complete executable argument vector. - cwd: Host working directory for the child process. - environment: Complete child-process environment. - timeout: Maximum runtime in seconds, clamped to a small positive value. - stdin: Optional bytes written to the child process. - stream_output: Optional text sink that receives redacted output as lines arrive. - - Returns: - Captured subprocess result. Timeouts are returned as results rather than raised. - - Raises: - asyncio.CancelledError: If the caller cancels execution after the process group is terminated. - OSError: If the subprocess cannot be created. - """ - process = await asyncio.create_subprocess_exec( - *argv, - cwd=str(cwd), - env=dict(environment), - stdin=asyncio.subprocess.PIPE if stdin is not None else None, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - start_new_session=True, - ) - stdout_chunks: list[bytes] = [] - stderr_chunks: list[bytes] = [] - communication = asyncio.create_task( - _communicate_streaming( - process, - stdin=stdin, - redact=_make_line_redactor(environment), - stdout_chunks=stdout_chunks, - stderr_chunks=stderr_chunks, - output=stream_output, - ) - ) - - async def _abort() -> None: - """Cancel communication, terminate the process group, and reap the task.""" - communication.cancel() - await _terminate_process_group(process) - await asyncio.gather(communication, return_exceptions=True) - - try: - stdout_bytes, stderr_bytes = await asyncio.wait_for( - communication, - timeout=max(0.1, timeout), - ) - return ComposeCommandResult( - argv=argv, - return_code=int(process.returncode or 0), - stdout=stdout_bytes.decode("utf-8", errors="replace"), - stderr=stderr_bytes.decode("utf-8", errors="replace"), - ) - except asyncio.CancelledError: - await _abort() - raise - except TimeoutError: - await _abort() - return ComposeCommandResult( - argv=argv, - return_code=SANDBOX_RUNTIME_RETURN_CODE, - stdout=b"".join(stdout_chunks).decode("utf-8", errors="replace"), - stderr=( - b"".join(stderr_chunks).decode("utf-8", errors="replace") + f"Command timed out after {timeout:.1f}s" - ), - timed_out=True, - ) - - -async def _communicate_streaming( - process: asyncio.subprocess.Process, - *, - stdin: bytes | None, - redact: Callable[[str], str], - stdout_chunks: list[bytes], - stderr_chunks: list[bytes], - output: IO[str] | None, -) -> tuple[bytes, bytes]: - """Drain both subprocess streams while writing redacted progress lines. - - Args: - process: Running subprocess with piped standard streams. - stdin: Optional bytes to write before waiting for process completion. - redact: Function that removes secrets from each emitted text line. - stdout_chunks: Mutable accumulator for raw standard-output bytes. - stderr_chunks: Mutable accumulator for raw standard-error bytes. - output: Optional text sink for redacted progress lines from both streams. - - Returns: - Complete raw standard-output and standard-error byte strings. - - Raises: - RuntimeError: If required subprocess pipes are unavailable. - """ - if process.stdout is None or process.stderr is None: - raise RuntimeError("Streaming subprocess pipes are unavailable") - readers = ( - asyncio.create_task( - _drain_stream( - process.stdout, - stdout_chunks, - redact=redact, - output=output, - ) - ), - asyncio.create_task( - _drain_stream( - process.stderr, - stderr_chunks, - redact=redact, - output=output, - ) - ), - ) - try: - if stdin is not None: - if process.stdin is None: - raise RuntimeError("Streaming subprocess stdin is unavailable") - process.stdin.write(stdin) - await process.stdin.drain() - process.stdin.close() - await process.stdin.wait_closed() - await process.wait() - await asyncio.gather(*readers) - finally: - for reader in readers: - if not reader.done(): - reader.cancel() - await asyncio.gather(*readers, return_exceptions=True) - return b"".join(stdout_chunks), b"".join(stderr_chunks) - - -async def _drain_stream( - stream: asyncio.StreamReader, - chunks: list[bytes], - *, - redact: Callable[[str], str], - output: IO[str] | None, -) -> None: - """Capture raw stream bytes and emit each decoded line after redaction. - - Args: - stream: Async subprocess stream to read until EOF. - chunks: Mutable raw-byte accumulator used for the command result. - redact: Function applied before a decoded line leaves the provider. - output: Optional text sink that receives redacted lines and is flushed immediately. - """ - if output is None: - while chunk := await stream.read(64 * 1024): - chunks.append(chunk) - return - - pending_line = bytearray() - scan_offset = 0 - while chunk := await stream.read(64 * 1024): - chunks.append(chunk) - pending_line.extend(chunk) - while (newline := pending_line.find(b"\n", scan_offset)) >= 0: - line = bytes(pending_line[: newline + 1]) - del pending_line[: newline + 1] - scan_offset = 0 - output.write(redact(line.decode("utf-8", errors="replace"))) - output.flush() - scan_offset = len(pending_line) - if pending_line: - output.write(redact(pending_line.decode("utf-8", errors="replace"))) - output.flush() - - -async def _retry_command( - operation: Callable[[], Awaitable[ComposeCommandResult]], -) -> ComposeCommandResult: - """Retry a cleanup command with short linear backoff. - - Args: - operation: Zero-argument async function that starts one command attempt. - - Returns: - First successful result or the final failed result after ``_CLEANUP_ATTEMPTS``. - """ - result: ComposeCommandResult | None = None - for attempt in range(_CLEANUP_ATTEMPTS): - result = await operation() - if result.ok: - return result - if attempt + 1 < _CLEANUP_ATTEMPTS: - await asyncio.sleep(_CLEANUP_RETRY_DELAY_SECONDS * (attempt + 1)) - if result is None: # pragma: no cover - attempts is a positive module constant - raise RuntimeError("Cleanup command was not attempted") - return result - - -async def _run_shielded( - operation: Awaitable[ComposeCleanupError | None], -) -> tuple[ComposeCleanupError | None, asyncio.CancelledError | None]: - """Let cleanup finish when its caller is cancelled. - - Args: - operation: Cleanup awaitable that returns an optional aggregated error. - - Returns: - Pair of the cleanup result and the first cancellation received by the caller. - The caller decides when to restore that cancellation. - """ - task = asyncio.ensure_future(operation) - cancellation: asyncio.CancelledError | None = None - while True: - try: - result = await asyncio.shield(task) - return result, cancellation - except asyncio.CancelledError as exc: - cancellation = cancellation or exc - if task.done(): - return task.result(), cancellation - - -async def _terminate_process_group(process: asyncio.subprocess.Process) -> None: - """Terminate a subprocess group, escalating from ``SIGTERM`` to ``SIGKILL``. - - Args: - process: Process whose session process group should be stopped and reaped. - """ - try: - os.killpg(process.pid, signal.SIGTERM) - except ProcessLookupError: - pass - try: - await asyncio.wait_for(process.wait(), timeout=2) - except TimeoutError: - try: - os.killpg(process.pid, signal.SIGKILL) - except ProcessLookupError: - pass - await process.wait() - - -def _make_line_redactor(environment: Mapping[str, str]) -> Callable[[str], str]: - """Build a redactor that scans the environment's secrets only once. - - The returned callable is applied to every streamed log line, so the secret - set and its (potentially large) alternation regex are compiled up front - rather than rebuilt per line. - - Args: - environment: Command environment whose secret-looking keys identify literal values to remove. - - Returns: - Function that redacts known values and inline authorization, API key, token, and password assignments. - - Example: - With ``{"API_KEY": "secret-value"}``, the returned function replaces both - ``secret-value`` and ``Authorization: Bearer value``-style credentials. - """ - secret_values = { - value for key, value in environment.items() if _SECRET_ENV_FRAGMENT.search(key) and len(value) >= 4 - } - parts = [] - for value in sorted(secret_values, key=len, reverse=True): - prefix = r"(? str: - """Redact one text fragment using the precompiled secret patterns. - - Args: - text: Decoded command output or diagnostic text. - - Returns: - Text with known and inline credential values replaced by ````. - """ - if secret_pattern is not None: - text = secret_pattern.sub("", text) - return _INLINE_SECRET.sub(_redact_inline_secret, text) - - return redact - - -def _redact(text: str, environment: Mapping[str, str]) -> str: - """Redact known environment secrets and inline credentials from text. - - Args: - text: Command output or diagnostics to sanitize. - environment: Environment used to discover literal secret values. - - Returns: - Sanitized text safe for errors, logs, and diagnostic files. - """ - return _make_line_redactor(environment)(text) - - -def _redact_inline_secret(match: re.Match[str]) -> str: - """Replace a matched inline credential value while preserving its prefix and quotes. - - Args: - match: Match produced by ``_INLINE_SECRET``. - - Returns: - Credential assignment with its value replaced by ````. - """ - quote = match.group("quote") or "" - return f"{match.group('prefix')}{quote}{quote}" diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/_compose_contracts.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/_compose_contracts.py deleted file mode 100644 index 2329938084..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/_compose_contracts.py +++ /dev/null @@ -1,87 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Contracts shared by the Docker Compose sandbox provider modules.""" - -from __future__ import annotations - -import logging -from collections.abc import Callable -from dataclasses import dataclass -from typing import Literal - -logger = logging.getLogger(f"{__package__}.compose") - - -class ComposeCleanupError(RuntimeError): - """The managed Compose project could not be completely stopped.""" - - -@dataclass(frozen=True) -class ComposeServiceTopology: - """Expected active services for one Compose project. - - The rendered active service set must match these two groups exactly. Long-running - services must be running and healthy when a health check is configured; one-shot - services must have exited successfully. - - Attributes: - target_service: Long-running service used for sandbox command execution and file transfer. - long_running_services: Services that must remain running after startup. - one_shot_services: Services that must finish successfully during startup. - - Example: - ``ComposeServiceTopology("agent", frozenset({"agent", "redis"}), frozenset({"init"}))`` - targets ``agent``, requires ``agent`` and ``redis`` to stay up, and requires ``init`` to exit zero. - """ - - target_service: str - long_running_services: frozenset[str] - one_shot_services: frozenset[str] = frozenset() - - def __post_init__(self) -> None: - """Normalize service collections and validate their lifecycle roles. - - Raises: - ValueError: If a service has two roles or the target is not long-running. - """ - object.__setattr__(self, "long_running_services", frozenset(self.long_running_services)) - object.__setattr__(self, "one_shot_services", frozenset(self.one_shot_services)) - overlap = self.long_running_services & self.one_shot_services - if overlap: - raise ValueError(f"Compose services cannot be both long-running and one-shot: {sorted(overlap)}") - if self.target_service not in self.long_running_services: - raise ValueError("target_service must be one of long_running_services") - - @property - def active_services(self) -> frozenset[str]: - """Return every service expected in the rendered Compose project.""" - return self.long_running_services | self.one_shot_services - - -@dataclass(frozen=True) -class ComposeCommandResult: - """Result returned for a Docker or Compose command. - - Attributes: - argv: Exact argument vector passed to the subprocess. - return_code: Process return code, or the sandbox runtime code after a timeout. - stdout: Captured standard output. - stderr: Captured standard error. - timed_out: Whether the provider terminated the command at its deadline. - """ - - argv: tuple[str, ...] - return_code: int - stdout: str - stderr: str - timed_out: bool = False - - @property - def ok(self) -> bool: - """Return whether the command completed successfully before its deadline.""" - return self.return_code == 0 and not self.timed_out - - -PullPolicy = Literal["always", "missing", "never"] -ProgressCallback = Callable[[str], None] diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/_compose_inspection.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/_compose_inspection.py deleted file mode 100644 index bfb5e53a49..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/_compose_inspection.py +++ /dev/null @@ -1,226 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Rendered Compose configuration and service-state inspection.""" - -from __future__ import annotations - -import asyncio -import json -import socket -from collections.abc import Mapping -from dataclasses import dataclass -from typing import Any - -from ._compose_contracts import ComposeServiceTopology - - -@dataclass(frozen=True, order=True) -class _PublishedPort: - """One rendered host-to-container port publication. - - Attributes: - service: Compose service that declares the publication. - host_ip: Host address on which Docker publishes the port. - published: Published host port number. - target: Container port number. - protocol: Lowercase transport protocol such as ``tcp`` or ``udp``. - """ - - service: str - host_ip: str - published: int - target: int - protocol: str - - -def _parse_json_rows(text: str) -> list[dict[str, Any]]: - """Parse Compose JSON-array, JSON-object, or JSON-lines output. - - Args: - text: Raw output from a Compose command such as ``ps --format json``. - - Returns: - Dictionary rows; non-object JSON values are ignored. - - Raises: - json.JSONDecodeError: If neither the complete payload nor an individual line is valid JSON. - - Example: - A JSON array and newline-delimited JSON objects both produce a list of service - dictionaries. - """ - stripped = text.strip() - if not stripped: - return [] - try: - payload = json.loads(stripped) - except json.JSONDecodeError: - rows: list[dict[str, Any]] = [] - for line in stripped.splitlines(): - value = json.loads(line) - if isinstance(value, dict): - rows.append(value) - return rows - if isinstance(payload, list): - return [row for row in payload if isinstance(row, dict)] - return [payload] if isinstance(payload, dict) else [] - - -def _parse_compose_config(text: str) -> dict[str, Any]: - """Parse and validate the service mapping from rendered Compose configuration. - - Args: - text: JSON object emitted by ``docker compose config --format json``. - - Returns: - Rendered service names mapped to their service configuration values. - - Raises: - json.JSONDecodeError: If ``text`` is not valid JSON. - TypeError: If the root or ``services`` value is not an object. - """ - payload = json.loads(text) - if not isinstance(payload, dict): - raise TypeError("Compose config JSON must be an object") - services = payload.get("services", {}) - if not isinstance(services, dict): - raise TypeError("Compose config services must be an object") - return services - - -def _published_ports(services: Mapping[str, Any]) -> list[_PublishedPort]: - """Extract fixed host-port publications from rendered Compose services. - - Args: - services: Validated rendered Compose service mapping. - - Returns: - Sorted, de-duplicated publications. Dynamically assigned host ports are omitted. - - Raises: - ValueError: If a published or target port is not numeric. - """ - published_ports: set[_PublishedPort] = set() - for service, service_config in services.items(): - if not isinstance(service_config, dict): - continue - ports = service_config.get("ports", []) - if not isinstance(ports, list): - continue - for port in ports: - if not isinstance(port, dict) or port.get("published") in {None, ""}: - continue - published = int(port["published"]) - if published == 0: - continue - published_ports.add( - _PublishedPort( - service=str(service), - host_ip=str(port.get("host_ip") or "0.0.0.0"), - published=published, - target=int(port.get("target") or published), - protocol=str(port.get("protocol") or "tcp").casefold(), - ) - ) - return sorted(published_ports) - - -async def _find_port_conflicts( - published_ports: list[_PublishedPort], -) -> list[_PublishedPort]: - """Probe host-port availability without blocking the event loop. - - Args: - published_ports: Rendered fixed host-port publications to probe. - - Returns: - Publications whose host address and port cannot be bound locally. - """ - availability = await asyncio.gather( - *(asyncio.to_thread(_published_port_available, published_port) for published_port in published_ports) - ) - return [ - published_port - for published_port, available in zip( - published_ports, - availability, - strict=True, - ) - if not available - ] - - -def _published_port_available(published_port: _PublishedPort) -> bool: - """Check whether one host address and port can be bound. - - Args: - published_port: Publication describing address family, protocol, and host port. - - Returns: - ``True`` when a temporary matching socket can bind the host endpoint. - """ - family = socket.AF_INET6 if ":" in published_port.host_ip else socket.AF_INET - socket_type = socket.SOCK_DGRAM if published_port.protocol == "udp" else socket.SOCK_STREAM - with socket.socket(family, socket_type) as probe: - if socket_type == socket.SOCK_STREAM: - probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - try: - probe.bind((published_port.host_ip, published_port.published)) - except OSError: - return False - return True - - -def _service_is_running(rows: list[dict[str, Any]], service: str) -> bool: - """Check whether any Compose state row reports a service as running. - - Args: - rows: Parsed Compose service-state rows. - service: Service name to locate. - - Returns: - ``True`` when a matching row has state ``running``. - """ - return any(str(row.get("Service")) == service and str(row.get("State", "")).casefold() == "running" for row in rows) - - -def _services_ready( - rows: list[dict[str, Any]], - topology: ComposeServiceTopology, -) -> str | None: - """Validate service rows against long-running and one-shot expectations. - - Args: - rows: Parsed Compose service-state rows. - topology: Exact service roles expected after startup. - - Returns: - ``None`` when every role is ready; otherwise a concise failure description. - """ - services: dict[str, list[dict[str, Any]]] = {} - for row in rows: - services.setdefault(str(row.get("Service")), []).append(row) - missing = sorted(topology.active_services - services.keys()) - if missing: - return f"Compose services missing after startup: {missing}" - unexpected = sorted(services.keys() - topology.active_services) - if unexpected: - return f"Unexpected Compose services after startup: {unexpected}" - for service in sorted(topology.long_running_services): - for row in services[service]: - if str(row.get("State", "")).casefold() != "running": - return f"Compose service {service!r} is not running: {row.get('State')}" - health = str(row.get("Health", "")).casefold() - if health and health != "healthy": - return f"Compose service {service!r} is not healthy: {row.get('Health')}" - for service in sorted(topology.one_shot_services): - for row in services[service]: - state = str(row.get("State", "")).casefold() - try: - exit_code = int(row.get("ExitCode", 1)) - except (TypeError, ValueError): - exit_code = 1 - if state != "exited" or exit_code != 0: - return f"Compose one-shot service {service!r} did not exit successfully" - return None diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/_compose_lifecycle.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/_compose_lifecycle.py deleted file mode 100644 index 9a392d9df4..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/_compose_lifecycle.py +++ /dev/null @@ -1,333 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Provider-independent lifecycle primitives for Docker Compose sandboxes.""" - -from __future__ import annotations - -import asyncio -import json -from collections.abc import Awaitable, Callable, Mapping -from pathlib import Path -from typing import Any - -from ..base import SandboxCreateError -from ._compose_cli import _ComposeCli, _redact -from ._compose_contracts import ComposeCommandResult, ComposeServiceTopology, logger -from ._compose_inspection import ( - _find_port_conflicts, - _parse_compose_config, - _parse_json_rows, - _published_ports, - _services_ready, -) -from ._compose_state import _ComposeCommandScope - - -async def _preflight( - cli: _ComposeCli, - command_scope: _ComposeCommandScope, - service_topology: ComposeServiceTopology, - environment: Mapping[str, str], - *, - command_timeout_seconds: float, - port_override_hints: Mapping[str, str], -) -> None: - """Validate rendered topology, project ownership, and published host ports. - - Args: - cli: Command gateway bound to the lifecycle command scope. - command_scope: Immutable Docker and Compose project settings. - service_topology: Exact active services and lifecycle roles. - environment: Fully merged environment used for Compose interpolation. - command_timeout_seconds: Deadline for preflight Compose commands. - port_override_hints: Service-specific hints shown for occupied host ports. - - Raises: - SandboxCreateError: If the configuration is invalid, the project already has - containers, service roles differ, or a published host port is unavailable. - """ - config_task = asyncio.create_task( - cli.run_compose( - ["config", "--format", "json"], - environment=environment, - timeout=command_timeout_seconds, - ) - ) - existing_task = asyncio.create_task( - cli.run_compose( - ["ps", "--all", "--quiet"], - environment=environment, - timeout=command_timeout_seconds, - ) - ) - try: - config, existing = await asyncio.gather(config_task, existing_task) - except BaseException: - config_task.cancel() - existing_task.cancel() - drain_task = asyncio.gather(config_task, existing_task, return_exceptions=True) - cancellation: asyncio.CancelledError | None = None - while not drain_task.done(): - try: - await asyncio.shield(drain_task) - except asyncio.CancelledError as exc: - cancellation = cancellation or exc - if cancellation is not None: - raise cancellation - raise - if not config.ok: - raise SandboxCreateError(cli.failure_message("Invalid Compose configuration", config, environment)) - if not existing.ok: - raise SandboxCreateError(cli.failure_message("Could not inspect managed project", existing, environment)) - if existing.stdout.strip(): - raise SandboxCreateError( - f"Managed Compose project {command_scope.project_name!r} already has containers; " - "refusing to adopt or remove them" - ) - try: - services = _parse_compose_config(config.stdout) - published_ports = _published_ports(services) - active_services = frozenset(str(service) for service in services) - except (TypeError, ValueError, json.JSONDecodeError) as exc: - raise SandboxCreateError(f"Could not inspect rendered Compose configuration: {exc}") from exc - expected_services = service_topology.active_services - if active_services != expected_services: - missing = sorted(expected_services - active_services) - unexpected = sorted(active_services - expected_services) - raise SandboxCreateError( - "Rendered Compose service topology does not match the provider configuration: " - f"missing={missing}, unexpected={unexpected}" - ) - conflicts = await _find_port_conflicts(published_ports) - if conflicts: - details = "\n".join( - f"- {port.service}: {port.host_ip}:{port.published} -> " - f"{port.target}/{port.protocol} " - f"(override {port_override_hints.get(port.service, 'its Compose port mapping')})" - for port in conflicts - ) - raise SandboxCreateError( - "Managed Compose host ports are unavailable:\n" - f"{details}\n" - "Stop the conflicting stack or override every occupied port." - ) - - -async def _assert_ready( - cli: _ComposeCli, - service_topology: ComposeServiceTopology, - environment: Mapping[str, str], - *, - command_timeout_seconds: float, -) -> None: - """Require every configured service to satisfy its lifecycle role. - - Args: - cli: Command gateway bound to the active lifecycle. - service_topology: Service roles expected after startup. - environment: Environment used to query Compose state. - command_timeout_seconds: Deadline for the state query. - - Raises: - SandboxCreateError: If a long-running or one-shot service is not ready. - """ - rows = await _compose_ps( - cli, - environment, - command_timeout_seconds=command_timeout_seconds, - ) - problem = _services_ready(rows, service_topology) - if problem is not None: - raise SandboxCreateError(problem) - - -async def _compose_ps( - cli: _ComposeCli, - environment: Mapping[str, str], - *, - command_timeout_seconds: float, -) -> list[dict[str, Any]]: - """Return parsed state rows for all project services. - - Args: - cli: Command gateway bound to the active lifecycle. - environment: Environment forwarded to ``docker compose ps``. - command_timeout_seconds: Deadline for the state query. - - Returns: - Parsed JSON objects emitted for service containers. - - Raises: - RuntimeError: If Compose cannot inspect the project. - json.JSONDecodeError: If Compose emits malformed JSON. - """ - result = await cli.run_compose( - ["ps", "--all", "--format", "json"], - environment=environment, - timeout=command_timeout_seconds, - ) - if not result.ok: - raise RuntimeError(cli.failure_message("Could not inspect Compose services", result, environment)) - return _parse_json_rows(result.stdout) - - -async def _capture_diagnostics( - cli: _ComposeCli, - environment: Mapping[str, str], - *, - command_timeout_seconds: float, - diagnostics_dir: Path | None, - reason: str, -) -> None: - """Best-effort write redacted project state and recent logs. - - Args: - cli: Command gateway bound to the active lifecycle. - environment: Environment used for Compose commands and secret redaction. - command_timeout_seconds: Deadline for each diagnostics command. - diagnostics_dir: Optional output directory for diagnostics. - reason: Filename-safe lifecycle label such as ``startup-failure`` or ``shutdown``. - - Diagnostics failures are logged and never replace the lifecycle error being investigated. - """ - if diagnostics_dir is None: - return - try: - diagnostics_dir.mkdir(parents=True, exist_ok=True) - ps_result, logs_result = await asyncio.gather( - cli.run_compose( - ["ps", "--all"], - environment=environment, - timeout=command_timeout_seconds, - ), - cli.run_compose( - ["logs", "--no-color", "--tail", "200"], - environment=environment, - timeout=command_timeout_seconds, - ), - ) - ps_text = _redact(f"{ps_result.stdout}\n{ps_result.stderr}", environment) - logs_text = _redact(f"{logs_result.stdout}\n{logs_result.stderr}", environment) - (diagnostics_dir / f"compose-{reason}-ps.txt").write_text( - ps_text, - encoding="utf-8", - ) - (diagnostics_dir / f"compose-{reason}-logs.txt").write_text( - logs_text, - encoding="utf-8", - ) - except Exception: # noqa: BLE001 - diagnostics must not mask lifecycle errors - logger.exception("Could not capture Compose diagnostics") - - -async def _managed_resource_names( - cli: _ComposeCli, - command_scope: _ComposeCommandScope, - kind: str, - environment: Mapping[str, str], - *, - command_timeout_seconds: float, -) -> tuple[list[str], str | None]: - """List Docker resources carrying this Compose project's label. - - Args: - cli: Command gateway bound to the active lifecycle. - command_scope: Immutable Compose project settings. - kind: Docker resource kind: ``container``, ``network``, or ``volume``. - environment: Environment forwarded to the Docker CLI. - command_timeout_seconds: Deadline for the resource query. - - Returns: - Pair of resource names and an optional redacted inspection error. - """ - args = [kind, "ls"] - if kind == "container": - args.append("--all") - args.extend( - [ - "--quiet", - "--filter", - f"label=com.docker.compose.project={command_scope.project_name}", - ] - ) - result = await cli.run_docker( - args, - environment=environment, - timeout=command_timeout_seconds, - ) - if not result.ok: - return [], cli.failure_message( - f"Could not inspect managed {kind}s", - result, - environment, - ) - return [line.strip() for line in result.stdout.splitlines() if line.strip()], None - - -async def _compose_down( - cli: _ComposeCli, - environment: Mapping[str, str], - *, - shutdown_timeout_seconds: float, - command_timeout_seconds: float, - remove_project_volumes: bool, -) -> ComposeCommandResult: - """Stop the managed project using the provider's cleanup policy. - - Args: - cli: Command gateway bound to the active lifecycle. - environment: Environment forwarded to Docker Compose. - shutdown_timeout_seconds: Grace period supplied to ``compose down``. - command_timeout_seconds: Additional deadline for the cleanup command. - remove_project_volumes: Whether ``compose down`` should remove project volumes. - - Returns: - Result of the final successful or exhausted cleanup attempt. - """ - down_args = [ - "down", - "--remove-orphans", - "--timeout", - str(max(1, int(shutdown_timeout_seconds))), - ] - if remove_project_volumes: - down_args.append("--volumes") - return await cli.retry_compose( - down_args, - environment=environment, - timeout=shutdown_timeout_seconds + command_timeout_seconds, - ) - - -async def _verify_project_destroyed( - environment: Mapping[str, str], - *, - remove_project_volumes: bool, - managed_resource_names: Callable[ - [str, Mapping[str, str]], - Awaitable[tuple[list[str], str | None]], - ], -) -> list[str]: - """Check that managed Docker resources no longer exist. - - Args: - environment: Environment forwarded to Docker inspection commands. - remove_project_volumes: Whether managed volumes must also be absent. - managed_resource_names: Resource-inspection callback bound to the active scope. - - Returns: - Human-readable verification failures. Volumes are checked only when volume removal is enabled. - """ - kinds = ["container", "network"] - if remove_project_volumes: - kinds.append("volume") - - errors: list[str] = [] - for kind in kinds: - names, query_error = await managed_resource_names(kind, environment) - if query_error is not None: - errors.append(query_error) - elif names: - errors.append(f"Managed Compose {kind}s remain after teardown: {', '.join(names)}") - return errors diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/_compose_provider.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/_compose_provider.py deleted file mode 100644 index cbb58e231f..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/_compose_provider.py +++ /dev/null @@ -1,935 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# ruff: noqa: I001 - the vendored SDK mirror uses different import-order settings. - -"""Docker Compose sandbox provider orchestration.""" - -from __future__ import annotations - -import asyncio -import contextlib -import os -import re -import tempfile -import time -import uuid -from collections.abc import Awaitable, Callable, Mapping, Sequence -from pathlib import Path -from typing import Any - -from ..base import ( - SANDBOX_RUNTIME_RETURN_CODE, - SandboxCreateError, - SandboxExecResult, - SandboxHandle, - SandboxSpec, - SandboxStatus, -) -from . import _compose_lifecycle, _compose_transfer -from ._compose_cli import _ComposeCli, _run_shielded -from ._compose_contracts import ( - ComposeCleanupError, - ComposeCommandResult, - ComposeServiceTopology, - ProgressCallback, - PullPolicy, - logger, -) -from ._compose_inspection import _service_is_running, _services_ready -from ._compose_state import _ComposeCommandScope, _ComposeProjectLock, _ComposeSession - - -class ComposeTeardownContext: - """Constrained project operations available to a trusted teardown hook.""" - - def __init__(self, provider: DockerComposeSandboxProvider, environment: Mapping[str, str]) -> None: - """Bind teardown operations to one provider and its command environment. - - Args: - provider: Provider that owns the Compose project being torn down. - environment: Environment to forward to teardown commands. - """ - self._provider = provider - self._environment = environment - - async def service_is_running(self, service: str) -> bool: - """Check whether a named service currently has a running container. - - Args: - service: Compose service name to inspect. - - Returns: - ``True`` when at least one matching service row is running. - """ - return _service_is_running(await self._provider._compose_ps(self._environment), service) - - async def stop_service(self, service: str) -> ComposeCommandResult: - """Gracefully stop a service, retrying transient command failures. - - Args: - service: Compose service name to stop. - - Returns: - Result of the final successful or exhausted stop attempt. - """ - return await self._provider._cli.retry_compose( - [ - "stop", - "--timeout", - str(max(1, int(self._provider.shutdown_timeout_seconds))), - service, - ], - environment=self._environment, - timeout=self._provider.shutdown_timeout_seconds + 10, - ) - - async def kill_service(self, service: str) -> ComposeCommandResult: - """Force-stop a service, retrying transient command failures. - - Args: - service: Compose service name to kill. - - Returns: - Result of the final successful or exhausted kill attempt. - """ - return await self._provider._cli.retry_compose( - ["kill", service], - environment=self._environment, - timeout=self._provider.command_timeout_seconds, - ) - - async def exec_service( - self, - service: str, - command: Sequence[str], - *, - timeout_seconds: float | None = None, - ) -> ComposeCommandResult: - """Execute an argument-vector command in a running service. - - Args: - service: Compose service name in which to execute the command. - command: Command and arguments passed directly to ``docker compose exec``. - timeout_seconds: Optional command deadline; defaults to the provider command timeout. - - Returns: - Captured command result, including timeout state. - - Example: - ``await context.exec_service("redis", ("redis-cli", "PING"))`` executes without shell parsing. - """ - return await self._provider._cli.run_compose( - ["exec", "--no-TTY", service, *command], - environment=self._environment, - timeout=timeout_seconds or self._provider.command_timeout_seconds, - ) - - def failure_message(self, prefix: str, result: ComposeCommandResult) -> str: - """Format a redacted teardown failure message. - - Args: - prefix: Human-readable description of the failed operation. - result: Command result whose captured output should be summarized. - - Returns: - Redacted message containing the prefix, timeout state, and command output. - """ - return self._provider._cli.failure_message(prefix, result, self._environment) - - -TeardownHook = Callable[[ComposeTeardownContext], Awaitable[None]] - - -class DockerComposeSandboxProvider: - """Own one exclusive Docker Compose project for a sandbox lifecycle.""" - - name = "docker-compose" - - def __init__( - self, - *, - compose_files: Sequence[str | Path], - service_topology: ComposeServiceTopology, - project_directory: str | Path | None = None, - project_name: str | None = None, - profiles: Sequence[str] = (), - build: bool = False, - pull_policy: PullPolicy = "missing", - startup_timeout_seconds: float = 600, - command_timeout_seconds: float = 60, - shutdown_timeout_seconds: float = 30, - lock_path: str | Path | None = None, - diagnostics_dir: str | Path | None = None, - environment_defaults: Mapping[str, str] | None = None, - port_override_hints: Mapping[str, str] | None = None, - teardown_hook: TeardownHook | None = None, - remove_project_volumes: bool = False, - progress_callback: ProgressCallback | None = None, - docker_bin: str = "docker", - ) -> None: - """Configure one reusable owner for a caller-described Compose project. - - Args: - compose_files: Ordered Compose files; later files override earlier files. - service_topology: Exact active services and their expected lifecycle roles. - project_directory: Base directory for relative Compose paths; defaults to the first file's parent. - project_name: Compose project name; a unique evaluator name is generated when omitted. - profiles: Compose profiles to enable, with duplicate names removed in first-seen order. - build: Whether ``compose up`` may build source images. - pull_policy: Image pull behavior passed to ``compose up``. - startup_timeout_seconds: Maximum time allowed for project startup. - command_timeout_seconds: Default deadline for Compose and Docker commands. - shutdown_timeout_seconds: Grace period supplied to service stop and project shutdown. - lock_path: Optional cross-process ownership lock file. - diagnostics_dir: Optional directory for startup progress, service state, and logs. - environment_defaults: Lowest-precedence environment values for Compose interpolation. - port_override_hints: Service-specific configuration hints shown for occupied host ports. - teardown_hook: Trusted project-specific cleanup run before ``compose down``. - remove_project_volumes: Whether shutdown removes and verifies project volumes. - progress_callback: Optional receiver for lifecycle progress messages. - docker_bin: Docker CLI executable name or path. - - Raises: - ValueError: If Compose files are empty, the project name is invalid, or the pull policy is unsupported. - - Example: - ``DockerComposeSandboxProvider(compose_files=("compose.yaml",), service_topology=topology)`` - starts existing images by default; pass ``build=True`` to build provisioned source. - """ - if isinstance(compose_files, (str, Path)) or not compose_files: - raise ValueError("compose_files must contain at least one path") - if pull_policy not in {"always", "missing", "never"}: - raise ValueError("pull_policy must be one of: always, missing, never") - self.compose_files = tuple(Path(path).expanduser().resolve() for path in compose_files) - self.project_directory = ( - Path(project_directory).expanduser().resolve() - if project_directory is not None - else self.compose_files[0].parent - ) - self.project_name = project_name or f"nemo-eval-{uuid.uuid4().hex[:12]}" - if re.fullmatch(r"[a-z0-9][a-z0-9_-]*", self.project_name) is None: - raise ValueError( - "project_name must start with a lowercase letter or digit and contain only " - "lowercase letters, digits, hyphens, or underscores" - ) - self.service_topology = service_topology - self.target_service = service_topology.target_service - self.profiles = tuple(dict.fromkeys(profiles)) - self.build = build - self.pull_policy: PullPolicy = pull_policy - self.startup_timeout_seconds = float(startup_timeout_seconds) - self.command_timeout_seconds = float(command_timeout_seconds) - self.shutdown_timeout_seconds = float(shutdown_timeout_seconds) - self.lock_path = ( - Path(lock_path) - if lock_path is not None - else Path(tempfile.gettempdir()) / (f"nemo-eval-compose-{self.project_name}.lock") - ) - self.environment_defaults = dict(environment_defaults or {}) - self.port_override_hints = dict(port_override_hints or {}) - self.teardown_hook = teardown_hook - self.remove_project_volumes = remove_project_volumes - self.progress_callback = progress_callback - self.docker_bin = docker_bin - self.diagnostics_dir = Path(diagnostics_dir).expanduser().resolve() if diagnostics_dir is not None else None - self._session: _ComposeSession | None = None - self._closed = False - self._cli = _ComposeCli(self._command_scope) - - async def create(self, spec: SandboxSpec) -> SandboxHandle: - """Validate, start, and claim exclusive ownership of the Compose project. - - Args: - spec: Sandbox request. Its environment overrides host and provider defaults; - provider options are not accepted by this provider. - - Returns: - Handle targeting the configured long-running service. - - Raises: - SandboxCreateError: If validation, locking, startup, readiness, or cleanup fails. - asyncio.CancelledError: If the caller cancels creation after cleanup finishes. - """ - if self._closed: - raise SandboxCreateError("DockerComposeSandboxProvider is closed") - if self._session is not None: - raise SandboxCreateError("DockerComposeSandboxProvider already owns a stack") - if spec.provider_options: - raise SandboxCreateError( - "DockerComposeSandboxProvider does not accept SandboxSpec.provider_options; " - "configure the provider through its constructor" - ) - command_scope = self._public_command_scope() - target_service = self.target_service - service_topology = self.service_topology - missing_files = [path for path in command_scope.compose_files if not path.is_file()] - if missing_files: - raise SandboxCreateError(f"Compose files do not exist: {missing_files}") - if not command_scope.project_directory.is_dir(): - raise SandboxCreateError(f"Compose project directory does not exist: {command_scope.project_directory}") - - environment = {**self.environment_defaults, **os.environ, **spec.env} - for key, value in self.environment_defaults.items(): - if not environment.get(key): - environment[key] = value - try: - project_lock = _ComposeProjectLock.acquire(self.lock_path) - except SandboxCreateError: - raise - except OSError as exc: - raise SandboxCreateError(f"Could not acquire Compose project lock {self.lock_path}: {exc}") from exc - session = _ComposeSession( - session_id=f"{command_scope.project_name}:{target_service}:{uuid.uuid4().hex}", - environment=environment, - lock=project_lock, - command_scope=command_scope, - target_service=target_service, - service_topology=service_topology, - ) - self._session = session - try: - await self._preflight(environment) - session.owns_project = True - up_args = [ - "up", - "--detach", - "--wait", - "--wait-timeout", - str(max(1, int(self.startup_timeout_seconds))), - "--build" if self.build else "--no-build", - "--pull", - self.pull_policy, - ] - build_mode = "build enabled" if self.build else "reusing existing images" - self._progress(f"Starting managed Compose project {command_scope.project_name!r} ({build_mode})...") - progress_log_path: Path | None = None - if self.diagnostics_dir is not None: - self.diagnostics_dir.mkdir(parents=True, exist_ok=True) - progress_log_path = self.diagnostics_dir / "compose-up.log" - self._progress(f"Compose startup log: {progress_log_path.as_uri()}") - startup_started_at = time.monotonic() - with contextlib.ExitStack() as stack: - stream_output = ( - stack.enter_context(progress_log_path.open("w", encoding="utf-8")) - if progress_log_path is not None - else None - ) - result = await self._cli.run_compose( - up_args, - environment=environment, - timeout=self.startup_timeout_seconds, - stream_output=stream_output, - ) - if not result.ok: - raise SandboxCreateError(self._cli.failure_message("Compose startup failed", result, environment)) - await self._assert_ready(environment) - self._progress( - f"Managed Compose project {command_scope.project_name!r} ready in " - f"{time.monotonic() - startup_started_at:.1f}s." - ) - except BaseException as exc: - cleanup_error = await self._shielded_cleanup(session, diagnostics_reason="startup-failure") - if cleanup_error is not None: - exc.add_note(f"Compose cleanup also failed: {cleanup_error}") - if isinstance(exc, (asyncio.CancelledError, KeyboardInterrupt, SystemExit)): - raise - if isinstance(exc, SandboxCreateError): - raise - raise SandboxCreateError(str(exc)) from exc - - handle = SandboxHandle( - sandbox_id=session.session_id, - provider_name=self.name, - raw=session, - ) - return handle - - async def exec( - self, - handle: SandboxHandle, - command: str, - *, - cwd: str | None = None, - env: dict[str, str] | None = None, - timeout_s: int | float | None = None, - stdin: bytes | None = None, - ) -> SandboxExecResult: - """Run a shell command in the configured target service. - - Args: - handle: Active handle returned by this provider. - command: Shell command evaluated by ``sh -lc`` in the target service. - cwd: Optional working directory inside the service container. - env: Optional command-specific environment variables. - timeout_s: Optional command deadline; defaults to ``command_timeout_seconds``. - stdin: Optional bytes forwarded to the command's standard input. - - Returns: - Sandbox result containing captured output, return code, and timeout classification. - """ - state = self._state(handle) - args = ["exec", "--no-TTY"] - if cwd is not None: - args.extend(["--workdir", cwd]) - for key, value in (env or {}).items(): - args.extend(["--env", f"{key}={value}"]) - args.extend([state.target_service, "sh", "-lc", command]) - result = await self._cli.run_compose( - args, - environment=state.environment, - timeout=float(timeout_s or self.command_timeout_seconds), - stdin=stdin, - ) - return SandboxExecResult( - stdout=result.stdout, - stderr=result.stderr, - return_code=(SANDBOX_RUNTIME_RETURN_CODE if result.timed_out else result.return_code), - error_type="timeout" if result.timed_out else None, - ) - - async def upload_file( - self, - handle: SandboxHandle, - source_path: Path, - target_path: str, - ) -> None: - """Upload one file under the target service user's parent-directory authority. - - The destination must name an exact, non-root file. Missing parents are created - as the target service user; if that user cannot create them, the upload fails - without changing parent or ancestor ownership or permissions. After copying, - only the regular non-symlink file leaf is assigned to the runtime identity and - made owner-writable. - - Upload is non-atomic: a copy followed by failed leaf repair may leave the file - in place, and retrying the same source and destination is supported. The caller - must keep the destination and its complete ancestor chain stable during the - operation. Stable symlink ancestors follow normal POSIX path resolution; this - API is not a security boundary against concurrent in-container path mutation. - - Args: - handle: Active handle returned by this provider. - source_path: Existing host file to copy. - target_path: Exact non-root file path inside the target service. - - Raises: - ValueError: If ``target_path`` is empty, root, or directory-shaped. - RuntimeError: If target preparation, copying, or ownership repair fails. - """ - await self._copy_to_service(handle, source_path, target_path, directory=False) - - async def upload_dir( - self, - handle: SandboxHandle, - source_dir: Path, - target_dir: str, - ) -> None: - """Upload directory contents into a dedicated service-owned target directory. - - The destination must be a non-root exact target that is not itself a symlink. - The provider merges the source contents into that directory and recursively - assigns the entire resulting tree to the target service user. Callers must not - use a shared or externally owned tree for this operation. - - The caller must keep the destination and its complete ancestor chain stable - during the upload. Stable symlink ancestors follow normal POSIX path resolution; - this API is not a security boundary against concurrent in-container path - mutation. - - Args: - handle: Active handle returned by this provider. - source_dir: Host directory whose contents should be copied. - target_dir: Destination directory inside the target service. The provider - recursively assigns this entire target tree to the service runtime user. - - Raises: - ValueError: If ``target_dir`` is empty or resolves to the container root. - RuntimeError: If target preparation, copying, or ownership repair fails. - - Example: - Uploading ``/tmp/seed/.`` to ``/workspace`` produces ``/workspace/file`` rather - than ``/workspace/seed/file``. - """ - await self._copy_to_service(handle, source_dir, target_dir, directory=True) - - async def download_file( - self, - handle: SandboxHandle, - source_path: str, - target_path: Path, - ) -> None: - """Download one target-service file to the host. - - Args: - handle: Active handle returned by this provider. - source_path: File path inside the target service. - target_path: Host destination file; missing parent directories are created. - - Raises: - RuntimeError: If the Compose copy command fails. - """ - await self._copy_from_service(handle, source_path, target_path, directory=False) - - async def download_dir( - self, - handle: SandboxHandle, - source_dir: str, - target_dir: Path, - ) -> None: - """Download directory contents into a host directory. - - Args: - handle: Active handle returned by this provider. - source_dir: Directory path inside the target service. - target_dir: Host directory to create or merge copied contents into. - - Raises: - RuntimeError: If the Compose copy command fails. - """ - await self._copy_from_service(handle, source_dir, target_dir, directory=True) - - async def status(self, handle: SandboxHandle) -> SandboxStatus: - """Return the aggregate lifecycle state of the managed project. - - Args: - handle: Handle whose project state should be inspected. - - Returns: - ``RUNNING`` when all configured service roles are ready, ``STOPPED`` when - the project is absent, ``ERROR`` for an unhealthy topology, or ``UNKNOWN`` - when inspection fails. - """ - state = self._state(handle) - if not state.owns_project: - return SandboxStatus.STOPPED - try: - rows = await self._compose_ps(state.environment) - except Exception: # noqa: BLE001 - status must collapse provider failures - return SandboxStatus.UNKNOWN - if not rows: - return SandboxStatus.STOPPED - return SandboxStatus.RUNNING if _services_ready(rows, state.service_topology) is None else SandboxStatus.ERROR - - async def close(self, handle: SandboxHandle) -> None: - """Tear down the active project and release its ownership lock. - - Args: - handle: Active handle returned by this provider. - - Raises: - ComposeCleanupError: If the teardown hook, Compose shutdown, or resource verification fails. - asyncio.CancelledError: If cancellation arrives while shielded cleanup is running. - """ - session = self._state(handle) - error = await self._shielded_cleanup(session) - if error is not None: - raise error - - async def aclose(self) -> None: - """Close provider-scoped resources and tear down any active project. - - The method is idempotent and permanently prevents subsequent ``create`` calls. - - Raises: - ComposeCleanupError: If the active project cannot be fully removed. - asyncio.CancelledError: If cancellation arrives while shielded cleanup is running. - """ - if self._closed: - return - self._closed = True - session = self._session - if session is None: - return - error = await self._shielded_cleanup(session) - if error is not None: - raise error - - def _command_scope(self) -> _ComposeCommandScope: - """Return the active lifecycle scope or current public configuration. - - Returns: - Frozen settings for the active lifecycle, or a fresh public-configuration - snapshot when the provider does not own a project. - """ - if self._session is not None: - return self._session.command_scope - return self._public_command_scope() - - def _public_command_scope(self) -> _ComposeCommandScope: - """Snapshot the provider's current public command configuration. - - Returns: - Immutable settings suitable for the next lifecycle. - """ - return _ComposeCommandScope( - docker_bin=self.docker_bin, - project_directory=self.project_directory, - compose_files=self.compose_files, - project_name=self.project_name, - profiles=self.profiles, - ) - - async def _preflight(self, environment: dict[str, str]) -> None: - """Validate rendered topology, project ownership, and published host ports. - - Args: - environment: Fully merged environment used for Compose interpolation. - - Raises: - SandboxCreateError: If the configuration is invalid, the project already has - containers, service roles differ, or a published host port is unavailable. - """ - session = self._session - service_topology = session.service_topology if session is not None else self.service_topology - await _compose_lifecycle._preflight( - self._cli, - self._command_scope(), - service_topology, - environment, - command_timeout_seconds=self.command_timeout_seconds, - port_override_hints=self.port_override_hints, - ) - - async def _assert_ready(self, environment: Mapping[str, str]) -> None: - """Require every configured service to satisfy its lifecycle role. - - Args: - environment: Environment used to query Compose state. - - Raises: - SandboxCreateError: If a long-running or one-shot service is not ready. - """ - session = self._session - service_topology = session.service_topology if session is not None else self.service_topology - await _compose_lifecycle._assert_ready( - self._cli, - service_topology, - environment, - command_timeout_seconds=self.command_timeout_seconds, - ) - - async def _compose_ps(self, environment: Mapping[str, str]) -> list[dict[str, Any]]: - """Return parsed state rows for all project services. - - Args: - environment: Environment forwarded to ``docker compose ps``. - - Returns: - Parsed JSON objects emitted for service containers. - - Raises: - RuntimeError: If Compose cannot inspect the project. - json.JSONDecodeError: If Compose emits malformed JSON. - """ - return await _compose_lifecycle._compose_ps( - self._cli, - environment, - command_timeout_seconds=self.command_timeout_seconds, - ) - - async def _cleanup_owned_project( - self, - session: _ComposeSession, - ) -> ComposeCleanupError | None: - """Run diagnostics, caller cleanup, Compose shutdown, and resource verification. - - Args: - session: Active lifecycle whose project and lock must be released. - - Returns: - Aggregated cleanup error when any teardown phase fails; otherwise ``None``. - - Raises: - asyncio.CancelledError: If the cleanup task itself is cancelled. - """ - if not session.owns_project: - self._retire_session(session) - return None - - environment = session.environment - errors: list[str] = [] - try: - await self._capture_diagnostics(environment, reason="shutdown") - if self.teardown_hook is not None: - await self.teardown_hook(ComposeTeardownContext(self, environment)) - except BaseException as exc: # noqa: BLE001 - Compose down must still run - if isinstance(exc, (asyncio.CancelledError, KeyboardInterrupt, SystemExit)): - raise - errors.append(f"Compose teardown hook failed: {type(exc).__name__}: {exc}") - finally: - try: - down = await _compose_lifecycle._compose_down( - self._cli, - environment, - shutdown_timeout_seconds=self.shutdown_timeout_seconds, - command_timeout_seconds=self.command_timeout_seconds, - remove_project_volumes=self.remove_project_volumes, - ) - if not down.ok: - errors.append(self._cli.failure_message("Compose down failed", down, environment)) - errors.extend(await self._verify_project_destroyed(environment)) - except BaseException as exc: # noqa: BLE001 - release ownership even if Docker fails - if isinstance(exc, (asyncio.CancelledError, KeyboardInterrupt, SystemExit)): - raise - errors.append(f"Compose teardown failed: {type(exc).__name__}: {exc}") - finally: - self._retire_session(session) - - if errors: - return ComposeCleanupError("; ".join(errors)) - return None - - async def _shielded_cleanup( - self, - session: _ComposeSession, - *, - diagnostics_reason: str | None = None, - ) -> ComposeCleanupError | None: - """Finish project cleanup even when the calling task is cancelled. - - Args: - session: Active lifecycle to clean up. - diagnostics_reason: Optional diagnostic label to capture inside the shield - before normal project cleanup starts. - - Returns: - Cleanup error when teardown completes with failures; otherwise ``None``. - - Raises: - asyncio.CancelledError: Re-raised after cleanup completes when the caller was cancelled. - """ - - process_exit: KeyboardInterrupt | SystemExit | None = None - - async def cleanup() -> ComposeCleanupError | None: - """Capture optional diagnostics, then tear down the owned project.""" - nonlocal process_exit - try: - if diagnostics_reason is not None: - await self._capture_diagnostics(session.environment, reason=diagnostics_reason) - except (KeyboardInterrupt, SystemExit) as exc: - process_exit = exc - - try: - return await self._cleanup_owned_project(session) - except (KeyboardInterrupt, SystemExit) as exc: - if process_exit is None: - process_exit = exc - else: - process_exit.add_note(f"Compose cleanup also exited: {exc}") - return None - - result, cancellation = await _run_shielded(cleanup()) - if cancellation is not None: - if result is not None: - cancellation.add_note(f"Compose cleanup also failed: {result}") - raise cancellation - if process_exit is not None: - raise process_exit - return result - - def _retire_session(self, session: _ComposeSession) -> None: - """Release one lifecycle's ownership resources and clear it when active. - - Args: - session: Lifecycle whose project ownership and lock should be released. - """ - session.owns_project = False - session.lock.release() - if self._session is session: - self._session = None - - async def _verify_project_destroyed( - self, - environment: Mapping[str, str], - ) -> list[str]: - """Check that managed Docker resources no longer exist. - - Args: - environment: Environment forwarded to Docker inspection commands. - - Returns: - Human-readable verification failures. Volumes are checked only when volume removal is enabled. - """ - return await _compose_lifecycle._verify_project_destroyed( - environment, - remove_project_volumes=self.remove_project_volumes, - managed_resource_names=self._managed_resource_names, - ) - - async def _managed_resource_names( - self, - kind: str, - environment: Mapping[str, str], - ) -> tuple[list[str], str | None]: - """List Docker resources carrying this Compose project's label. - - Args: - kind: Docker resource kind: ``container``, ``network``, or ``volume``. - environment: Environment forwarded to the Docker CLI. - - Returns: - Pair of resource names and an optional redacted inspection error. - """ - return await _compose_lifecycle._managed_resource_names( - self._cli, - self._command_scope(), - kind, - environment, - command_timeout_seconds=self.command_timeout_seconds, - ) - - async def _run_target_root( - self, - session: _ComposeSession, - command: Sequence[str], - ) -> ComposeCommandResult: - """Run a command as root in the configured target service. - - Args: - session: Active lifecycle identifying the target service and environment. - command: Executable and arguments to append after the target service name. - - Returns: - Captured result for the privileged Compose exec command. - """ - return await _compose_transfer._run_target_root( - self._cli, - session, - command, - command_timeout_seconds=self.command_timeout_seconds, - ) - - async def _copy_to_service( - self, - handle: SandboxHandle, - source: Path, - target: str, - *, - directory: bool, - ) -> None: - """Copy a host path into the target service and repair ownership. - - Args: - handle: Active handle returned by this provider. - source: Host file or directory to copy. - target: Destination path inside the target service. - directory: When ``True``, create the full target and copy only ``source`` contents; - otherwise create only the file's parent and copy the file itself. - - Raises: - RuntimeError: If target preparation, copying, or ownership repair fails. - SandboxCreateError: If the target service runtime identity cannot be determined. - - Example: - With ``directory=True``, source ``/tmp/work`` is passed as ``/tmp/work/.`` so - Docker merges its contents directly into the prepared target directory. - """ - state = self._state(handle) - await _compose_transfer._copy_to_service( - self._cli, - state, - source, - target, - directory=directory, - command_timeout_seconds=self.command_timeout_seconds, - ) - - async def _copy_from_service( - self, - handle: SandboxHandle, - source: str, - target: Path, - *, - directory: bool, - ) -> None: - """Copy a target-service path to a prepared host destination. - - Args: - handle: Active handle returned by this provider. - source: File or directory path inside the target service. - target: Host destination path. - directory: When ``True``, create the target directory and copy source contents; - otherwise create only the target file's parent. - - Raises: - RuntimeError: If the Compose copy command fails. - """ - state = self._state(handle) - await _compose_transfer._copy_from_service( - self._cli, - state, - source, - target, - directory=directory, - command_timeout_seconds=self.command_timeout_seconds, - ) - - async def _capture_diagnostics( - self, - environment: Mapping[str, str], - *, - reason: str, - ) -> None: - """Best-effort write redacted project state and recent logs. - - Args: - environment: Environment used for Compose commands and secret redaction. - reason: Filename-safe lifecycle label such as ``startup-failure`` or ``shutdown``. - - Diagnostics failures are logged and never replace the lifecycle error being investigated. - """ - await _compose_lifecycle._capture_diagnostics( - self._cli, - environment, - command_timeout_seconds=self.command_timeout_seconds, - diagnostics_dir=self.diagnostics_dir, - reason=reason, - ) - - async def _target_identity(self, session: _ComposeSession) -> str: - """Read the runtime ``UID:GID`` of the target service user. - - Args: - session: Active lifecycle identifying the target service and command environment. - - Returns: - Numeric identity formatted as ``UID:GID`` for ``chown``. - - Raises: - SandboxCreateError: If the identity command fails or emits an unexpected value. - """ - return await _compose_transfer._target_identity( - self._cli, - session, - command_timeout_seconds=self.command_timeout_seconds, - ) - - def _state(self, handle: SandboxHandle) -> _ComposeSession: - """Validate a handle and return its provider-private state. - - Args: - handle: Sandbox handle supplied to a provider operation. - - Returns: - Compose state stored on the handle. - - Raises: - ValueError: If the handle belongs to another provider or is not the active stack. - """ - if handle.provider_name != self.name or not isinstance(handle.raw, _ComposeSession): - raise ValueError("Sandbox handle does not belong to this Compose provider") - if self._session is not handle.raw or handle.sandbox_id != handle.raw.session_id: - raise ValueError("Sandbox handle is not the active Compose session") - return handle.raw - - def _progress(self, message: str) -> None: - """Publish a lifecycle progress message. - - Args: - message: Human-readable progress text sent to the callback or module logger. - """ - if self.progress_callback is not None: - self.progress_callback(message) - else: - logger.info(message) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/_compose_state.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/_compose_state.py deleted file mode 100644 index 620e3652e5..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/_compose_state.py +++ /dev/null @@ -1,116 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""State and ownership leases for Docker Compose sandbox lifecycles.""" - -from __future__ import annotations - -import errno -import os -from dataclasses import dataclass -from pathlib import Path - -from ..base import SandboxCreateError -from ._compose_contracts import ComposeServiceTopology - - -@dataclass -class _ComposeProjectLock: - """Exclusive POSIX lock lease for one managed Compose project. - - Attributes: - path: Host lock-file path. - fd: Open file descriptor holding the lock, or ``None`` after release. - """ - - path: Path - fd: int | None = None - - @classmethod - def acquire(cls, path: Path) -> _ComposeProjectLock: - """Acquire and return a nonblocking project lock lease. - - Args: - path: Host lock-file path to create and lock. - - Returns: - Lease whose open descriptor holds the exclusive lock. - - Raises: - SandboxCreateError: If POSIX locking is unavailable or another process holds the lock. - OSError: If the lock file cannot be created or locked for another reason. - """ - try: - import fcntl - except ImportError as exc: - raise SandboxCreateError("DockerComposeSandboxProvider requires POSIX fcntl file locking") from exc - - path.parent.mkdir(parents=True, exist_ok=True) - fd = os.open(path, os.O_CREAT | os.O_RDWR, 0o600) - try: - fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - except OSError as exc: - os.close(fd) - if exc.errno in {errno.EACCES, errno.EAGAIN}: - raise SandboxCreateError(f"Another Compose sandbox holds {path}") from exc - raise - return cls(path=path, fd=fd) - - def release(self) -> None: - """Release and close the lease when it is still held.""" - if self.fd is None: - return - fd = self.fd - try: - import fcntl - - fcntl.flock(fd, fcntl.LOCK_UN) - finally: - try: - os.close(fd) - finally: - self.fd = None - - -@dataclass(frozen=True) -class _ComposeCommandScope: - """Current project settings needed to construct Docker CLI commands. - - Attributes: - docker_bin: Docker CLI executable name or path. - project_directory: Host working directory and Compose project directory. - compose_files: Ordered Compose configuration files. - project_name: Explicit Compose project name. - profiles: Ordered enabled Compose profiles. - """ - - docker_bin: str - project_directory: Path - compose_files: tuple[Path, ...] - project_name: str - profiles: tuple[str, ...] - - -@dataclass -class _ComposeSession: - """State and ownership resources for one provider lifecycle. - - Attributes: - session_id: Unique identifier used by the public sandbox handle. - environment: Environment used for all commands in this lifecycle. - lock: Exclusive project lock held until cleanup completes. - command_scope: Immutable Docker and Compose project settings for this lifecycle. - target_service: Service used for sandbox command execution and file transfer. - service_topology: Service roles used for lifecycle readiness checks. - owns_project: Whether startup reached the point requiring Compose teardown. - target_identity: Cached ``UID:GID`` of the target service runtime user. - """ - - session_id: str - environment: dict[str, str] - lock: _ComposeProjectLock - command_scope: _ComposeCommandScope - target_service: str - service_topology: ComposeServiceTopology - owns_project: bool = False - target_identity: str | None = None diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/_compose_transfer.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/_compose_transfer.py deleted file mode 100644 index a939c7683e..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/_compose_transfer.py +++ /dev/null @@ -1,380 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# ruff: noqa: I001 - the vendored SDK mirror uses different import-order settings. - -"""File-transfer preparation and ownership repair for Compose sandboxes.""" - -from __future__ import annotations - -import os -import posixpath -import re -from collections.abc import Sequence -from pathlib import Path - -from ..base import SandboxCreateError -from ._compose_cli import _ComposeCli -from ._compose_contracts import ComposeCommandResult -from ._compose_state import _ComposeSession - -_FILE_TARGET_OPERATION = "nemo-compose-file-target" -_REPAIR_FILE_OPERATION = "nemo-compose-file-repair" -_DIRECTORY_TARGET_OPERATION = "nemo-compose-directory-target" -_PREPARE_FILE_TARGET_SCRIPT = """\ -parent=$1 -target=$2 -mkdir -p "$parent" || exit 1 -if [ -L "$target" ] || [ -d "$target" ]; then - exit 1 -fi -if [ -e "$target" ] && [ ! -f "$target" ]; then - exit 1 -fi -""" -_REPAIR_FILE_SCRIPT = """\ -target=$1 -identity=$2 -[ -f "$target" ] && [ ! -L "$target" ] || exit 1 -chown -h "$identity" "$target" || exit 1 -chmod u+w "$target" || exit 1 -[ -f "$target" ] && [ ! -L "$target" ] || exit 1 -""" -_PREPARE_DIRECTORY_TARGET_SCRIPT = """\ -target=$1 -[ ! -L "$target" ] || exit 1 -mkdir -p "$target" || exit 1 -[ -d "$target" ] && [ ! -L "$target" ] || exit 1 -""" - - -async def _run_target_root( - cli: _ComposeCli, - session: _ComposeSession, - command: Sequence[str], - *, - command_timeout_seconds: float, -) -> ComposeCommandResult: - """Run a command as root in the configured target service. - - Args: - cli: Command gateway bound to the active Compose lifecycle. - session: Active lifecycle identifying the target service and environment. - command: Executable and arguments to append after the target service name. - command_timeout_seconds: Deadline for the privileged command. - - Returns: - Captured result for the privileged Compose exec command. - """ - return await cli.run_compose( - ["exec", "--no-TTY", "--user", "0", session.target_service, *command], - environment=session.environment, - timeout=command_timeout_seconds, - ) - - -async def _prepare_file_target( - cli: _ComposeCli, - session: _ComposeSession, - parent: str, - target: str, - *, - command_timeout_seconds: float, -) -> None: - """Prepare and validate a file target as the configured service user. - - Args: - cli: Command gateway bound to the active Compose lifecycle. - session: Active lifecycle identifying the target service and environment. - parent: Normalized absolute parent path for the uploaded file. - target: Normalized absolute file path to validate. - command_timeout_seconds: Deadline for target preparation. - - Raises: - RuntimeError: If the service user cannot prepare a safe exact target. - """ - result = await cli.run_compose( - [ - "exec", - "--no-TTY", - session.target_service, - "sh", - "-c", - _PREPARE_FILE_TARGET_SCRIPT, - _FILE_TARGET_OPERATION, - parent, - target, - ], - environment=session.environment, - timeout=command_timeout_seconds, - ) - if not result.ok: - raise RuntimeError( - cli.failure_message( - "Compose upload target preparation failed", - result, - session.environment, - ) - ) - - -async def _repair_uploaded_file( - cli: _ComposeCli, - session: _ComposeSession, - target: str, - identity: str, - *, - command_timeout_seconds: float, -) -> None: - """Validate and repair only the exact uploaded regular-file leaf.""" - ownership = await _run_target_root( - cli, - session, - [ - "sh", - "-c", - _REPAIR_FILE_SCRIPT, - _REPAIR_FILE_OPERATION, - target, - identity, - ], - command_timeout_seconds=command_timeout_seconds, - ) - if not ownership.ok: - raise RuntimeError( - cli.failure_message( - "Compose upload ownership repair failed", - ownership, - session.environment, - ) - ) - - -async def _prepare_directory_target( - cli: _ComposeCli, - session: _ComposeSession, - target: str, - *, - command_timeout_seconds: float, -) -> None: - """Prepare a dedicated non-symlink directory target as root.""" - prepared = await _run_target_root( - cli, - session, - [ - "sh", - "-c", - _PREPARE_DIRECTORY_TARGET_SCRIPT, - _DIRECTORY_TARGET_OPERATION, - target, - ], - command_timeout_seconds=command_timeout_seconds, - ) - if not prepared.ok: - raise RuntimeError( - cli.failure_message( - "Compose upload target preparation failed", - prepared, - session.environment, - ) - ) - - -async def _copy_to_service( - cli: _ComposeCli, - session: _ComposeSession, - source: Path, - target: str, - *, - directory: bool, - command_timeout_seconds: float, -) -> None: - """Copy a host path into the target service and repair ownership. - - Args: - cli: Command gateway bound to the active Compose lifecycle. - session: Active lifecycle identifying the target service and environment. - source: Host file or directory to copy. - target: Destination path inside the target service. - directory: When ``True``, create the full target and copy only ``source`` contents; - otherwise create only the file's parent and copy the file itself. - command_timeout_seconds: Deadline for each Compose operation. - - Raises: - RuntimeError: If target preparation, copying, or ownership repair fails. - SandboxCreateError: If the target service runtime identity cannot be determined. - - Example: - With ``directory=True``, source ``/tmp/work`` is passed as ``/tmp/work/.`` so - Docker merges its contents directly into the prepared target directory. - """ - container_target = _normalized_upload_target(target, directory=directory) - remote_directory = container_target if directory else posixpath.dirname(container_target) - if directory: - await _prepare_directory_target( - cli, - session, - remote_directory, - command_timeout_seconds=command_timeout_seconds, - ) - else: - await _prepare_file_target( - cli, - session, - remote_directory, - container_target, - command_timeout_seconds=command_timeout_seconds, - ) - copy_source = f"{source}{os.sep}." if directory else str(source) - result = await cli.run_compose( - ["cp", copy_source, f"{session.target_service}:{container_target}"], - environment=session.environment, - timeout=command_timeout_seconds, - ) - if not result.ok: - raise RuntimeError(cli.failure_message("Compose upload failed", result, session.environment)) - if session.target_identity is None: - session.target_identity = await _target_identity( - cli, - session, - command_timeout_seconds=command_timeout_seconds, - ) - if directory: - ownership = await _run_target_root( - cli, - session, - ["chown", "-R", session.target_identity, "--", container_target], - command_timeout_seconds=command_timeout_seconds, - ) - if not ownership.ok: - raise RuntimeError( - cli.failure_message( - "Compose upload ownership repair failed", - ownership, - session.environment, - ) - ) - else: - await _repair_uploaded_file( - cli, - session, - container_target, - session.target_identity, - command_timeout_seconds=command_timeout_seconds, - ) - - -async def _copy_from_service( - cli: _ComposeCli, - session: _ComposeSession, - source: str, - target: Path, - *, - directory: bool, - command_timeout_seconds: float, -) -> None: - """Copy a target-service path to a prepared host destination. - - Args: - cli: Command gateway bound to the active Compose lifecycle. - session: Active lifecycle identifying the target service and environment. - source: File or directory path inside the target service. - target: Host destination path. - directory: When ``True``, create the target directory and copy source contents; - otherwise create only the target file's parent. - command_timeout_seconds: Deadline for the Compose copy command. - - Raises: - RuntimeError: If the Compose copy command fails. - """ - if directory: - target.mkdir(parents=True, exist_ok=True) - else: - target.parent.mkdir(parents=True, exist_ok=True) - copy_source = posixpath.join(source, ".") if directory else source - result = await cli.run_compose( - ["cp", f"{session.target_service}:{copy_source}", str(target)], - environment=session.environment, - timeout=command_timeout_seconds, - ) - if not result.ok: - raise RuntimeError(cli.failure_message("Compose download failed", result, session.environment)) - - -async def _target_identity( - cli: _ComposeCli, - session: _ComposeSession, - *, - command_timeout_seconds: float, -) -> str: - """Read the runtime ``UID:GID`` of the target service user. - - Args: - cli: Command gateway bound to the active Compose lifecycle. - session: Active lifecycle identifying the target service and command environment. - command_timeout_seconds: Deadline for the identity command. - - Returns: - Numeric identity formatted as ``UID:GID`` for ``chown``. - - Raises: - SandboxCreateError: If the identity command fails or emits an unexpected value. - """ - result = await cli.run_compose( - [ - "exec", - "--no-TTY", - session.target_service, - "sh", - "-c", - 'printf "%s:%s" "$(id -u)" "$(id -g)"', - ], - environment=session.environment, - timeout=command_timeout_seconds, - ) - identity = result.stdout.strip() - if not result.ok or not re.fullmatch(r"\d+:\d+", identity): - raise SandboxCreateError( - cli.failure_message( - "Could not determine target service identity", - result, - session.environment, - ) - ) - return identity - - -def _absolute_container_path(path: str) -> str: - """Normalize a Docker container path against its root directory. - - Docker copy commands interpret relative container paths from ``/``, while commands - executed in a container interpret them from the image or service working directory. - Normalizing once keeps preparation, copy, and ownership repair on the same target. - - Args: - path: Absolute or root-relative POSIX container path. - - Returns: - Normalized absolute POSIX path. - - Raises: - ValueError: If ``path`` is empty. - - Example: - ``work/output.txt`` becomes ``/work/output.txt``. - """ - if not path: - raise ValueError("Container path cannot be empty") - return posixpath.normpath(f"/{path.lstrip('/')}") - - -def _normalized_upload_target(target: str, *, directory: bool) -> str: - """Normalize and validate an exact upload destination.""" - if not target: - raise ValueError("Container path cannot be empty") - normalized = _absolute_container_path(target) - if normalized == "/": - kind = "Directory" if directory else "File" - raise ValueError(f"{kind} upload target cannot be the container root") - if not directory and target.endswith("/"): - raise ValueError("File upload target must name an exact file") - return normalized diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/compose.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/compose.py deleted file mode 100644 index 0f54969bf3..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/compose.py +++ /dev/null @@ -1,39 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Public Docker Compose sandbox provider façade.""" - -from __future__ import annotations - -from ._compose_contracts import ( - ComposeCleanupError, - ComposeCommandResult, - ComposeServiceTopology, - ProgressCallback, - PullPolicy, -) -from ._compose_provider import ( - ComposeTeardownContext, - DockerComposeSandboxProvider, - TeardownHook, -) - -for _public_class in ( - ComposeCleanupError, - ComposeCommandResult, - ComposeServiceTopology, - ComposeTeardownContext, - DockerComposeSandboxProvider, -): - _public_class.__module__ = __name__ - -__all__ = [ - "ComposeCleanupError", - "ComposeCommandResult", - "ComposeServiceTopology", - "ComposeTeardownContext", - "DockerComposeSandboxProvider", - "ProgressCallback", - "PullPolicy", - "TeardownHook", -] diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/docker.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/docker.py deleted file mode 100644 index d4a4e0aa5d..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/sandbox/providers/docker.py +++ /dev/null @@ -1,265 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Docker-backed sandbox provider. - -Runs each sandbox as one persistent container (``docker run -d`` a keep-alive process), -execs commands with ``docker exec``, and moves files across the boundary with ``docker cp`` -— the same transfer verb that maps to ``kubectl cp`` for the Kubernetes provider that -follows. Shells out to the ``docker`` CLI (stdlib ``subprocess``/asyncio only); no Python -Docker SDK dependency. - -Isolation note: the container does **not** default to ``--network none``, because the agent -harness legitimately needs egress to reach its model endpoint. Network mode is a provider -option (``network``), defaulting to Docker's default bridge. Endpoint-scoped egress control -(allow the model API, deny everything else) is future work — it belongs to a policy-capable -backend (e.g. NVIDIA OpenShell), not this provider. -""" - -from __future__ import annotations - -import asyncio -import contextlib -import logging -import os -import posixpath -import shlex -import signal -import uuid -from dataclasses import dataclass -from pathlib import Path - -from nemo_platform.beta.evaluator.agent_eval.runtimes.environment import _redact_for_logging -from nemo_platform.beta.evaluator.agent_eval.runtimes.sandbox.base import ( - SANDBOX_RUNTIME_RETURN_CODE, - SandboxCreateError, - SandboxExecResult, - SandboxHandle, - SandboxResources, - SandboxSpec, - SandboxStatus, -) - -logger = logging.getLogger(__name__) - -_CONTAINER_NAME_PREFIX = "nemo-eval-sbx-" -_KEEP_ALIVE_COMMAND = "sh -c 'exec sleep infinity'" -DEFAULT_EXEC_TIMEOUT_S = 180.0 -DEFAULT_START_TIMEOUT_S = 120.0 - - -@dataclass -class _DockerContainer: - """Provider-private state stashed on ``SandboxHandle.raw``.""" - - name: str - image: str - env: dict[str, str] - - -def _resource_flags(resources: SandboxResources) -> list[str]: - """Translate neutral resources into ``docker run`` flags (unmappable fields ignored).""" - flags: list[str] = [] - if resources.cpu is not None: - flags += ["--cpus", str(resources.cpu)] - if resources.memory_mib is not None: - flags += ["--memory", f"{resources.memory_mib}m"] - if resources.gpu: - # Request all GPUs; a specific count/type is a future refinement. - flags += ["--gpus", "all"] - return flags - - -class DockerSandboxProvider: - """Sandbox provider backed by the local Docker CLI.""" - - name = "docker" - - def __init__( - self, - *, - docker_bin: str = "docker", - network: str | None = None, - default_timeout_s: float | None = DEFAULT_EXEC_TIMEOUT_S, - start_timeout_s: float | None = DEFAULT_START_TIMEOUT_S, - extra_run_args: list[str] | None = None, - ) -> None: - self._docker = docker_bin - self._network = network - self._default_timeout_s = default_timeout_s - self._start_timeout_s = start_timeout_s - self._extra_run_args = list(extra_run_args or []) - - async def _run( - self, - argv: list[str], - *, - timeout_s: float | None, - stdin: bytes | None = None, - ) -> tuple[int, str, str]: - """Run a ``docker`` CLI command. Returns (return_code, stdout, stderr). - - Single chokepoint every CLI call goes through (mocked at this boundary in tests). - Enforces the timeout with ``asyncio.wait_for`` and kills the whole process group so - no child lingers. Raises :class:`TimeoutError` on timeout. - """ - proc = await asyncio.create_subprocess_exec( - *argv, - stdin=asyncio.subprocess.PIPE if stdin is not None else None, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - start_new_session=True, - ) - try: - stdout_b, stderr_b = await asyncio.wait_for(proc.communicate(input=stdin), timeout=timeout_s) - except (asyncio.TimeoutError, TimeoutError) as exc: - with contextlib.suppress(ProcessLookupError): - os.killpg(os.getpgid(proc.pid), signal.SIGKILL) - with contextlib.suppress(Exception): - await proc.wait() - # Redact argv: it can carry `-e KEY=` (e.g. NVIDIA_API_KEY), and this error is - # caught and persisted into error.json, so a raw argv would leak credentials into artifacts. - raise TimeoutError(f"docker command timed out after {timeout_s:g}s: {_redact_for_logging(argv)}") from exc - code = proc.returncode if proc.returncode is not None else SANDBOX_RUNTIME_RETURN_CODE - return code, stdout_b.decode(errors="replace"), stderr_b.decode(errors="replace") - - async def create(self, spec: SandboxSpec) -> SandboxHandle: - if spec.image is None: - raise SandboxCreateError("spec.image is required for the docker provider") - - name = _CONTAINER_NAME_PREFIX + uuid.uuid4().hex - argv: list[str] = [self._docker, "run", "-d", "--name", name] - if self._network is not None: - argv += ["--network", self._network] - if spec.workdir is not None: - argv += ["-w", spec.workdir] - for key, value in spec.env.items(): - argv += ["-e", f"{key}={value}"] - argv += _resource_flags(spec.resources) - argv += self._extra_run_args - # Keep the container alive so exec/cp can target it across its lifetime; the harness - # is driven by exec, not by the container's entrypoint. - argv += [spec.image, "sh", "-c", "exec sleep infinity"] - - try: - code, _out, err = await self._run(argv, timeout_s=self._start_timeout_s) - except TimeoutError as exc: - # ``docker run -d`` may have created the container before we timed out (or SIGKILLed it); - # best-effort remove by name so a slow start never leaks an orphan. - await self._force_remove(name) - raise SandboxCreateError(f"docker run timed out for image={spec.image!r}: {exc}") from exc - if code != 0: - await self._force_remove(name) # clean up any partially-created container - raise SandboxCreateError(f"docker run failed (code={code}) for image={spec.image!r}: {err.strip()}") - - return SandboxHandle( - sandbox_id=name, - provider_name=self.name, - raw=_DockerContainer(name=name, image=spec.image, env=dict(spec.env)), - ) - - async def exec( - self, - handle: SandboxHandle, - command: str, - *, - cwd: str | None = None, - env: dict[str, str] | None = None, - timeout_s: int | float | None = None, - stdin: bytes | None = None, - ) -> SandboxExecResult: - container = _container(handle) - argv: list[str] = [self._docker, "exec"] - if stdin is not None: - argv.append("-i") - if cwd is not None: - argv += ["-w", cwd] - if env: - for key, value in env.items(): - argv += ["-e", f"{key}={value}"] - argv += [container.name, "sh", "-c", command] - - effective_timeout = timeout_s if timeout_s is not None else self._default_timeout_s - try: - code, out, err = await self._run(argv, timeout_s=effective_timeout, stdin=stdin) - except TimeoutError as exc: - return SandboxExecResult( - stdout=None, stderr=str(exc), return_code=SANDBOX_RUNTIME_RETURN_CODE, error_type="timeout" - ) - return SandboxExecResult(stdout=out, stderr=err, return_code=code, error_type=None) - - async def upload_file(self, handle: SandboxHandle, source_path: Path, target_path: str) -> None: - container = _container(handle) - parent = posixpath.dirname(target_path) - if parent: - result = await self.exec(handle, f"mkdir -p {shlex.quote(parent)}") - if not result.ok: - raise RuntimeError(f"docker upload: mkdir {parent!r} failed: {result.stderr}") - await self._cp(f"{source_path}", f"{container.name}:{target_path}") - - async def upload_dir(self, handle: SandboxHandle, source_dir: Path, target_dir: str) -> None: - container = _container(handle) - result = await self.exec(handle, f"mkdir -p {shlex.quote(target_dir)}") - if not result.ok: - raise RuntimeError(f"docker upload_dir: mkdir {target_dir!r} failed: {result.stderr}") - # A trailing "/." copies directory *contents* into target_dir (not nested under it). - await self._cp(f"{source_dir}{os.sep}.", f"{container.name}:{target_dir}") - - async def download_file(self, handle: SandboxHandle, source_path: str, target_path: Path) -> None: - container = _container(handle) - target_path.parent.mkdir(parents=True, exist_ok=True) - await self._cp(f"{container.name}:{source_path}", f"{target_path}") - - async def download_dir(self, handle: SandboxHandle, source_dir: str, target_dir: Path) -> None: - container = _container(handle) - target_dir.mkdir(parents=True, exist_ok=True) - await self._cp(f"{container.name}:{posixpath.join(source_dir, '.')}", f"{target_dir}") - - async def _cp(self, source: str, dest: str) -> None: - code, _out, err = await self._run([self._docker, "cp", source, dest], timeout_s=self._default_timeout_s) - if code != 0: - raise RuntimeError(f"docker cp {source!r} -> {dest!r} failed (code={code}): {err.strip()}") - - async def status(self, handle: SandboxHandle) -> SandboxStatus: - container = _container(handle) - try: - code, out, _err = await self._run( - [self._docker, "inspect", "-f", "{{.State.Status}}", container.name], - timeout_s=self._default_timeout_s, - ) - except TimeoutError: - return SandboxStatus.UNKNOWN - if code != 0: - return SandboxStatus.STOPPED - state = out.strip().lower() - if state == "running": - return SandboxStatus.RUNNING - if state in {"created", "restarting"}: - return SandboxStatus.STARTING - if state in {"exited", "dead", "removing", "paused"}: - return SandboxStatus.STOPPED - return SandboxStatus.UNKNOWN - - async def close(self, handle: SandboxHandle) -> None: - await self._force_remove(_container(handle).name) - - async def _force_remove(self, name: str) -> None: - """Best-effort ``docker rm -f`` — teardown must never raise, or it leaks the container and can - mask the in-block exception it runs alongside. Failures are logged, not propagated.""" - try: - code, _out, err = await self._run([self._docker, "rm", "-f", name], timeout_s=self._default_timeout_s) - except Exception as exc: # noqa: BLE001 - teardown is best-effort - logger.warning("docker rm -f %s errored during teardown: %s", name, exc) - return - if code != 0: - logger.warning("docker rm -f %s failed during teardown (code=%d): %s", name, code, err.strip()) - - async def aclose(self) -> None: - return None - - -def _container(handle: SandboxHandle) -> _DockerContainer: - raw = handle.raw - if not isinstance(raw, _DockerContainer): - raise TypeError(f"handle.raw is not a docker container handle: {type(raw).__name__}") - return raw diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/scores.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/scores.py deleted file mode 100644 index d4014a35cc..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/scores.py +++ /dev/null @@ -1,97 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Per-trial metric scoring records and scoring diagnostics.""" - -from __future__ import annotations - -from enum import Enum -from typing import Any - -from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrialStatus -from nemo_platform.beta.evaluator.metrics.protocol import MetricOutput -from pydantic import BaseModel, ConfigDict, Field - - -class AgentEvalScoreStatus(str, Enum): - """Status of metric scoring for one trial.""" - - COMPLETED = "completed" - FAILED = "failed" - PARTIAL = "partial" - - -#: Diagnostic detail key stamped when a score is ``FAILED`` because the *trial* failed — the agent -#: produced nothing to score — rather than because the metric itself raised. Both are reported as -#: ``FAILED``, but they mean different things to a reader: a failed trial did not pass, a -#: failed metric is a failed *measurement*. Consumers that must tell them apart read this key via -#: :func:`is_trial_failure`. -TRIAL_STATUS_DETAIL = "trial_status" - - -class AgentEvalDiagnosticSeverity(str, Enum): - """Severity level for a scoring diagnostic.""" - - ERROR = "error" - WARNING = "warning" - INFO = "info" - - -class AgentEvalDiagnostic(BaseModel): - """Diagnostic emitted while scoring one trial with one metric.""" - - model_config = ConfigDict(extra="forbid") - - severity: AgentEvalDiagnosticSeverity = Field(description="Severity of the diagnostic.") - message: str = Field(description="Human-readable diagnostic message.") - source: str | None = Field( - default=None, - description="Component or stage that produced the diagnostic, if known.", - ) - details: dict[str, Any] = Field( - default_factory=dict, - description="Structured supporting details for the diagnostic.", - ) - - -class AgentEvalTaskScore(BaseModel): - """Per-task, per-trial, per-metric scoring record: metric outputs, diagnostics, status, and metadata.""" - - model_config = ConfigDict(extra="forbid") - - id: str = Field(description="Stable identifier for this score record.") - run_id: str = Field(description="Identifier of the run this score belongs to.") - task_id: str = Field(description="Identifier of the task that was scored.") - trial_id: str = Field(description="Identifier of the trial that was scored.") - metric_type: str = Field(description="Task-local metric type (metric.type) that produced this score.") - status: AgentEvalScoreStatus = Field(description="Status of this metric score.") - outputs: list[MetricOutput] = Field( - default_factory=list, - description="Named metric outputs emitted for this trial/metric pair.", - ) - diagnostics: list[AgentEvalDiagnostic] = Field( - default_factory=list, - description="Diagnostics emitted while scoring this trial with this metric.", - ) - metadata: dict[str, Any] = Field( - default_factory=dict, - description="Free-form metadata associated with the score.", - ) - - -def is_trial_failure(score: AgentEvalTaskScore) -> bool: - """True when a ``FAILED`` score records a failed *trial* rather than a metric that raised. - - The evaluator short-circuits a failed trial into a failed score without running the metric, and - stamps :data:`TRIAL_STATUS_DETAIL` on the diagnostic; a metric that raised is stamped with its - exception type instead. The distinction is what lets pass@k charge a dead rollout to the agent - while leaving an unusable measurement out of the denominator entirely. - - Matches on the detail's *value*, not merely the key's presence: ``trial_status`` is a natural - thing for a hand-built score to carry for its own reasons, and only ``failed`` means the attempt - is one the agent is answerable for. - """ - return score.status is AgentEvalScoreStatus.FAILED and any( - diagnostic.details.get(TRIAL_STATUS_DETAIL) == AgentEvalTrialStatus.FAILED.value - for diagnostic in score.diagnostics - ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/tasks.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/tasks.py deleted file mode 100644 index b80dabb31b..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/tasks.py +++ /dev/null @@ -1,239 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Task definitions, semantic views, and run configuration for agent evaluation.""" - -from __future__ import annotations - -from enum import Enum -from pathlib import Path -from typing import Any, Protocol, runtime_checkable - -from nemo_platform.beta.evaluator.metrics.protocol import Metric -from nemo_platform.beta.evaluator.metrics.utils import metric_type_name -from nemo_platform.beta.evaluator.values import RunConfig, RunConfigOnline, RunConfigOnlineModel -from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator, model_validator - - -class SemanticReducer(str, Enum): - """Reduction strategy used to combine task-scoped view signals into one score.""" - - SINGLE = "single" - ALL = "all" - ANY = "any" - MEAN = "mean" - WEIGHTED_MEAN = "weighted_mean" - - -class ViewSignal(BaseModel): - """Task-scoped metric output that contributes to a semantic view.""" - - model_config = ConfigDict(extra="forbid") - - metric: str = Field(description="Task-local metric type (metric.type) whose output feeds this signal.") - output: str = Field(description="Name of the metric output, as declared by the metric's output_spec().") - weight: float | None = Field( - default=None, - description="Relative weight for this signal when the view reducer is 'weighted_mean'; unused otherwise.", - ) - - @field_validator("metric", "output") - @classmethod - def _non_empty(cls, value: str) -> str: - if not value: - raise ValueError("view signal metric and output must not be empty") - return value - - -class SemanticView(BaseModel): - """Task-scoped reporting view that maps a task's own metric outputs into a named score.""" - - model_config = ConfigDict(extra="forbid") - - reducer: SemanticReducer = Field( - description="Strategy used to reduce this view's signals into a single task-level score.", - ) - signals: list[ViewSignal] = Field( - min_length=1, - description="Ordered metric outputs contributing to this view; at least one is required.", - ) - - -class AgentEvalTask(BaseModel): - """Standalone agent-eval task: the unit of work being evaluated.""" - - # TODO: Tasks may need to define a set of required_capabilities or something that allow the - # runtime to skip trying to complete a task that isn't possible. - model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") - - id: str = Field(description="Stable task identifier, unique within the supplied task collection.") - intent: str = Field(description="Human-readable description of the desired agent behavior.") - inputs: dict[str, Any] = Field( - description="What the agent receives or starts from, e.g. instruction, filesystem seed, or state refs.", - ) - reference: dict[str, Any] = Field( - default_factory=dict, - description="Grader-only ground truth (held-out tests, expected outputs, rubric data). Surfaced to " - "metrics as row.data['reference'] but never seeded into the agent's workspace or shown to the " - "agent, so a metric can grade against artifacts the agent cannot influence.", - ) - metrics: list[Metric] = Field( - default_factory=list, - description="Ordered concrete SDK metric instances that score this task; metric types must be unique.", - ) - views: dict[str, SemanticView] = Field( - default_factory=dict, - description="Optional reporting views mapping this task's metric outputs into named semantic scores.", - ) - metadata: dict[str, Any] = Field( - default_factory=dict, - description="Free-form metadata associated with the task.", - ) - - @field_validator("id") - @classmethod - def _id_must_not_be_empty(cls, value: str) -> str: - if not value: - raise ValueError("task id must not be empty") - return value - - def agent_prompt(self) -> str: - """The intent-free prompt handed to the agent under evaluation. - - Exactly the task's natural-language instruction (``inputs["instruction"]``), with no - runtime-added framing. ``intent`` is deliberately never used: it is the eval-side description - of the desired behavior (what the grader checks for), so exposing it to the agent is a - reward-hacking hole. - - Raises ``ValueError`` when ``inputs["instruction"]`` is missing or empty; a task with no - instruction cannot be evaluated, so the runner fails that task rather than running an agent on - an empty prompt. - """ - instruction = self.inputs.get("instruction") - if instruction: - return str(instruction) - raise ValueError(f"task {self.id!r} has no instruction: set inputs['instruction']") - - @field_serializer("metrics", when_used="json") - def _serialize_metrics(self, metrics: list[Metric]) -> list[dict[str, Any]]: - """Serialize local metric instances as descriptors for run bundles.""" - serialized: list[dict[str, Any]] = [] - for metric in metrics: - outputs = [ - { - "name": output.name, - "description": output.description, - "value_schema": output.value_schema.__name__, - } - for output in metric.output_spec() - ] - serialized.append({"type": metric_type_name(metric), "outputs": outputs}) - return serialized - - @model_validator(mode="after") - def _validate_metric_references(self) -> AgentEvalTask: - metric_types = [metric_type_name(metric) for metric in self.metrics] - duplicate_metric_types = sorted( - {metric_type for metric_type in metric_types if metric_types.count(metric_type) > 1} - ) - if duplicate_metric_types: - raise ValueError(f"duplicate task metric types: {duplicate_metric_types}") - - outputs_by_metric = { - metric_type_name(metric): {output.name for output in metric.output_spec()} for metric in self.metrics - } - for view_name, view in self.views.items(): - for signal in view.signals: - if signal.metric not in outputs_by_metric: - raise ValueError(f"view {view_name!r} references unknown metric {signal.metric!r}") - if signal.output not in outputs_by_metric[signal.metric]: - raise ValueError( - f"view {view_name!r} references unknown output {signal.output!r} for metric {signal.metric!r}" - ) - return self - - -class AgentEvalTaskset(BaseModel): - """A named set of SDK-native tasks (with optional metadata) to evaluate. - - Produced by an :class:`AgentEvalTasksetLoader`; the evaluator scores - ``tasks`` directly and never consumes a loader. - """ - - model_config = ConfigDict(extra="forbid") - - tasks: list[AgentEvalTask] = Field( - default_factory=list, - min_length=1, - description="Tasks in this set; at least one is required and task ids must be unique.", - ) - metadata: dict[str, Any] = Field(default_factory=dict, description="Free-form taskset metadata for the run.") - - @model_validator(mode="after") - def _task_ids_unique(self) -> AgentEvalTaskset: - ids = [task.id for task in self.tasks] - duplicates = sorted({task_id for task_id in ids if ids.count(task_id) > 1}) - if duplicates: - raise ValueError(f"duplicate taskset task ids: {duplicates}") - return self - - -@runtime_checkable -class AgentEvalTasksetLoader(Protocol): - """Protocol for adapting an external taskset into agent-eval. - - A loader is a named adapter that loads SDK-native tasks (optionally from an - external ``source``). Per the design, loaders are resolved upstream into an - :class:`AgentEvalTaskset`; the evaluator scores those tasks and never consumes - a loader directly. - """ - - @property - def name(self) -> str: - """Stable taskset name used in diagnostics, metadata, and user-facing output.""" - ... - - def load( - self, - *, - source: str | Path | None = None, - limit: int | None = None, - evidence_dir: Path | None = None, - ) -> AgentEvalTaskset: - """Load tasks into an :class:`AgentEvalTaskset`. - - ``source`` is an optional path/URI to load from, ``limit`` an optional - positive cap on the number of tasks, and ``evidence_dir`` an optional - directory holding task evidence inputs. - """ - ... - - -class AgentEvalRunConfig(BaseModel): - """Configuration for a standalone agent-eval run.""" - - model_config = ConfigDict(extra="forbid") - - work_dir: Path | None = Field( - default=None, - description="Directory the run works in: runtimes write trial evidence beneath it, and it is " - "the default target for AgentEvalResult.persist so the bundle contains that evidence. Purely " - "in-memory when omitted.", - ) - run_id: str | None = Field(default=None, description="Explicit run identifier; generated when omitted.") - prompt_template: str | dict[str, Any] | None = Field( - default=None, - description="Optional prompt template applied when generating trials online.", - ) - params: RunConfig | RunConfigOnline | RunConfigOnlineModel | None = Field( - default=None, - description="Inference/run parameters used when producing trials online.", - ) - parallelism: int = Field(default=4, ge=1, description="Maximum number of tasks scored concurrently.") - labels: dict[str, str] = Field( - default_factory=dict, - description="Caller-supplied tags recorded on the run's metadata (e.g. benchmark, mode, backend, " - "scenario). Free-form by design and never derived: nothing is inferred from task metadata, so a " - "label is present only if the caller set it.", - ) - fail_fast: bool = Field(default=False, description="Stop the run on the first scoring failure when True.") diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/trials.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/trials.py deleted file mode 100644 index 68f0f50c40..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/trials.py +++ /dev/null @@ -1,248 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Trial artifacts, the runtime/serde interfaces that produce them, and the -runtime-agnostic helpers for shaping trials from artifacts (status mapping + -the standard evidence-key builder).""" - -from __future__ import annotations - -from collections.abc import Sequence -from enum import Enum -from pathlib import Path -from typing import Any, Protocol, runtime_checkable - -from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask -from nemo_platform.beta.evaluator.values import Agent, Model -from nemo_platform.beta.evaluator.values.evidence import ( - EVIDENCE_FINAL_STATE, - EVIDENCE_FORMAT_ATIF, - EVIDENCE_FORMAT_JSON, - EVIDENCE_INITIAL_STATE, - EVIDENCE_LOGS, - EVIDENCE_TRACE, - EVIDENCE_VERIFIER_LOGS, - CandidateEvidence, - EvidenceDescriptor, -) -from nemo_platform.beta.evaluator.values.results import AggregateScore -from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator, model_validator - - -class AgentEvalTrialStatus(str, Enum): - """Lifecycle status for a trial: completed, failed, or partial.""" - - COMPLETED = "completed" - FAILED = "failed" - PARTIAL = "partial" - - -class AgentOutput(BaseModel): - """Captured final output from the evaluated agent, model, or imported baseline for a trial.""" - - model_config = ConfigDict(extra="forbid") - - output_text: str | None = Field( - default=None, - description="User-visible final text produced by the agent, if any.", - ) - response: JsonValue | None = Field( - default=None, - description="Final response payload produced by the agent, if any. Any JSON value — a " - "structured object, or a raw JSON string/array for agents that don't return an object.", - ) - metadata: dict[str, Any] = Field( - default_factory=dict, - description="Free-form metadata associated with the agent output.", - ) - - -class AgentEvalTrial(BaseModel): - """Durable trial artifact for one task: output, evidence, status, and metadata.""" - - model_config = ConfigDict(extra="forbid") - - id: str = Field(description="Stable identifier for this trial.") - task_id: str = Field(description="Identifier of the AgentEvalTask this trial was produced for.") - status: AgentEvalTrialStatus = Field(description="Lifecycle status of the trial.") - output: AgentOutput | None = Field( - default=None, - description="Final agent output captured for the trial; required when status is completed.", - ) - evidence: CandidateEvidence | None = Field( - default=None, - description="Named evidence descriptors (final state, traces, logs, ...) captured for the trial.", - ) - metadata: dict[str, Any] = Field( - default_factory=dict, - description="Free-form metadata associated with the trial.", - ) - - @field_validator("id", "task_id") - @classmethod - def _non_empty(cls, value: str) -> str: - if not value: - raise ValueError("trial id and task_id must not be empty") - return value - - @model_validator(mode="after") - def _completed_trial_requires_output(self) -> AgentEvalTrial: - if self.status == AgentEvalTrialStatus.COMPLETED and self.output is None: - raise ValueError("completed trial requires output") - return self - - -@runtime_checkable -class AgentTaskRunner(Protocol): - """Online execution interface that runs tasks and produces trials.""" - - async def run_tasks( - self, - tasks: Sequence[AgentEvalTask], - config: AgentEvalRunConfig | None = None, - ) -> Sequence[AgentEvalTrial]: ... - - def runner_info(self) -> RunnerInfo: - """Identify this runner and the settings that shape its results, for run provenance. - - Required: every run has a producer, and the result records it on - ``AgentEvalResult.metadata.target`` so a run can be understood after the fact. Return a stable - short ``name`` (``"gym"``, ``"harbor"``) rather than a class name. ``config`` must not contain - secrets — it is persisted with the run bundle. - """ - ... - - -class RunnerInfo(BaseModel): - """Identity of whatever produced a run's trials, recorded for provenance.""" - - model_config = ConfigDict(extra="forbid") - - name: str = Field(description="Identifier of the runner/target, e.g. 'gym', 'harbor', or a model name.") - kind: str = Field( - default="runner", - description="What produced the trials: 'runner', 'model', 'agent', or 'imported' for stored trials.", - ) - version: str | None = Field(default=None, description="Version of the backing tool, when known.") - config: dict[str, Any] = Field( - default_factory=dict, - description="Runner-specific settings that affect results, recorded so a run can be understood " - "after the fact. Must not contain secrets.", - ) - - -def callable_identity(target: object) -> str: - """Module-qualified identity of a callable, for :attr:`RunnerInfo.config`. - - A bare ``__qualname__`` is ambiguous across modules — two runs using different callables that - share a name would record identical provenance — so qualify it with the defining module. - """ - module = getattr(target, "__module__", None) - name = getattr(target, "__qualname__", None) or type(target).__name__ - return f"{module}.{name}" if module else name - - -@runtime_checkable -class RunAggregationsProvider(Protocol): - """Optional companion to :class:`AgentTaskRunner`: a runner that computed its own run-level - aggregations (a backend's pass@k, reward profile, environment-specific metrics) exposes them here, - mapped onto the SDK's typed aggregate scores. The evaluator calls this after ``run_tasks``; - implementers stash their numbers during the run and convert them here. - - Returned scores are merged into ``summary.scores``, so a backend's own figures sit alongside the - SDK's and are addressable by name the same way. Implementers must namespace names under - ``runner..`` so an imported figure is never mistaken for one the SDK computed. Runners - with no run-level aggregations simply don't implement this protocol. - """ - - def run_aggregate_scores(self) -> Sequence[AggregateScore]: ... - - -@runtime_checkable -class AgentTrialSerde(Protocol): - """Read/write a single stored trial artifact as an :class:`AgentEvalTrial`. - - The offline counterpart to :class:`AgentTaskRunner`: instead of *executing* an - agent it adapts a stored artifact (a run dir/file) to and from a trial, so prior - runs can be re-scored. The SDK ships only the protocol; concrete codecs (which - know a particular on-disk layout) live with their producers. - """ - - def read(self) -> AgentEvalTrial: ... - - def write(self, trial: AgentEvalTrial) -> None: ... - - -AgentEvalTarget = Model | Agent | AgentTaskRunner - - -def resolve_trial_status(agent_ok: bool) -> AgentEvalTrialStatus: - """Map an agent-phase outcome to a *scorable* trial status. - - ``AgentEvaluator`` excludes ``FAILED`` trials from scoring, so an - executed-but-unsuccessful agent uses ``PARTIAL`` (still scored as ``0`` for - pass-rate gating); ``FAILED`` is reserved for trial-*production* failures, - which a runtime surfaces by raising rather than emitting an unscorable trial. - """ - return AgentEvalTrialStatus.COMPLETED if agent_ok else AgentEvalTrialStatus.PARTIAL - - -def standard_evidence_descriptors( - *, - logs_dir: str | Path, - final_state_dir: str | Path, - trace_path: str | Path | None = None, - initial_state_ref: str | None = None, - verifier_logs_dir: str | Path | None = None, - primary_log: str | None = None, -) -> dict[str, EvidenceDescriptor]: - """Build the documented evidence map for an agent-eval trial. - - Standard keys: ``initial_state`` (task input filesystem, when staged), - ``trace`` (trajectory, ATIF-normalized when available), ``logs`` (agent log - dir), ``final_state`` (workspace), and ``verifier_logs`` (only when present). - Callers may add their own extension keys to the returned mapping. - """ - descriptors: dict[str, EvidenceDescriptor] = {} - - if initial_state_ref: - descriptors[EVIDENCE_INITIAL_STATE] = EvidenceDescriptor( - kind="filesystem", - format="dir", - ref=str(initial_state_ref), - metadata={"role": EVIDENCE_INITIAL_STATE}, - ) - - if trace_path is not None: - trace_name = Path(trace_path).name.lower() - is_atif = trace_name.startswith("atif") or ".atif." in trace_name - descriptors[EVIDENCE_TRACE] = EvidenceDescriptor( - kind=EVIDENCE_TRACE, - format=EVIDENCE_FORMAT_ATIF if is_atif else EVIDENCE_FORMAT_JSON, - ref=str(trace_path), - ) - - logs_metadata = {"primary_log": primary_log} if primary_log else {} - descriptors[EVIDENCE_LOGS] = EvidenceDescriptor( - kind="logs", - format="dir", - ref=str(logs_dir), - metadata=logs_metadata, - ) - - descriptors[EVIDENCE_FINAL_STATE] = EvidenceDescriptor( - kind="filesystem", - format="dir", - ref=str(final_state_dir), - metadata={"role": EVIDENCE_FINAL_STATE}, - ) - - if verifier_logs_dir is not None and Path(verifier_logs_dir).exists(): - descriptors[EVIDENCE_VERIFIER_LOGS] = EvidenceDescriptor( - kind="logs", - format="dir", - ref=str(verifier_logs_dir), - metadata={"role": "verifier"}, - ) - - return descriptors diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/workspace_seeds.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/workspace_seeds.py deleted file mode 100644 index 93cc4d1e84..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/workspace_seeds.py +++ /dev/null @@ -1,171 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Workspace seed files for agent-eval tasks. - -A task can stage starter files into the agent's workspace before it runs, under -``inputs[SEED_FILES_INPUT_KEY]`` as a ``{relative_path: source}`` map. Each *source* is a -JSON-serializable value whose ``kind`` selects a registered :class:`SeedHandler` (a bare string is -sugar for inline text). - -The SDK ships handlers only for the kinds it can resolve with **no external dependency** — ``inline`` -and ``path``. Other kinds are contributed by consumers via :func:`register_seed_handler`; the SDK has -no knowledge of them (e.g. a platform ``fileset`` handler lives in the plugin and resolves against the -Files service at run time). An unregistered kind raises :class:`WorkspaceSeedError`. -""" - -from __future__ import annotations - -import base64 -from collections.abc import Mapping -from pathlib import Path -from typing import Any, Literal, Protocol, runtime_checkable - -from pydantic import BaseModel, ConfigDict, Field - -#: ``inputs`` key holding the ``{relative_path: seed}`` map of files to stage into the workspace. -SEED_FILES_INPUT_KEY = "files" - - -class WorkspaceSeedError(ValueError): - """A workspace seed could not be parsed, resolved, or written (bad value, unknown kind, ...). - - Subclasses ``ValueError`` so a runner's per-task error handling still catches it. - """ - - -class InlineSeed(BaseModel): - """File contents carried in the task itself. Portable to any runner.""" - - model_config = ConfigDict(extra="forbid") - - kind: Literal["inline"] = "inline" - content: str = Field(description="File contents; UTF-8 text, or base64-encoded bytes when encoding='base64'.") - encoding: Literal["text", "base64"] = "text" - - -class PathSeed(BaseModel): - """A file on the authoring host. Resolvable only where that path exists (local runs).""" - - model_config = ConfigDict(extra="forbid") - - kind: Literal["path"] = "path" - path: str = Field(description="Filesystem path on the authoring host (absolute or relative to the cwd).") - - -@runtime_checkable -class SeedHandler(Protocol): - """Parses + resolves one seed ``kind`` into the bytes to stage. - - Consumers implement this for kinds the SDK doesn't ship (e.g. a platform ``fileset`` handler) and - wire them in with :func:`register_seed_handler`. ``resolve`` runs at seeding time, so a handler - that needs external services (a client, credentials) acquires them there. - """ - - kind: str - - def parse(self, value: Mapping[str, Any]) -> BaseModel: - """Validate a raw seed mapping into this kind's typed model.""" - ... - - def resolve(self, seed: BaseModel) -> bytes: - """Resolve a parsed seed to the bytes to write into the workspace.""" - ... - - -_HANDLERS: dict[str, SeedHandler] = {} - - -def register_seed_handler(handler: SeedHandler) -> None: - """Register a :class:`SeedHandler` under its ``kind`` (replacing any handler already registered).""" - _HANDLERS[handler.kind] = handler - - -def _handler_for(kind: str) -> SeedHandler: - handler = _HANDLERS.get(kind) - if handler is None: - raise WorkspaceSeedError(f"no handler registered for seed kind {kind!r}") - return handler - - -class _InlineSeedHandler: - kind = "inline" - - def parse(self, value: Mapping[str, Any]) -> BaseModel: - return InlineSeed.model_validate(value) - - def resolve(self, seed: BaseModel) -> bytes: - assert isinstance(seed, InlineSeed) - if seed.encoding == "base64": - try: - return base64.b64decode(seed.content, validate=True) - except (ValueError, TypeError) as exc: - raise WorkspaceSeedError(f"inline seed is not valid base64: {exc}") from exc - return seed.content.encode("utf-8") - - -class _PathSeedHandler: - kind = "path" - - def parse(self, value: Mapping[str, Any]) -> BaseModel: - return PathSeed.model_validate(value) - - def resolve(self, seed: BaseModel) -> bytes: - assert isinstance(seed, PathSeed) - source = Path(seed.path).expanduser() - try: - return source.read_bytes() - except OSError as exc: - raise WorkspaceSeedError(f"path seed {seed.path!r} could not be read: {exc}") from exc - - -register_seed_handler(_InlineSeedHandler()) -register_seed_handler(_PathSeedHandler()) - - -def parse_seed(value: str | Mapping[str, Any]) -> BaseModel: - """Coerce a seed map value into its validated model. A bare string is inline UTF-8 text.""" - if isinstance(value, str): - return InlineSeed(content=value) - if not isinstance(value, Mapping): - raise WorkspaceSeedError(f"seed must be a string or mapping, got {type(value).__name__}") - kind = value.get("kind") - if not isinstance(kind, str): - raise WorkspaceSeedError("seed mapping is missing a string 'kind'") - handler = _handler_for(kind) - try: - return handler.parse(value) - except WorkspaceSeedError: - raise - except Exception as exc: # noqa: BLE001 - normalize a handler's validation error into our type - raise WorkspaceSeedError(f"invalid {kind!r} seed: {exc}") from exc - - -def _resolve_seed_bytes(seed: BaseModel) -> bytes: - """Resolve a parsed seed to bytes via its registered handler.""" - kind = getattr(seed, "kind", None) - if not isinstance(kind, str): - raise WorkspaceSeedError("parsed seed has no string 'kind'") - return _handler_for(kind).resolve(seed) - - -def seed_workspace(workspace_dir: str | Path, files: Mapping[str, Any] | None) -> list[str]: - """Write the ``files`` seed map into ``workspace_dir``; return the seeded relative paths. - - Each value is parsed into a seed model and resolved to bytes by its registered handler. Paths that - escape the workspace (absolute, or ``..`` traversal) are rejected so a task can only stage files - inside its own sandbox. ``None``/empty seeds nothing. - """ - if not isinstance(files, Mapping): - return [] - root = Path(workspace_dir).resolve() - written: list[str] = [] - for rel_path, value in files.items(): - target = (root / str(rel_path)).resolve() - if target != root and root not in target.parents: - raise WorkspaceSeedError(f"seed file path escapes the workspace: {rel_path!r}") - data = _resolve_seed_bytes(parse_seed(value)) - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(data) - written.append(str(rel_path)) - return written diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_inference.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_inference.py deleted file mode 100644 index 181b29bbdf..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_inference.py +++ /dev/null @@ -1,800 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Shared HTTP inference for generic and NeMo Agent Toolkit targets. - -Public agent variants are normalized into one transport description and then -executed as either a blocking JSON request or a JSON SSE stream. The typed -``invoke_agent`` path preserves status and evidence, while -``make_agent_inference_request`` retains the legacy OpenAI-like dictionary -contract and failure behavior. -""" - -# ruff: noqa: I001 - the vendored SDK mirror uses different import-order settings. - -from __future__ import annotations - -import hashlib -import json -import re -import shutil -from collections.abc import Awaitable, Callable, Mapping -from enum import Enum -from functools import partial -from pathlib import Path -from typing import Any, Protocol, TypeAlias, runtime_checkable -from urllib.parse import urlparse - -import httpx -from httpx import Timeout -from pydantic import BaseModel, ConfigDict, Field - -from nemo_platform.beta.evaluator.agent_stream_translation import ( - SseFrame, - AgentStreamTranslation, - AgentStreamTranslationContext, - AgentStreamTranslator, -) -from nemo_platform.beta.evaluator.inference import get_logger, requests_log_var -from nemo_platform.beta.evaluator.resilience.api import run_with_resilience -from nemo_platform.beta.evaluator.resilience.classifier import endpoint_identity -from nemo_platform.beta.evaluator.templates import render_template -from nemo_platform.beta.evaluator.values.agents import ( - Agent, - GenericAgent, - NatAgentConfig, - NemoAgentToolkitAgent, - StreamAggregation, -) -from nemo_platform.beta.evaluator.values.evidence import ( - EVIDENCE_FORMAT_ATIF, - EVIDENCE_FORMAT_JSON, - EVIDENCE_FORMAT_TEXT, - EVIDENCE_HTTP_METADATA, - EVIDENCE_RAW_STREAM, - EVIDENCE_REQUEST_HEADERS, - EVIDENCE_REQUEST_PAYLOAD, - EVIDENCE_STREAM_EVENTS, - EVIDENCE_TRACE, - EVIDENCE_TRANSLATION_ERROR, - CandidateEvidence, - EvidenceDescriptor, -) - -# Default timeout for agent requests (seconds). -_DEFAULT_TIMEOUT = 120.0 - - -class AgentInvocationStatus(str, Enum): - """Agent invocation outcome before it is adapted into an agent-eval trial.""" - - COMPLETED = "completed" - PARTIAL = "partial" - FAILED = "failed" - - -class AgentInvocationResult(BaseModel): - """Typed agent response with optional evidence and partial-run status.""" - - model_config = ConfigDict(extra="forbid") - - status: AgentInvocationStatus - response: dict[str, Any] - output_text: str | None = None - evidence: CandidateEvidence | None = None - metadata: dict[str, Any] = Field(default_factory=dict) - - -class AgentInferenceContext(BaseModel): - """Per-invocation persistence and identity supplied by an evaluator.""" - - model_config = ConfigDict(extra="forbid") - - evidence_dir: Path | None = None - metadata: dict[str, Any] = Field(default_factory=dict) - - -class _HttpAgentInvocation(BaseModel): - """Resolved transport request shared by every HTTP agent variant.""" - - model_config = ConfigDict(extra="forbid", frozen=True) - - endpoint: str - payload: dict[str, Any] - query_params: dict[str, str] = Field(default_factory=dict) - response_path: str - trajectory_path: str | None = None - stream: bool = False - response_path_field: str = "response_path" - response_aggregation: StreamAggregation = "last" - - -# SSE field names look like ``data``, ``intermediate_data``, ``observability_trace``; -# require the pre-colon token to match before treating a line as a frame, so a bare -# JSON line (e.g. ``{"value": 1}``) is not mis-split at an interior colon. -_SSE_CHANNEL_PATTERN = re.compile(r"^[A-Za-z_][\w-]*$") - - -class _StreamCapture(BaseModel): - model_config = ConfigDict(extra="forbid") - - event_count: int = 0 - raw_lines: list[str] = Field(default_factory=list) - frames: list[SseFrame] = Field(default_factory=list) - final_payload: Any | None = None - # Raw extracted response value (any JSON type); preserved so the OpenAI-like - # response keeps the original type instead of an unconditional ``str()`` cast. - final_value: Any | None = None - final_trajectory: Any | None = None - output_text: str | None = None - status_code: int | None = None - response_headers: dict[str, str] = Field(default_factory=dict) - error: str | None = None - - -@runtime_checkable -class AgentInferenceFn(Protocol): - """Callable protocol for agent inference function dependency injection.""" - - def __call__( - self, - agent: Agent, - request: dict, - *, - client: httpx.AsyncClient | None = None, - max_retries: int | None, - api_key: str | None = None, - default_headers: dict[str, str] | None = None, - timeout: float | None = None, - ) -> Awaitable[dict | AgentInvocationResult]: ... - - -AgentInferenceFnFactory: TypeAlias = Callable[[AgentInferenceContext], AgentInferenceFn] - - -def make_agent_inference_fn( - context: AgentInferenceContext, - *, - stream_translator: AgentStreamTranslator | None = None, - capture_evidence: bool = False, -) -> AgentInferenceFn: - """Bind evaluator-owned context and stream policy to ``invoke_agent``.""" - return partial( - invoke_agent, - evidence_dir=context.evidence_dir, - invocation_context=dict(context.metadata), - stream_translator=stream_translator, - capture_evidence=capture_evidence, - ) - - -# --------------------------------------------------------------------------- -# Public entrypoint -# --------------------------------------------------------------------------- - - -def new_agent_inference_client(timeout: float | None = None) -> httpx.AsyncClient: - return httpx.AsyncClient(timeout=Timeout(timeout or _DEFAULT_TIMEOUT)) - - -async def make_agent_inference_request( - agent: Agent, - request: dict, - *, - client: httpx.AsyncClient | None = None, - max_retries: int | None = 3, - api_key: str | None = None, - default_headers: dict[str, str] | None = None, - timeout: float | None = None, -) -> dict: - """Run inference and return the legacy OpenAI-like response dictionary.""" - result = await invoke_agent( - agent, - request, - client=client, - max_retries=max_retries, - api_key=api_key, - default_headers=default_headers, - timeout=timeout, - ) - if result.status is not AgentInvocationStatus.COMPLETED: - endpoint = result.metadata.get("endpoint", agent.url) - raise RuntimeError( - f"Agent at {endpoint} completed the SSE stream without producing a final value. " - "Verify that the agent endpoint is functioning correctly." - ) - return result.response - - -async def invoke_agent( - agent: Agent, - request: dict, - *, - client: httpx.AsyncClient | None = None, - max_retries: int | None = 3, - api_key: str | None = None, - default_headers: dict[str, str] | None = None, - timeout: float | None = None, - evidence_dir: str | Path | None = None, - stream_translator: AgentStreamTranslator | None = None, - invocation_context: Mapping[str, Any] | None = None, - capture_evidence: bool = False, -) -> AgentInvocationResult: - """Invoke an agent and preserve structured status and evidence.""" - invocation = _resolve_http_agent_invocation(agent, request) - return await _invoke_http_agent( - agent, - invocation, - client=client, - max_retries=max_retries, - api_key=api_key, - default_headers=default_headers, - timeout=timeout, - evidence_dir=evidence_dir, - stream_translator=stream_translator, - invocation_context=invocation_context, - capture_evidence=capture_evidence, - ) - - -# --------------------------------------------------------------------------- -# Compatibility wrappers -# --------------------------------------------------------------------------- - - -async def _make_generic_agent_request( - agent: GenericAgent, - request: dict, - *, - client: httpx.AsyncClient | None = None, - max_retries: int | None = 3, - api_key: str | None = None, - default_headers: dict[str, str] | None = None, - timeout: float | None = None, -) -> dict: - return await make_agent_inference_request( - agent, - request, - client=client, - max_retries=max_retries, - api_key=api_key, - default_headers=default_headers, - timeout=timeout, - ) - - -async def _make_nat_agent_request( - agent: NemoAgentToolkitAgent, - request: dict, - *, - client: httpx.AsyncClient | None = None, - max_retries: int | None = 3, - api_key: str | None = None, - default_headers: dict[str, str] | None = None, - timeout: float | None = None, -) -> dict: - return await make_agent_inference_request( - agent, - request, - client=client, - max_retries=max_retries, - api_key=api_key, - default_headers=default_headers, - timeout=timeout, - ) - - -def _resolve_http_agent_invocation(agent: Agent, request: dict[str, Any]) -> _HttpAgentInvocation: - """Normalize a public agent target into one HTTP transport request.""" - if isinstance(agent, GenericAgent): - context: dict[str, Any] = {**request, "request": request} - rendered_body = render_template(agent.body, context=context) - payload = rendered_body if isinstance(rendered_body, dict) else {"args": rendered_body} - return _HttpAgentInvocation( - endpoint=agent.url, - payload=payload, - response_path=agent.response_path, - trajectory_path=agent.trajectory_path, - stream=agent.stream, - response_aggregation=agent.response_aggregation, - ) - - config = agent.nat or NatAgentConfig() - endpoint = _nat_endpoint(agent, config) - payload = request if config.request_mode == "passthrough" else {"input_message": _derive_input_message(request)} - return _HttpAgentInvocation( - endpoint=endpoint, - payload=payload, - query_params=config.query_params, - response_path=config.response_path, - stream=True, - response_path_field="nat.response_path", - response_aggregation=config.response_aggregation, - ) - - -async def _invoke_http_agent( - agent: Agent, - invocation: _HttpAgentInvocation, - *, - client: httpx.AsyncClient | None = None, - max_retries: int | None = 3, - api_key: str | None = None, - default_headers: dict[str, str] | None = None, - timeout: float | None = None, - evidence_dir: str | Path | None = None, - stream_translator: AgentStreamTranslator | None = None, - invocation_context: Mapping[str, Any] | None = None, - capture_evidence: bool = False, -) -> AgentInvocationResult: - log = get_logger() - resolved_api_key = api_key or agent.api_key - effective_timeout = timeout or _DEFAULT_TIMEOUT - - headers: dict[str, str] = {**(default_headers or {}), "Content-Type": "application/json"} - if resolved_api_key: - headers["Authorization"] = f"Bearer {resolved_api_key}" - - endpoint_key = endpoint_identity(invocation.endpoint, model_id=agent.name, auth_identity=resolved_api_key) - max_attempts = max(1, (max_retries if max_retries is not None else 0) + 1) - inference_client = client or new_agent_inference_client(timeout=effective_timeout) - retain_stream_details = capture_evidence or stream_translator is not None - - if not invocation.stream: - - async def _invoke_post() -> dict[str, Any]: - response = await inference_client.post( - invocation.endpoint, - json=invocation.payload, - headers=headers, - params=invocation.query_params, - timeout=effective_timeout, - ) - response.raise_for_status() - return response.json() - - log.info("Making agent request to %s", invocation.endpoint) - try: - result_data = await run_with_resilience(endpoint_key, _invoke_post, max_attempts=max_attempts) - except Exception: - log.exception("Agent request to %s failed after %d attempts", invocation.endpoint, max_attempts) - raise - finally: - if client is None: - await inference_client.aclose() - - response_value = _extract_jsonpath( - result_data, - invocation.response_path, - field_name=invocation.response_path_field, - ) - response = _openai_response(str(response_value)) - if invocation.trajectory_path: - trajectory = _extract_jsonpath( - result_data, - invocation.trajectory_path, - field_name="trajectory_path", - required=False, - ) - if trajectory is not None: - response["trajectory"] = trajectory - requests_log_var.get([]).append({"request": invocation.payload, "response": result_data}) - log.info("Agent request to %s completed", invocation.endpoint) - return AgentInvocationResult( - status=AgentInvocationStatus.COMPLETED, - response=response, - output_text=_openai_response_text(response), - ) - - async def _invoke_stream() -> _StreamCapture: - capture = _StreamCapture() - # In "concat" mode each data frame carries a token-level delta, so the - # final output is the ordered join of every matched value rather than - # the last one. Accumulate parts here and materialize after the stream. - aggregate = invocation.response_aggregation == "concat" - value_parts: list[str] = [] - try: - async with inference_client.stream( - "POST", - invocation.endpoint, - json=invocation.payload, - headers=headers, - params=invocation.query_params, - timeout=effective_timeout, - ) as response: - capture.status_code = response.status_code if isinstance(response.status_code, int) else None - capture.response_headers = _string_headers(response.headers) - response.raise_for_status() - async for raw_line in response.aiter_lines(): - if retain_stream_details: - capture.raw_lines.append(raw_line) - frame = _parse_sse_frame(raw_line) - if frame is None: - continue - capture.event_count += 1 - if retain_stream_details: - capture.frames.append(frame) - if frame.channel != "data" or frame.payload == "[DONE]": - continue - capture.final_payload = frame.payload - value = _extract_jsonpath( - frame.payload, - invocation.response_path, - field_name=invocation.response_path_field, - required=False, - ) - if value is not None: - if aggregate: - value_parts.append(value if isinstance(value, str) else str(value)) - else: - capture.final_value = value - # Preserve the original type in the response; expose - # ``output_text`` only when the value is already textual. - capture.output_text = value if isinstance(value, str) else None - if invocation.trajectory_path: - trajectory = _extract_jsonpath( - frame.payload, - invocation.trajectory_path, - field_name="trajectory_path", - required=False, - ) - if trajectory is not None: - capture.final_trajectory = trajectory - capture.error = capture.error or _stream_error(frame.payload) - except Exception as exc: - if capture.event_count == 0: - raise - capture.error = f"{type(exc).__name__}: {exc}" - if aggregate and value_parts: - capture.final_value = "".join(value_parts) - capture.output_text = capture.final_value - return capture - - log.info("Making streaming agent request to %s", invocation.endpoint) - try: - capture = await run_with_resilience(endpoint_key, _invoke_stream, max_attempts=max_attempts) - except Exception as exc: - log.exception("Streaming agent request to %s failed after %d attempts", invocation.endpoint, max_attempts) - # When evidence capture or a stream translator is enabled, surface an HTTP - # failure that occurred before the first stream frame as a PARTIAL result - # with http_metadata evidence instead of raising, so the trial stays - # inspectable. - # The legacy dict-returning path keeps capture disabled, so it still raises. - http_error = _http_status_error(exc) if capture_evidence or stream_translator is not None else None - if http_error is None: - raise - capture = _StreamCapture( - status_code=http_error.response.status_code, - response_headers=_string_headers(http_error.response.headers), - error=f"HTTP {http_error.response.status_code}", - ) - finally: - if client is None: - await inference_client.aclose() - - # COMPLETED only when a non-empty value was extracted and no terminal stream - # error occurred. An extracted-but-empty value (e.g. "") stays PARTIAL. - has_output = capture.final_value is not None and capture.final_value != "" - status = AgentInvocationStatus.COMPLETED if has_output and capture.error is None else AgentInvocationStatus.PARTIAL - response = _openai_response(capture.final_value) - if capture.final_trajectory is not None: - response["trajectory"] = capture.final_trajectory - evidence = ( - _stream_evidence(capture, invocation.payload, headers) - if capture_evidence or stream_translator is not None - else None - ) - translation_metadata: dict[str, Any] = {} - if stream_translator is not None and capture.frames: - values = dict(invocation_context or {}) - context = AgentStreamTranslationContext( - agent_name=agent.name, - endpoint=invocation.endpoint, - request_payload=invocation.payload, - final_payload=capture.final_payload, - output_text=capture.output_text, - run_id=_optional_string(values.get("run_id")), - task_id=_optional_string(values.get("task_id")), - invocation_id=_optional_string(values.get("invocation_id")), - conversation_id=_optional_string(invocation.payload.get("conversation_id")), - http_status=capture.status_code, - stream_error=capture.error, - ) - try: - raw_translation = stream_translator(capture.frames, context=context) - translation = AgentStreamTranslation.model_validate( - raw_translation.model_dump(mode="python") - if isinstance(raw_translation, AgentStreamTranslation) - else raw_translation - ) - schema_version = translation.trajectory.get("schema_version") - if schema_version != "ATIF-v1.7": - raise ValueError( - f"Agent stream translators must return a canonical ATIF-v1.7 trajectory, got {schema_version}" - ) - reserved = { - EVIDENCE_TRACE, - EVIDENCE_RAW_STREAM, - EVIDENCE_STREAM_EVENTS, - EVIDENCE_REQUEST_PAYLOAD, - EVIDENCE_REQUEST_HEADERS, - EVIDENCE_HTTP_METADATA, - } - collisions = reserved.intersection(translation.evidence) - if collisions: - raise ValueError(f"translator evidence uses reserved names: {sorted(collisions)}") - descriptors = dict(evidence.descriptors) if evidence is not None else {} - descriptors[EVIDENCE_TRACE] = EvidenceDescriptor( - kind=EVIDENCE_TRACE, - format=EVIDENCE_FORMAT_ATIF, - data=translation.trajectory, - ) - descriptors.update(translation.evidence) - evidence = CandidateEvidence( - descriptors=descriptors, - metadata=dict(evidence.metadata) if evidence is not None else {}, - ) - translation_metadata = translation.metadata - except Exception as exc: - status = AgentInvocationStatus.FAILED - descriptors = dict(evidence.descriptors) if evidence is not None else {} - descriptors[EVIDENCE_TRANSLATION_ERROR] = EvidenceDescriptor( - kind="error", - format=EVIDENCE_FORMAT_JSON, - data={"error_type": type(exc).__name__, "error": str(exc)}, - ) - evidence = CandidateEvidence(descriptors=descriptors) - translation_metadata = { - EVIDENCE_TRANSLATION_ERROR: str(exc), - f"{EVIDENCE_TRANSLATION_ERROR}_type": type(exc).__name__, - } - if evidence is not None and evidence_dir is not None: - evidence = _persist_stream_evidence(evidence, Path(evidence_dir)) - - # Record request/response for audit - requests_log = requests_log_var.get([]) - requests_log.append({"request": invocation.payload, "response": capture.final_payload}) - - log.info("Streaming agent request to %s completed", invocation.endpoint) - return AgentInvocationResult( - status=status, - response=response, - output_text=capture.output_text, - evidence=evidence, - metadata={ - "endpoint": invocation.endpoint, - "event_count": capture.event_count, - "final_payload": capture.final_payload, - "http_status": capture.status_code, - "stream_error": capture.error, - **translation_metadata, - }, - ) - - -def _nat_endpoint(agent: NemoAgentToolkitAgent, config: NatAgentConfig) -> str: - if urlparse(config.endpoint).scheme: - return config.endpoint - return f"{agent.url.rstrip('/')}/{config.endpoint.lstrip('/')}" - - -def _parse_sse_frame(raw_line: str) -> SseFrame | None: - line = raw_line.strip() - if not line or line.startswith("event:") or ":" not in line: - return None - channel, payload_text = line.split(":", 1) - channel = channel.strip() - # Only treat the line as a frame when the pre-colon token is a valid SSE - # field name; otherwise it is a bare payload line (e.g. raw JSON) and is skipped. - if not _SSE_CHANNEL_PATTERN.match(channel): - return None - payload_text = payload_text.strip() - try: - payload = json.loads(payload_text) - except json.JSONDecodeError: - payload = payload_text - return SseFrame(channel=channel, payload=payload, raw=raw_line) - - -def _http_status_error(exc: BaseException) -> httpx.HTTPStatusError | None: - """Return the first ``HTTPStatusError`` in the exception's ``__cause__`` chain. - - A non-retryable HTTP error re-raises directly, while a retryable one that - exhausts attempts is wrapped by the resilience scheduler with the original - error chained via ``from exc``; walk the chain to find either. - """ - current: BaseException | None = exc - while current is not None: - if isinstance(current, httpx.HTTPStatusError): - return current - current = current.__cause__ - return None - - -def _stream_error(payload: Any) -> str | None: - if not isinstance(payload, dict): - return None - error = payload.get("error") - if error is None and isinstance(payload.get("value"), dict): - error = payload["value"].get("error") - if isinstance(error, dict): - code = error.get("code") - message = error.get("message") - if code and message: - return f"{code}: {message}" - return str(message or code or error) - if error is not None: - return str(error) - return None - - -def _openai_response(content: Any) -> dict[str, Any]: - return {"choices": [{"message": {"role": "assistant", "content": content}}]} - - -def _openai_response_text(response: dict[str, Any]) -> str | None: - try: - content = response["choices"][0]["message"]["content"] - except (KeyError, IndexError, TypeError): - return None - return content if isinstance(content, str) else None - - -def _string_headers(value: Any) -> dict[str, str]: - if not isinstance(value, Mapping): - return {} - return {str(key): str(item) for key, item in value.items()} - - -def _optional_string(value: Any) -> str | None: - return str(value) if value is not None else None - - -def _redact_headers(headers: dict[str, str]) -> dict[str, str]: - sensitive = {"authorization", "cookie", "proxy-authorization", "set-cookie", "x-api-key"} - return {key: "" if key.lower() in sensitive else value for key, value in headers.items()} - - -def _stream_evidence( - capture: _StreamCapture, - payload: dict[str, Any], - headers: dict[str, str], -) -> CandidateEvidence: - raw_stream = "\n".join(capture.raw_lines) + ("\n" if capture.raw_lines else "") - values: dict[str, tuple[str, str, Any]] = { - EVIDENCE_RAW_STREAM: ("agent_stream", EVIDENCE_FORMAT_TEXT, raw_stream), - EVIDENCE_STREAM_EVENTS: ( - "agent_stream_events", - EVIDENCE_FORMAT_JSON, - [frame.model_dump(mode="json") for frame in capture.frames], - ), - EVIDENCE_REQUEST_PAYLOAD: (EVIDENCE_REQUEST_PAYLOAD, EVIDENCE_FORMAT_JSON, payload), - EVIDENCE_REQUEST_HEADERS: (EVIDENCE_REQUEST_HEADERS, EVIDENCE_FORMAT_JSON, _redact_headers(headers)), - EVIDENCE_HTTP_METADATA: ( - EVIDENCE_HTTP_METADATA, - EVIDENCE_FORMAT_JSON, - { - "status_code": capture.status_code, - "headers": _redact_headers(capture.response_headers), - "error": capture.error, - }, - ), - } - descriptors: dict[str, EvidenceDescriptor] = {} - for name, (kind, format_name, data) in values.items(): - descriptors[name] = EvidenceDescriptor(kind=kind, format=format_name, data=data) - return CandidateEvidence(descriptors=descriptors) - - -def _evidence_filename( - name: str, - descriptor: EvidenceDescriptor, - *, - reserved_filenames: set[str], - used_filenames: set[str], -) -> str: - suffix = "txt" if descriptor.format in {EVIDENCE_FORMAT_TEXT, "txt"} else "json" - canonical_trace = name == EVIDENCE_TRACE and descriptor.format == EVIDENCE_FORMAT_ATIF - if canonical_trace: - filename = "atif_trace.json" - else: - stem = "".join(char if char.isalnum() or char in "-_." else "-" for char in name) - stem = stem.strip("-_.")[:96] or "evidence" - filename = f"{stem}.{suffix}" - - filename_key = filename.casefold() - if not canonical_trace and (filename_key in reserved_filenames or filename_key in used_filenames): - digest = hashlib.sha256(name.encode("utf-8")).hexdigest()[:16] - stem = filename.rsplit(".", maxsplit=1)[0][:79] - filename = f"{stem}-{digest}.{suffix}" - filename_key = filename.casefold() - - if filename_key in used_filenames: - raise ValueError(f"evidence descriptors map to the same filename: {filename!r}") - used_filenames.add(filename_key) - return filename - - -def _persist_stream_evidence(evidence: CandidateEvidence, root: Path) -> CandidateEvidence: - """Replace one SDK-owned invocation directory with file-backed evidence.""" - if root.exists(): - shutil.rmtree(root) - root.mkdir(parents=True, exist_ok=True) - - canonical_trace = evidence.descriptors.get(EVIDENCE_TRACE) - reserved_filenames = ( - {"atif_trace.json"} - if canonical_trace is not None - and canonical_trace.data is not None - and canonical_trace.format == EVIDENCE_FORMAT_ATIF - else set() - ) - used_filenames: set[str] = set() - persisted: dict[str, EvidenceDescriptor] = {} - for name, descriptor in evidence.descriptors.items(): - if descriptor.data is None: - persisted[name] = descriptor - continue - filename = _evidence_filename( - name, - descriptor, - reserved_filenames=reserved_filenames, - used_filenames=used_filenames, - ) - path = root / filename - if descriptor.format in {EVIDENCE_FORMAT_TEXT, "txt"}: - path.write_text(str(descriptor.data), encoding="utf-8") - else: - path.write_text( - json.dumps(descriptor.data, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - persisted[name] = descriptor.model_copy(update={"ref": str(path.resolve()), "data": None}) - return evidence.model_copy(update={"descriptors": persisted}) - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def _derive_input_message(request: dict) -> str: - """Derive a single input_message string from an inference request. - - Handles both chat-style (``messages``) and completion-style (``prompt``) - requests. - """ - if "messages" in request: - messages = request["messages"] - # Use the last user message content, or concatenate all messages - for msg in reversed(messages): - if msg.get("role") == "user": - return str(msg["content"]) - # Fallback: concatenate all message contents - return "\n".join(str(msg.get("content", "")) for msg in messages) - - if "prompt" in request: - return str(request["prompt"]) - - raise ValueError("Agent inference request must contain 'messages' or 'prompt'.") - - -def _extract_jsonpath( - data: dict[str, Any], - path: str, - *, - field_name: str = "path", - required: bool = True, -) -> Any: - """Extract a value from data using a JSONPath expression.""" - # Imported here rather than at module scope: this is jsonpath_ng's only use in the module, and - # the module sits on the agent_eval run-time path via agent_eval/evaluator.py. - from jsonpath_ng import parse as jsonpath_parse - - expr = jsonpath_parse(path) - matches = expr.find(data) - if not matches: - if required: - raise ValueError(f"JSONPath '{path}' ({field_name}) did not match any value in agent response: {data}") - return None - return matches[-1].value diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_stream_translation.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_stream_translation.py deleted file mode 100644 index 835c4dea2e..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_stream_translation.py +++ /dev/null @@ -1,76 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Typed extension point for translating agent stream frames into ATIF evidence.""" - -# ruff: noqa: I001 - the vendored SDK mirror uses different import-order settings. - -from __future__ import annotations - -from collections.abc import Sequence -from typing import Any, Protocol, runtime_checkable - -from pydantic import BaseModel, ConfigDict, Field, field_validator - -from nemo_platform.beta.evaluator.values.atif import Trajectory -from nemo_platform.beta.evaluator.values.evidence import EvidenceDescriptor - - -class SseFrame(BaseModel): - """One parsed field from an agent's JSON SSE response.""" - - model_config = ConfigDict(extra="forbid") - - channel: str - payload: Any - raw: str - - -class AgentStreamTranslationContext(BaseModel): - """Non-secret invocation context supplied to an agent stream translator.""" - - model_config = ConfigDict(extra="forbid") - - agent_name: str - endpoint: str - request_payload: dict[str, Any] - final_payload: Any | None = None - output_text: str | None = None - run_id: str | None = None - task_id: str | None = None - invocation_id: str | None = None - conversation_id: str | None = None - http_status: int | None = None - stream_error: str | None = None - - -class AgentStreamTranslation(BaseModel): - """Canonical ATIF plus optional client-owned derived evidence. - - The trajectory stays as the producer-emitted dictionary so fields outside - the evaluator SDK's lightweight ATIF read model are not discarded. - """ - - model_config = ConfigDict(extra="forbid") - - trajectory: dict[str, Any] - evidence: dict[str, EvidenceDescriptor] = Field(default_factory=dict) - metadata: dict[str, Any] = Field(default_factory=dict) - - @field_validator("trajectory") - @classmethod - def _validate_atif(cls, value: dict[str, Any]) -> dict[str, Any]: - Trajectory.model_validate(value) - return value - - -@runtime_checkable -class AgentStreamTranslator(Protocol): - """Translate captured agent stream frames into canonical ATIF evidence.""" - - def __call__( - self, - frames: Sequence[SseFrame], - *, - context: AgentStreamTranslationContext, - ) -> AgentStreamTranslation: ... diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/constants.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/constants.py deleted file mode 100644 index 6d57625658..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/constants.py +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -PLACEHOLDER_INFERENCE_API_KEY = "XXX" diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/dataset_schemas/common.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/dataset_schemas/common.py deleted file mode 100644 index 9fca522698..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/dataset_schemas/common.py +++ /dev/null @@ -1,177 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Shared schema, path, and validation primitives for evaluator dataset-schema helpers.""" - -from __future__ import annotations - -from typing import Any, Literal - -from jsonschema.exceptions import SchemaError -from jsonschema.validators import validator_for - -JSON_SCHEMA_DIALECT = "https://json-schema.org/draft/2020-12/schema" -ARRAY_TOKEN = "[]" - - -class TemplateSchemaInferenceError(ValueError): - """Raised when a template cannot be mapped to a canonical evaluator schema safely.""" - - -class SchemaCompatibilityError(ValueError): - """Raised when metric schemas cannot be merged or validated.""" - - -class _MissingType: - pass - - -_MISSING = _MissingType() - - -def validate_json_schema(schema: dict | None) -> None: - """Validate a JSON Schema document if present. - - Raises `ValueError` when the schema is malformed according to the declared JSON Schema - dialect. - """ - if schema is None: - return - validator = validator_for(schema) - try: - validator.check_schema(schema) - except SchemaError as e: - raise ValueError(f"invalid JSON Schema: {e.message}") from e - - -def empty_object_schema() -> dict: - return {"$schema": JSON_SCHEMA_DIALECT, "type": "object", "properties": {}, "required": []} - - -def encode_type(types: list[str]) -> str | list[str]: - return types[0] if len(types) == 1 else types - - -def primitive_type_name(value: Any) -> Literal["boolean", "integer", "number", "string", "null"]: - if value is None: - return "null" - if isinstance(value, bool): - return "boolean" - if isinstance(value, int): - return "integer" - if isinstance(value, float): - return "number" - return "string" - - -def value_kind(value: Any) -> str: - if isinstance(value, dict): - return "object" - if isinstance(value, list): - return "array" - return primitive_type_name(value) - - -def allowed_types(schema: dict) -> set[str]: - """Return the set of JSON Schema types implied by a schema fragment. - - This includes inferred object/array kinds when ``type`` is omitted but - ``properties`` or ``items`` are present. - """ - schema_type = schema.get("type") - if isinstance(schema_type, str): - return {schema_type} - if isinstance(schema_type, list): - return {item for item in schema_type if isinstance(item, str)} - if "properties" in schema: - return {"object"} - if "items" in schema: - return {"array"} - return set() - - -def schema_kind(schema: dict) -> Literal["any", "object", "array", "primitive"]: - """Classify a schema fragment into the subset used by path traversal helpers. - - ``object`` and ``array`` include nullable unions (for example ``["object", - "null"]``), while empty/untyped fragments are classified as ``any``. - """ - allowed = allowed_types(schema) - if not allowed: - return "any" - if "object" in allowed and allowed <= {"object", "null"}: - return "object" - if "array" in allowed and allowed <= {"array", "null"}: - return "array" - return "primitive" - - -def display_path(path: str) -> str: - return path or "" - - -def split_path(path: str) -> list[str]: - """Split a dot path into segments, expanding array suffixes. - - Examples: - "messages[].content" -> ["messages", "[]", "content"] - "messages[][].content" -> ["messages", "[]", "[]", "content"] - """ - if not path: - return [] - - parts: list[str] = [] - for segment in path.split("."): - array_depth = 0 - while segment.endswith(ARRAY_TOKEN): - segment = segment[: -len(ARRAY_TOKEN)] - array_depth += 1 - if segment: - parts.append(segment) - parts.extend([ARRAY_TOKEN] * array_depth) - return parts - - -def get_value_at_path(data: dict[str, Any], path: str) -> Any: - """Resolve a dotted path from an input row, returning ``_MISSING`` if absent. - - Array tokens (``[]``) are not traversed against concrete row values and are - treated as unresolved for dataset-schema inference purposes. - """ - current: Any = data - for segment in split_path(path): - if segment == ARRAY_TOKEN: - return _MISSING - if not isinstance(current, dict) or segment not in current: - return _MISSING - current = current[segment] - return current - - -def get_schema_at_path(schema: dict, path: str) -> tuple[dict | None, bool]: - """Resolve a schema fragment for a dotted path and whether it is required. - - Requiredness accumulates as traversal descends through object properties. - Returns ``(None, False)`` when the path cannot be resolved from the schema. - """ - current = schema - required = True - for segment in split_path(path): - current_kind = schema_kind(current) - if segment == ARRAY_TOKEN: - if current_kind != "array": - return None, False - items = current.get("items") - if not isinstance(items, dict): - return None, False - current = items - continue - - if current_kind != "object": - return None, False - properties = current.get("properties", {}) - if segment not in properties: - return None, False - required = required and segment in current.get("required", []) - current = properties[segment] - return current, required diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/dataset_schemas/compatibility.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/dataset_schemas/compatibility.py deleted file mode 100644 index 77043275c2..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/dataset_schemas/compatibility.py +++ /dev/null @@ -1,353 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Schema projection, compatibility checking, and merge helpers for evaluator inputs.""" - -from __future__ import annotations - -from collections.abc import Iterable -from typing import Any - -from nemo_platform.beta.evaluator.dataset_schemas.common import ( - _MISSING, - JSON_SCHEMA_DIALECT, - SchemaCompatibilityError, - allowed_types, - display_path, - empty_object_schema, - encode_type, - get_schema_at_path, - get_value_at_path, - schema_kind, - validate_json_schema, -) -from nemo_platform.beta.evaluator.dataset_schemas.templates import ( - infer_required_schema_from_template, -) -from nemo_platform.beta.evaluator.values.dataset_schemas import FieldMapping - - -def validate_dataset_schema_requirement( - dataset_schema: dict, - required_schema: dict, - field_mapping: FieldMapping | None = None, -) -> list[str]: - """Project a dataset schema into canonical fields and check compatibility. - - This is the main adapter entrypoint for comparing raw dataset schemas against the canonical - evaluator field schema expected by a metric. - """ - mapped_path_errors = _validate_column_mapping_paths(dataset_schema, required_schema, field_mapping) - projected_dataset_schema = project_dataset_schema_for_column_mapping(dataset_schema, required_schema, field_mapping) - compatibility_errors = check_dataset_schema_compatibility(projected_dataset_schema, required_schema) - if not mapped_path_errors: - return compatibility_errors - - assert field_mapping is not None - missing_canonical_fields = { - canonical_name - for canonical_name, dataset_path in field_mapping.mapping().items() - if get_schema_at_path(dataset_schema, dataset_path)[0] is None - } - filtered_errors = [ - error - for error in compatibility_errors - if not _is_redundant_missing_field_error(error, missing_canonical_fields) - ] - return [*mapped_path_errors, *filtered_errors] - - -def validate_prompt_template_against_dataset_schema( - dataset_schema: dict, - prompt_template: str | dict, - field_mapping: FieldMapping | None = None, - *, - ignored_roots: set[str] | None = None, - optional_fields: set[str] | None = None, -) -> list[str]: - """Infer a schema from a prompt template and validate it against a dataset schema.""" - prompt_schema = infer_required_schema_from_template( - prompt_template, - ignored_roots=ignored_roots, - optional_fields=optional_fields, - ) - return validate_dataset_schema_requirement(dataset_schema, prompt_schema, field_mapping) - - -def merge_metric_required_schemas(named_schemas: Iterable[tuple[str, dict]]) -> dict: - """Merge metric-required schemas for benchmark validation. - - Primitive `integer` and `number` requirements are widened to `number`. Other incompatible - primitive type combinations raise `SchemaCompatibilityError`. - """ - merged: dict | None = None - for metric_name, schema in named_schemas: - validate_json_schema(schema) - merged = schema if merged is None else _merge_schema_nodes(merged, schema, metric_name, path="") - - if merged is None: - merged = empty_object_schema() - validate_json_schema(merged) - return merged - - -def check_dataset_schema_compatibility(dataset_schema: dict, required_schema: dict) -> list[str]: - """Return compatibility errors between a dataset schema and a metric-required schema.""" - validate_json_schema(dataset_schema) - validate_json_schema(required_schema) - return _check_schema_node(dataset_schema, required_schema, path="") - - -def prune_schema_properties(schema: dict, excluded_fields: set[str]) -> dict: - """Drop top-level fields that are supplied at runtime rather than by the dataset.""" - if not excluded_fields: - return schema - if schema_kind(schema) != "object": - return schema - - properties = schema.get("properties", {}) - required = schema.get("required", []) - pruned = { - "$schema": schema.get("$schema", JSON_SCHEMA_DIALECT), - "type": schema.get("type", "object"), - "properties": {name: value for name, value in properties.items() if name not in excluded_fields}, - "required": [name for name in required if name not in excluded_fields], - } - if "additionalProperties" in schema: - pruned["additionalProperties"] = schema["additionalProperties"] - return pruned - - -def project_dataset_schema_for_column_mapping( - dataset_schema: dict, - required_schema: dict, - column_mapping: FieldMapping | None = None, -) -> dict: - """Project a raw dataset schema into canonical evaluator field names. - - Each required evaluator field is resolved either directly by name or through the provided - column mapping. Missing dataset paths are omitted from the projected schema so that the - compatibility checker can produce the final user-facing errors. - """ - validate_json_schema(dataset_schema) - validate_json_schema(required_schema) - - if schema_kind(required_schema) != "object": - return dataset_schema - - mapping = column_mapping.mapping() if column_mapping is not None else {} - projected = empty_object_schema() - for property_name in required_schema.get("properties", {}): - dataset_path = mapping.get(property_name, property_name) - dataset_property_schema, is_required = get_schema_at_path(dataset_schema, dataset_path) - if dataset_property_schema is None: - continue - projected["properties"][property_name] = dataset_property_schema - if property_name in required_schema.get("required", []) and is_required: - projected["required"].append(property_name) - - if dataset_schema.get("additionalProperties") is False: - projected["additionalProperties"] = False - return projected - - -def _validate_column_mapping_paths( - dataset_schema: dict, - required_schema: dict, - column_mapping: FieldMapping | None = None, -) -> list[str]: - if schema_kind(required_schema) != "object" or column_mapping is None: - return [] - - errors: list[str] = [] - for canonical_name in required_schema.get("properties", {}): - dataset_path = column_mapping.mapping().get(canonical_name) - if dataset_path is None: - continue - dataset_property_schema, _ = get_schema_at_path(dataset_schema, dataset_path) - if dataset_property_schema is None: - errors.append( - f"field_mapping.{canonical_name} refers to dataset field '{dataset_path}', but that field is not present in the dataset schema" - ) - return errors - - -def _is_redundant_missing_field_error(error: str, missing_canonical_fields: set[str]) -> bool: - for field_name in missing_canonical_fields: - if error == f"dataset schema missing required field '{field_name}'": - return True - if error == f"dataset schema missing field definition '{field_name}'": - return True - return False - - -def apply_column_mapping_to_row(row: dict[str, Any], column_mapping: FieldMapping | None = None) -> dict[str, Any]: - """Augment a dataset row with canonical evaluator fields.""" - if column_mapping is None: - return dict(row) - - mapped = dict(row) - for canonical_name, dataset_path in column_mapping.mapping().items(): - value = get_value_at_path(row, dataset_path) - if value is not _MISSING: - mapped[canonical_name] = value - return mapped - - -def _merge_schema_nodes(left: dict, right: dict, metric_name: str, path: str) -> dict: - left_kind = schema_kind(left) - right_kind = schema_kind(right) - left_nullable = "null" in allowed_types(left) - right_nullable = "null" in allowed_types(right) - - if left_kind == "any": - return right - if right_kind == "any": - return left - if left_kind != right_kind: - raise SchemaCompatibilityError( - f"benchmark metrics require incompatible schemas at '{display_path(path)}' for metric '{metric_name}': {left_kind} vs {right_kind}" - ) - - if left_kind == "object": - merged_properties: dict[str, dict] = {} - left_properties = left.get("properties", {}) - right_properties = right.get("properties", {}) - property_names = sorted(set(left_properties) | set(right_properties)) - for property_name in property_names: - prop_path = f"{path}.{property_name}" if path else property_name - if property_name in left_properties and property_name in right_properties: - merged_properties[property_name] = _merge_schema_nodes( - left_properties[property_name], - right_properties[property_name], - metric_name, - prop_path, - ) - else: - if property_name in left_properties: - merged_properties[property_name] = left_properties[property_name] - else: - merged_properties[property_name] = right_properties[property_name] - - merged_required = sorted(set(left.get("required", [])) | set(right.get("required", []))) - merged = { - "$schema": JSON_SCHEMA_DIALECT, - "type": _merge_container_types("object", left_nullable, right_nullable), - "properties": merged_properties, - "required": merged_required, - } - if left.get("additionalProperties") is False and right.get("additionalProperties") is False: - merged["additionalProperties"] = False - return merged - - if left_kind == "array": - return { - "$schema": JSON_SCHEMA_DIALECT, - "type": _merge_container_types("array", left_nullable, right_nullable), - "items": _merge_schema_nodes(left.get("items", {}), right.get("items", {}), metric_name, f"{path}[]"), - } - - allowed = _merge_primitive_types(allowed_types(left), allowed_types(right)) - return {"$schema": JSON_SCHEMA_DIALECT, "type": encode_type(allowed)} - - -def _merge_primitive_types(left: set[str], right: set[str]) -> list[str]: - left_non_null = left - {"null"} - right_non_null = right - {"null"} - combined_non_null = left_non_null | right_non_null - - if combined_non_null <= {"integer", "number"}: - merged = {"number"} - elif len(combined_non_null) == 1: - merged = set(combined_non_null) - else: - raise SchemaCompatibilityError(f"incompatible primitive schema types: {sorted(left)} vs {sorted(right)}") - - if "null" in left and "null" in right: - merged.add("null") - if len(merged) == 1: - return sorted(merged) - if merged == {"number", "null"}: - return ["null", "number"] - if merged == {"integer", "null"}: - return ["integer", "null"] - raise SchemaCompatibilityError(f"incompatible primitive schema types: {sorted(left)} vs {sorted(right)}") - - -def _check_schema_node(dataset: dict, required: dict, path: str) -> list[str]: - dataset_allowed_types = allowed_types(dataset) - required_allowed_types = allowed_types(required) - required_kind = schema_kind(required) - dataset_kind = schema_kind(dataset) - errors: list[str] = [] - - if required_kind == "any": - return [] - if dataset_kind == "any": - return [ - f"dataset field '{display_path(path)}' has unconstrained schema and cannot satisfy required schema {required!r}" - ] - if "null" in dataset_allowed_types and "null" not in required_allowed_types: - return [ - f"dataset field '{display_path(path)}' is incompatible: expected {sorted(required_allowed_types)}, found {sorted(dataset_allowed_types)}" - ] - if required_kind == "primitive": - if not _dataset_types_fit_requirement(dataset_allowed_types, required_allowed_types): - return [ - f"dataset field '{display_path(path)}' is incompatible: expected {sorted(required_allowed_types)}, found {sorted(dataset_allowed_types)}" - ] - return [] - if dataset_kind != required_kind: - return [f"dataset field '{display_path(path)}' is incompatible: expected {required_kind}, found {dataset_kind}"] - - if required_kind == "array": - dataset_items = dataset.get("items") - required_items = required.get("items") - if required_items and dataset_items is None: - return [f"dataset field '{display_path(path)}' is missing array item schema"] - if dataset_items is None or required_items is None: - return [] - return _check_schema_node(dataset_items, required_items, f"{path}[]") - - if required_kind == "object": - dataset_required = set(dataset.get("required", [])) - dataset_properties = dataset.get("properties", {}) - required_properties = required.get("properties", {}) - - for property_name in required.get("required", []): - if property_name not in dataset_required: - child_path = f"{path}.{property_name}" if path else property_name - errors.append(f"dataset schema missing required field '{display_path(child_path)}'") - - for property_name, property_schema in required_properties.items(): - child_path = f"{path}.{property_name}" if path else property_name - dataset_property_schema = dataset_properties.get(property_name) - if dataset_property_schema is None: - if property_name in required.get("required", []): - errors.append(f"dataset schema missing field definition '{display_path(child_path)}'") - continue - errors.extend(_check_schema_node(dataset_property_schema, property_schema, child_path)) - return errors - - return [] - - -def _dataset_types_fit_requirement(dataset_types: set[str], required_types: set[str]) -> bool: - dataset_non_null = dataset_types - {"null"} - required_non_null = required_types - {"null"} - return all( - any(_type_is_compatible(dataset_type, required_type) for required_type in required_non_null) - for dataset_type in dataset_non_null - ) - - -def _type_is_compatible(dataset_type: str, required_type: str) -> bool: - return dataset_type == required_type or (dataset_type == "integer" and required_type == "number") - - -def _merge_container_types( - base_type: str, - left_nullable: bool, - right_nullable: bool, -) -> str | list[str]: - return [base_type, "null"] if left_nullable and right_nullable else base_type diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/dataset_schemas/templates.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/dataset_schemas/templates.py deleted file mode 100644 index 1bae46a29a..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/dataset_schemas/templates.py +++ /dev/null @@ -1,389 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Template parsing helpers for deriving canonical evaluator input schemas.""" - -from __future__ import annotations - -from collections.abc import Iterable -from dataclasses import dataclass -from typing import Any - -from jinja2 import nodes -from jinja2.sandbox import SandboxedEnvironment -from jinja2.visitor import NodeVisitor -from nemo_platform.beta.evaluator.dataset_schemas.common import ( - ARRAY_TOKEN, - TemplateSchemaInferenceError, - empty_object_schema, - validate_json_schema, -) - -_UNSUPPORTED_TEMPLATE_NODES = ( - nodes.CallBlock, - nodes.Import, - nodes.Include, - nodes.Macro, -) - - -@dataclass(frozen=True) -class _PathReference: - root: str - segments: tuple[str, ...] - - -_jinja_env = SandboxedEnvironment() - - -class _TemplateReferenceVisitor(NodeVisitor): - def __init__(self, *, ignored_roots: set[str]) -> None: - self.references: set[_PathReference] = set() - self.ignored_roots = ignored_roots - self.local_names: set[str] = set() - - def generic_visit(self, node: nodes.Node, *args: Any, **kwargs: Any) -> None: - if isinstance(node, _UNSUPPORTED_TEMPLATE_NODES): - raise TemplateSchemaInferenceError( - f"unsupported Jinja construct for dataset schema inference: {type(node).__name__}" - ) - super().generic_visit(node, *args, **kwargs) - - def visit_Output(self, node: nodes.Output, *args: Any, **kwargs: Any) -> None: - for child in node.nodes: - self.references.update( - _collect_expression_references( - child, - ignored_roots=self.ignored_roots, - ignored_names=self.local_names, - ) - ) - self.visit(child) - - def visit_For(self, node: nodes.For, *args: Any, **kwargs: Any) -> None: - references = _collect_expression_references( - node.iter, - ignored_roots=self.ignored_roots, - ignored_names=self.local_names, - ) - if any(reference.root not in self.ignored_roots for reference in references): - raise TemplateSchemaInferenceError("unsupported Jinja construct for dataset schema inference: For") - loop_local_names = _collect_target_names(node.target) - previous_local_names = set(self.local_names) - self.local_names.update(loop_local_names) - try: - for body_node in node.body: - self.visit(body_node) - for else_node in node.else_: - self.visit(else_node) - finally: - self.local_names = previous_local_names - - -def infer_required_schema_from_template( - template: str | dict | list, - *, - ignored_roots: set[str] | None = None, - optional_fields: set[str] | None = None, -) -> dict: - """Infer a canonical evaluator input schema from a Jinja template structure. - - This inference intentionally accepts only a narrow subset of Jinja that maps cleanly to a - canonical evaluator input contract. Dynamic indexing is rejected for schema inference. Control - flow is rejected unless it only references roots explicitly listed in `ignored_roots`, which - are treated as runtime-only context rather than dataset-provided inputs. Function/filter - expressions are supported for dependency extraction, but callable identifiers are not treated - as dataset-provided inputs. - - We intentionally do not support function-call argument unpacking (`*args` / `**kwargs`) for - schema inference because this usage is uncommon in evaluator prompts and makes dependency - extraction ambiguous. For example, `{{ sample.output_json.get(*item.lookup_path) }}` and - `{{ sample.output_json.get(**item.lookup_kwargs) }}` are not considered supported schema - inference patterns. - """ - effective_ignored_roots = set(ignored_roots or ()) - effective_optional_fields = set(optional_fields or ()) - references = list(_extract_template_references(template, ignored_roots=effective_ignored_roots)) - schema = _build_schema_from_references(references, ignored_roots=effective_ignored_roots) - _drop_optional_fields_from_required(schema, optional_fields=effective_optional_fields) - validate_json_schema(schema) - return schema - - -def _extract_template_references(template: str | dict | list, *, ignored_roots: set[str]) -> set[_PathReference]: - if isinstance(template, dict): - references: set[_PathReference] = set() - for value in template.values(): - references.update(_extract_template_references(value, ignored_roots=ignored_roots)) - return references - if isinstance(template, list): - references = set() - for value in template: - references.update(_extract_template_references(value, ignored_roots=ignored_roots)) - return references - if not isinstance(template, str): - return set() - - parsed = _jinja_env.parse(template) - visitor = _TemplateReferenceVisitor(ignored_roots=ignored_roots) - visitor.visit(parsed) - return visitor.references - - -def _collect_expression_references( - node: nodes.Node, - *, - ignored_roots: set[str], - ignored_names: set[str], -) -> set[_PathReference]: - if isinstance(node, nodes.TemplateData | nodes.Const): - return set() - if isinstance(node, nodes.Filter): - references = _collect_expression_references( - node.node, - ignored_roots=ignored_roots, - ignored_names=ignored_names, - ) - for arg in node.args: - references.update( - _collect_expression_references( - arg, - ignored_roots=ignored_roots, - ignored_names=ignored_names, - ) - ) - for keyword in node.kwargs: - references.update( - _collect_expression_references( - keyword.value, - ignored_roots=ignored_roots, - ignored_names=ignored_names, - ) - ) - return references - if isinstance(node, nodes.Call): - references = _collect_callable_base_references(node.node, ignored_names=ignored_names) - references.update( - _collect_call_argument_references( - args=node.args, - kwargs=node.kwargs, - ignored_roots=ignored_roots, - ignored_names=ignored_names, - ) - ) - return references - if isinstance(node, nodes.If): - references = _collect_expression_references( - node.test, - ignored_roots=ignored_roots, - ignored_names=ignored_names, - ) - for body_node in node.body: - references.update( - _collect_expression_references( - body_node, - ignored_roots=ignored_roots, - ignored_names=ignored_names, - ) - ) - for else_node in node.else_: - references.update( - _collect_expression_references( - else_node, - ignored_roots=ignored_roots, - ignored_names=ignored_names, - ) - ) - if any(reference.root not in ignored_roots for reference in references): - raise TemplateSchemaInferenceError("conditionals are not supported for dataset schema inference") - return references - if isinstance(node, nodes.Name | nodes.Getattr | nodes.Getitem): - reference = _path_reference_from_node(node, ignored_names=ignored_names) - return set() if reference is None else {reference} - if isinstance(node, nodes.List | nodes.Tuple): - references: set[_PathReference] = set() - for item in node.items: - references.update( - _collect_expression_references( - item, - ignored_roots=ignored_roots, - ignored_names=ignored_names, - ) - ) - return references - if isinstance(node, nodes.Dict): - references: set[_PathReference] = set() - for item in node.items: - references.update( - _collect_expression_references( - item.key, - ignored_roots=ignored_roots, - ignored_names=ignored_names, - ) - ) - references.update( - _collect_expression_references( - item.value, - ignored_roots=ignored_roots, - ignored_names=ignored_names, - ) - ) - return references - if isinstance(node, nodes.Pair): - references = _collect_expression_references( - node.key, - ignored_roots=ignored_roots, - ignored_names=ignored_names, - ) - references.update( - _collect_expression_references( - node.value, - ignored_roots=ignored_roots, - ignored_names=ignored_names, - ) - ) - return references - if isinstance(node, _UNSUPPORTED_TEMPLATE_NODES): - raise TemplateSchemaInferenceError( - f"unsupported Jinja construct for dataset schema inference: {type(node).__name__}" - ) - raise TemplateSchemaInferenceError( - f"unsupported Jinja expression for dataset schema inference: {type(node).__name__}" - ) - - -def _collect_call_argument_references( - *, - args: list[nodes.Node], - kwargs: list[nodes.Keyword], - ignored_roots: set[str], - ignored_names: set[str], -) -> set[_PathReference]: - references: set[_PathReference] = set() - for arg in args: - references.update( - _collect_expression_references( - arg, - ignored_roots=ignored_roots, - ignored_names=ignored_names, - ) - ) - for keyword in kwargs: - references.update( - _collect_expression_references( - keyword.value, - ignored_roots=ignored_roots, - ignored_names=ignored_names, - ) - ) - return references - - -def _collect_callable_base_references(node: nodes.Node, *, ignored_names: set[str]) -> set[_PathReference]: - # For `input.upper()`, require `input`; for `upper(input)`, do not require `upper`. - if isinstance(node, nodes.Getattr | nodes.Getitem): - reference = _path_reference_from_node(node.node, ignored_names=ignored_names) - return set() if reference is None else {reference} - return set() - - -def _path_reference_from_node(node: nodes.Node, *, ignored_names: set[str]) -> _PathReference | None: - if isinstance(node, nodes.Name): - if node.name in ignored_names: - return None - return _PathReference(root=node.name, segments=()) - if isinstance(node, nodes.Getattr): - base = _path_reference_from_node(node.node, ignored_names=ignored_names) - if base is None: - return None - return _PathReference(root=base.root, segments=base.segments + (node.attr,)) - if isinstance(node, nodes.Getitem): - base = _path_reference_from_node(node.node, ignored_names=ignored_names) - if base is None: - return None - if not isinstance(node.arg, nodes.Const): - raise TemplateSchemaInferenceError("dynamic indexing is not supported for dataset schema inference") - if isinstance(node.arg.value, int): - segment = ARRAY_TOKEN - elif isinstance(node.arg.value, str): - segment = node.arg.value - else: - raise TemplateSchemaInferenceError("unsupported index type for dataset schema inference") - return _PathReference(root=base.root, segments=base.segments + (segment,)) - raise TemplateSchemaInferenceError( - f"unsupported Jinja expression for dataset schema inference: {type(node).__name__}" - ) - - -def _collect_target_names(node: nodes.Node) -> set[str]: - if isinstance(node, nodes.Name): - return {node.name} - if isinstance(node, nodes.Tuple): - names: set[str] = set() - for item in node.items: - names.update(_collect_target_names(item)) - return names - return set() - - -def _build_schema_from_references( - references: Iterable[_PathReference], - *, - ignored_roots: set[str], -) -> dict: - schema = empty_object_schema() - - for reference in references: - path = _normalize_reference(reference, ignored_roots=ignored_roots) - if path is not None: - _add_required_path(schema, path) - return schema - - -def _normalize_reference( - reference: _PathReference, - *, - ignored_roots: set[str], -) -> tuple[str, ...] | None: - if reference.root == "sample" and reference.segments == ("output_text",): - return ("output",) - if reference.root in ignored_roots: - return None - if reference.root == "item": - return reference.segments or None - return (reference.root,) + reference.segments - - -def _add_required_path(schema: dict, path: tuple[str, ...]) -> None: - current = schema - index = 0 - while index < len(path): - segment = path[index] - if segment == ARRAY_TOKEN: - current["type"] = "array" - current.setdefault("items", {}) - current = current["items"] - index += 1 - continue - - current.setdefault("type", "object") - properties = current.setdefault("properties", {}) - required = current.setdefault("required", []) - if segment not in properties: - properties[segment] = {} - if segment not in required: - required.append(segment) - - current = properties[segment] - if index + 1 < len(path) and path[index + 1] != ARRAY_TOKEN: - current.setdefault("type", "object") - index += 1 - - -def _drop_optional_fields_from_required(schema: dict, *, optional_fields: set[str]) -> None: - if not optional_fields: - return - required = schema.get("required") - if not isinstance(required, list): - return - schema["required"] = [field for field in required if field not in optional_fields] diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/datasets/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/datasets/__init__.py deleted file mode 100644 index 44c008af78..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/datasets/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Dataset loading and template utilities for evaluator SDK runtime.""" - -from nemo_platform.beta.evaluator.datasets.loader import ( - DatasetLoadError, - discover_files, - is_glob_pattern, - load_dataset, - load_dataset_as_dicts, - load_file, -) -from nemo_platform.beta.evaluator.templates import ( - render_request, - render_template, -) - -__all__ = [ - "DatasetLoadError", - "is_glob_pattern", - "discover_files", - "load_file", - "load_dataset", - "load_dataset_as_dicts", - "render_request", - "render_template", -] diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/datasets/loader.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/datasets/loader.py deleted file mode 100644 index 6c5641e0f0..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/datasets/loader.py +++ /dev/null @@ -1,441 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Dataset loader module for evaluator SDK runtime.""" - -import gzip -import io -import json -import logging -from pathlib import Path -from typing import Any - -import pyarrow as pa -import pyarrow.csv as pa_csv -import pyarrow.feather as pa_feather -import pyarrow.json as pa_json -import pyarrow.parquet as pa_parquet - -from nemo_platform.beta.evaluator.values.datasets import DatasetInput, DatasetRows - -_log = logging.getLogger(__name__) - -SUPPORTED_EXTENSIONS = { - ".json", - ".jsonl", - ".csv", - ".parquet", - ".feather", - ".arrow", - ".orc", -} - -COMPRESSION_EXTENSIONS = {".gz", ".gzip"} - - -class DatasetLoadError(Exception): - """Raised when a dataset cannot be loaded.""" - - -def rows_from_dataset(dataset: DatasetInput) -> list[dict[str, Any]]: - """Convert supported dataset containers into a list of row dictionaries. - - Args: - dataset: Materialized dataset payload or container to normalize. - - Returns: - The dataset rows as plain dictionaries. - - Raises: - TypeError: If the dataset container type is unsupported. - """ - if isinstance(dataset, DatasetRows): - return dataset.rows - if isinstance(dataset, list): - return dataset - if isinstance(dataset, pa.Table): - return dataset.to_pylist() - raise TypeError(f"Unsupported dataset type: {type(dataset).__name__}") - - -def normalize_dataset( - dataset: DatasetInput | str | Path, - pattern: str | None, -) -> DatasetInput: - """Normalize dataset inputs into an in-memory evaluation payload. - - Args: - dataset: Dataset input accepted by evaluator execution. - pattern: Optional file selector used when ``dataset`` points to a directory. - - Returns: - A materialized dataset payload suitable for row extraction. - - Raises: - TypeError: If the dataset input type is unsupported. - FileNotFoundError: If the provided dataset path does not exist. - ValueError: If ``pattern`` is used with a file path instead of a directory. - """ - if isinstance(dataset, (DatasetRows, pa.Table, list)): - return dataset - if not isinstance(dataset, (str, Path)): - raise TypeError(f"Unsupported dataset type: {type(dataset).__name__}") - - path = Path(dataset) - if not path.exists(): - if pattern is None and is_glob_pattern(str(path)): - base_path, path_pattern = split_glob_path(path) - return load_dataset_as_dicts(base_path, path_pattern) - raise FileNotFoundError(f"Dataset path does not exist: {path}") - - if path.is_dir(): - return load_dataset_as_dicts(path, pattern) - - if pattern is not None: - raise ValueError("pattern can only be used when dataset points to a directory") - - return load_dataset_as_dicts(path.parent, path.name) - - -def prepare_dataset_rows( - dataset: DatasetInput | str | Path, - pattern: str | None, - max_size: int | None, -) -> list[dict[str, Any]]: - """Materialize one dataset input into row dictionaries with optional truncation. - - Args: - dataset: Dataset input accepted by evaluator execution. - pattern: Optional file selector used when ``dataset`` points to a directory. - max_size: Optional maximum number of rows to retain from the materialized dataset. - - Returns: - The prepared row dictionaries used by local metric execution. - """ - rows = rows_from_dataset(normalize_dataset(dataset, pattern)) - if max_size is not None: - rows = rows[:max_size] - return rows - - -def is_glob_pattern(pattern: str) -> bool: - """Check whether a file pattern contains glob metacharacters. - - Args: - pattern: User-provided file selector. - - Returns: - ``True`` when ``pattern`` should be interpreted as a glob. - """ - glob_chars = {"*", "?", "[", "]"} - return any(c in pattern for c in glob_chars) - - -def split_glob_path(path: Path) -> tuple[Path, str]: - """Split a glob path into a concrete base directory and relative glob pattern. - - Args: - path: File path that contains at least one glob metacharacter. - - Returns: - A base directory before the first glob segment and the remaining relative - glob pattern. - - Raises: - ValueError: If ``path`` does not contain glob metacharacters. - """ - parts = path.parts - for index, part in enumerate(parts): - if is_glob_pattern(part): - if index == 0: - base_path = Path(path.anchor) if path.is_absolute() else Path(".") - else: - base_path = Path(*parts[:index]) - return base_path, str(Path(*parts[index:])) - raise ValueError(f"Path does not contain a glob pattern: {path}") - - -def discover_files(base_path: Path, pattern: str | None) -> list[Path]: - """Resolve dataset files under a base path. - - Args: - base_path: Directory that contains dataset files. - pattern: Optional explicit file name or glob pattern. - - Returns: - List of discovered files, sorted by path. - - Raises: - DatasetLoadError: If files cannot be found or selected paths are invalid. - """ - if not base_path.exists(): - raise DatasetLoadError(f"Dataset directory not found: {base_path}") - - if pattern is None: - files = sorted(f for f in base_path.rglob("*") if f.is_file()) - if not files: - raise DatasetLoadError(f"No files found in {base_path}") - return files - - file_path = base_path / pattern - if file_path.exists(): - if not file_path.is_file(): - raise DatasetLoadError(f"Path is not a file: {file_path}") - return [file_path] - - if is_glob_pattern(pattern): - files = sorted(base_path.glob(pattern)) - if not files: - raise DatasetLoadError(f"No files found matching pattern '{pattern}' in {base_path}") - return [f for f in files if f.is_file()] - - raise DatasetLoadError(f"File not found: {file_path}") - - -def _discover_files(base_path: Path, pattern: str | None) -> list[Path]: - """Backward-compatible alias for :func:`discover_files`.""" - return discover_files(base_path, pattern) - - -def _get_file_format(path: Path) -> str | None: - """Infer logical dataset format from a file path. - - For compressed paths such as ``data.jsonl.gz``, this function inspects the - extension before the compression suffix. - - Args: - path: Candidate dataset file path. - - Returns: - Supported data extension (for example ``.jsonl``), or ``None``. - """ - suffixes = path.suffixes - if suffixes and suffixes[-1].lower() in COMPRESSION_EXTENSIONS: - if len(suffixes) < 2: - return None - data_ext = suffixes[-2].lower() - else: - data_ext = path.suffix.lower() if path.suffix else None - - if data_ext in SUPPORTED_EXTENSIONS: - return data_ext - return None - - -def _is_compressed(path: Path) -> bool: - """Check whether a file path uses a supported gzip extension. - - Args: - path: Candidate dataset file path. - - Returns: - ``True`` when the file suffix indicates gzip compression. - """ - return path.suffix.lower() in COMPRESSION_EXTENSIONS - - -def _load_json_file(source: Path | io.BytesIO) -> pa.Table: - """Load JSON content as a PyArrow table. - - The loader supports both JSON arrays and JSONL streams. Arrays are parsed - via ``json.loads`` and converted with ``Table.from_pylist``; line-delimited - JSON is delegated to ``pyarrow.json.read_json``. - - Args: - source: JSON source as filesystem path or in-memory bytes buffer. - - Returns: - Parsed rows as a ``pyarrow.Table``. - - Raises: - ValueError: If JSON array parsing produces a non-list payload. - UnicodeDecodeError: If bytes cannot be decoded as UTF-8 text. - json.JSONDecodeError: If JSON content is invalid. - """ - if isinstance(source, Path): - content = source.read_bytes() - else: - content = source.read() - source.seek(0) - - text = content.decode("utf-8").strip() - if text.startswith("["): - data = json.loads(text) - if not isinstance(data, list): - raise ValueError("Expected JSON array") - return pa.Table.from_pylist(data) - - if isinstance(source, io.BytesIO): - source.seek(0) - return pa_json.read_json(source) - return pa_json.read_json(source) - - -def _load_content(source: Path | io.BytesIO, file_format: str) -> pa.Table: - """Load tabular content for a known file format. - - Args: - source: File path or in-memory bytes buffer. - file_format: Normalized format extension returned by ``_get_file_format``. - - Returns: - Parsed table for the given input format. - - Raises: - ValueError: If the format is not supported. - """ - if file_format in (".json", ".jsonl"): - return _load_json_file(source) - if file_format == ".csv": - return pa_csv.read_csv(source) - if file_format == ".parquet": - return pa_parquet.read_table(source) - if file_format in (".feather", ".arrow"): - return pa_feather.read_table(source) - if file_format == ".orc": - import pyarrow.orc as pa_orc - - return pa_orc.read_table(source) - raise ValueError(f"Unsupported format: {file_format}") - - -def load_file(path: Path) -> pa.Table | None: - """Load one file into a table, returning ``None`` for skipped/failed files. - - Args: - path: Dataset file path. - - Returns: - Parsed table, or ``None`` when format is unsupported or parsing fails. - """ - file_format = _get_file_format(path) - if file_format is None: - _log.debug("Skipping unsupported file format", extra={"path": str(path)}) - return None - - try: - if _is_compressed(path): - with gzip.open(path, "rb") as gz_file: - content = gz_file.read() - return _load_content(io.BytesIO(content), file_format) - return _load_content(path, file_format) - except Exception as e: # pragma: no cover - defensive logging path - _log.warning("Failed to load file", extra={"path": str(path), "error": str(e)}) - return None - - -def _load_file(path: Path) -> pa.Table | None: - """Backward-compatible alias for :func:`load_file`.""" - return load_file(path) - - -def load_dataset(base_path: Path, pattern: str | None) -> pa.Table: - """Load matching dataset files and concatenate them into one table. - - Args: - base_path: Directory containing dataset files. - pattern: Optional file name or glob pattern. - - Returns: - Single table containing rows from all successfully loaded files. - - Raises: - DatasetLoadError: If no files are discovered or no data can be loaded. - """ - files = discover_files(base_path, pattern) - - tables: list[pa.Table] = [] - for file_path in files: - table = load_file(file_path) - if table is not None: - tables.append(table) - _log.debug("Loaded file", extra={"file": file_path.name, "rows": table.num_rows}) - - if not tables: - raise DatasetLoadError(f"No data could be loaded from {base_path} (pattern: {pattern})") - - if len(tables) == 1: - return tables[0] - return pa.concat_tables(tables, promote_options="default") - - -def load_dataset_as_dicts(base_path: Path, pattern: str | None) -> list[dict]: - """Load a dataset and return rows as dictionaries. - - Args: - base_path: Directory containing dataset files. - pattern: Optional file name or glob pattern. - - Returns: - List of row dictionaries ready for metric execution. - """ - try: - table = load_dataset(base_path, pattern) - return table.to_pylist() - except DatasetLoadError as e: - rows = _load_json_from_fileset_path_as_dicts_fallback(Path(base_path), pattern) - if rows: - _log.warning( - "Falling back to Python JSON loader for dataset path due to pyarrow parsing failure", - extra={"base_path": str(base_path), "pattern": pattern, "rows": len(rows), "error": str(e)}, - ) - return rows - raise - - -def _load_json_file_as_dicts(path: Path) -> list[dict]: - """Load JSON/JSONL files as list[dict] without pyarrow schema inference.""" - if _is_compressed(path): - with gzip.open(path, "rb") as gz_file: - text = gz_file.read().decode("utf-8") - else: - text = path.read_text(encoding="utf-8") - - stripped = text.lstrip() - if stripped.startswith("["): - # startswith("[") means content is JSON array (not JSONL). - data = json.loads(stripped) - if isinstance(data, list): - return [row for row in data if isinstance(row, dict)] - if isinstance(data, dict): - return [data] - return [] - - rows: list[dict] = [] - for line in text.splitlines(): - line = line.strip() - if not line: - continue - obj = json.loads(line) - if isinstance(obj, dict): - rows.append(obj) - return rows - - -def _load_json_from_fileset_path_as_dicts_fallback( - fileset_path: Path, - pattern: str | None = None, -) -> list[dict]: - """Fallback loader for JSON/JSONL when pyarrow table loading fails. - - Supports .json, .jsonl, and .jsonl.gz (detected as .jsonl by _get_file_format). - """ - - if not fileset_path.exists(): - raise DatasetLoadError(f"Fileset directory not found: {fileset_path}") - - files = _discover_files(fileset_path, pattern) - rows: list[dict] = [] - for file_path in files: - file_format = _get_file_format(file_path) - - if file_format not in {".json", ".jsonl"}: - continue - try: - rows.extend(_load_json_file_as_dicts(file_path)) - except Exception as e: - _log.warning( - "Fallback JSON loader failed for file", - extra={"path": str(file_path), "error": str(e)}, - ) - return rows diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/enums.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/enums.py deleted file mode 100644 index 26266fb0fc..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/enums.py +++ /dev/null @@ -1,64 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Enums for evaluator SDK runtime.""" - -from enum import Enum - - -class MetricType(str, Enum): - """The predefined metric types.""" - - BLEU = "bleu" - ROUGE = "rouge" - F1 = "f1" - EXACT_MATCH = "exact-match" - STRING_CHECK = "string-check" - NUMBER_CHECK = "number-check" - LLM_JUDGE = "llm-judge" - TOOL_CALLING = "tool-calling" - REMOTE = "remote" - NEMO_AGENT_TOOLKIT_REMOTE = "nemo-agent-toolkit-remote" - - TOPIC_ADHERENCE = "topic_adherence" - TOOL_CALL_ACCURACY = "tool_call_accuracy" - AGENT_GOAL_ACCURACY = "agent_goal_accuracy" - - ANSWER_ACCURACY = "answer_accuracy" - CONTEXT_RELEVANCE = "context_relevance" - RESPONSE_GROUNDEDNESS = "response_groundedness" - - CONTEXT_RECALL = "context_recall" - CONTEXT_PRECISION = "context_precision" - CONTEXT_ENTITY_RECALL = "context_entity_recall" - RESPONSE_RELEVANCY = "response_relevancy" - FAITHFULNESS = "faithfulness" - NOISE_SENSITIVITY = "noise_sensitivity" - TUNABLE_RAG_EVALUATOR = "tunable-rag-evaluator" - - SYSTEM = "system" - - -class TaskStatus(str, Enum): - """Status of an evaluation task.""" - - PENDING = "pending" - RUNNING = "running" - COMPLETED = "completed" - FAILED = "failed" - SKIPPED = "skipped" - - -class ModelFormat(str, Enum): - """Inference format for a model.""" - - NVIDIA_NIM = "nim" - OPEN_AI = "openai" - LLAMA_STACK = "llama_stack" - - -class AgentFormat(str, Enum): - """Inference format for an agent.""" - - GENERIC = "generic" - NEMO_AGENT_TOOLKIT = "nemo_agent_toolkit" diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/_protocols.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/_protocols.py deleted file mode 100644 index 387b27aa80..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/_protocols.py +++ /dev/null @@ -1,20 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Internal optional capabilities used by the v4 evaluator implementation.""" - -from __future__ import annotations - -from typing import runtime_checkable - -from nemo_platform.beta.evaluator.values.params import RunConfig -from typing_extensions import Protocol - - -@runtime_checkable -class JobParamsConfigurableMetric(Protocol): - """Optional metric capability for applying runtime job params.""" - - def apply_evaluation_job_params(self, params: RunConfig) -> None: - """Apply runtime execution params directly on the metric instance.""" - ... diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/backends/base.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/backends/base.py deleted file mode 100644 index 8967aba487..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/backends/base.py +++ /dev/null @@ -1,92 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Backend protocol for completed-result evaluator execution.""" - -from __future__ import annotations - -from collections.abc import Sequence -from pathlib import Path -from typing import Any, Protocol - -from nemo_platform.beta.evaluator.inference import PostprocessResponse, PreprocessRequest -from nemo_platform.beta.evaluator.metrics.protocol import Metric -from nemo_platform.beta.evaluator.values import ( - Agent, - DatasetInput, - FieldMapping, - Model, - RunConfig, - RunConfigOnline, - RunConfigOnlineModel, -) -from nemo_platform.beta.evaluator.values.multi_metric_results import BenchmarkEvaluationResult -from nemo_platform.beta.evaluator.values.results import AggregateFieldName - -BackendParams = RunConfig | RunConfigOnline | RunConfigOnlineModel - - -class EvaluationBackend(Protocol): - async def evaluate_dataset( - self, - *, - metrics: Sequence[Metric], - dataset: DatasetInput | str | Path, - params: BackendParams, - target: Model | Agent | None = None, - field_mapping: FieldMapping | None = None, - prompt_template: str | dict[str, Any] | None = None, - aggregate_fields: tuple[AggregateFieldName, ...] | None = None, - preprocess_hooks: tuple[PreprocessRequest, ...] | None = None, - postprocess_hooks: tuple[PostprocessResponse, ...] | None = None, - ) -> BenchmarkEvaluationResult: - """Evaluate multiple metrics directly and return the completed result. - - Args: - metrics: Metrics to prepare and execute together. - dataset: Inline dataset rows, a dataset file, or a dataset directory/glob path. - params: Validated run configuration for the selected target mode. - target: Optional model or agent used to generate candidate responses before scoring. - field_mapping: Optional mapping from canonical evaluator fields to dataset columns. - prompt_template: Optional prompt template for online target generation. - aggregate_fields: Optional aggregate score fields to keep in the returned result. - preprocess_hooks: Optional request preprocess hooks for online execution. - postprocess_hooks: Optional response postprocess hooks for online execution. - - Returns: - The completed multi-metric evaluation result. - """ - ... - - -class SyncEvaluationBackend(Protocol): - def evaluate_dataset( - self, - *, - metrics: Sequence[Metric], - dataset: DatasetInput | str | Path, - params: BackendParams, - target: Model | Agent | None = None, - field_mapping: FieldMapping | None = None, - prompt_template: str | dict[str, Any] | None = None, - aggregate_fields: tuple[AggregateFieldName, ...] | None = None, - preprocess_hooks: tuple[PreprocessRequest, ...] | None = None, - postprocess_hooks: tuple[PostprocessResponse, ...] | None = None, - ) -> BenchmarkEvaluationResult: - """Evaluate multiple metrics directly and return the completed result. - - Args: - metrics: Metrics to prepare and execute together. - dataset: Inline dataset rows, a dataset file, or a dataset directory/glob path. - params: Validated run configuration for the selected target mode. - target: Optional model or agent used to generate candidate responses before scoring. - field_mapping: Optional mapping from canonical evaluator fields to dataset columns. - prompt_template: Optional prompt template for online target generation. - aggregate_fields: Optional aggregate score fields to keep in the returned result. - preprocess_hooks: Optional request preprocess hooks for online execution. - postprocess_hooks: Optional response postprocess hooks for online execution. - - Returns: - The completed multi-metric result. - """ - ... diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/backends/local/backend.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/backends/local/backend.py deleted file mode 100644 index 9f89fc5259..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/backends/local/backend.py +++ /dev/null @@ -1,165 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Local backend implementation for completed-result evaluator execution.""" - -from __future__ import annotations - -from collections.abc import Sequence -from logging import getLogger -from pathlib import Path -from typing import Any - -from nemo_platform.beta.evaluator.dataset_schemas.compatibility import apply_column_mapping_to_row -from nemo_platform.beta.evaluator.datasets.loader import prepare_dataset_rows -from nemo_platform.beta.evaluator.execution.backends.base import BackendParams -from nemo_platform.beta.evaluator.execution.benchmark_execution import evaluate_benchmark as sdk_evaluate_benchmark -from nemo_platform.beta.evaluator.execution.metric_execution import _merge_online_hooks, evaluate_metric -from nemo_platform.beta.evaluator.execution.utils import prepare_metric_for_execution, unique_metric_keys -from nemo_platform.beta.evaluator.inference import PostprocessResponse, PreprocessRequest -from nemo_platform.beta.evaluator.metrics.protocol import Metric -from nemo_platform.beta.evaluator.resolvers import LocalModelResolver, LocalSecretResolver -from nemo_platform.beta.evaluator.values import Agent, DatasetInput, FieldMapping, Model -from nemo_platform.beta.evaluator.values.multi_metric_results import BenchmarkEvaluationResult, namespace_result -from nemo_platform.beta.evaluator.values.results import AggregateFieldName, EvaluationResult - -log = getLogger(__name__) - - -def _prepare_rows( - dataset: DatasetInput | str | Path, - params: BackendParams, - field_mapping: FieldMapping | None, -) -> list[dict[str, Any]]: - """Load dataset rows, apply sampling limits, and project mapped fields.""" - rows = prepare_dataset_rows( - dataset, - None, - params.limit_samples, - ) - if field_mapping is None: - return rows - return [apply_column_mapping_to_row(row, field_mapping) for row in rows] - - -class LocalBackend: - """Local backend that executes metrics in-process.""" - - def __init__(self) -> None: - """Create a local backend with local resolver defaults.""" - self.secret_resolver = LocalSecretResolver() - self.model_resolver = LocalModelResolver() - - async def _evaluate_one( - self, - *, - metric: Metric, - metric_key: str, - params: BackendParams, - target: Model | Agent | None, - prompt_template: str | dict[str, Any] | None, - aggregate_fields: tuple[AggregateFieldName, ...] | None, - preprocess_hooks: tuple[PreprocessRequest, ...] | None, - postprocess_hooks: tuple[PostprocessResponse, ...] | None, - rows: list[dict[str, Any]], - ) -> EvaluationResult: - """Prepare one metric and execute it through the local runtime. - - Args: - metric: Metric to execute. - metric_key: Public metric key used to namespace the result. - params: Validated run configuration for the selected target mode. - target: Optional model or agent used to generate candidate responses before scoring. - prompt_template: Optional prompt template for online target generation. - aggregate_fields: Optional aggregate score fields to keep in the returned result. - preprocess_hooks: Optional request preprocess hooks for online execution. - postprocess_hooks: Optional response postprocess hooks for online execution. - rows: Precomputed dataset rows shared across metrics in the request. - - Returns: - A namespaced single-metric evaluation result. - """ - prepared_metric = await prepare_metric_for_execution( - metric, - params=params, - model_resolver=self.model_resolver, - secret_resolver=self.secret_resolver, - ) - - result = await evaluate_metric( - metric=prepared_metric, - target=target, - rows=rows, - prompt_template=prompt_template, - params=params, - preprocess_hooks=preprocess_hooks, - postprocess_hooks=postprocess_hooks, - ) - - return namespace_result(metric_key, result, aggregate_fields) - - async def evaluate_dataset( - self, - *, - metrics: Sequence[Metric], - dataset: DatasetInput | str | Path, - params: BackendParams, - target: Model | Agent | None = None, - field_mapping: FieldMapping | None = None, - prompt_template: str | dict[str, Any] | None = None, - aggregate_fields: tuple[AggregateFieldName, ...] | None = None, - preprocess_hooks: tuple[PreprocessRequest, ...] | None = None, - postprocess_hooks: tuple[PostprocessResponse, ...] | None = None, - ) -> BenchmarkEvaluationResult: - """Execute multiple metrics locally using the shared streaming pipeline. - - Delegates to :func:`sdk_evaluate_benchmark` so that each dataset row runs - target inference exactly once, regardless of metric count. - - Args: - metrics: Metrics to prepare and execute together. - dataset: Inline dataset rows, a dataset file, or a dataset directory/glob path. - params: Validated run configuration for the selected target mode. - target: Optional model or agent used to generate candidate responses before scoring. - field_mapping: Optional mapping from canonical evaluator fields to dataset columns. - prompt_template: Optional prompt template for online target generation. - aggregate_fields: Optional aggregate score fields to keep in the returned result. - preprocess_hooks: Optional request preprocess hooks for online execution. - postprocess_hooks: Optional response postprocess hooks for online execution. - - Returns: - A completed multi-metric result. - """ - rows = _prepare_rows(dataset, params, field_mapping) - metric_keys = unique_metric_keys(metrics) - prepared_metrics = [ - await prepare_metric_for_execution( - metric, - params=params, - model_resolver=self.model_resolver, - secret_resolver=self.secret_resolver, - ) - for metric in metrics - ] - metrics_built: list[tuple[str, Metric]] = list(zip(metric_keys, prepared_metrics, strict=True)) - if target is not None: - merged_preprocess_hooks, merged_postprocess_hooks = _merge_online_hooks( - params=params, - target=target, - preprocess_hooks=preprocess_hooks, - postprocess_hooks=postprocess_hooks, - ) - else: - merged_preprocess_hooks = tuple(preprocess_hooks or ()) - merged_postprocess_hooks = tuple(postprocess_hooks or ()) - return await sdk_evaluate_benchmark( - metrics=metrics_built, - rows=rows, - target=target, - params=params, - prompt_template=prompt_template, - preprocess_hooks=merged_preprocess_hooks, - postprocess_hooks=merged_postprocess_hooks, - aggregate_fields=aggregate_fields, - logger=log, - ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/benchmark_execution.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/benchmark_execution.py deleted file mode 100644 index 7e5a335b46..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/benchmark_execution.py +++ /dev/null @@ -1,694 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Streaming benchmark execution pipeline for evaluator SDK runtime. - -Generates each sample once and fans it out to every metric worker so that -multi-metric online benchmarks do not duplicate target inference per metric. - -Pipeline shape: -1. Producer workers generate per-row samples and broadcast each sample - directly to every per-metric queue (``_run_producer_workers``). -2. Metric workers consume their queue and emit row-level metric results. -3. The orchestrator awaits one failure-propagating ``asyncio.TaskGroup`` and - assembles a :class:`BenchmarkEvaluationResult`. - -Shutdown signaling for metric workers flows through ``_put_pipeline_sentinels`` -with a cancellation-safe ``put_nowait`` fallback so that task-group cancellation -does not deadlock on a full per-metric queue. -""" - -from __future__ import annotations - -import asyncio -import logging -from collections.abc import Sequence -from dataclasses import dataclass -from logging import getLogger -from types import MappingProxyType -from typing import Any, Protocol, cast - -import httpx -from nemo_platform.beta.evaluator.agent_inference import AgentInferenceFn, new_agent_inference_client -from nemo_platform.beta.evaluator.execution.config import fail_fast_from_params -from nemo_platform.beta.evaluator.execution.metric_execution import ( - generate_online_sample, - generate_online_sample_agent, -) -from nemo_platform.beta.evaluator.execution.samples import build_metric_input, build_offline_sample -from nemo_platform.beta.evaluator.execution.scoring import ( - corpus_output_spec, - nan_metric_result, -) -from nemo_platform.beta.evaluator.execution.values import EvaluationError, EvaluationPhase -from nemo_platform.beta.evaluator.inference import ( - InferenceFn, - PostprocessResponse, - PreprocessRequest, - make_inference_request, - new_inference_client, - requests_log_var, -) -from nemo_platform.beta.evaluator.metrics.aggregation import ( - add_corpus_scores, - aggregate_metrics, - rubric_definitions_from_metric, -) -from nemo_platform.beta.evaluator.metrics.protocol import ( - CorpusMetric, - Metric, - MetricOutputSpec, - MetricResult, - validate_metric_result, -) -from nemo_platform.beta.evaluator.resilience.api import use_resilience_session -from nemo_platform.beta.evaluator.resilience.errors import first_failure_cause, iter_leaf_causes -from nemo_platform.beta.evaluator.values import ( - Agent, - AgentBase, - AggregatedMetricResult, - AggregateFieldName, - EvaluationResult, - MetricDiagnostic, - Model, - RowScore, - RunConfig, - RunConfigOnline, - RunConfigOnlineModel, -) -from nemo_platform.beta.evaluator.values.multi_metric_results import ( - BenchmarkEvaluationResult, - namespace_result, -) -from openai import AsyncOpenAI - -_log = getLogger(__name__) - -_QUEUE_END: object = object() - -BenchmarkParams = RunConfig | RunConfigOnline | RunConfigOnlineModel - - -class ProgressReporter(Protocol): - """Progress-callback surface used by the benchmark pipeline. - - The pipeline always invokes ``increment_work()`` with no positional - arguments. The protocol accepts an optional ``increment`` and an arbitrary - return type so richer service-side implementations (e.g., ones that batch - updates or return an HTTP response) satisfy the structural contract. - """ - - def increment_work(self, increment: int = 1, /) -> Any: - """Signal that ``increment`` units of work completed.""" - ... - - -@dataclass(frozen=True) -class _SampleEvent: - """One generated row sample ready to fan out to metric workers.""" - - row_index: int - item: MappingProxyType - sample: MappingProxyType - requests: list[dict] - - -@dataclass -class _MetricPipeline: - """Queue + result storage for one benchmark metric.""" - - metric_ref: str - metric: Metric - output_spec: list[MetricOutputSpec] - queue: asyncio.Queue - results: list[MetricResult | None] - - -def _normalize_metric_result(metric_result: MetricResult, expected_outputs: list[MetricOutputSpec]) -> MetricResult: - """Validate output names and normalize output ordering to the declared spec.""" - validated = validate_metric_result(metric_result, expected_outputs) - actual_outputs = {output.name: output for output in validated.outputs} - return MetricResult( - outputs=[actual_outputs[output.name] for output in expected_outputs], - diagnostics=validated.diagnostics, - ) - - -def _benchmark_error_from_exception(exc: BaseException) -> EvaluationError | None: - """Return the deterministic benchmark error leaf from an exception tree.""" - errors = [leaf for leaf in iter_leaf_causes(exc) if isinstance(leaf, EvaluationError)] - if not errors: - return None - return min(errors, key=lambda error: (error.index, error.metric_key or "")) - - -def _initialize_row_scores(items: list[dict]) -> list[RowScore]: - """Create deterministic per-row output slots aligned with ``items``.""" - return [RowScore(row_index=idx, item=item, sample={}, metrics={}, requests=[]) for idx, item in enumerate(items)] - - -def _build_metric_pipelines( - metrics: Sequence[tuple[str, Metric]], - *, - item_count: int, - queue_capacity: int, -) -> list[_MetricPipeline]: - """Attach one bounded queue per pre-built metric. - - Callers pass an ordered ``(metric_ref, metric)`` sequence so that service - and SDK callers can build concrete metrics with their own factories before - entering the pipeline. - """ - pipelines: list[_MetricPipeline] = [] - for metric_ref, metric in metrics: - output_spec = list(metric.output_spec()) - if not output_spec: - raise RuntimeError(f"Metric '{metric_ref}' does not declare any outputs") - pipelines.append( - _MetricPipeline( - metric_ref=metric_ref, - metric=metric, - output_spec=output_spec, - queue=asyncio.Queue(maxsize=queue_capacity), - results=[None] * item_count, - ) - ) - return pipelines - - -def _finalize_row_request_logs( - *, - row_scores: list[RowScore], - row_metric_requests: list[dict[str, list[dict]]], - metric_refs_in_order: list[str], -) -> None: - """Append metric requests to each row in deterministic metric order.""" - for row_idx, row_score in enumerate(row_scores): - by_metric = row_metric_requests[row_idx] - for metric_ref in metric_refs_in_order: - row_score.requests.extend(by_metric.get(metric_ref, [])) - - -def _metric_errors_for_ref(metric_errors: dict[str, str] | None, metric_ref: str) -> dict[str, str] | None: - """Return the metric-specific error payload for a per-metric row. - - Combined benchmark rows can contain errors for multiple metric refs. When - building an individual metric result, keep only that metric's error so - per-metric exports and summaries report the same failed row status without - leaking sibling metric failures. - """ - if not metric_errors: - return None - error = metric_errors.get(metric_ref) - if error is None: - return None - return {metric_ref: error} - - -def _metric_diagnostics_for_ref( - metric_diagnostics: dict[str, list[MetricDiagnostic]] | None, metric_ref: str -) -> dict[str, list[MetricDiagnostic]] | None: - """Return the metric-specific diagnostics payload for a per-metric row.""" - if not metric_diagnostics: - return None - diagnostics = metric_diagnostics.get(metric_ref) - if diagnostics is None: - return None - return {metric_ref: diagnostics} - - -async def _finalize_benchmark_metric_result( - *, - metric: Metric, - results: Sequence[MetricResult | None], - row_scores: list[RowScore], -) -> EvaluationResult: - """Build one metric's benchmark result while preserving NaN fallback rows. - - Benchmark lenient failures materialize as NaN metric results and - must remain in aggregate statistics. Corpus-level scoring is different: it - should only see successful rows so failed rows with empty samples do not - skew corpus metrics. - """ - output_spec = metric.output_spec() - metric_results = [result for result in results if result is not None] - rubric_definitions = rubric_definitions_from_metric(metric) - if rubric_definitions: - aggregated = aggregate_metrics(metric_results, output_spec, rubric_definitions=rubric_definitions) - else: - aggregated = aggregate_metrics(metric_results, output_spec) - if isinstance(metric, CorpusMetric): - # Ignored sample-generation failures intentionally keep metric_errors - # empty to match the previous service benchmark row artifacts, so - # exclude their placeholder samples from corpus scoring explicitly. - corpus_rows = [ - row_score - for row_score in row_scores - if not row_score.metric_errors and "inference_error" not in row_score.sample - ] - if corpus_rows: - corpus_result = await metric.compute_corpus_scores( - inputs=[ - build_metric_input(row_score.item, row_score.sample, row_score.row_index) - for row_score in corpus_rows - ], - ) - if corpus_result is not None: - add_corpus_scores(aggregated, corpus_result, corpus_output_spec(metric, output_spec)) - return EvaluationResult(row_scores=row_scores, aggregate_scores=aggregated) - - -async def _put_pipeline_sentinels( - *, - pipelines: list[_MetricPipeline], - worker_count: int, -) -> None: - """Signal metric workers to stop by pushing one sentinel per worker. - - During TaskGroup cancellation, awaiting on a full queue can deadlock - shutdown. Best-effort sentinel insertion is sufficient because sibling - tasks are being cancelled anyway. - """ - current = asyncio.current_task() - cancelling = current is not None and current.cancelling() > 0 - for pipeline in pipelines: - for _ in range(worker_count): - if cancelling: - try: - pipeline.queue.put_nowait(_QUEUE_END) - except asyncio.QueueFull: - pass - else: - await pipeline.queue.put(_QUEUE_END) - - -async def _run_producer_workers( - *, - items: list[dict], - target: Model | Agent | None, - inference_fn: InferenceFn | AgentInferenceFn | None, - client: AsyncOpenAI | httpx.AsyncClient | None = None, - params: BenchmarkParams, - prompt_template: str | dict[str, Any] | None, - row_scores: list[RowScore], - pipelines: list[_MetricPipeline], - worker_count: int, - default_headers: dict[str, str] | None, - preprocess_hooks: Sequence[PreprocessRequest], - postprocess_hooks: Sequence[PostprocessResponse], - logger: logging.Logger, -) -> None: - """Generate sample events concurrently and broadcast to every metric queue. - - Each worker pops a row index, runs inference for that row, writes the - row's sample and request log into ``row_scores``, then pushes the resulting - event onto every metric pipeline queue. The only backpressure point is the - per-metric queues, whose shutdown path is cancellation-safe via - :func:`_put_pipeline_sentinels`. - """ - index_queue: asyncio.Queue[int] = asyncio.Queue() - for idx in range(len(items)): - index_queue.put_nowait(idx) - - is_online = target is not None - tolerate_failure = not fail_fast_from_params(params) - online_params = params if isinstance(params, RunConfigOnline) else None - online_model_params = params if isinstance(params, RunConfigOnlineModel) else None - - async def _produce_worker() -> None: - while True: - try: - idx = index_queue.get_nowait() - except asyncio.QueueEmpty: - return - - item = items[idx] - requests_log: list[dict] = [] - requests_log_var.set(requests_log) - try: - if is_online: - assert target is not None - if prompt_template is None: - raise ValueError("prompt_template is required for online benchmark evaluation") - if isinstance(target, AgentBase): - agent_target = cast(Agent, target) - sample = await generate_online_sample_agent( - agent=agent_target, - row=item, - index=idx, - prompt_template=prompt_template, - params=online_params, - preprocess_hooks=preprocess_hooks, - postprocess_hooks=postprocess_hooks, - agent_inference_fn=cast(AgentInferenceFn | None, inference_fn), - client=cast(httpx.AsyncClient | None, client), - default_headers=default_headers, - ) - else: - model_inference_fn = ( - cast(InferenceFn, inference_fn) if inference_fn is not None else make_inference_request - ) - sample = await generate_online_sample( - target=target, - row=item, - index=idx, - prompt_template=prompt_template, - params=online_model_params, - preprocess_hooks=preprocess_hooks, - postprocess_hooks=postprocess_hooks, - inference_fn=model_inference_fn, - client=cast(AsyncOpenAI | None, client), - default_headers=default_headers, - ) - else: - sample = build_offline_sample(item) - # TODO: Consider matching ComputeMetricPipeline by applying - # offline postprocess hooks after reconciling service - # progress tracking so sample hooks do not double count. - except Exception as e: - if not tolerate_failure: - raise EvaluationError( - index=idx, - message=str(e), - phase=EvaluationPhase.SAMPLE_GENERATION, - ) from e - logger.warning( - "Online sample generation failed, marking row as NaN-eligible", - extra={"item_index": idx, "error": str(e)}, - ) - # TODO: Consider short-circuiting metric workers for ignored - # sample-generation failures instead of forwarding the - # inference_error placeholder sample to each metric. - sample = {"output_text": None, "response": {}, "inference_error": str(e)} - finally: - index_queue.task_done() - - row_scores[idx].sample = dict(sample) - row_scores[idx].requests.extend(requests_log) - - event = _SampleEvent( - row_index=idx, - item=MappingProxyType(item), - sample=MappingProxyType(sample), - requests=requests_log, - ) - for pipeline in pipelines: - await pipeline.queue.put(event) - - producer_workers = min(worker_count, len(items)) - async with asyncio.TaskGroup() as producer_tg: - for _ in range(producer_workers): - producer_tg.create_task(_produce_worker()) - - -async def _metric_worker( - *, - params: BenchmarkParams, - pipeline: _MetricPipeline, - row_scores: list[RowScore], - row_metric_requests: list[dict[str, list[dict]]], - logger: logging.Logger, - progress: ProgressReporter | None = None, -) -> None: - """Consume one metric queue and compute row-level metric results.""" - tolerate_failure = not fail_fast_from_params(params) - while True: - queued_event = await pipeline.queue.get() - if queued_event is _QUEUE_END: - pipeline.queue.task_done() - return - if not isinstance(queued_event, _SampleEvent): - raise ValueError(f"Expected _SampleEvent, got: {type(queued_event).__name__}") - - event = queued_event - requests_log: list[dict] = [] - requests_log_var.set(requests_log) - try: - metric_result = _normalize_metric_result( - await pipeline.metric.compute_scores( - build_metric_input(dict(event.item), dict(event.sample), event.row_index) - ), - pipeline.output_spec, - ) - except Exception as e: - if not tolerate_failure: - raise EvaluationError( - index=event.row_index, - message=str(e), - phase=EvaluationPhase.METRIC_SCORING, - metric_key=pipeline.metric_ref, - ) from e - error_message = str(e) - logger.warning( - "Evaluation failed, marking as NaN", - extra={"metric_ref": pipeline.metric_ref, "item_index": event.row_index, "error": error_message}, - ) - metric_result = nan_metric_result(pipeline.output_spec) - # Record the swallowed metric exception on the row while keeping - # the NaN score result. Example: if the "judge" metric raises - # "bad output", the row gets metric_errors={"judge": "bad output"}. - metric_errors = row_scores[event.row_index].metric_errors or {} - metric_errors[pipeline.metric_ref] = error_message - row_scores[event.row_index].metric_errors = metric_errors - finally: - row_metric_requests[event.row_index][pipeline.metric_ref] = list(requests_log) - pipeline.queue.task_done() - - pipeline.results[event.row_index] = metric_result - row_scores[event.row_index].metrics[pipeline.metric_ref] = metric_result.outputs - if metric_result.diagnostics: - row_diagnostics = row_scores[event.row_index].metric_diagnostics or {} - row_diagnostics[pipeline.metric_ref] = metric_result.diagnostics - row_scores[event.row_index].metric_diagnostics = row_diagnostics - if progress is not None: - progress.increment_work() - - -async def _run_streaming_pipeline( - *, - items: list[dict], - target: Model | Agent | None, - inference_fn: InferenceFn | AgentInferenceFn | None, - client: AsyncOpenAI | httpx.AsyncClient | None = None, - params: BenchmarkParams, - prompt_template: str | dict[str, Any] | None, - row_scores: list[RowScore], - pipelines: list[_MetricPipeline], - row_metric_requests: list[dict[str, list[dict]]], - worker_count: int, - default_headers: dict[str, str] | None, - preprocess_hooks: Sequence[PreprocessRequest], - postprocess_hooks: Sequence[PostprocessResponse], - progress: ProgressReporter | None, - logger: logging.Logger, -) -> None: - """Run producer + metric consumers under one failure-propagating task group. - - Producers broadcast directly to every per-metric queue, so the only - shutdown-signaling path goes through :func:`_put_pipeline_sentinels`, which - is cancellation-safe. A producer only blocks on ``pipeline.queue.put`` when - that specific metric queue is full, which surfaces backpressure one stage - earlier than a buffered intermediate queue would. - """ - - async def _produce_and_signal() -> None: - try: - await _run_producer_workers( - items=items, - target=target, - inference_fn=inference_fn, - client=client, - params=params, - prompt_template=prompt_template, - row_scores=row_scores, - pipelines=pipelines, - worker_count=worker_count, - default_headers=default_headers, - preprocess_hooks=preprocess_hooks, - postprocess_hooks=postprocess_hooks, - logger=logger, - ) - finally: - await _put_pipeline_sentinels(pipelines=pipelines, worker_count=worker_count) - - try: - async with asyncio.TaskGroup() as tg: - for pipeline in pipelines: - for _ in range(worker_count): - tg.create_task( - _metric_worker( - params=params, - pipeline=pipeline, - row_scores=row_scores, - row_metric_requests=row_metric_requests, - logger=logger, - progress=progress, - ) - ) - tg.create_task(_produce_and_signal()) - except BaseException as exc: - root = first_failure_cause(exc) - logger.error( - "Benchmark streaming pipeline failed", - extra={ - "root_error_type": type(root).__name__, - "root_error": str(root), - "raw_error_type": type(exc).__name__, - }, - ) - raise - - -async def evaluate_benchmark( - *, - metrics: Sequence[tuple[str, Metric]], - rows: list[dict], - target: Model | Agent | None = None, - inference_fn: InferenceFn | AgentInferenceFn | None = None, - params: BenchmarkParams, - prompt_template: str | dict[str, Any] | None = None, - preprocess_hooks: Sequence[PreprocessRequest] = (), - postprocess_hooks: Sequence[PostprocessResponse] = (), - default_headers: dict[str, str] | None = None, - progress: ProgressReporter | None = None, - aggregate_fields: tuple[AggregateFieldName, ...] | None = None, - logger: logging.Logger | None = None, -) -> BenchmarkEvaluationResult: - """Run one benchmark evaluation using the shared streaming pipeline. - - Each dataset row runs inference exactly once regardless of how many metrics - are configured; each sample is fanned out to every metric worker through a - bounded per-metric queue. Row-level failures (inference *or* metric scoring) - are mapped to NaN only when ``params`` is an online params object with - ``ignore_request_failure=True``; otherwise row failures abort the run. - - Args: - metrics: Ordered ``(metric_ref, metric)`` tuples. ``metric_ref`` is the - public identifier used to namespace aggregate score names. - rows: Dataset rows to evaluate. - target: Model or agent used for online inference. Pass ``None`` for - offline benchmarks; metric workers then receive the offline sample - built from each row. - inference_fn: Optional inference callable. Defaults to SDK's - ``make_inference_request`` / ``make_agent_inference_request`` based - on the ``target`` type. - params: Task-level execution parameters. Online-only fields - (``ignore_request_failure``, ``request_timeout``, ``max_retries``) - are read only when ``params`` is an :class:`RunConfigOnline`. - prompt_template: Jinja template used to render per-row requests; required - when ``target`` is set. - preprocess_hooks: Request preprocessors applied before each online - inference call. - postprocess_hooks: Response postprocessors applied after each online - inference call. - default_headers: Optional headers appended to every online target request - (mirrors :class:`ComputeMetricPipeline.default_headers`). - progress: Optional progress reporter; ``increment_work`` fires once per - metric/row result. - aggregate_fields: Optional aggregate score fields to keep in the returned result. - logger: Optional logger for pipeline-level warnings and errors. - - Returns: - A :class:`BenchmarkEvaluationResult` combining per-row scores, per-metric - aggregates (with ``metric_ref.`` namespaced score names), and a flattened - top-level aggregate view. - - Raises: - ValueError: If ``metrics`` is empty. - EvaluationError: If a row fails during strict benchmark - execution (``fail_fast=True``). - RuntimeError: If any metric result slot is missing after pipeline - completion (an internal invariant check). - """ - log = logger or _log - if not metrics: - raise ValueError("metrics must contain at least one (metric_ref, metric) tuple") - - queue_capacity = max(1, params.parallelism * 2) - worker_count = min(len(rows), max(1, params.parallelism)) if rows else max(1, params.parallelism) - - row_scores = _initialize_row_scores(rows) - row_metric_requests: list[dict[str, list[dict]]] = [dict() for _ in range(len(rows))] - pipelines = _build_metric_pipelines(metrics, item_count=len(rows), queue_capacity=queue_capacity) - - client = None - client_close_fn = None - if isinstance(target, Model): - client = new_inference_client(target) - client_close_fn = client.close - elif isinstance(target, AgentBase): - client = new_agent_inference_client() - client_close_fn = client.aclose - - async with use_resilience_session(): - try: - await _run_streaming_pipeline( - items=rows, - target=target, - inference_fn=inference_fn, - client=client, - params=params, - prompt_template=prompt_template, - row_scores=row_scores, - pipelines=pipelines, - row_metric_requests=row_metric_requests, - worker_count=worker_count, - default_headers=default_headers, - preprocess_hooks=preprocess_hooks, - postprocess_hooks=postprocess_hooks, - progress=progress, - logger=log, - ) - except Exception as exc: - if benchmark_error := _benchmark_error_from_exception(exc): - # TaskGroup wraps worker failures in ExceptionGroup. Re-raise - # the typed SDK error as the public exception while preserving - # the original row-level failure as its cause. - raise benchmark_error from benchmark_error.__cause__ - raise - - if client_close_fn: - await client_close_fn() - - row_generation_requests = [list(row.requests) for row in row_scores] - - _finalize_row_request_logs( - row_scores=row_scores, - row_metric_requests=row_metric_requests, - metric_refs_in_order=[metric_ref for metric_ref, _ in metrics], - ) - - per_metric: dict[str, EvaluationResult] = {} - for pipeline in pipelines: - if any(r is None for r in pipeline.results): - raise RuntimeError(f"Internal error: missing metric results for '{pipeline.metric_ref}'") - completed_rows = [ - RowScore( - row_index=row.row_index, - item=row.item, - sample=row.sample, - metrics={pipeline.metric_ref: row.metrics.get(pipeline.metric_ref, [])}, - requests=[ - *row_generation_requests[row_idx], - *row_metric_requests[row_idx].get(pipeline.metric_ref, []), - ], - metric_errors=_metric_errors_for_ref(row.metric_errors, pipeline.metric_ref), - metric_diagnostics=_metric_diagnostics_for_ref(row.metric_diagnostics, pipeline.metric_ref), - ) - for row_idx, row in enumerate(row_scores) - ] - raw_result = await _finalize_benchmark_metric_result( - metric=pipeline.metric, - results=pipeline.results, - row_scores=completed_rows, - ) - per_metric[pipeline.metric_ref] = namespace_result(pipeline.metric_ref, raw_result, aggregate_fields) - - top_aggregate_scores = AggregatedMetricResult( - scores=[score for result in per_metric.values() for score in result.aggregate_scores.scores] - ) - return BenchmarkEvaluationResult( - row_scores=row_scores, - aggregate_scores=top_aggregate_scores, - per_metric=per_metric, - ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/config.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/config.py deleted file mode 100644 index ec24d763c2..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/config.py +++ /dev/null @@ -1,51 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Public configuration types for the v4 evaluator API.""" - -from __future__ import annotations - -from typing import TypeAlias - -from nemo_platform.beta.evaluator.values import ( - Agent, - AgentBase, - Model, - RunConfig, - RunConfigOnline, - RunConfigOnlineModel, -) - -_RunConfigT: TypeAlias = RunConfig | RunConfigOnline | RunConfigOnlineModel - - -def resolve_params( - params: _RunConfigT | None = None, - target: Model | Agent | None = None, -) -> _RunConfigT: - """Return params after validating that they match the selected target mode.""" - if isinstance(target, Model): - if params is None or type(params) is RunConfig: - raise TypeError("model target requires RunConfigOnlineModel") - if type(params) is RunConfigOnline: - return RunConfigOnlineModel.model_validate(params.model_dump()) - if not isinstance(params, RunConfigOnlineModel): - raise TypeError("model target requires RunConfigOnlineModel") - return params - if isinstance(target, AgentBase): - if type(params) is not RunConfigOnline: - raise TypeError("agent target requires RunConfigOnline") - return params - if params is None: - return RunConfig() - if type(params) is not RunConfig: - raise TypeError("offline evaluation requires RunConfig") - return params - - -def fail_fast_from_params(params: _RunConfigT) -> bool: - """ - Return whether row failures should abort execution for the given params. - When params is not an online params, return fail_fast is True - always fail fast. - """ - return not (isinstance(params, RunConfigOnline) and params.ignore_request_failure) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/evaluator.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/evaluator.py deleted file mode 100644 index 197d22acd6..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/evaluator.py +++ /dev/null @@ -1,316 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Public evaluator entrypoint for completed-result execution.""" - -from __future__ import annotations - -import asyncio -import inspect -from collections.abc import Sequence -from pathlib import Path -from typing import Any, TypeGuard, overload - -import nemo_platform.beta.evaluator.inference as inference -from nemo_platform.beta.evaluator.execution.metric_execution import run_sync -from nemo_platform.beta.evaluator.metrics.protocol import Metric -from nemo_platform.beta.evaluator.values.agents import Agent -from nemo_platform.beta.evaluator.values.dataset_schemas import FieldMapping -from nemo_platform.beta.evaluator.values.datasets import DatasetInput -from nemo_platform.beta.evaluator.values.models import Model -from nemo_platform.beta.evaluator.values.multi_metric_results import BenchmarkEvaluationResult -from nemo_platform.beta.evaluator.values.params import RunConfig, RunConfigOnline, RunConfigOnlineModel -from nemo_platform.beta.evaluator.values.results import AggregateFieldName - -from .backends.base import BackendParams, EvaluationBackend, SyncEvaluationBackend -from .backends.local.backend import LocalBackend -from .config import resolve_params - -BackendClient = EvaluationBackend | SyncEvaluationBackend - - -def _validate_backend_client(client: BackendClient) -> None: - """Validate that a backend client exposes callable evaluator methods. - - Do not use runtime-checkable protocols for this check. ``EvaluationBackend`` - and ``SyncEvaluationBackend`` share method names, and runtime protocol - checks cannot distinguish async methods from sync methods. - - Args: - client: Backend client to validate. - - Raises: - TypeError: If the backend client does not expose the evaluator backend methods. - """ - if not callable(getattr(client, "evaluate_dataset", None)): - raise TypeError("client must provide a callable evaluate_dataset method") - - -def _is_async_backend(client: BackendClient) -> TypeGuard[EvaluationBackend]: - """Return whether the validated backend client exposes an async evaluator method.""" - return inspect.iscoroutinefunction(client.evaluate_dataset) - - -def _is_sync_backend(client: BackendClient) -> TypeGuard[SyncEvaluationBackend]: - """Return whether the validated backend client exposes a sync evaluator method.""" - return not inspect.iscoroutinefunction(client.evaluate_dataset) - - -class _SyncBackendAdapter: - """Expose a sync evaluator backend through the async backend contract.""" - - def __init__(self, backend: SyncEvaluationBackend) -> None: - """Store the sync backend to execute off the event loop.""" - self._backend = backend - - async def evaluate_dataset( - self, - *, - metrics: Sequence[Metric], - dataset: DatasetInput | str | Path, - params: BackendParams, - target: Model | Agent | None = None, - field_mapping: FieldMapping | None = None, - prompt_template: str | dict[str, Any] | None = None, - aggregate_fields: tuple[AggregateFieldName, ...] | None = None, - preprocess_hooks: tuple[inference.PreprocessRequest, ...] | None = None, - postprocess_hooks: tuple[inference.PostprocessResponse, ...] | None = None, - ) -> BenchmarkEvaluationResult: - """Evaluate multiple metrics by running the sync backend in a worker thread.""" - return await asyncio.to_thread( - self._backend.evaluate_dataset, - metrics=metrics, - dataset=dataset, - params=params, - target=target, - field_mapping=field_mapping, - prompt_template=prompt_template, - aggregate_fields=aggregate_fields, - preprocess_hooks=preprocess_hooks, - postprocess_hooks=postprocess_hooks, - ) - - -class Evaluator: - """Evaluator convenience API for backends that return completed results. - - ``Evaluator`` evaluates metrics locally by default. When constructed with an - evaluator backend object, it delegates completed-result execution to that - backend. Sync backends are adapted to the async backend contract. - - Examples: - Local evaluation uses `run` directly: - - ```python - evaluator = Evaluator() - result = await evaluator.run( - metrics=[ExactMatchMetric(reference="{{item.reference}}")], - dataset=[{"reference": "Paris", "output_text": "Paris"}], - ) - ``` - """ - - def __init__(self, client: BackendClient | None = None) -> None: - """Create an evaluator for completed-result backends. - - Args: - client: Optional evaluator backend. Async backends are used directly; - sync backends are adapted to the async backend contract. When - omitted, the evaluator runs metrics in-process via ``LocalBackend``. - """ - if client is None: - self._backend: EvaluationBackend = LocalBackend() - return - - _validate_backend_client(client) - - # One contract method, so its flavour decides: there is no mixed sync/async case left to - # reject. The final branch is unreachable and exists so the checker can narrow. - if _is_async_backend(client): - self._backend = client - elif _is_sync_backend(client): - self._backend = _SyncBackendAdapter(client) - else: # pragma: no cover - raise TypeError("client must provide a callable evaluate_dataset method") - - @overload - async def run( - self, - metrics: Sequence[Metric], - dataset: DatasetInput | str | Path, - *, - config: RunConfig | None = None, - target: None = None, - field_mapping: FieldMapping | None = None, - prompt_template: None = None, - aggregate_fields: tuple[AggregateFieldName, ...] | None = None, - preprocess_hooks: Sequence[inference.PreprocessRequest] | None = None, - postprocess_hooks: Sequence[inference.PostprocessResponse] | None = None, - ) -> BenchmarkEvaluationResult: ... - - @overload - async def run( - self, - metrics: Sequence[Metric], - dataset: DatasetInput | str | Path, - *, - config: RunConfigOnlineModel, - target: Model, - field_mapping: FieldMapping | None = None, - prompt_template: str | dict[str, Any] | None = None, - aggregate_fields: tuple[AggregateFieldName, ...] | None = None, - preprocess_hooks: Sequence[inference.PreprocessRequest] | None = None, - postprocess_hooks: Sequence[inference.PostprocessResponse] | None = None, - ) -> BenchmarkEvaluationResult: ... - - @overload - async def run( - self, - metrics: Sequence[Metric], - dataset: DatasetInput | str | Path, - *, - config: RunConfigOnline, - target: Agent, - field_mapping: FieldMapping | None = None, - prompt_template: str | dict[str, Any], - aggregate_fields: tuple[AggregateFieldName, ...] | None = None, - preprocess_hooks: Sequence[inference.PreprocessRequest] | None = None, - postprocess_hooks: Sequence[inference.PostprocessResponse] | None = None, - ) -> BenchmarkEvaluationResult: ... - - async def run( - self, - metrics: Sequence[Metric], - dataset: DatasetInput | str | Path, - *, - config: RunConfig | RunConfigOnline | RunConfigOnlineModel | None = None, - target: Model | Agent | None = None, - field_mapping: FieldMapping | None = None, - prompt_template: str | dict[str, Any] | None = None, - aggregate_fields: tuple[AggregateFieldName, ...] | None = None, - preprocess_hooks: Sequence[inference.PreprocessRequest] | None = None, - postprocess_hooks: Sequence[inference.PostprocessResponse] | None = None, - ) -> BenchmarkEvaluationResult: - """Evaluate metrics and return the finished result. - - Args: - metrics: Metrics to execute together over each dataset row. - dataset: Inline dataset rows, a dataset file, or a dataset directory/glob path. - config: Optional run-level execution configuration. Offline calls default to ``RunConfig``. - target: Optional model or agent used for online generation. Omit for offline scoring. - field_mapping: Optional mapping from canonical evaluator fields to dataset columns. - prompt_template: Optional prompt template to use for online target generation. - aggregate_fields: Optional aggregate score fields to keep in the returned result. - preprocess_hooks: Optional request preprocess hooks for online execution. - postprocess_hooks: Optional response postprocess hooks for online execution. - - Returns: - The completed multi-metric result. - """ - params = resolve_params(config, target) - normalized_preprocess_hooks = tuple(preprocess_hooks) if preprocess_hooks is not None else None - normalized_postprocess_hooks = tuple(postprocess_hooks) if postprocess_hooks is not None else None - return await self._backend.evaluate_dataset( - metrics=list(metrics), - dataset=dataset, - params=params, - target=target, - field_mapping=field_mapping, - prompt_template=prompt_template, - aggregate_fields=aggregate_fields, - preprocess_hooks=normalized_preprocess_hooks, - postprocess_hooks=normalized_postprocess_hooks, - ) - - @overload - def run_sync( - self, - metrics: Sequence[Metric], - dataset: DatasetInput | str | Path, - *, - config: RunConfig | None = None, - target: None = None, - field_mapping: FieldMapping | None = None, - prompt_template: None = None, - aggregate_fields: tuple[AggregateFieldName, ...] | None = None, - preprocess_hooks: Sequence[inference.PreprocessRequest] | None = None, - postprocess_hooks: Sequence[inference.PostprocessResponse] | None = None, - ) -> BenchmarkEvaluationResult: ... - - @overload - def run_sync( - self, - metrics: Sequence[Metric], - dataset: DatasetInput | str | Path, - *, - config: RunConfigOnlineModel, - target: Model, - field_mapping: FieldMapping | None = None, - prompt_template: str | dict[str, Any] | None = None, - aggregate_fields: tuple[AggregateFieldName, ...] | None = None, - preprocess_hooks: Sequence[inference.PreprocessRequest] | None = None, - postprocess_hooks: Sequence[inference.PostprocessResponse] | None = None, - ) -> BenchmarkEvaluationResult: ... - - @overload - def run_sync( - self, - metrics: Sequence[Metric], - dataset: DatasetInput | str | Path, - *, - config: RunConfigOnline, - target: Agent, - field_mapping: FieldMapping | None = None, - prompt_template: str | dict[str, Any], - aggregate_fields: tuple[AggregateFieldName, ...] | None = None, - preprocess_hooks: Sequence[inference.PreprocessRequest] | None = None, - postprocess_hooks: Sequence[inference.PostprocessResponse] | None = None, - ) -> BenchmarkEvaluationResult: ... - - def run_sync( - self, - metrics: Sequence[Metric], - dataset: DatasetInput | str | Path, - *, - config: RunConfig | RunConfigOnline | RunConfigOnlineModel | None = None, - target: Model | Agent | None = None, - field_mapping: FieldMapping | None = None, - prompt_template: str | dict[str, Any] | None = None, - aggregate_fields: tuple[AggregateFieldName, ...] | None = None, - preprocess_hooks: Sequence[inference.PreprocessRequest] | None = None, - postprocess_hooks: Sequence[inference.PostprocessResponse] | None = None, - ) -> BenchmarkEvaluationResult: - """Synchronously evaluate metrics and return the finished result. - - Args: - metrics: Metrics to execute together over each dataset row. - dataset: Inline dataset rows, a dataset file, or a dataset directory/glob path. - config: Optional run-level execution configuration. Offline calls default to ``RunConfig``. - target: Optional model or agent used for online generation. Omit for offline scoring. - field_mapping: Optional mapping from canonical evaluator fields to dataset columns. - prompt_template: Optional prompt template to use for online target generation. - aggregate_fields: Optional aggregate score fields to keep in the returned result. - preprocess_hooks: Optional request preprocess hooks for online execution. - postprocess_hooks: Optional response postprocess hooks for online execution. - - Returns: - The completed multi-metric result. - """ - - async def _call() -> BenchmarkEvaluationResult: - params = resolve_params(config, target) - normalized_preprocess_hooks = tuple(preprocess_hooks) if preprocess_hooks is not None else None - normalized_postprocess_hooks = tuple(postprocess_hooks) if postprocess_hooks is not None else None - return await self._backend.evaluate_dataset( - metrics=list(metrics), - dataset=dataset, - params=params, - target=target, - field_mapping=field_mapping, - prompt_template=prompt_template, - aggregate_fields=aggregate_fields, - preprocess_hooks=normalized_preprocess_hooks, - postprocess_hooks=normalized_postprocess_hooks, - ) - - return run_sync(_call) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/job_poll.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/job_poll.py deleted file mode 100644 index 7122a263bc..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/job_poll.py +++ /dev/null @@ -1,98 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Async job polling aligned with ``nmp.testing.e2e.jobs.poll_until_terminal``. - -The E2E helper lives in ``nmp_testing``; the evaluator SDK cannot depend on that -package at runtime, so this module mirrors its timeout semantics (pending time -excluded from the job timeout, separate image-pull cap). -""" - -from __future__ import annotations - -import asyncio -import json -import logging -import time -from collections.abc import Awaitable, Callable, Mapping -from typing import TypeVar - -log = logging.getLogger(__name__) - -_StatusT = TypeVar("_StatusT") - - -async def _async_pause(seconds: float) -> None: - await asyncio.sleep(seconds) - - -async def async_poll_until_terminal( - get_status: Callable[[], Awaitable[_StatusT]], - *, - status_value: Callable[[_StatusT], str], - details_value: Callable[[_StatusT], Mapping[str, object] | None] | None = None, - job_name: str, - terminal: frozenset[str], - timeout: float, - pending_timeout: float, - poll_interval: float, -) -> _StatusT: - """Poll *get_status* until it returns a value in *terminal* or a timeout fires. - - *status_value* must return a **lowercase** status string for each response - returned by *get_status*. - - *details_value* may return status details to include as a nested JSON object - in each progress log line. - - Time spent in ``pending`` status is not counted against *timeout*; it is - instead capped by the separate *pending_timeout*. - - Returns: - The terminal response returned by *get_status*. - - Raises: - TimeoutError: When *timeout* is exceeded (excluding pending time) or - *pending_timeout* is exceeded while in pending status. - """ - elapsed = 0.0 - pending_elapsed = 0.0 - - while True: - poll_start = time.monotonic() - status_response = await get_status() - status = status_value(status_response) - log_payload: dict[str, object] = { - "job_name": job_name, - "status": status or "", - "elapsed_s": round(elapsed, 1), - "poll_interval_s": poll_interval, - } - if status == "pending": - log_payload["pending_elapsed_s"] = round(pending_elapsed, 1) - if details_value is not None: - details = details_value(status_response) - if details: - log_payload["status_details"] = dict(details) - log.info(json.dumps(log_payload, separators=(",", ":"))) - - if status in terminal: - return status_response - - poll_duration = time.monotonic() - poll_start - if status == "pending": - pending_elapsed += poll_duration - if pending_elapsed >= pending_timeout: - raise TimeoutError(f"'{job_name}' stuck in pending after {pending_timeout}s.") - else: - elapsed += poll_duration - if elapsed >= timeout: - raise TimeoutError(f"'{job_name}' timed out after {timeout}s. Status: {status}") - - sleep_start = time.monotonic() - await _async_pause(poll_interval) - sleep_duration = time.monotonic() - sleep_start - if status == "pending": - pending_elapsed += sleep_duration - else: - elapsed += sleep_duration diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/metric_execution.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/metric_execution.py deleted file mode 100644 index 62d2b336bc..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/metric_execution.py +++ /dev/null @@ -1,901 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Metric evaluation orchestration for evaluator SDK runtime.""" - -# ruff: noqa: I001 - the vendored SDK mirror uses different import-order settings. - -from __future__ import annotations - -import asyncio -import json -import threading -from collections.abc import Callable, Coroutine, Sequence -from functools import partial -from logging import getLogger -from types import MappingProxyType -from typing import Any, TypeVar, cast, overload -from urllib.parse import urlparse - -import httpx -import nemo_platform.beta.evaluator.inference as inference -from nemo_platform.beta.evaluator.agent_inference import ( - AgentInferenceFn, - AgentInvocationResult, - invoke_agent, - make_agent_inference_request, - new_agent_inference_client, -) -from nemo_platform.beta.evaluator.enums import ModelFormat -from nemo_platform.beta.evaluator.execution.config import fail_fast_from_params, resolve_params -from nemo_platform.beta.evaluator.execution.pipeline import ( - GeneratedSampleEvent, - GeneratedSampleScoringPipeline, - PipelineRuntime, -) -from nemo_platform.beta.evaluator.execution.samples import build_offline_sample -from nemo_platform.beta.evaluator.execution.scoring import ( - empty_evaluation_result, - finalize_evaluation_result, - nan_metric_result, - score_row, -) -from nemo_platform.beta.evaluator.execution.values import EvaluationError, EvaluationPhase -from nemo_platform.beta.evaluator.inference import InferenceMetricBase -from nemo_platform.beta.evaluator.metrics.protocol import ( - Metric, - MetricResult, -) -from nemo_platform.beta.evaluator.metrics.utils import metric_type_name -from nemo_platform.beta.evaluator.resilience.api import run_indexed_tasks, use_resilience_session -from nemo_platform.beta.evaluator.resilience.errors import get_evaluation_error -from nemo_platform.beta.evaluator.templates import render_request -from nemo_platform.beta.evaluator.values import ( - Agent, - AgentBase, - EvaluationResult, - Model, - RowScore, - RunConfig, - RunConfigOnline, - RunConfigOnlineModel, -) -from openai import AsyncOpenAI - -log = getLogger(__name__) -T = TypeVar("T") -_QUEUE_END = object() -_DATASET_INPUT_FAILURE_HINT = ( - "To prevent failure of evaluation, fix the dataset row or set " - "params.ignore_request_failure=true to skip invalid rows." -) -_INFERENCE_FAILURE_HINT = ( - "To prevent failure of evaluation from inference request failures, check the model endpoint, " - "credentials, request timeout, and retry settings, or set params.ignore_request_failure=true " - "to mark failed rows as NaN." -) - - -def _has_empty_message_content(row: dict[str, object]) -> bool: - """Return True when a row includes a chat message with empty string content.""" - messages = row.get("messages") - if not isinstance(messages, list): - return False - return any( - isinstance(message, dict) and cast(dict[str, object], message).get("content") == "" for message in messages - ) - - -def _has_empty_prompt(row: dict[str, object]) -> bool: - """Return True when a row includes an empty prompt string.""" - return row.get("prompt") == "" - - -def _format_exception_summary(error: Exception) -> str: - """Return a concise one-line cause summary for user-facing row errors.""" - cause = str(error).strip() - if not cause and error.__cause__ is not None: - cause = str(error.__cause__).strip() - if not cause: - cause = type(error).__name__ - return " ".join(cause.split()) - - -# --------------------------------------------------------------------------- -# Sync bridge -# --------------------------------------------------------------------------- - - -def run_sync(awaitable_factory: Callable[[], Coroutine[Any, Any, T]]) -> T: - """Run an async factory from synchronous code. - - This helper uses ``asyncio.run`` directly when there is no running event - loop. If a loop is already active (for example in notebook environments), - it runs the coroutine inside a dedicated thread to avoid nested-loop errors. - - Args: - awaitable_factory: Zero-argument callable that returns a coroutine. - - Returns: - The resolved coroutine result. - - Raises: - BaseException: Any exception raised by the coroutine. - """ - try: - asyncio.get_running_loop() - except RuntimeError: - has_running_loop = False - else: - has_running_loop = True - - if not has_running_loop: - return asyncio.run(awaitable_factory()) - - results: list[T] = [] - errors: list[BaseException] = [] - - def _runner() -> None: - """Execute the coroutine inside a thread-local event loop. - - Returns: - ``None``. Results and exceptions are captured in outer-scope lists. - """ - try: - # A separate thread gives notebook-style environments a fresh event loop - # without nesting asyncio.run() inside the caller's running loop. - results.append(asyncio.run(awaitable_factory())) - except BaseException as exc: # pragma: no cover - re-raised on caller thread - errors.append(exc) - - thread = threading.Thread(target=_runner, name="nemo-evaluator-sdk-sync-runner") - thread.start() - thread.join() - - if errors: - raise errors[0] - - return results[0] - - -# --------------------------------------------------------------------------- -# Online helpers -# --------------------------------------------------------------------------- - - -def _is_completions_endpoint(url: str) -> bool: - """Return whether the configured model URL targets completions rather than chat.""" - path = urlparse(url).path.rstrip("/") - return path.endswith("/completions") and not path.endswith("/chat/completions") - - -def _default_online_request_template(row: dict[str, Any], model: Model) -> dict: - """Pick the request template used for online sample generation if possible to infer from the row.""" - prompt_candidates = ("prompt", "input", "question", "query") - prompt_candidates_text = ", ".join(prompt_candidates) - inference_error = ( - "Unable to infer prompt template from row. " - f"Use a custom prompt_template or provide one of these row fields: {prompt_candidates_text}." - ) - - if _is_completions_endpoint(model.url): - for field_name in prompt_candidates: - if field_name in row: - return {"prompt": f"{{{{item.{field_name}}}}}"} - raise ValueError(inference_error) - - if "messages" in row: - return {"messages": "{{item.messages}}"} - for field_name in prompt_candidates: - if field_name in row: - return {"messages": [{"role": "user", "content": f"{{{{item.{field_name}}}}}"}]} - raise ValueError(inference_error) - - -def _resolve_online_prompt_template( - prompt_template: str | dict[str, Any] | None, - model: Model, - first_row: dict[str, Any], -) -> str | dict[str, Any]: - if prompt_template is not None: - return prompt_template - resolved = _default_online_request_template(first_row, model) - log.warning( - "No prompt_template provided for online evaluation. " - "Setting prompt_template is required when providing an online model for evaluation. " - "Making best effort to infer it from the first row.\n" - "Inferred prompt_template from the first row:\n%s", - json.dumps(resolved, indent=2), - extra={"prompt_template": resolved}, - ) - return resolved - - -def _merge_online_hooks( - *, - params: RunConfig | RunConfigOnline | RunConfigOnlineModel | None, - target: Model | Agent | None, - preprocess_hooks: Sequence[inference.PreprocessRequest] | None, - postprocess_hooks: Sequence[inference.PostprocessResponse] | None, -) -> tuple[list[inference.PreprocessRequest], list[inference.PostprocessResponse]]: - """Build deterministic hook lists for SDK local online generation. - - Online sample generation should only use run-level generation hooks. - Metric hooks belong to metric.compute_scores(input) and must not - affect the evaluated-model generation stage. - """ - - # build the default hooks shared by sdk and service. - # new_hooks() always returns at least the log hook in each list: - # preprocess -> [..., log_hook] - # postprocess -> [log_hook, ...] - built_preprocess_hooks, built_postprocess_hooks = inference.new_hooks( - params if isinstance(params, RunConfigOnlineModel) else None, - model_format=target.format if isinstance(target, Model) else None, - ) - if not built_preprocess_hooks or not built_postprocess_hooks: - raise ValueError( - f"inference.new_hooks() must return at least the log hook in each list. built_preprocess_hooks: {len(built_preprocess_hooks)}, built_postprocess_hooks: {len(built_postprocess_hooks)}" - ) - - built_preprocess_core = built_preprocess_hooks[:-1] - preprocess_log_hook = built_preprocess_hooks[-1] - postprocess_log_hook = built_postprocess_hooks[0] - built_postprocess_tail = built_postprocess_hooks[1:] - - # peels off the two log hooks, then splices in caller-supplied - # preprocess_hooks / postprocess_hooks in the middle - return ( - [ - *built_preprocess_core, - *(preprocess_hooks or ()), - preprocess_log_hook, - ], - [ - postprocess_log_hook, - *(postprocess_hooks or ()), - *built_postprocess_tail, - ], - ) - - -def _maybe_set_nim_default_max_tokens( - *, - request: dict[str, Any], - model: Model, - params: RunConfigOnlineModel | None, -) -> None: - """Apply the NIM max token default only when neither params nor request set one.""" - if model.format != ModelFormat.NVIDIA_NIM: - return - - inference_params = params.inference if isinstance(params, RunConfigOnlineModel) else None - if inference_params is not None and ( - inference_params.max_tokens is not None or inference_params.max_completion_tokens is not None - ): - return - if "max_tokens" in request or "max_completion_tokens" in request: - return - request["max_tokens"] = 4096 - - -def _process_online_response( - response: dict[str, Any], - *, - index: int, - postprocess_hooks: Sequence[inference.PostprocessResponse] | None, -) -> tuple[dict[str, Any], str | None]: - """Apply response hooks and extract model text output.""" - processed_response = response - for hook in postprocess_hooks or (): - processed_response = hook.postprocess(processed_response, id=str(index)) - output_text = inference.process_output(processed_response, hooks=[], id=str(index)) - return processed_response, output_text - - -# --------------------------------------------------------------------------- -# Sample generation -# --------------------------------------------------------------------------- - - -@overload -async def generate_online_sample( - *, - target: Model, - row: dict[str, Any], - index: int, - prompt_template: str | dict[str, Any], - params: RunConfigOnlineModel | None = None, - inference_fn: inference.InferenceFn, - client: AsyncOpenAI | None = None, - preprocess_hooks: Sequence[inference.PreprocessRequest] | None = None, - postprocess_hooks: Sequence[inference.PostprocessResponse] | None = None, - default_headers: dict[str, str] | None = None, - template_context: dict[str, Any] | None = None, -) -> dict[str, Any]: ... - - -@overload -async def generate_online_sample( - *, - target: Agent, - row: dict[str, Any], - index: int, - prompt_template: str | dict[str, Any], - params: RunConfigOnline | None = None, - inference_fn: AgentInferenceFn, - client: httpx.AsyncClient | None = None, - preprocess_hooks: Sequence[inference.PreprocessRequest] | None = None, - postprocess_hooks: Sequence[inference.PostprocessResponse] | None = None, - default_headers: dict[str, str] | None = None, - template_context: dict[str, Any] | None = None, -) -> dict[str, Any]: ... - - -async def generate_online_sample( - *, - target: Model | Agent, - row: dict[str, Any], - index: int, - prompt_template: str | dict[str, Any], - params: RunConfigOnline | RunConfigOnlineModel | None = None, - inference_fn: inference.InferenceFn | AgentInferenceFn, - client: AsyncOpenAI | httpx.AsyncClient | None = None, - preprocess_hooks: Sequence[inference.PreprocessRequest] | None = None, - postprocess_hooks: Sequence[inference.PostprocessResponse] | None = None, - default_headers: dict[str, str] | None = None, - template_context: dict[str, Any] | None = None, -) -> dict[str, Any]: - """Generate one sample payload using shared request rendering and inference logic. - - Two valid call shapes, pinned by the ``@overload``s above: - - - ``target: Model`` pairs with ``inference_fn: InferenceFn``. NIM - max-token defaults are applied and ``default_headers`` is forwarded - to the inference fn. - - ``target: Agent`` pairs with ``inference_fn: AgentInferenceFn``. - ``default_headers`` is forwarded to the inference fn. - """ - request = render_request(prompt_template, context={**row, "item": row, **(template_context or {})}) - if isinstance(target, Model): - model_params = params if isinstance(params, RunConfigOnlineModel) else None - _maybe_set_nim_default_max_tokens(request=request, model=target, params=model_params) - request = inference.preprocess_request(request, list(preprocess_hooks or ()), id=str(index)) - - max_retries = params.max_retries if params is not None else 3 - timeout = params.request_timeout if params is not None else None - - # ``InferenceFn`` and ``AgentInferenceFn`` are structurally identical at - # runtime (both expose only ``__call__``), so ``isinstance`` can't - # discriminate them. ``target`` is the real discriminator and the - # overloads statically pin the pairing — ``cast`` just records that. - if isinstance(target, Model): - model_fn = cast(inference.InferenceFn, inference_fn) - response = await model_fn( - target, - request, - max_retries, - client=cast(AsyncOpenAI | None, client), - default_headers=default_headers, - timeout=timeout, - ) - else: - agent_fn = cast(AgentInferenceFn, inference_fn) - response = await agent_fn( - target, - request, - client=cast(httpx.AsyncClient | None, client), - max_retries=max_retries, - default_headers=default_headers, - timeout=timeout, - ) - - if isinstance(response, AgentInvocationResult): - invocation = response - response_payload = response.response - else: - invocation = None - response_payload = cast(dict[str, Any], response) - processed_response, processed_output_text = _process_online_response( - response_payload, - index=index, - postprocess_hooks=postprocess_hooks, - ) - output_text = processed_output_text - if invocation is not None and not isinstance(output_text, str): - output_text = invocation.output_text - - sample: dict[str, Any] = {} - if output_text: - sample["output_text"] = output_text - if processed_response: - sample["response"] = processed_response - # Agent runtimes return trajectory information alongside the response; surface - # it at the top of the sample so metric evaluators can read it without digging - # through the nested response payload. - if isinstance(processed_response, dict) and "trajectory" in processed_response: - sample["trajectory"] = processed_response["trajectory"] - if invocation is not None: - sample["invocation_status"] = invocation.status.value - sample["invocation_metadata"] = invocation.metadata - if invocation.evidence is not None: - sample["evidence"] = invocation.evidence - return sample - - -async def generate_online_sample_agent( - *, - agent: Agent, - row: dict[str, Any], - index: int, - prompt_template: str | dict[str, Any], - params: RunConfigOnline | None = None, - preprocess_hooks: Sequence[inference.PreprocessRequest] | None = None, - postprocess_hooks: Sequence[inference.PostprocessResponse] | None = None, - agent_inference_fn: AgentInferenceFn | None = None, - client: httpx.AsyncClient | None = None, - default_headers: dict[str, str] | None = None, -) -> dict[str, Any]: - """Generate one agent sample through the unified online sample helper.""" - return await generate_online_sample( - target=agent, - row=row, - index=index, - prompt_template=prompt_template, - params=params, - inference_fn=agent_inference_fn or invoke_agent, - client=client, - preprocess_hooks=preprocess_hooks, - postprocess_hooks=postprocess_hooks, - default_headers=default_headers, - ) - - -# --------------------------------------------------------------------------- -# Concrete pipeline -# --------------------------------------------------------------------------- - - -class ComputeMetricPipeline: - """Pipeline configuration for row-based metric execution. - - Used by both online evaluation, where rows are turned into generated samples - before scoring, and offline evaluation, where rows are scored without model - inference. - - Overloaded constructors enforce the valid pairings between ``target`` and - ``inference_fn`` at type-check time: - - - ``target: Agent`` pairs with ``inference_fn: AgentInferenceFn``. - - ``target: Model`` pairs with ``inference_fn: inference.InferenceFn``. - - ``target: None`` is offline mode — no inference function is used. - - Attributes: - rows: Input dataset rows to evaluate. - parallelism: Maximum row-level worker fanout for the shared pipeline. - In online mode this limits concurrent sample generation and scorer - workers; in offline mode it still limits scorer fanout. - metric: Runtime metric implementation used to score each row. - target: Optional target used to generate per-row samples before scoring. - If None, the pipeline runs without inference and starts from the - offline sample built from the row. - metric_key: Metric identifier used for `RowScore.metrics` and for - synthesized NaN/error results. - prompt_template: Request template used to render online inference - requests. Required when `target` is set. - params: Evaluation parameters used for inference requests and failure policy. - inference_fn: Inference function used for online sample - generation. Must match ``target`` (see overloads); ``None`` is only - valid when ``target`` is ``None`` (offline). - default_headers: Optional default headers passed to online inference - requests. - preprocess_hooks: Hooks applied before online inference requests are - sent. - postprocess_hooks: Hooks applied after online inference responses are - received, and also to the offline sample when no target is configured. - """ - - rows: list[dict[str, Any]] - parallelism: int - metric: Metric - target: Model | Agent | None - metric_key: str - prompt_template: str | dict[str, Any] | None - params: RunConfig | RunConfigOnline | RunConfigOnlineModel - inference_fn: inference.InferenceFn | AgentInferenceFn | None - client: AsyncOpenAI | httpx.AsyncClient | None - default_headers: dict[str, str] | None - preprocess_hooks: list[inference.PreprocessRequest] - postprocess_hooks: list[inference.PostprocessResponse] - - @overload - def __init__( - self, - *, - rows: list[dict[str, Any]], - parallelism: int, - metric: Metric, - target: Agent, - metric_key: str, - prompt_template: str | dict[str, Any], - inference_fn: AgentInferenceFn, - client: httpx.AsyncClient | None = None, - params: RunConfigOnline, - default_headers: dict[str, str] | None = None, - preprocess_hooks: Sequence[inference.PreprocessRequest] | None = None, - postprocess_hooks: Sequence[inference.PostprocessResponse] | None = None, - ) -> None: ... - - @overload - def __init__( - self, - *, - rows: list[dict[str, Any]], - parallelism: int, - metric: Metric, - target: Model, - metric_key: str, - prompt_template: str | dict[str, Any], - inference_fn: inference.InferenceFn, - client: AsyncOpenAI | None = None, - params: RunConfigOnlineModel, - default_headers: dict[str, str] | None = None, - preprocess_hooks: Sequence[inference.PreprocessRequest] | None = None, - postprocess_hooks: Sequence[inference.PostprocessResponse] | None = None, - ) -> None: ... - - @overload - def __init__( - self, - *, - rows: list[dict[str, Any]], - parallelism: int, - metric: Metric, - target: None, - metric_key: str, - params: RunConfig, - prompt_template: None = None, - inference_fn: None = None, - client: None = None, - default_headers: None = None, - preprocess_hooks: Sequence[inference.PreprocessRequest] | None = None, - postprocess_hooks: Sequence[inference.PostprocessResponse] | None = None, - ) -> None: ... - - def __init__( - self, - *, - rows: list[dict[str, Any]], - parallelism: int, - metric: Metric, - target: Model | Agent | None, - metric_key: str, - prompt_template: str | dict[str, Any] | None = None, - params: RunConfig | RunConfigOnline | RunConfigOnlineModel, - inference_fn: inference.InferenceFn | AgentInferenceFn | None = None, - client: AsyncOpenAI | httpx.AsyncClient | None = None, - default_headers: dict[str, str] | None = None, - preprocess_hooks: Sequence[inference.PreprocessRequest] | None = None, - postprocess_hooks: Sequence[inference.PostprocessResponse] | None = None, - ) -> None: - self.rows = rows - self.parallelism = parallelism - self.metric = metric - self.target = target - self.metric_key = metric_key - self.prompt_template = prompt_template - self.params = params - self.inference_fn = inference_fn - self.client = client - self.default_headers = default_headers - self.preprocess_hooks = list(preprocess_hooks) if preprocess_hooks is not None else [] - self.postprocess_hooks = list(postprocess_hooks) if postprocess_hooks is not None else [] - - async def generate_sample(self, index: int, row: dict[str, Any]) -> dict[str, Any]: - """Generate the sample payload for one dataset row, including offline row-derived fields.""" - if self.target is None: - response = build_offline_sample(row) - for hook in self.postprocess_hooks or (): - response = hook.postprocess(response, id=f"{index}") - return response - - if self.prompt_template is None: - raise ValueError("prompt_template is required for service online evaluation") - - # The actual discriminator is ``self.target`` (Agent vs Model). The - # overloaded ``__init__`` pins the valid pairings statically, and this - # guard makes the Agent-side contract fail loudly at runtime if a - # caller bypasses those types. - if isinstance(self.target, AgentBase): - if self.inference_fn is None: - raise TypeError("expected AgentInferenceFn for Agent target") - - # Safe by the Agent↔AgentInferenceFn overload (see above). - agent_target = cast(Agent, self.target) - agent_fn = cast(AgentInferenceFn, self.inference_fn) - agent_params = self.params if isinstance(self.params, RunConfigOnline) else None - return await generate_online_sample( - target=agent_target, - row=row, - index=index, - prompt_template=self.prompt_template, - inference_fn=agent_fn, - client=cast(httpx.AsyncClient | None, self.client), - params=agent_params, - preprocess_hooks=self.preprocess_hooks, - postprocess_hooks=self.postprocess_hooks, - default_headers=self.default_headers, - ) - - model_params = self.params if isinstance(self.params, RunConfigOnlineModel) else None - model_fn: inference.InferenceFn = ( - # Safe by the Model↔InferenceFn overload (see above). - cast(inference.InferenceFn, self.inference_fn) - if self.inference_fn is not None - else inference.make_inference_request - ) - return await generate_online_sample( - target=self.target, - row=row, - index=index, - prompt_template=self.prompt_template, - inference_fn=model_fn, - client=cast(AsyncOpenAI | None, self.client), - params=model_params, - preprocess_hooks=self.preprocess_hooks, - postprocess_hooks=self.postprocess_hooks, - default_headers=self.default_headers, - ) - - def handle_generation_error( - self, - index: int, - row: dict[str, object], - error: Exception, - generation_requests: list[dict[str, object]], - ) -> tuple[int, MetricResult | None, RowScore]: - """Convert inference failures into NaN rows when the job allows ignoring them.""" - # Prefer row-derived root causes when we can identify them. Generic - # inference failures fall back to the original exception summary. - if _has_empty_message_content(row): - error_message = ( - f"Row {index} has empty message content and failed inference: " - f"{_format_exception_summary(error)}. {_DATASET_INPUT_FAILURE_HINT}" - ) - elif _has_empty_prompt(row): - error_message = ( - f"Row {index} has empty prompt and failed inference: " - f"{_format_exception_summary(error)}. {_DATASET_INPUT_FAILURE_HINT}" - ) - else: - error_message = ( - f"Row {index} failed inference: {_format_exception_summary(error)}. {_INFERENCE_FAILURE_HINT}" - ) - - if fail_fast_from_params(self.params): - raise EvaluationError( - index, - error_message, - phase=EvaluationPhase.SAMPLE_GENERATION, - metric_key=self.metric_key, - ) from error - - log.warning("Inference failed, marking as NaN", extra={"item_index": index, "error": error_message}) - sample = {"output_text": None, "response": {}, "inference_error": error_message} - nan_result = nan_metric_result(self.metric.output_spec()) - - return ( - index, - nan_result, - RowScore( - row_index=index, - item=row, - sample=sample, - metrics={self.metric_key: nan_result.outputs}, - requests=generation_requests, - metric_errors={self.metric_key: error_message}, - ), - ) - - async def score_row( - self, - index: int, - row: dict[str, object], - sample: dict[str, object], - generation_requests: list[dict[str, object]], - ) -> tuple[int, MetricResult | None, RowScore]: - """Score one online row using the generated sample payload.""" - return await score_row( - metric=self.metric, - metric_key=self.metric_key, - row=row, - sample=sample, - index=index, - fail_fast=fail_fast_from_params(self.params), - generation_requests=generation_requests, - logger=log, - ) - - -# --------------------------------------------------------------------------- -# Pipeline execution -# --------------------------------------------------------------------------- - - -async def _generate_pipeline_item(index: int, runtime: PipelineRuntime) -> None: - """Run the producer half of the shared queue pipeline for one row index.""" - row = runtime.pipeline.rows[index] - generation_requests: list[dict[str, Any]] = [] - inference.requests_log_var.set(generation_requests) - - try: - sample = await runtime.pipeline.generate_sample(index, row) - except Exception as error: - runtime.results[index] = runtime.pipeline.handle_generation_error(index, row, error, generation_requests) - return - - await runtime.sample_queue.put( - GeneratedSampleEvent( - row_index=index, - item=MappingProxyType(row), - sample=MappingProxyType(sample), - requests_log=generation_requests, - ) - ) - - -async def _score_pipeline_samples(runtime: PipelineRuntime) -> None: - """Drain generated samples from the queue and score them until the sentinel is received.""" - while True: - queued = await runtime.sample_queue.get() - if queued is _QUEUE_END: - runtime.sample_queue.task_done() - return - - if not isinstance(queued, GeneratedSampleEvent): - raise ValueError(f"Expected GeneratedSampleEvent, got: {type(queued).__name__}") - - event = queued - try: - runtime.results[event.row_index] = await runtime.pipeline.score_row( - event.row_index, - dict(event.item), - dict(event.sample), - event.requests_log, - ) - finally: - runtime.sample_queue.task_done() - - -async def run_generated_sample_scoring_pipeline( - pipeline: GeneratedSampleScoringPipeline, -) -> list[tuple[int, MetricResult | None, RowScore]]: - """Run a pipeline object through the shared generated-sample queue flow.""" - if not pipeline.rows: - return [] - - worker_count = min(len(pipeline.rows), max(1, pipeline.parallelism)) - queue_capacity = max(1, worker_count * 2) - runtime = PipelineRuntime( - pipeline=pipeline, - sample_queue=asyncio.Queue(maxsize=queue_capacity), - results=[None] * len(pipeline.rows), - ) - - async with use_resilience_session(): - async with asyncio.TaskGroup() as tg: - for _ in range(worker_count): - tg.create_task(_score_pipeline_samples(runtime)) - try: - await run_indexed_tasks( - list(range(len(pipeline.rows))), - partial(_generate_pipeline_item, runtime=runtime), - parallelism=worker_count, - ) - finally: - for _ in range(worker_count): - await runtime.sample_queue.put(_QUEUE_END) - - if any(result is None for result in runtime.results): - raise RuntimeError("Internal error: missing row evaluation result after online execution") - - return [result for result in runtime.results if result is not None] - - -# --------------------------------------------------------------------------- -# High-level orchestration -# --------------------------------------------------------------------------- - - -async def evaluate_metric( - metric: Metric, - *, - rows: list[dict[str, Any]], - target: Model | Agent | None = None, - prompt_template: str | dict[str, Any] | None = None, - params: RunConfig | RunConfigOnline | RunConfigOnlineModel | None = None, - preprocess_hooks: Sequence[inference.PreprocessRequest] | None = None, - postprocess_hooks: Sequence[inference.PostprocessResponse] | None = None, -) -> EvaluationResult: - """Generate model outputs for prepared rows and evaluate a prepared metric.""" - if not rows: - log.warning("No rows found in dataset, returning empty evaluation result") - return empty_evaluation_result() - - params = resolve_params(params, target) - - client_close_fn = None - - merged_preprocess_hooks, merged_postprocess_hooks = _merge_online_hooks( - params=params, - target=target, - preprocess_hooks=preprocess_hooks, - postprocess_hooks=postprocess_hooks, - ) - if isinstance(target, Model): - params = cast(RunConfigOnlineModel, params) - inference_fn = ( - metric.inference_fn if isinstance(metric, InferenceMetricBase) else inference.make_inference_request - ) - resolved_prompt_template = _resolve_online_prompt_template(prompt_template, target, rows[0]) - client = inference.new_inference_client(target) - client_close_fn = client.close - pipeline = ComputeMetricPipeline( - rows=rows, - parallelism=params.parallelism, - metric=metric, - target=target, - metric_key=metric_type_name(metric), - prompt_template=resolved_prompt_template, - params=params, - inference_fn=inference_fn, - client=client, - default_headers=None, - preprocess_hooks=merged_preprocess_hooks, - postprocess_hooks=merged_postprocess_hooks, - ) - elif isinstance(target, AgentBase): - params = cast(RunConfigOnline, params) - if prompt_template is None: - raise ValueError("prompt_template is required for agent online evaluation") - - client = new_agent_inference_client() - client_close_fn = client.aclose - - pipeline = ComputeMetricPipeline( - rows=rows, - parallelism=params.parallelism, - metric=metric, - target=target, - metric_key=metric_type_name(metric), - prompt_template=prompt_template, - params=params, - inference_fn=make_agent_inference_request, - client=client, - preprocess_hooks=merged_preprocess_hooks, - postprocess_hooks=merged_postprocess_hooks, - ) - else: - pipeline = ComputeMetricPipeline( - rows=rows, - parallelism=params.parallelism, - metric=metric, - target=None, - metric_key=metric_type_name(metric), - params=params, - preprocess_hooks=merged_preprocess_hooks, - postprocess_hooks=merged_postprocess_hooks, - ) - - try: - completed = await run_generated_sample_scoring_pipeline(pipeline) - except Exception as e: - evaluation_error = get_evaluation_error(e) - if isinstance(evaluation_error, EvaluationError) and evaluation_error.__cause__ is not None: - raise evaluation_error from evaluation_error.__cause__ - raise evaluation_error from e - finally: - if client_close_fn: - await client_close_fn() - - return await finalize_evaluation_result(metric, completed) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/pipeline.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/pipeline.py deleted file mode 100644 index e44a45020a..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/pipeline.py +++ /dev/null @@ -1,64 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Pipeline infrastructure types for online execution.""" - -from __future__ import annotations - -import asyncio -from dataclasses import dataclass -from types import MappingProxyType -from typing import Any, Protocol - -from nemo_platform.beta.evaluator.metrics.protocol import MetricResult -from nemo_platform.beta.evaluator.values.results import RowScore - - -@dataclass -class GeneratedSampleEvent: - """Carries one generated sample row from producer workers to scorer workers.""" - - row_index: int - item: MappingProxyType - sample: MappingProxyType - requests_log: list[dict[str, Any]] - - -class GeneratedSampleScoringPipeline(Protocol): - """Internal contract for pipelines that generate a sample before scoring.""" - - rows: list[dict[str, Any]] - parallelism: int - - async def generate_sample(self, index: int, row: dict[str, Any]) -> dict[str, Any]: - """Prepare the generated sample payload for one input row.""" - ... - - def handle_generation_error( - self, - index: int, - row: dict[str, Any], - error: Exception, - generation_requests: list[dict[str, Any]], - ) -> tuple[int, MetricResult | None, RowScore]: - """Convert a generation failure into a completed row result or raise.""" - ... - - async def score_row( - self, - index: int, - row: dict[str, Any], - sample: dict[str, Any], - generation_requests: list[dict[str, Any]], - ) -> tuple[int, MetricResult | None, RowScore]: - """Score one prepared sample and return the completed row result.""" - ... - - -@dataclass -class PipelineRuntime: - """Mutable queue-worker state shared by the generic pipeline helpers.""" - - pipeline: GeneratedSampleScoringPipeline - sample_queue: asyncio.Queue[GeneratedSampleEvent | object] - results: list[tuple[int, MetricResult | None, RowScore] | None] diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/runs.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/runs.py deleted file mode 100644 index 30837a2c84..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/runs.py +++ /dev/null @@ -1,143 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Job-oriented run handle types for evaluator backends.""" - -from __future__ import annotations - -from typing import Any, Protocol - -from nemo_platform.beta.evaluator.values.multi_metric_results import BenchmarkEvaluationResult -from nemo_platform.beta.evaluator.values.results import EvaluationResult - - -class JobEvaluationRun(Protocol): - """Structural contract for job-backed single-metric run handles.""" - - async def status(self) -> str: - """Return the current lifecycle status for the run. - - Returns: - A backend-defined status string such as `created`, `active`, or - `completed`. - """ - ... - - async def result( - self, - *, - timeout_s: float | None = None, - poll_interval_s: float = 1.0, - ) -> EvaluationResult: - """Wait for the run to finish and return its metric result. - - Args: - timeout_s: Optional maximum wait time before raising `TimeoutError`. - poll_interval_s: Poll interval used by backends that require - repeated status checks. - - Returns: - The completed single-metric evaluation result. - """ - ... - - def result_sync( - self, - *, - timeout_s: float | None = None, - poll_interval_s: float = 1.0, - ) -> EvaluationResult: - """Synchronously wait for the run to finish and return its result. - - Args: - timeout_s: Optional maximum wait time before raising `TimeoutError`. - poll_interval_s: Poll interval used by backends that require - repeated status checks. - - Returns: - The completed single-metric evaluation result. - """ - ... - - def job(self) -> Any | None: - """Return the primary backend job object, if one exists. - - Returns: - A backend-specific job object for single-job runs, or `None` for - purely local completed runs. - """ - ... - - def jobs(self) -> list[Any]: - """Return all backend job objects associated with the run. - - Returns: - One or more backend-specific job objects. - """ - ... - - -class JobBenchmarkEvaluationRun(Protocol): - """Structural contract for job-backed multi-metric run handles.""" - - async def status(self) -> str: - """Return the current lifecycle status for the combined run. - - Returns: - A backend-defined status string aggregated across all underlying - metric runs. - """ - ... - - async def result( - self, - *, - timeout_s: float | None = None, - poll_interval_s: float = 1.0, - ) -> BenchmarkEvaluationResult: - """Wait for all metrics to finish and return the combined result. - - Args: - timeout_s: Optional maximum wait time before raising `TimeoutError`. - poll_interval_s: Poll interval used by backends that require - repeated status checks. - - Returns: - The completed multi-metric evaluation result. - """ - ... - - def result_sync( - self, - *, - timeout_s: float | None = None, - poll_interval_s: float = 1.0, - ) -> BenchmarkEvaluationResult: - """Synchronously wait for all metrics to finish and return the result. - - Args: - timeout_s: Optional maximum wait time before raising `TimeoutError`. - poll_interval_s: Poll interval used by backends that require - repeated status checks. - - Returns: - The completed multi-metric evaluation result. - """ - ... - - def job(self) -> Any | None: - """Return a primary backend job object when one exists. - - Returns: - A backend-specific job object, or `None` when the run is backed by - multiple jobs or no remote jobs at all. - """ - ... - - def jobs(self) -> list[Any]: - """Return all backend job objects associated with the run. - - Returns: - One or more backend-specific job objects. - """ - ... diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/samples.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/samples.py deleted file mode 100644 index 0d3619cf50..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/samples.py +++ /dev/null @@ -1,43 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Sample payload adapters for evaluator SDK execution.""" - -from typing import Any - -from nemo_platform.beta.evaluator.metrics.protocol import CandidateOutput, DatasetRow, MetricInput - -_CANDIDATE_SAMPLE_FIELDS = frozenset({"output_text", "response", "trajectory", "evidence"}) - - -def build_offline_sample(row: dict[str, Any]) -> dict[str, Any]: - """Build the sample payload for an offline row. - - Field mapping can normalize an offline prediction into the canonical - ``output`` row field. Surface that value as ``sample.output_text`` so - protocol metrics see the same candidate location as online evaluations. - """ - output = row.get("output") - if isinstance(output, str): - return {"output_text": output} - return {} - - -def build_metric_input(row: dict[str, Any], sample: dict[str, Any], index: int | None = None) -> MetricInput: - """Build the metric protocol input from dataset row and generated sample payloads.""" - output_text = sample.get("output_text") - metadata = { - key: value - for key, value in sample.items() - if key not in _CANDIDATE_SAMPLE_FIELDS or (key == "output_text" and not isinstance(output_text, str)) - } - return MetricInput( - row=DatasetRow(row_index=index, data=row), - candidate=CandidateOutput( - output_text=output_text if isinstance(output_text, str) else None, - response=sample.get("response"), - trajectory=sample.get("trajectory"), - evidence=sample.get("evidence"), - metadata=metadata, - ), - ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/scoring.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/scoring.py deleted file mode 100644 index 101544696b..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/scoring.py +++ /dev/null @@ -1,201 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Shared row-scoring and result-finalization primitives used during execution.""" - -from collections.abc import Iterable, Sequence -from logging import Logger, getLogger -from typing import Any - -from nemo_platform.beta.evaluator.execution.samples import build_metric_input -from nemo_platform.beta.evaluator.execution.values import EvaluationError, EvaluationPhase -from nemo_platform.beta.evaluator.inference import requests_log_var -from nemo_platform.beta.evaluator.metrics.aggregation import ( - add_corpus_scores, - aggregate_metrics, - is_aggregateable_output_spec, - rubric_definitions_from_metric, -) -from nemo_platform.beta.evaluator.metrics.protocol import ( - CorpusMetric, - Metric, - MetricOutput, - MetricOutputSpec, - MetricResult, - validate_metric_result, -) -from nemo_platform.beta.evaluator.metrics.utils import metric_type_name -from nemo_platform.beta.evaluator.values import ( - EvaluationResult, - RowScore, -) - -logger = getLogger(__name__) - - -def nan_metric_result(outputs: Iterable[MetricOutputSpec]) -> MetricResult: - """Build the NaN output payload used for ignored scoring failures. - - Only aggregateable outputs receive NaN placeholders; non-score outputs are - not synthesized for failed rows. - """ - return MetricResult( - outputs=[ - MetricOutput(name=output.name, value=float("nan")) - for output in outputs - if is_aggregateable_output_spec(output) - ] - ) - - -def corpus_output_spec(metric: Metric, fallback: list[MetricOutputSpec] | None = None) -> list[MetricOutputSpec]: - """Return corpus-level output specs when a metric declares them.""" - corpus_spec = getattr(metric, "corpus_output_spec", None) - if callable(corpus_spec): - return list(corpus_spec()) - return list(fallback if fallback is not None else metric.output_spec()) - - -CompletedRowEvaluation = tuple[int, MetricResult | None, RowScore] - - -def empty_evaluation_result() -> EvaluationResult: - """Return the canonical empty evaluation result payload.""" - return EvaluationResult(row_scores=[], aggregate_scores=aggregate_metrics([], [])) - - -async def finalize_evaluation_result( - metric: Metric, - eval_results: Sequence[CompletedRowEvaluation], - *, - skip_errored: bool = False, -) -> EvaluationResult: - """Build the final evaluation result from eval_results row-level outputs. - - Callers are expected to pass ``eval_results`` in the original row order; the - upstream pipeline (``run_indexed_tasks`` / ``run_generated_sample_scoring_pipeline``) - already writes results by index, so no re-sorting is performed here. - - When ``skip_errored`` is true, rows whose ``RowScore.metric_errors`` is - populated are excluded from aggregation so the NaN placeholder produced - for ignored failures does not contribute to ``nan_count``, and the same - rows are excluded from the ``items``/``samples`` passed to - :meth:`CorpusMetric.compute_corpus_scores` so corpus-level aggregation - (e.g. BLEU/ROUGE-corpus) isn't skewed by failed rows with empty samples. - Errored rows still appear in ``row_scores``. - """ - valid_eval_results = [ - (result, row_score) - for _, result, row_score in eval_results - if result is not None and not (skip_errored and row_score.metric_errors) - ] - metric_results = [result for result, _ in valid_eval_results] - # Keep all rows in the reported ``row_scores`` (including errored ones); only - # aggregation and corpus inputs honor ``skip_errored``. - row_scores = [row_score for _, _, row_score in eval_results] - - output_spec = metric.output_spec() - rubric_definitions = rubric_definitions_from_metric(metric) - if rubric_definitions: - aggregated_result = aggregate_metrics(metric_results, output_spec, rubric_definitions=rubric_definitions) - else: - aggregated_result = aggregate_metrics(metric_results, output_spec) - - if valid_eval_results and isinstance(metric, CorpusMetric): - corpus_metric_result = await metric.compute_corpus_scores( - inputs=[ - build_metric_input(row_score.item, row_score.sample, row_score.row_index) - for _, row_score in valid_eval_results - ], - ) - if corpus_metric_result: - add_corpus_scores(aggregated_result, corpus_metric_result, corpus_output_spec(metric, output_spec)) - - return EvaluationResult( - row_scores=row_scores, - aggregate_scores=aggregated_result, - ) - - -async def score_row( - metric: Metric, - row: dict[str, Any], - sample: dict[str, Any], - index: int, - metric_key: str, - fail_fast: bool, - generation_requests: list[dict[str, Any]], - logger: Logger | None = None, -) -> tuple[int, MetricResult | None, RowScore]: - """Score an already-prepared sample for one row. - - Args: - metric: Metric object used for scoring. - row: Input row from the dataset. - sample: Prepared sample payload passed to the metric. - index: Row position in the original dataset. - metric_key: Key used to place score output in ``RowScore.metrics``. - fail_fast: Whether metric errors should raise immediately. When - ``True``, the exception is wrapped in ``EvaluationError`` and - raised. When ``False``, a metric exception yields a NaN score row. - generation_requests: Requests collected before metric scoring, - such as online generation requests. - logger: Optional logger override for row-scoring logs. - Returns: - Tuple of ``(index, metric_result_or_none, row_score_payload)``. - - Raises: - EvaluationError: If row evaluation fails and ``fail_fast`` is ``True``. - """ - - metric_requests: list[dict[str, Any]] = [] - requests_log_var.set(metric_requests) - active_logger = logger or globals()["logger"] - - try: - output_spec = metric.output_spec() - result = validate_metric_result( - await metric.compute_scores(build_metric_input(row, sample, index)), output_spec - ) - active_logger.debug( - "Computed metric", - extra={ - "item_index": index, - "metric_type": metric_type_name(metric), - "outputs": [output.model_dump() for output in result.outputs], - }, - ) - return ( - index, - result, - RowScore( - row_index=index, - item=row, - sample=sample, - metrics={metric_key: result.outputs}, - requests=[*generation_requests, *metric_requests], - metric_diagnostics={metric_key: result.diagnostics} if result.diagnostics else None, - ), - ) - except Exception as e: - if fail_fast: - raise EvaluationError( - index, - str(e), - phase=EvaluationPhase.METRIC_SCORING, - metric_key=metric_key, - ) from e - active_logger.warning("Evaluation failed, marking as NaN", extra={"item_index": index, "error": str(e)}) - result = nan_metric_result(metric.output_spec()) - return ( - index, - result, - RowScore( - row_index=index, - item=row, - sample=sample, - metrics={metric_key: result.outputs}, - requests=[*generation_requests, *metric_requests], - metric_errors={metric_key: str(e)}, - ), - ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/utils.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/utils.py deleted file mode 100644 index 6f0e9a4471..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/utils.py +++ /dev/null @@ -1,100 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Local metric preparation helpers for evaluator SDK runtime.""" - -from __future__ import annotations - -import copy -from collections.abc import Sequence -from typing import TypeGuard, cast - -from nemo_platform.beta.evaluator.execution._protocols import JobParamsConfigurableMetric -from nemo_platform.beta.evaluator.metrics.protocol import Metric, MetricWithModels, MetricWithPreflight, MetricWithSecrets -from nemo_platform.beta.evaluator.metrics.utils import metric_type_name -from nemo_platform.beta.evaluator.resolver_protocols import ModelResolver, SecretResolver -from nemo_platform.beta.evaluator.values.params import RunConfig, RunConfigOnline, RunConfigOnlineModel -from pydantic import BaseModel - - -def unique_metric_keys(metrics: Sequence[Metric]) -> list[str]: - """Assign stable unique keys to a sequence of metrics. - - Args: - metrics: Metrics submitted in one evaluator call. - - Returns: - Unique metric keys in the same order as the input metrics. - """ - - seen: dict[str, int] = {} - keys: list[str] = [] - for metric in metrics: - base = metric_type_name(metric) - seen[base] = seen.get(base, 0) + 1 - suffix = seen[base] - keys.append(base if suffix == 1 else f"{base}_{suffix}") - return keys - - -def is_metric(metrics: object) -> TypeGuard[Metric]: - """Return whether a value is the single-metric form.""" - if isinstance(metrics, Metric): - return True - return False - - -def is_metric_sequence(metrics: object) -> TypeGuard[Sequence[Metric]]: - """Return whether a value is the benchmark/multi-metric form.""" - if not isinstance(metrics, Metric) and isinstance(metrics, Sequence) and not isinstance(metrics, (str, bytes)): - return all(isinstance(metric, Metric) for metric in metrics) - return False - - -def copy_metric(metric: Metric) -> Metric: - """Create a best-effort isolated copy of a metric instance. - - Preparation mutates metrics in place (runtime params, resolver hydration, - preflight state), so backends copy first to avoid side effects on the - caller's original metric object. - """ - if isinstance(metric, BaseModel): - return cast(Metric, metric.model_copy(deep=True)) - - try: - return copy.deepcopy(metric) - except Exception as exc: - raise TypeError( - f"Cannot copy metric {type(metric).__name__}; use a Pydantic model or ensure the metric " - "supports copy.deepcopy() (for example, by implementing __deepcopy__)." - ) from exc - - -async def prepare_metric_for_execution( - metric: Metric, - *, - params: RunConfig | RunConfigOnline | RunConfigOnlineModel, - model_resolver: ModelResolver, - secret_resolver: SecretResolver, -) -> Metric: - """Copy and prepare one metric for execution. - - Args: - metric: User-provided metric instance. - params: Materialized execution params for this run. - model_resolver: Resolver used for any ``ModelRef`` fields. - secret_resolver: Resolver used for any ``SecretRef`` fields. - - Returns: - A copied metric ready for execution. - """ - prepared_metric = copy_metric(metric) - if isinstance(prepared_metric, JobParamsConfigurableMetric): - prepared_metric.apply_evaluation_job_params(params) - if isinstance(prepared_metric, MetricWithModels): - await prepared_metric.resolve_models(model_resolver) - if isinstance(prepared_metric, MetricWithSecrets): - await prepared_metric.resolve_secrets(secret_resolver) - if isinstance(prepared_metric, MetricWithPreflight): - await prepared_metric.preflight() - return prepared_metric diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/values.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/values.py deleted file mode 100644 index 2e5fba8c27..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/values.py +++ /dev/null @@ -1,53 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Value types used during pipeline execution.""" - -from enum import Enum - - -class EvaluationPhase(str, Enum): - """Phase where a row failure is raised instead of converted to NaN.""" - - SAMPLE_GENERATION = "sample_generation" - METRIC_SCORING = "metric_scoring" - - -class EvaluationError(Exception): - """Raised when evaluation fails with ``fail_fast=True``. - - ``metric_key`` is the public metric identifier. For single-metric - pipelines it is the metric type name; for multi-metric benchmark - pipelines it is the fully-qualified metric ref. - """ - - def __init__( - self, - index: int, - message: str, - *, - phase: EvaluationPhase = EvaluationPhase.METRIC_SCORING, - metric_key: str | None = None, - ) -> None: - """Create an evaluation error that keeps sample index context. - - Strict mode means ``fail_fast=True``: row failures abort evaluation - instead of being converted into NaN row results. - - Args: - index: Position of the failing sample in the input list. - message: Original exception message. - phase: Execution phase where the row failed. - metric_key: Public metric identifier for the failing metric. - """ - self.index = index - self.message = message - self.phase = phase - self.metric_key = metric_key - super().__init__(self._format_message()) - - def _format_message(self) -> str: - """Return a concise user-facing error message.""" - phase = self.phase.value.replace("_", " ") - metric = f" for metric {self.metric_key!r}" if self.metric_key else "" - return f"Evaluation failed during {phase}{metric} on row {self.index}: {self.message}" diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/inference.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/inference.py deleted file mode 100644 index f26a4441d7..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/inference.py +++ /dev/null @@ -1,456 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -import logging -from abc import ABC, abstractmethod -from collections.abc import Awaitable -from contextvars import ContextVar -from typing import Any, Dict, Optional, Protocol, runtime_checkable -from urllib.parse import parse_qsl, urlparse, urlunparse - -import openai -from openai import AsyncOpenAI -from openai.types import Completion -from openai.types.chat import ChatCompletion -from pydantic import BaseModel, PrivateAttr - -from nemo_platform.beta.evaluator.constants import PLACEHOLDER_INFERENCE_API_KEY -from nemo_platform.beta.evaluator.enums import ModelFormat -from nemo_platform.beta.evaluator.resilience.api import run_with_resilience -from nemo_platform.beta.evaluator.resilience.classifier import endpoint_identity -from nemo_platform.beta.evaluator.resilience.scheduler import ResilienceCancelledError -from nemo_platform.beta.evaluator.values import Model, ReasoningParams -from nemo_platform.beta.evaluator.values.models import filter_auth_headers -from nemo_platform.beta.evaluator.values.params import InferenceParams - -# We use a context variable to store the requests log for the current request. -requests_log_var = ContextVar("requests_log") - -# We use a context variable for the name of the logger to use -logger_var = ContextVar("logger_name") - - -def get_logger() -> logging.Logger: - return logging.getLogger(logger_var.get(__name__)) - - -def merge_default_headers(model: Model, default_headers: dict | None) -> dict[str, str] | None: - """Merge model-level and per-call default headers for one inference request.""" - if model.default_headers is None and default_headers is None: - return None - - return { - **(model.default_headers or {}), - **(default_headers or {}), - } - - -def redact_request_for_logging(request_body: dict[str, Any]) -> dict[str, Any]: - """Return a copy of the request payload that is safe to persist in request logs.""" - redacted_request = dict(request_body) - if "extra_headers" in redacted_request: - filtered_headers = filter_auth_headers(redacted_request["extra_headers"]) - if filtered_headers: - redacted_request["extra_headers"] = filtered_headers - else: - redacted_request.pop("extra_headers") - return redacted_request - - -class ClientInferenceError(RuntimeError): - def __init__(self, e: openai.APIStatusError, context: str | None = None): - self.status_code: int = e.status_code - error_detail = getattr(e.response, "text", str(e)) - # Build the base message - message = f"Unable to complete inference because a {e.status_code} error occurred." - # Add context if provided - if context: - message = f"{message} {context}" - # Add details only if there's actual content - if error_detail and error_detail.strip(): - message = f"{message} Details: {error_detail}" - super().__init__(message) - - -@runtime_checkable -class InferenceFn(Protocol): - """Callable protocol for inference function dependency injection.""" - - def __call__( - self, - model: Model, - request: dict, - max_retries: int | None, - *, - client: AsyncOpenAI | None = None, - api_key: str | None = None, - default_headers: dict | None = None, - timeout: float | None = None, - ) -> Awaitable[dict]: ... - - -class InferenceHookParams(Protocol): - """Fields that new_hooks() actually reads from its params argument.""" - - @property - def system_prompt(self) -> str | None: ... - - @property - def inference(self) -> InferenceParams | None: ... - - @property - def structured_output(self) -> dict | None: ... - - @property - def reasoning(self) -> ReasoningParams | None: ... - - -class InferenceMetricBase(BaseModel): - """Reusable inference transport state for V2 metrics.""" - - _inference_fn: InferenceFn | None = PrivateAttr(default=None) - - @property - def inference_fn(self) -> InferenceFn: - """Return the effective inference function for this metric.""" - return self._inference_fn or make_inference_request - - def set_inference_fn(self, inference_fn: InferenceFn) -> None: - """Inject the inference function to use for this metric.""" - self._inference_fn = inference_fn - - -class PreprocessRequest(ABC): - """Interface for preprocessing inference request.""" - - @abstractmethod - def preprocess(self, request: Dict, id: Optional[str] = None) -> Dict: - pass - - -class PostprocessResponse(ABC): - """Interface for postprocessing inference response.""" - - @abstractmethod - def postprocess(self, response: Dict, id: Optional[str] = None) -> Dict: - pass - - -class LogHook(PreprocessRequest, PostprocessResponse): - """ - Log the inference request and response - """ - - def __init__(self, logger: logging.Logger | None = None): - self.logger = logger or logging.getLogger(__name__) - - def preprocess(self, request: Dict, id: Optional[str] = None) -> Dict: - if id: - self.logger.debug("Request %s: %s", id, request) - else: - self.logger.debug("Request: %s", request) - return request - - def postprocess(self, response: Dict, id: Optional[str] = None) -> Dict: - if id: - self.logger.debug("Response %s: %s", id, response) - else: - self.logger.debug("Response: %s", response) - return response - - -class AddInferenceParameter(PreprocessRequest): - def __init__(self, params: Dict[str, Any]): - if not params: - raise ValueError("params cannot be empty") - self.params = params - - def preprocess(self, request: Dict, id: Optional[str] = None) -> Dict: - return deep_merge(request, self.params) - - -class InjectSystemMessage(PreprocessRequest): - def __init__(self, system_message: str, logger: logging.Logger | None = None): - if not system_message: - raise ValueError("system_message cannot be empty") - self.system_message = system_message - self.logger = logger or logging.getLogger(__name__) - - def preprocess(self, request: Dict, id: Optional[str] = None) -> Dict: - """ - Prepend system message into the payload for existing message or insert as a new system message. - """ - if request.get("messages"): - msg = request["messages"][0] - if msg.get("role") == "system": - # Prefix the first message with the custom message - request["messages"][0]["content"] = f"{self.system_message} {msg['content']}" - else: - # Add new system message - request["messages"].insert(0, {"role": "system", "content": self.system_message}) - elif request.get("prompt"): - request["prompt"] = f"{self.system_message} {request['prompt']}" - else: - prefix = "Request:" - if id: - prefix = f"Request {id}:" - self.logger.warning( - f"{prefix} Custom system message was not added to request due to unexpected format: missing prompt or messages" - ) - return request - - -class TransformReasoningOutput(PostprocessResponse): - """ - TransformReasoningOutput postprocess hook is primarily targeted for handling reasoning with - Nemotron models which include reasoning context within the model output. Reasoning context is - denoted with token context and is removed from the output and moved to a new - response field `reasoning_content`. - """ - - def __init__(self, end_reasoning_token: Optional[str] = None): - self.end_reasoning_token = end_reasoning_token - - def postprocess(self, response: Dict, id: Optional[str] = None) -> Dict: - """ - Move reasoning tokens from output to a new field - """ - for i, choice in enumerate(response.get("choices", [])): - msg = choice.get("message") - if msg and msg.get("role") == "assistant": - content = msg.get("content") - if not isinstance(content, str): - # Content can be None with function calling - continue - - split_content = content.rsplit(self.end_reasoning_token, 1) - if len(split_content) == 2: - # Add last token back after split - response["choices"][i]["message"]["reasoning_content"] = split_content[0] + self.end_reasoning_token - response["choices"][i]["message"]["content"] = split_content[1] - elif choice.get("text"): - split_content = choice["text"].rsplit(self.end_reasoning_token, 1) - if len(split_content) == 2: - choice["reasoning_content"] = split_content[0] + self.end_reasoning_token - choice["text"] = split_content[1] - return response - - -def new_hooks( - params: InferenceHookParams | None, - model_format: ModelFormat | None = ModelFormat.NVIDIA_NIM, - logger: logging.Logger | None = None, -) -> tuple[list[PreprocessRequest], list[PostprocessResponse]]: - """Build the standard online generation hooks used by SDK and service flows.""" - from nemo_platform.beta.evaluator.structured_output import InferenceStructuredOutput, default_structured_output_mode - - log_hook = LogHook(logger) - preprocess_hooks: list[PreprocessRequest] = [] - postprocess_hooks: list[PostprocessResponse] = [log_hook] - - # System prompt injection - must be first to prepend system message - if params and params.system_prompt: - preprocess_hooks.append(InjectSystemMessage(params.system_prompt, logger)) - - if params and params.inference: - preprocess_hooks.append(AddInferenceParameter(params.inference.model_dump(mode="json", exclude_none=True))) - - if params and params.structured_output: - preprocess_hooks.append( - InferenceStructuredOutput( - default_structured_output_mode(model_format or "nim"), - params.structured_output, - ) - ) - - preprocess_hooks.append(log_hook) - - if params and params.reasoning and params.reasoning.end_token: - postprocess_hooks.append(TransformReasoningOutput(params.reasoning.end_token)) - - return preprocess_hooks, postprocess_hooks - - -def new_inference_client(model: Model, api_key: str | None = None) -> AsyncOpenAI: - """ - Initialize a new client for inference - """ - - # Make sure the base_url does not end in /completions or /chat/completions - base_url = model.url - parsed_url = urlparse(model.url) - for suffix in ["/chat/completions", "/completions"]: - if parsed_url.path.endswith(suffix): - base_url = urlunparse(parsed_url._replace(path=parsed_url.path[: -1 * len(suffix)], query="")) - parsed_url = urlparse(base_url) - - return AsyncOpenAI( - base_url=base_url, - # Sometimes, a fake key is still required for the OpenAI client to work. - api_key=api_key or model.api_key or PLACEHOLDER_INFERENCE_API_KEY, - # Defer retry to resilience - max_retries=0, - ) - - -async def make_inference_request( - model: Model, - request: dict, - max_retries: int | None = 3, - *, - client: AsyncOpenAI | None = None, - api_key: str | None = None, - default_headers: dict | None = None, - timeout: float | None = None, -) -> dict: - """ - Helper to run inference on a model with a given prompt. - - Only OpenAI format is supported (nim and openai formats). - - Args: - model: The Model to run inference on. - request: The request to run. Can be a completion or a chat request. - max_retries: Maximum number of retries for the request. - client: AsyncOpenAI client to use for inference requests. - api_key: Optional explicit API key. If provided, overrides the placeholder. - If not provided, uses placeholder (caller must resolve api_key_secret). - timeout: Optional request timeout in seconds. If None, client default behavior is used. - - Returns: - The result of the inference. - """ - log = get_logger() - - model_id = model.name - extra_headers = merge_default_headers(model, default_headers) - - parsed_url = urlparse(model.url) - extra_query = dict(parse_qsl(parsed_url.query)) - if client: - inference_client = client - else: - inference_client = new_inference_client(model, api_key=api_key) - base_url = str(inference_client.base_url) - - # To distinguish between completions and chat completions, we look at the request body. - # TODO: add typing for the request. - max_attempts = max(1, (max_retries if max_retries is not None else 0) + 1) - - request_body = {"model": model_id, **request} - if timeout: - request_body["timeout"] = timeout - - endpoint_key = endpoint_identity(base_url, model_id=model_id, auth_identity=inference_client.api_key) - try: - log.info("Making request to %s: %s", base_url, {"model": model_id, **request}) - - requests_log = requests_log_var.get([]) - if extra_query: - request_body["extra_query"] = extra_query - if extra_headers: - request_body["extra_headers"] = extra_headers - - fn = inference_client.chat.completions.create if "messages" in request else inference_client.completions.create - # ty cannot bind `fn`'s ParamSpec because `fn` is selected at runtime - # (chat vs text completions), so the spread body cannot be validated here. - completion: ChatCompletion | Completion = await run_with_resilience( - endpoint_key, - fn, # ty: ignore[invalid-argument-type] - max_attempts=max_attempts, - **request_body, # ty: ignore[invalid-argument-type] - ) - logged_request_body = redact_request_for_logging(request_body) - requests_log.append({"request": logged_request_body, "response": completion.model_dump()}) - return completion.model_dump() - - except openai.APIConnectionError as e: - log.warning(f"Error connecting to inference server at {base_url}, cause: {e.__cause__}") - raise RuntimeError(f"Error connecting to inference server at {base_url}") from e - except openai.RateLimitError as e: - log.warning(f"Rate limit exceeded when issuing inference requests for {model_id}") - raise RuntimeError(f"Rate limit exceeded when issuing inference requests for {model_id}") from e - except openai.BadRequestError as e: - if "guided_json is unsupported" in str(e): - raise ClientInferenceError(e, "Verify whether the model version supports structured outputs.") - raise ClientInferenceError(e, f"base_url: {base_url}, model_id: {model_id}") - except openai.APIStatusError as e: - exception = ClientInferenceError(e, f"base_url: {base_url}, model_id: {model_id}") - log.warning(exception) - raise exception - except ResilienceCancelledError: - # Preserve cancellation semantics for callers coordinating task/group shutdown. - raise - except Exception as e: - # TODO: it maybe is sharing too much information to expose this error if it - # ends up propagating back to the user - # RRA: Better to err on sharing too much information than too little. - log.exception(f"Unexpected error making completion request to {model_id}") - raise RuntimeError(f"Unexpected error making completion request to {model_id}") from e - finally: - if not client: - # Close instantiated client scoped to function - await inference_client.close() - - -def preprocess_request(request: dict, hooks: list[PreprocessRequest], id: Optional[str] = None) -> Dict: - """ - Applies preprocessing hooks to request. Hooks are applied in order. - """ - for hook in hooks: - request = hook.preprocess(request, id=id) - return request - - -def process_output(response: dict, hooks: list[PostprocessResponse], id: Optional[str] = None) -> str: - """ - Applies postprocessing hooks to response before extracting the text from the full LM response. - - Args: - response (dict): The full LLM response. - id (str): Optional identifier of the response - hooks (List[PostprocessResponse]): Optional hooks to apply for postprocessing of the response. Hooks are applied in order. - Returns: - str: The text extracted from the response based on endpoint type. - """ - for hook in hooks: - response = hook.postprocess(response, id=id) - - if not ("choices" in response and len(response["choices"]) > 0): - raise ValueError("Invalid response format: No choices found in the response.") - - if ("message" in response["choices"][0]) and ("content" in response["choices"][0]["message"]): - # Return text from chat-completion response - return response["choices"][0]["message"]["content"] - elif "text" in response["choices"][0]: - # Return text from completion response - return response["choices"][0]["text"] - else: - # If neither field is present, raise an error - raise ValueError(f"Invalid response format: No text found in the response {response}.") - - -def deep_merge(request: Dict[str, Any], params: Dict[str, Any]) -> Dict[str, Any]: - """Merge provider request params into an existing request. - - Nested dictionaries are merged recursively so provider-specific payloads such as - ``extra_body.nvext`` can combine user-specified options with evaluator-added fields. - - Example: - request = {"extra_body": {"nvext": {"max_thinking_tokens": 256}}} - params = {"extra_body": {"nvext": {"guided_json": {...}}}} - - A plain ``request.update(params)`` would replace the entire ``extra_body`` payload and drop - ``max_thinking_tokens``. This helper preserves both keys under ``extra_body.nvext``. - - Non-dict values are overwritten by the newer params. - """ - merged = request.copy() - - for key, value in params.items(): - current = merged.get(key) - if isinstance(current, dict) and isinstance(value, dict): - merged[key] = deep_merge(current, value) - else: - merged[key] = value - return merged diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/aggregation.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/aggregation.py deleted file mode 100644 index 22698fab8f..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/aggregation.py +++ /dev/null @@ -1,479 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Aggregation data structures and computations for metric results.""" - -from __future__ import annotations - -import math -from collections import OrderedDict, defaultdict -from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Protocol, cast, runtime_checkable - -from nemo_platform.beta.evaluator.metrics.protocol import ( - BooleanValue, - ContinuousScore, - DiscreteScore, - MetricOutput, - MetricOutputSpec, - MetricResult, -) -from nemo_platform.beta.evaluator.values.results import ( - AggregatedMetricResult, - AggregateRangeScore, - AggregateRubricScore, - AggregateScore, - Histogram, - HistogramBin, - MetricScore, - Percentiles, - RubricScoreStat, - ScoreStats, -) - -if TYPE_CHECKING: - # Importing the score configuration module loads jsonschema. Aggregation's lightweight helpers - # (notably compute_percentiles) do not need it, so keep this typing-only edge deferred. - from nemo_platform.beta.evaluator.values.scores import Score - - -def is_aggregateable_output_spec(output_spec: MetricOutputSpec) -> bool: - """Return whether an output should contribute aggregate statistics.""" - return issubclass(output_spec.value_schema, (ContinuousScore, DiscreteScore, BooleanValue)) - - -@runtime_checkable -class MetricWithScores(Protocol): - """Metric config protocol for metrics that carry rubric score definitions.""" - - @property - def scores(self) -> Sequence[Score]: ... - - -def _coerce_aggregate_output(output: MetricOutput, output_spec: MetricOutputSpec) -> MetricScore | None: - """Convert one declared aggregateable metric output into MetricScore form.""" - if not is_aggregateable_output_spec(output_spec): - return None - if ( - issubclass(output_spec.value_schema, BooleanValue) - and isinstance(output.value, float) - and math.isnan(output.value) - ): - return MetricScore(name=output.name, value=output.value) - coerced = cast(ContinuousScore | DiscreteScore | BooleanValue, output_spec.coerce_output(output)) - value = coerced.root - if isinstance(value, bool): - value = 1.0 if value else 0.0 - return MetricScore(name=output.name, value=value) - - -def _attach_rubric_stats( - score: MetricScore, - output_by_name: Mapping[str, MetricOutput], - rubric_definitions: Mapping[str, Sequence[RubricScoreStat]], -) -> MetricScore: - """Attach per-row rubric bucket stats from companion label outputs.""" - rubric_definition = rubric_definitions.get(score.name) - if not rubric_definition: - return score - - label_output = output_by_name.get(f"{score.name}.label") - selected_label = label_output.value if label_output is not None else None - if isinstance(score.value, float) and math.isnan(score.value): - selected_label = None - - rubric_distribution = [ - RubricScoreStat( - label=rubric.label, - description=rubric.description, - value=rubric.value, - count=int(isinstance(selected_label, str) and selected_label == rubric.label), - ) - for rubric in rubric_definition - ] - return MetricScore( - name=score.name, - value=score.value, - stats=ScoreStats(rubric_distribution=rubric_distribution), - ) - - -def _aggregateable_scores( - result: MetricResult, - output_specs: list[MetricOutputSpec], - rubric_definitions: Mapping[str, Sequence[RubricScoreStat]] | None = None, -) -> list[MetricScore]: - """Extract score-like outputs from a metric result using declared output specs.""" - specs_by_name = {output_spec.name: output_spec for output_spec in output_specs} - output_by_name = {output.name: output for output in result.outputs} - scores: list[MetricScore] = [] - for output in result.outputs: - output_spec = specs_by_name.get(output.name) - if output_spec is None: - continue - score = _coerce_aggregate_output(output, output_spec) - if score is not None: - score = _attach_rubric_stats(score, output_by_name, rubric_definitions or {}) - scores.append(score) - return scores - - -def add_corpus_scores( - aggregated_result: AggregatedMetricResult, - corpus_result: MetricResult, - output_specs: list[MetricOutputSpec], -) -> None: - """Append corpus-level scores using aggregate-score schema fields. - - Args: - aggregated_result: Aggregate result object to mutate. - corpus_result: Corpus-level metric output with one or more scores. - - Returns: - ``None``. The ``aggregated_result`` object is updated in place. - """ - for score in _aggregateable_scores(corpus_result, output_specs): - value = score.value - # Corpus-level metrics contribute one already-aggregated value, so - # expose them through the same aggregate schema with count=1. - corpus_score = AggregateRangeScore( - name=score.name, - count=1, - nan_count=0, - sum=value, - mean=value, - min=value, - max=value, - std_dev=0.0, - variance=0.0, - percentiles=Percentiles( - p10=value, - p20=value, - p30=value, - p40=value, - p50=value, - p60=value, - p70=value, - p80=value, - p90=value, - p100=value, - ), - histogram=Histogram(bins=[HistogramBin(lower_bound=value, upper_bound=value, count=1)]), - ) - aggregated_result.scores.append(corpus_score) - - -def _compute_percentile(sorted_values: list[float], percentile: float) -> float: - """Compute a percentile using linearly interpolated rank position. - - The rank position is computed as ``(p/100) * (n + 1) - 1`` and then - interpolated between neighboring points when the position is fractional. - - Args: - sorted_values: Score values sorted in ascending order. - percentile: Percentile in the inclusive range ``[0, 100]``. - - Returns: - The interpolated percentile value, or ``0.0`` when no values exist. - """ - if not sorted_values: - return 0.0 - n = len(sorted_values) - pos = (percentile / 100.0) * (n + 1) - 1 - if pos <= 0: - return sorted_values[0] - if pos >= n - 1: - return sorted_values[-1] - lower_idx = int(pos) - frac = pos - lower_idx - return sorted_values[lower_idx] + frac * (sorted_values[lower_idx + 1] - sorted_values[lower_idx]) - - -def compute_percentiles(sorted_values: list[float]) -> Percentiles: - """Compute the fixed percentile set used by SDK aggregate output. - - Args: - sorted_values: Score values sorted in ascending order. - - Returns: - ``Percentiles`` containing p10 through p100. - """ - return Percentiles( - p10=_compute_percentile(sorted_values, 10), - p20=_compute_percentile(sorted_values, 20), - p30=_compute_percentile(sorted_values, 30), - p40=_compute_percentile(sorted_values, 40), - p50=_compute_percentile(sorted_values, 50), - p60=_compute_percentile(sorted_values, 60), - p70=_compute_percentile(sorted_values, 70), - p80=_compute_percentile(sorted_values, 80), - p90=_compute_percentile(sorted_values, 90), - p100=_compute_percentile(sorted_values, 100), - ) - - -def _compute_histogram(values: list[float], num_bins: int = 10) -> Histogram: - """Build a fixed-width histogram for numeric score distribution. - - The algorithm uses equal-width bins between the global minimum and maximum. - All bins except the last are half-open ``[lower, upper)``, while the final - bin is closed ``[lower, upper]`` so the maximum value is always counted. - - Args: - values: Numeric score values to bucket. - num_bins: Number of equally sized bins. - - Returns: - Histogram object containing ordered bin counts. - """ - if not values: - return Histogram(bins=[]) - - min_val = min(values) - max_val = max(values) - if min_val == max_val: - return Histogram(bins=[HistogramBin(lower_bound=min_val, upper_bound=max_val, count=len(values))]) - - bin_width = (max_val - min_val) / num_bins - bins: list[HistogramBin] = [] - for i in range(num_bins): - lower = min_val + i * bin_width - upper = min_val + (i + 1) * bin_width - if i == num_bins - 1: - count = sum(1 for v in values if lower <= v <= upper) - else: - count = sum(1 for v in values if lower <= v < upper) - bins.append(HistogramBin(lower_bound=lower, upper_bound=upper, count=count)) - return Histogram(bins=bins) - - -def aggregate_metrics( - items: list[MetricResult], - output_specs: list[MetricOutputSpec], - rubric_definitions: Mapping[str, Sequence[RubricScoreStat]] | None = None, -) -> AggregatedMetricResult: - """Aggregate row-level metric results into range or rubric summaries. - - This function performs two logical passes: - 1. Incremental accumulation of count/sum/min/max and rubric occurrences. - 2. Finalization of derived statistics (variance, percentiles, histogram, - and rubric mode category) once complete value sets are known. - - Args: - items: Row-level metric results to aggregate. - output_specs: Declared outputs for the metric. Only continuous, - discrete, and boolean output values contribute to aggregate scores. - rubric_definitions: Optional rubric bucket definitions keyed by numeric - output name. This is aggregation metadata, not metric protocol - metadata, and is usually derived from LLM judge score config. - - Returns: - Aggregate metric result with one aggregate score per score name. - """ - score_values: dict[str, list[float]] = defaultdict(list) - - aggregated_results: dict[str, MetricScore] = {} - rubric_distribution: dict[str, dict[str, dict]] = defaultdict(lambda: defaultdict(OrderedDict)) - has_rubric: dict[str, bool] = {} - - for item in items: - for score in _aggregateable_scores(item, output_specs, rubric_definitions): - if score.name not in aggregated_results: - # Keep one running accumulator per score name; distribution - # details are materialized in a second pass once all values exist. - aggregated_results[score.name] = MetricScore( - name=score.name, - value=0.0, - stats=ScoreStats( - count=0, - sum=0, - sum_squared=0, - min=None, - max=None, - mean=0, - variance=None, - stddev=None, - nan_count=0, - ), - ) - has_rubric[score.name] = bool(score.stats and score.stats.rubric_distribution) - if score.stats and score.stats.rubric_distribution: - for rubric_stat in score.stats.rubric_distribution: - rubric_distribution[score.name][rubric_stat.label] = { - "label": rubric_stat.label, - "value": rubric_stat.value, - "count": 0, - } - - if score.stats and score.stats.rubric_distribution: - for rubric_stat in score.stats.rubric_distribution: - if rubric_stat.count: - rubric_distribution[score.name][rubric_stat.label]["count"] += 1 - - results = aggregated_results[score.name] - assert results.stats is not None - - # int values can never be NaN in Python, so skipping int and - # float checks is sufficient. - # float('nan') is the only way to get a NaN in Python. - if isinstance(score.value, float) and math.isnan(score.value): - results.stats.nan_count = (results.stats.nan_count or 0) + 1 - continue - - score_values[score.name].append(score.value) - results.stats.count = (results.stats.count or 0) + 1 - results.stats.sum = (results.stats.sum or 0) + score.value - results.stats.sum_squared = (results.stats.sum_squared or 0) + score.value**2 - results.stats.mean = results.stats.sum / results.stats.count - - if results.stats.min is None or score.value < results.stats.min: - results.stats.min = score.value - if results.stats.max is None or score.value > results.stats.max: - results.stats.max = score.value - - results.value = results.stats.mean - - for score_name, values in score_values.items(): - if not values: - continue - results = aggregated_results[score_name] - assert results.stats is not None - - n = len(values) - mean = results.stats.mean or 0 - # Report both conventions under explicit names rather than leaving the divisor implicit. - # `variance`/`stddev` stay population (divide by n): these rows are the full evaluation set, - # not a sample intended to estimate a larger population. The sample (n-1) figures are also - # provided for callers estimating the spread of the process the values were drawn from, and - # are undefined for a single value. - sum_sq_dev = sum((v - mean) ** 2 for v in values) - variance = sum_sq_dev / n if n > 0 else 0 - results.stats.variance = variance - results.stats.stddev = math.sqrt(variance) - sample_variance = sum_sq_dev / (n - 1) if n > 1 else None - results.stats.sample_variance = sample_variance - results.stats.sample_stddev = math.sqrt(sample_variance) if sample_variance is not None else None - - aggregated_scores: list[AggregateScore] = [] - for score_name, metric_score in aggregated_results.items(): - stats = metric_score.stats - assert stats is not None - - values = score_values[score_name] - base_name = metric_score.name - base_count = stats.count or 0 - base_nan_count = stats.nan_count or 0 - base_mean = stats.mean - base_sum = stats.sum if stats.sum is not None else (None if base_mean is None else (base_mean * base_count)) - base_min = stats.min if stats.min is not None else base_mean - base_max = stats.max if stats.max is not None else base_mean - base_variance = stats.variance if stats.variance is not None else (None if base_mean is None else 0.0) - base_std_dev = stats.stddev if stats.stddev is not None else (None if base_mean is None else 0.0) - # Sample stats stay None when undefined (fewer than two values) rather than defaulting to 0.0. - base_sample_variance = stats.sample_variance - base_sample_std_dev = stats.sample_stddev - # Derived from the same helper that produces p50, so `median` and `percentiles.p50` agree - # exactly wherever both are present (rubric scores carry no percentiles but still get a median). - base_median = _compute_percentile(sorted(values), 50) if values else None - - if base_count == 0: - base_sum = None - base_mean = None - base_min = None - base_max = None - base_variance = None - base_std_dev = None - base_sample_variance = None - base_sample_std_dev = None - base_median = None - - if has_rubric.get(score_name): - rubric_dist = [ - RubricScoreStat(label=r["label"], value=r["value"], count=r["count"]) - for r in rubric_distribution[score_name].values() - ] - mode_category = None - if rubric_dist: - # Break ties deterministically so aggregate output is stable. - max_count = max(r.count for r in rubric_dist) - tied = sorted(r.label for r in rubric_dist if r.count == max_count) - mode_category = tied[0] - - aggregated_scores.append( - AggregateRubricScore( - name=base_name, - count=base_count, - nan_count=base_nan_count, - sum=base_sum, - mean=base_mean, - min=base_min, - max=base_max, - median=base_median, - variance=base_variance, - std_dev=base_std_dev, - sample_variance=base_sample_variance, - sample_std_dev=base_sample_std_dev, - rubric_distribution=rubric_dist, - mode_category=mode_category, - ) - ) - else: - if values: - # Range scores get richer distribution metadata than rubric scores. - sorted_values = sorted(values) - percentiles = compute_percentiles(sorted_values) - histogram = _compute_histogram(values) - else: - percentiles = None - histogram = Histogram(bins=[]) - - aggregated_scores.append( - AggregateRangeScore( - name=base_name, - count=base_count, - nan_count=base_nan_count, - sum=base_sum, - mean=base_mean, - min=base_min, - max=base_max, - median=base_median, - variance=base_variance, - std_dev=base_std_dev, - sample_variance=base_sample_variance, - sample_std_dev=base_sample_std_dev, - percentiles=percentiles, - histogram=histogram, - ) - ) - - return AggregatedMetricResult(scores=aggregated_scores) - - -def rubric_definitions_from_scores(scores: Sequence[Score]) -> dict[str, list[RubricScoreStat]]: - """Return declared rubric buckets keyed by score name.""" - from nemo_platform.beta.evaluator.values.scores import RubricScore - - definitions: dict[str, list[RubricScoreStat]] = {} - for score in scores: - if not isinstance(score, RubricScore): - continue - definitions[score.name] = [ - RubricScoreStat( - label=rubric.label, - description=rubric.description, - value=rubric.value, - count=0, - ) - for rubric in score.rubric - ] - return definitions - - -def rubric_definitions_from_metric(metric: object) -> dict[str, list[RubricScoreStat]]: - """Return rubric bucket definitions for metrics that carry score config.""" - if not isinstance(metric, MetricWithScores): - return {} - scores = metric.scores - if not isinstance(scores, Sequence) or isinstance(scores, (str, bytes)): - return {} - return rubric_definitions_from_scores(scores) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/bleu.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/bleu.py deleted file mode 100644 index f9b756f48f..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/bleu.py +++ /dev/null @@ -1,94 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""BLEU metric runtime implementation.""" - -import sacrebleu -from nemo_platform.beta.evaluator.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult -from nemo_platform.beta.evaluator.metrics.template_rendering import ( - TemplateSample, - build_template_context, - render_default_output_text_candidate_or_raise, - render_template_or_raise, - template_metric_repr, -) -from nemo_platform.beta.evaluator.values.metrics import BLEU - -__all__ = ["BLEUMetric"] - - -class BLEUMetric(BLEU): - """BLEU metric for sentence- and corpus-level n-gram overlap. - - Evaluator-driven runs render references from dataset fields exposed through - ``item``. Candidate text can come from explicit dataset fields or from - ``sample.output_text`` when the evaluator generates model outputs online. - """ - - def output_spec(self) -> list[MetricOutputSpec]: - """Return row-level outputs emitted by this metric.""" - return [MetricOutputSpec.continuous_score("sentence")] - - def corpus_output_spec(self) -> list[MetricOutputSpec]: - """Return corpus-level outputs emitted by this metric.""" - return [MetricOutputSpec.continuous_score("corpus")] - - def _render_references(self, item: dict, sample: TemplateSample) -> list[str]: - """Render all reference templates for one row.""" - context = build_template_context(item, sample) - metric_repr = template_metric_repr(self) - references: list[str] = [] - for index, reference in enumerate(self.references): - rendered_reference = render_template_or_raise( - template_name=f"references[{index}]", - template=reference, - context=context, - item=item, - sample=sample, - metric_repr=metric_repr, - ) - if not isinstance(rendered_reference, str): - raise TypeError("The reference must be a string.") - references.append(rendered_reference) - return references - - def _render_candidate(self, item: dict, sample: TemplateSample) -> str: - """Render the candidate text for one row.""" - if self.candidate: - context = build_template_context(item, sample) - prediction = render_template_or_raise( - template_name="candidate", - template=self.candidate, - context=context, - item=item, - sample=sample, - metric_repr=template_metric_repr(self), - ) - else: - prediction = render_default_output_text_candidate_or_raise( - sample=sample, - metric_name=self.__class__.__name__, - ) - - if not isinstance(prediction, str): - raise TypeError("The candidate must be a string.") - return prediction - - async def compute_scores(self, input: MetricInput) -> MetricResult: - """Compute the scores for the metric.""" - item = input.row.data - sample = input.candidate - references = self._render_references(item, sample) - candidate = self._render_candidate(item, sample) - score = sacrebleu.sentence_bleu(candidate, references).score.real - return MetricResult(outputs=[MetricOutput(name="sentence", value=score)]) - - async def compute_corpus_scores(self, inputs: list[MetricInput]) -> MetricResult | None: - """Compute the corpus-level BLEU metric.""" - item_sample_pairs = [(input.row.data, input.candidate) for input in inputs] - references_raw = [self._render_references(item, sample) for item, sample in item_sample_pairs] - # NOTE: because of the bug in sacrebleu, we need to flatten the references - references = [[reference_set[0] for reference_set in references_raw]] - candidates = [self._render_candidate(item, sample) for item, sample in item_sample_pairs] - score = sacrebleu.corpus_bleu(candidates, references) - return MetricResult(outputs=[MetricOutput(name="corpus", value=score.score.real)]) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/exact_match.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/exact_match.py deleted file mode 100644 index 1c65fc0b2b..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/exact_match.py +++ /dev/null @@ -1,52 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Exact-match metric runtime implementation.""" - -from nemo_platform.beta.evaluator.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult -from nemo_platform.beta.evaluator.metrics.template_rendering import render_reference_and_candidate, template_metric_repr -from nemo_platform.beta.evaluator.metrics.utils import normalize_text -from nemo_platform.beta.evaluator.values.metrics import ExactMatch - -__all__ = ["ExactMatchMetric"] - - -class ExactMatchMetric(ExactMatch): - """Exact-match metric runtime for evaluator-driven execution. - - Evaluator-driven runs expose dataset values through ``item`` and generated - model outputs through ``sample.output_text`` for online execution. - """ - - def output_spec(self) -> list[MetricOutputSpec]: - """Return outputs emitted by this metric.""" - return [MetricOutputSpec.continuous_score(self.type.value)] - - async def compute_scores(self, input: MetricInput) -> MetricResult: - """Compute structured metric output for one item/sample pair. - - The algorithm renders reference and candidate text from templates, then - normalizes both strings (case, punctuation, articles, and whitespace) - before equality comparison. - - Args: - input: Original dataset row paired with candidate output. - - Returns: - ``MetricResult`` with one exact-match score entry. - - Raises: - TypeError: If rendered reference or candidate is not a string. - ValueError: If template rendering fails or ``candidate`` is omitted - and ``sample.output_text`` is missing. - """ - ground_truth, prediction = render_reference_and_candidate( - metric_repr=template_metric_repr(self), - metric_name=self.__class__.__name__, - reference_template=self.reference, - candidate_template=self.candidate, - item=input.row.data, - sample=input.candidate, - ) - score = int(normalize_text(prediction) == normalize_text(ground_truth)) - return MetricResult(outputs=[MetricOutput(name=self.type.value, value=score)]) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/f1.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/f1.py deleted file mode 100644 index 2a2d7f5933..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/f1.py +++ /dev/null @@ -1,50 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""F1 metric runtime implementation.""" - -import collections - -from nemo_platform.beta.evaluator.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult -from nemo_platform.beta.evaluator.metrics.template_rendering import render_reference_and_candidate, template_metric_repr -from nemo_platform.beta.evaluator.metrics.utils import normalize_text -from nemo_platform.beta.evaluator.values.metrics import F1 - -__all__ = ["F1Metric"] - - -class F1Metric(F1): - """F1 metric for token-overlap similarity scoring.""" - - def output_spec(self) -> list[MetricOutputSpec]: - """Return outputs emitted by this metric.""" - return [MetricOutputSpec.continuous_score(self.type.value)] - - async def compute_scores(self, input: MetricInput) -> MetricResult: - """Compute the scores for the metric.""" - ground_truth, prediction = render_reference_and_candidate( - metric_repr=template_metric_repr(self), - metric_name=self.__class__.__name__, - reference_template=self.reference, - candidate_template=self.candidate, - item=input.row.data, - sample=input.candidate, - ) - - prediction_tokens = normalize_text(prediction).split() - ground_truth_tokens = normalize_text(ground_truth).split() - - # If either token list is empty, the F1 is 1.0 if they agree, 0.0 otherwise. - if len(ground_truth_tokens) == 0 or len(prediction_tokens) == 0: - score = float(ground_truth_tokens == prediction_tokens) - return MetricResult(outputs=[MetricOutput(name=self.type.value, value=score)]) - - common = collections.Counter(prediction_tokens) & collections.Counter(ground_truth_tokens) - num_same = sum(common.values()) - if num_same == 0: - return MetricResult(outputs=[MetricOutput(name=self.type.value, value=0.0)]) - - precision = 1.0 * num_same / len(prediction_tokens) - recall = 1.0 * num_same / len(ground_truth_tokens) - score = (2 * precision * recall) / (precision + recall) - return MetricResult(outputs=[MetricOutput(name=self.type.value, value=score)]) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/hooks.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/hooks.py deleted file mode 100644 index d90b00a19a..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/hooks.py +++ /dev/null @@ -1,41 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Hook capability mixin for V2 SDK metrics.""" - -from __future__ import annotations - -from collections.abc import Sequence -from typing import Self - -import nemo_platform.beta.evaluator.inference as inference -from pydantic import BaseModel, PrivateAttr - - -class HooksBase(BaseModel): - """Reusable hook state and helpers for V2 metrics.""" - - _preprocess_hooks: list[inference.PreprocessRequest] = PrivateAttr(default_factory=list) - _postprocess_hooks: list[inference.PostprocessResponse] = PrivateAttr(default_factory=list) - - def with_hooks( - self, - *, - preprocess: Sequence[inference.PreprocessRequest] | None = None, - postprocess: Sequence[inference.PostprocessResponse] | None = None, - ) -> Self: - """Attach preprocess and postprocess hooks to this metric instance.""" - self._preprocess_hooks = list(preprocess or []) - self._postprocess_hooks = list(postprocess or []) - return self - - def _apply_preprocess_hooks(self, request: dict, *, id: str | None = None) -> dict: - """Apply preprocess hooks in order or return the request unchanged.""" - return inference.preprocess_request(request, hooks=self._preprocess_hooks, id=id) - - def _apply_postprocess_hooks(self, response: dict, *, id: str | None = None) -> dict: - """Apply postprocess hooks in order or return the response unchanged.""" - processed = response - for hook in self._postprocess_hooks: - processed = hook.postprocess(processed, id=id) - return processed diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/llm_judge.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/llm_judge.py deleted file mode 100644 index f22e5222c7..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/llm_judge.py +++ /dev/null @@ -1,441 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""LLM judge metric runtime implementation.""" - -import logging -from copy import copy, deepcopy -from typing import Any, Literal, Protocol, Self - -import nemo_platform.beta.evaluator.inference as inference -from nemo_platform.beta.evaluator.enums import ModelFormat -from nemo_platform.beta.evaluator.inference import InferenceFn, InferenceHookParams -from nemo_platform.beta.evaluator.inference import new_hooks as _new_inference_hooks -from nemo_platform.beta.evaluator.metrics.hooks import HooksBase -from nemo_platform.beta.evaluator.metrics.protocol import ( - MetricInput, - MetricOutput, - MetricOutputSpec, - MetricResult, -) -from nemo_platform.beta.evaluator.metrics.resolution import collect_model_refs, resolve_model_refs -from nemo_platform.beta.evaluator.metrics.template_rendering import ( - TemplateSample, - build_template_context, - sample_template_payload, -) -from nemo_platform.beta.evaluator.resolver_protocols import ModelResolver, SecretResolver -from nemo_platform.beta.evaluator.structured_output import InferenceStructuredOutput, detect_structured_output_mode -from nemo_platform.beta.evaluator.templates import render_request -from nemo_platform.beta.evaluator.values.common import SecretRef, SupportedJobTypes -from nemo_platform.beta.evaluator.values.llm_judge_defaults import ( - default_judge_prompt_template_chat, - default_judge_prompt_template_completions, - default_judge_prompt_template_for_model, -) -from nemo_platform.beta.evaluator.values.metrics import LLM_JUDGE_SCORES_CONTEXT_KEY, LLMJudge -from nemo_platform.beta.evaluator.values.models import Model, ModelRef -from nemo_platform.beta.evaluator.values.params import InferenceParams, ReasoningParams, RunConfig, RunConfigOnline -from nemo_platform.beta.evaluator.values.results import MetricScore -from nemo_platform.beta.evaluator.values.scores import ( - JSONScoreParser, - RangeScore, - RubricScore, - Score, - ScoreParser, - ScoreParserJSON, - ScoreParserRegex, -) -from openai import AsyncOpenAI -from pydantic import PrivateAttr -from pydantic_core import PydanticUndefined - -__all__ = [ - "InferenceParams", - "LLMJudgeMetric", - "Model", - "ModelRef", - "ReasoningParams", - "Score", - "default_judge_prompt_template_chat", - "default_judge_prompt_template_completions", - "generate_structured_output", - "new_hooks", -] - -_logger = logging.getLogger(__name__) - - -class _LLMJudgeHookParams(InferenceHookParams, Protocol): - model: Model | ModelRef - scores: list[Score] - prompt_template: str | dict | None - - -class LLMJudgeMetric(HooksBase, LLMJudge): - """Runtime metric implementation for LLM-as-a-judge scoring.""" - - _use_max_completion_tokens: bool = False - _api_key: str | None = None - _client: AsyncOpenAI | None = PrivateAttr(default=None) - _inference_fn: InferenceFn | None = None - _parsers: dict[str, ScoreParser] = PrivateAttr(default_factory=dict) - _score_dumps: dict[str, dict[str, Any]] = PrivateAttr(default_factory=dict) - _prompt_template_is_default: bool = PrivateAttr(default=False) - job_type: Literal[SupportedJobTypes.ONLINE, SupportedJobTypes.OFFLINE] = SupportedJobTypes.ONLINE - - @property - def client(self) -> AsyncOpenAI: - """Lazily instantiates the client on first access.""" - if self._client is None: - self._client = inference.new_inference_client(self._require_model(), api_key=self._api_key) - return self._client - - def _require_model(self) -> Model: - """Return the resolved model or fail clearly when a ModelRef remains unresolved.""" - if isinstance(self.model, Model): - return self.model - raise ValueError( - f"Model reference '{self.model.root}' has not been resolved. " - "Register it with LocalBackend.model_resolver.register_model() before local execution." - ) - - def __deepcopy__(self, memo: dict[int, Any] | None = None) -> Self: - """ - Override Pydantic __deepcopy__ which returns a deep copy of the model with support to instantiate a new - AsyncOpenAI client. - """ - cls = type(self) - m = cls.__new__(cls) - object.__setattr__(m, "__dict__", deepcopy(self.__dict__, memo=memo)) - object.__setattr__(m, "__pydantic_extra__", deepcopy(self.__pydantic_extra__, memo=memo)) - # This next line doesn't need a deepcopy because __pydantic_fields_set__ is a set[str], - # and attempting a deepcopy would be marginally slower. - object.__setattr__(m, "__pydantic_fields_set__", copy(self.__pydantic_fields_set__)) - - if not hasattr(self, "__pydantic_private__") or self.__pydantic_private__ is None: - object.__setattr__(m, "__pydantic_private__", None) - else: - # Runtime auth/client state must be recreated for the copied model. - private_attrs = deepcopy( - { - k: v - for k, v in self.__pydantic_private__.items() - if v is not PydanticUndefined and k not in {"_client", "_api_key"} - }, - memo=memo, - ) - private_attrs["_client"] = None - private_attrs["_api_key"] = None - object.__setattr__(m, "__pydantic_private__", private_attrs) - - return m - - def set_inference_fn(self, inference_fn: InferenceFn) -> None: - """Set the inference function to use for LLM calls.""" - self._inference_fn = inference_fn - - def apply_evaluation_job_params(self, params: RunConfig) -> None: - """Apply execution job type before resolving generated prompt defaults.""" - self.job_type = SupportedJobTypes.ONLINE if isinstance(params, RunConfigOnline) else SupportedJobTypes.OFFLINE - self._ensure_default_prompt_template() - - def output_spec(self) -> list[MetricOutputSpec]: - """Return outputs emitted by this metric.""" - specs: list[MetricOutputSpec] = [] - for score in self.scores: - if isinstance(score, RubricScore): - specs.append(MetricOutputSpec.continuous_score(score.name, description=score.description)) - specs.append( - MetricOutputSpec.label( - f"{score.name}.label", - description=f"Selected rubric label for {score.name}", - ) - ) - else: - specs.append(MetricOutputSpec.continuous_score(score.name, description=score.description)) - return specs - - def _handle_none_output_error(self, response: dict) -> ValueError: - error_message = "LLM judge returned no usable textual content for score parsing" - message = response.get("choices", [{}])[0].get("message") - if isinstance(message, dict): - has_reasoning = any( - isinstance(value, str) and bool(value.strip()) - for value in (message.get("reasoning"), message.get("reasoning_content")) - ) - if has_reasoning: - error_message = ( - f"{error_message}. The response contains reasoning output but no final text content; " - "the `max_tokens` budget may have been used entirely by reasoning. " - "Try increasing inference `max_tokens` " - "or configuring `inference.extra_body.nvext.max_thinking_tokens` for NIM endpoints" - ) - return ValueError(f"{error_message}. Response: {response}.") - - def _validate_output_text(self, output_text: str | None, response: dict) -> str: - """Ensure the judge returned textual content that can be parsed.""" - if isinstance(output_text, str): - return output_text - raise self._handle_none_output_error(response) - - def _handle_invalid_output(self, error: Exception, fallback: MetricResult, message: str) -> MetricResult: - if self.ignore_request_failure: - _logger.warning("%s: %s", message, str(error)) - return fallback - raise error - - def _nan_result(self) -> MetricResult: - outputs: list[MetricOutput] = [] - for score in self.scores: - outputs.append(MetricOutput(name=score.name, value=float("nan"))) - if isinstance(score, RubricScore): - outputs.append(MetricOutput(name=f"{score.name}.label", value="")) - return MetricResult(outputs=outputs) - - async def resolve_models(self, model_resolver: ModelResolver) -> None: - """Resolve judge model references before the metric is used for evaluation.""" - await resolve_model_refs(self, model_resolver) - self._client = None - self._api_key = None - self._ensure_default_prompt_template() - preprocess_hooks, postprocess_hooks = new_hooks(self) - self.with_hooks(preprocess=preprocess_hooks, postprocess=postprocess_hooks) - - def model_refs(self) -> dict[str, ModelRef]: - """Return judge model references present on this metric.""" - return collect_model_refs(self) - - async def resolve_secrets(self, secret_resolver: SecretResolver) -> None: - """Resolve API key secret if configured and reinitialize AsyncOpenAI client. Must be called before using the metric.""" - model = self._require_model() - if model.api_key_secret: - secret_name = model.api_key_secret.root - self._api_key = await secret_resolver.resolve_secret(model.api_key_secret) - if not self._api_key: - raise ValueError(f"Missing secret '{secret_name}' for API key authentication with LLM judge.") - self._client = inference.new_inference_client(model, api_key=self._api_key) - - async def preflight(self) -> None: - """Resolve structured-output mode once before parallel inference starts.""" - model = self._require_model() - if model.format != ModelFormat.NVIDIA_NIM or not self.structured_output: - return - - structured_hook: InferenceStructuredOutput | None = None - for hook in self._preprocess_hooks: - if isinstance(hook, InferenceStructuredOutput): - structured_hook = hook - break - - if structured_hook is None: - return - - mode = await detect_structured_output_mode( - format=model.format, - model=model, - inference_fn=self.inference_fn, - api_key=self._api_key, - probe_schema={ - "type": "object", - "properties": {"__nmp_probe_score": {"type": "integer"}}, - "required": ["__nmp_probe_score"], - "additionalProperties": False, - }, - ) - structured_hook.set_mode(mode) - _logger.info("NIM structured output mode selected: %s", mode.value) - - def secrets(self) -> dict[str, SecretRef]: - """Return secret env mappings required by this metric.""" - if isinstance(self.model, ModelRef): - return {} - if self.model.api_key_secret and self.model.api_key_env: - return {self.model.api_key_env: self.model.api_key_secret} - return {} - - @property - def inference_fn(self) -> InferenceFn: - """Get the inference function, defaulting to the global one if not injected.""" - return self._inference_fn or inference.make_inference_request - - def model_post_init(self, context: Any, /) -> None: - # Pydantic runs model_post_init() during BaseModel construction, before any - # custom __init__ logic would execute. Derive structured_output here so the - # first parser initialization validates against the finalized JSON schema. - self._prompt_template_is_default = self.prompt_template is None - self.structured_output = generate_structured_output(self) - self._initialize_score_parsers() - preprocess_hooks, postprocess_hooks = new_hooks(self) - self.with_hooks(preprocess=preprocess_hooks, postprocess=postprocess_hooks) - return super().model_post_init(context) - - def _ensure_default_prompt_template(self) -> None: - """Set the default prompt template for the configured judge model.""" - if not self._prompt_template_is_default: - return - if isinstance(self.model, ModelRef): - return - self.prompt_template = default_judge_prompt_template_for_model(self.model, self.job_type) - - def _initialize_score_parsers(self) -> None: - if not self.scores: - return - - for score in self.scores: - if not score.parser: - raise ValueError(f"parser is required for LLM-as-a-Judge score {score.name}: {score}") - - parser_type = score.parser.type - if parser_type == ScoreParserJSON.parser_type: - parser = ScoreParserJSON(score=score, structured_output=self.structured_output) - elif parser_type == ScoreParserRegex.parser_type: - parser = ScoreParserRegex(score=score) - else: - raise ValueError(f"unknown parser type for LLM-as-a-Judge score {score.name}: {parser_type}") - - self._parsers[score.name] = parser - self._score_dumps[score.name] = score.model_dump(mode="json", exclude={"parser"}) - - def _render_request(self, item: dict, sample: TemplateSample) -> dict: - sample_payload = sample_template_payload(sample) - overlapping_keys = set(item.keys()) & set(sample_payload.keys()) - if overlapping_keys: - _logger.warning( - "Dataset columns %s overlap with model response keys. " - "Model response values will be used. " - "To access your dataset values, use 'item.' in your template.", - overlapping_keys, - ) - - context = build_template_context(item, sample) - if self._score_dumps: - context[LLM_JUDGE_SCORES_CONTEXT_KEY] = self._score_dumps - self._ensure_default_prompt_template() - if self.prompt_template is None: - model_ref = self.model.root if isinstance(self.model, ModelRef) else "" - raise ValueError( - f"Model reference '{model_ref}' has not been resolved. " - "Register it with LocalBackend.model_resolver.register_model() before local execution." - ) - request = render_request(self.prompt_template, context=context) - - if "max_tokens" not in request: - request["max_tokens"] = 1024 - if self._use_max_completion_tokens: - request["max_completion_tokens"] = request["max_tokens"] - del request["max_tokens"] - - return self._apply_preprocess_hooks(request) - - def _retry_with_max_completion_tokens(self, request: dict) -> dict: - if not self._use_max_completion_tokens: - _logger.warning( - "Model does not support 'max_tokens' parameter. Switching to 'max_completion_tokens' for all future requests." - ) - self._use_max_completion_tokens = True - request["max_completion_tokens"] = request["max_tokens"] - del request["max_tokens"] - return request - - async def compute_scores(self, input: MetricInput) -> MetricResult: - """Compute structured score output for one item/sample pair.""" - item = input.row.data - sample = input.candidate - request = self._render_request(item, sample) - - try: - response = await self.inference_fn(self._require_model(), request, 3, client=self.client) - except inference.ClientInferenceError as error: - if "max_tokens" in request and "'max_tokens' is not supported with this model" in error.args[0]: - request = self._retry_with_max_completion_tokens(request) - response = await self.inference_fn(self._require_model(), request, 3, client=self.client) - else: - return self._handle_invalid_output( - error, - self._nan_result(), - "Inference failed with LLM judge, marking as NaN", - ) - - try: - output_text = self._validate_output_text( - inference.process_output(response, hooks=self._postprocess_hooks), - response, - ) - except ValueError as error: - return self._handle_invalid_output( - error, - self._nan_result(), - "LLM judge returned invalid output, marking as NaN", - ) - - result = MetricResult(outputs=[]) - for score_name, parser in self._parsers.items(): - score = parser.parse(output_text) - _logger.debug("Parsed score %s: %s", score_name, score.value) - result.outputs.append(MetricOutput(name=score.name, value=score.value)) - label = _selected_rubric_label(score) - if label is not None: - result.outputs.append(MetricOutput(name=f"{score.name}.label", value=label)) - return result - - -def _selected_rubric_label(score: MetricScore) -> str | None: - """Return the selected rubric label recorded by the parser, if any.""" - if not score.stats or not score.stats.rubric_distribution: - return None - for rubric_stat in score.stats.rubric_distribution: - if rubric_stat.count: - return rubric_stat.label - return "" - - -def new_hooks(params: _LLMJudgeHookParams | None): - """Initialize preprocess and postprocess hooks for the LLM judge.""" - model_format = params.model.format if params and isinstance(params.model, Model) else ModelFormat.NVIDIA_NIM - return _new_inference_hooks(params, model_format=model_format) - - -def generate_structured_output(params: _LLMJudgeHookParams) -> dict | None: - """Derive JSON schema for LLM structured output from score criteria.""" - if params.structured_output: - return params.structured_output - - properties: dict[str, dict[str, Any]] = {} - for score in params.scores: - if not isinstance(score.parser, JSONScoreParser): - continue - - key = score.parser.json_path - if isinstance(score, RubricScore): - schema = { - "type": "string", - "enum": [rubric.label for rubric in score.rubric], - } - elif isinstance(score, RangeScore): - schema = { - "type": "integer" if isinstance(score.minimum, int) else "number", - "minimum": score.minimum, - "maximum": score.maximum, - } - else: - continue - - existing = properties.get(key) - if existing is not None and existing != schema: - raise ValueError( - f"conflicting auto-generated structured_output for json_path '{key}'; " - "provide explicit structured_output" - ) - properties[key] = schema - - if not properties: - return None - - return { - "schema": { - "type": "object", - "properties": properties, - "required": list(properties.keys()), - } - } diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/llm_judge_defaults.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/llm_judge_defaults.py deleted file mode 100644 index cc98849ff8..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/llm_judge_defaults.py +++ /dev/null @@ -1,26 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Compatibility re-exports for LLM judge default prompt helpers.""" - -from nemo_platform.beta.evaluator.values.llm_judge_defaults import ( - DEFAULT_JUDGE_PROMPT_TEMPLATE_WITH_TARGET_MODEL, - DEFAULT_JUDGE_SYSTEM_PROMPT_TEMPLATE, - DEFAULT_PROMPT_TEMPLATE, - LLM_JUDGE_SCORES_CONTEXT_KEY, - default_judge_prompt_template_chat, - default_judge_prompt_template_completions, - default_judge_prompt_template_for_model, - is_chat_inference, -) - -__all__ = [ - "DEFAULT_JUDGE_PROMPT_TEMPLATE_WITH_TARGET_MODEL", - "DEFAULT_JUDGE_SYSTEM_PROMPT_TEMPLATE", - "DEFAULT_PROMPT_TEMPLATE", - "LLM_JUDGE_SCORES_CONTEXT_KEY", - "default_judge_prompt_template_chat", - "default_judge_prompt_template_completions", - "default_judge_prompt_template_for_model", - "is_chat_inference", -] diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/number_check.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/number_check.py deleted file mode 100644 index b4fffe4c6a..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/number_check.py +++ /dev/null @@ -1,95 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Number-check metric runtime implementation.""" - -import math -import re - -from nemo_platform.beta.evaluator.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult -from nemo_platform.beta.evaluator.metrics.template_rendering import ( - TemplateSample, - build_template_context, - render_template_or_raise, - template_metric_repr, -) -from nemo_platform.beta.evaluator.values.metrics import NumberCheck, NumberCheckOperation - -__all__ = ["NumberCheckMetric", "NumberCheckOperation"] - - -def _parse_number_answer(answer: str) -> int | float: - """Parse the last numeric value from text; return ``NaN`` when parsing fails.""" - numbers = re.findall(r"[+-]?[\.\d]*\d+", answer) - if not numbers: - return float("nan") - - last_number = numbers[-1] - decimal = last_number.count(".") - if decimal == 1: - return float(last_number) - if decimal == 0: - return int(last_number) - return float("nan") - - -class NumberCheckMetric(NumberCheck): - """Numeric-comparison metric with template-driven operands.""" - - def output_spec(self) -> list[MetricOutputSpec]: - """Return outputs emitted by this metric.""" - return [MetricOutputSpec.continuous_score(self.type.value)] - - async def compute_scores(self, input: MetricInput) -> MetricResult: - """Compute the scores for the metric.""" - item = input.row.data - sample: TemplateSample = input.candidate - context = build_template_context(item, sample) - metric_repr = template_metric_repr(self) - left_value = render_template_or_raise( - template_name="left_template", - template=self.left_template, - context=context, - item=item, - sample=sample, - metric_repr=metric_repr, - ) - right_value = render_template_or_raise( - template_name="right_template", - template=self.right_template, - context=context, - item=item, - sample=sample, - metric_repr=metric_repr, - ) - - left_number = _parse_number_answer(str(left_value)) - right_number = _parse_number_answer(str(right_value)) - # Preserve the legacy behavior: if either side fails to parse as a - # number, return NaN instead of raising. - if math.isnan(left_number): - return MetricResult(outputs=[MetricOutput(name=self.type.value, value=left_number)]) - if math.isnan(right_number): - return MetricResult(outputs=[MetricOutput(name=self.type.value, value=right_number)]) - - # Perform the requested numeric comparison on the parsed operands. - if self.operation in ["equals", "=="]: - score = left_number == right_number - elif self.operation in ["!=", "<>", "not equals"]: - score = left_number != right_number - elif self.operation in [">=", "gte", "greater than or equal"]: - score = left_number >= right_number - elif self.operation in [">", "gt", "greater than"]: - score = left_number > right_number - elif self.operation in ["<=", "lte", "less than or equal"]: - score = left_number <= right_number - elif self.operation in ["<", "lt", "less than"]: - score = left_number < right_number - elif self.operation == "absolute difference": - if self.epsilon is None: - raise ValueError("epsilon value is required with operation absolute difference") - score = abs(left_number - right_number) <= self.epsilon - else: - raise ValueError(f"Unsupported operation: {self.operation}") - - return MetricResult(outputs=[MetricOutput(name=self.type.value, value=1.0 if score else 0.0)]) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/protocol.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/protocol.py deleted file mode 100644 index 09c0911f6e..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/protocol.py +++ /dev/null @@ -1,115 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Runtime protocol for implementing Evaluator metrics.""" - -from __future__ import annotations - -from typing import Protocol, runtime_checkable - -from nemo_platform.beta.evaluator.resolver_protocols import ModelResolver, SecretResolver -from nemo_platform.beta.evaluator.values.common import SecretRef -from nemo_platform.beta.evaluator.values.models import ModelRef -from nemo_platform.beta.evaluator.values.protocol import ( - BooleanValue, - CandidateOutput, - ContinuousScore, - DatasetRow, - DiscreteScore, - Label, - MetricDescriptor, - MetricDiagnostic, - MetricInput, - MetricOutput, - MetricOutputSpec, - MetricResult, - MetricTypeName, - validate_metric_result, -) - -__all__ = [ - "BooleanValue", - "CandidateOutput", - "ContinuousScore", - "CorpusMetric", - "DatasetRow", - "DiscreteScore", - "Label", - "Metric", - "MetricDescriptor", - "MetricDiagnostic", - "MetricInput", - "MetricOutput", - "MetricOutputSpec", - "MetricResult", - "MetricTypeName", - "MetricWithModels", - "MetricWithPreflight", - "MetricWithSecrets", - "validate_metric_result", -] - - -@runtime_checkable -class Metric(Protocol): - """Shared row-scoring primitive for SDK runtime metrics.""" - - @property - def type(self) -> MetricTypeName: - """Return the public metric key/type identifier.""" - ... - - def output_spec(self) -> list[MetricOutputSpec]: - """Return declared row-level outputs emitted by this metric.""" - ... - - async def compute_scores(self, input: MetricInput) -> MetricResult: - """Compute structured output for one row/candidate pair.""" - ... - - -@runtime_checkable -class CorpusMetric(Protocol): - """Protocol for metrics that also emit corpus-level scores.""" - - async def compute_corpus_scores(self, inputs: list[MetricInput]) -> MetricResult | None: - """Compute corpus-level scores across all evaluated rows.""" - ... - - -@runtime_checkable -class MetricWithSecrets(Protocol): - """Protocol for metrics that require secrets.""" - - def secrets(self) -> dict[str, SecretRef]: - """Return environment variables mapped to secret references.""" - ... - - async def resolve_secrets(self, secret_resolver: SecretResolver) -> None: - """Resolve secrets before the metric is used for evaluation.""" - ... - - -@runtime_checkable -class MetricWithModels(Protocol): - """Protocol for metrics that require model resolution.""" - - def model_refs(self) -> dict[str, ModelRef]: - """Return metric field names mapped to model references. - - Example: ``{"model": ModelRef("workspace/model")}``. - """ - ... - - async def resolve_models(self, model_resolver: ModelResolver) -> None: - """Resolve model references before the metric is used for evaluation.""" - ... - - -@runtime_checkable -class MetricWithPreflight(Protocol): - """Protocol for metrics that need one-time setup before parallel evaluation starts.""" - - async def preflight(self) -> None: - """Run one-time preflight before processing rows.""" - ... diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/ragas/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/ragas/__init__.py deleted file mode 100644 index 184a85cd6f..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/ragas/__init__.py +++ /dev/null @@ -1,41 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -from nemo_platform.beta.evaluator.metrics.ragas.metrics import ( - AgentGoalAccuracyMetric, - AnswerAccuracyMetric, - ContextEntityRecallMetric, - ContextPrecisionMetric, - ContextRecallMetric, - ContextRelevanceMetric, - FaithfulnessMetric, - NoiseSensitivityMetric, - ResponseGroundednessMetric, - ResponseRelevancyMetric, - ToolCallAccuracyMetric, - TopicAdherenceMetric, -) -from nemo_platform.beta.evaluator.values.common import SecretRef -from nemo_platform.beta.evaluator.values.models import Model -from nemo_platform.beta.evaluator.values.params import InferenceParams, ReasoningParams - -__all__ = [ - # Metrics - "AgentGoalAccuracyMetric", - "AnswerAccuracyMetric", - "ContextEntityRecallMetric", - "ContextPrecisionMetric", - "ContextRecallMetric", - "ContextRelevanceMetric", - "FaithfulnessMetric", - "NoiseSensitivityMetric", - "ResponseGroundednessMetric", - "ResponseRelevancyMetric", - "ToolCallAccuracyMetric", - "TopicAdherenceMetric", - # Params - "Model", - "InferenceParams", - "ReasoningParams", - "SecretRef", -] diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/ragas/base.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/ragas/base.py deleted file mode 100644 index 28a027f7ab..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/ragas/base.py +++ /dev/null @@ -1,588 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -from __future__ import annotations - -import asyncio -import inspect -import json -import logging -import math -from functools import cache -from typing import TYPE_CHECKING, Any, cast - -import httpx -import nemo_platform.beta.evaluator.constants as constants -from nemo_platform.beta.evaluator.enums import MetricType -from nemo_platform.beta.evaluator.inference import get_logger, requests_log_var -from nemo_platform.beta.evaluator.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult - -# Lazy imports for RAGAS - these are getter functions that defer the expensive -# RAGAS/langchain imports (~20-30s) until first use, improving startup time. -from nemo_platform.beta.evaluator.metrics.ragas.imports import ( - get_evaluate_function, - get_evaluation_dataset_class, - get_langchain_embeddings_wrapper_class, - get_langchain_llm_wrapper_class, - get_run_config_class, -) -from nemo_platform.beta.evaluator.metrics.resolution import collect_model_refs, resolve_model_refs -from nemo_platform.beta.evaluator.resolver_protocols import ModelResolver, SecretResolver -from nemo_platform.beta.evaluator.templates import render_request -from nemo_platform.beta.evaluator.values import ( - MetricBase, - Model, - ModelRef, - SecretRef, -) -from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, ValidationError - -# Type-only imports for static analysis (not imported at runtime) -if TYPE_CHECKING: - from langchain_core.callbacks import BaseCallbackHandler - - from ragas import EvaluationDataset - from ragas.llms.base import LangchainLLMWrapper - -# RAGAS configuration constants -RAGAS_MAX_WAIT = 600 # 10 minutes in seconds -RAGAS_LOG_TENACITY = True -RAGAS_SEED = 42 -DEFAULT_JUDGE_TIMEOUT = 120 # 2 minutes in seconds -DEFAULT_JUDGE_MAX_RETRIES = 3 -DEFAULT_JUDGE_MAX_WORKER = 1 -log = logging.getLogger(__name__) - -RAGAS_OUTPUT_NAME_TO_SDK_OUTPUT_NAME: dict[str, str] = { - "agent_goal_accuracy": MetricType.AGENT_GOAL_ACCURACY.value, - "nv_accuracy": MetricType.ANSWER_ACCURACY.value, - "context_entity_recall": MetricType.CONTEXT_ENTITY_RECALL.value, - "context_precision": MetricType.CONTEXT_PRECISION.value, - "context_recall": MetricType.CONTEXT_RECALL.value, - "nv_context_relevance": MetricType.CONTEXT_RELEVANCE.value, - "faithfulness": MetricType.FAITHFULNESS.value, - "noise_sensitivity": MetricType.NOISE_SENSITIVITY.value, - "nv_response_groundedness": MetricType.RESPONSE_GROUNDEDNESS.value, - "answer_relevancy": MetricType.RESPONSE_RELEVANCY.value, - "tool_call_accuracy": MetricType.TOOL_CALL_ACCURACY.value, - "topic_adherence": MetricType.TOPIC_ADHERENCE.value, -} - - -def _strip_ragas_mode_suffix(name: str) -> str: - """Strip RAGAS's ``(mode=)`` suffix from a score name. - - RAGAS keys mode-bearing metrics (e.g. ``NoiseSensitivity`` with mode - relevant/irrelevant, ``TopicAdherence`` with mode precision/recall/f1) as - ``"(mode=)"`` (see ``ragas.evaluation``). The SDK declares the bare - metric-type name in ``output_spec``, so the suffix is removed before mapping the - RAGAS output name back to the declared SDK output name. - """ - base, separator, remainder = name.partition("(mode=") - if separator and remainder.endswith(")"): - return base - return name - - -# Lazy loaders for langchain classes (cached to avoid repeated imports) -@cache -def _get_langchain_chat_openai(): - """Lazy load ChatOpenAI from langchain.""" - from langchain_openai import ChatOpenAI - - return ChatOpenAI - - -@cache -def _get_nvidia_embeddings(): - """Lazy load NVIDIAEmbeddings from langchain.""" - from langchain_nvidia_ai_endpoints import NVIDIAEmbeddings - - return NVIDIAEmbeddings - - -@cache -def _get_base_callback_handler(): - """Lazy load BaseCallbackHandler from langchain_core.""" - from langchain_core.callbacks import BaseCallbackHandler - - return BaseCallbackHandler - - -@cache -def _get_output_parser_exception_type() -> type[BaseException] | None: - """Lazy load OutputParserException from langchain_core when available.""" - try: - from langchain_core.exceptions import OutputParserException - except Exception: - return None - return OutputParserException - - -class BaseRAGASMetric(MetricBase): - """Base class for all RAGAS metrics in v2. - - Generic over the params type to provide proper type inference in subclasses. - """ - - model_config = ConfigDict(arbitrary_types_allowed=True) - - # Note: Subclasses must define 'type: Literal[MetricType.XXX] = MetricType.XXX' - input_template: dict[str, Any] | None = Field( - default=None, - description="Optional Jinja template for rendering the input payload for RAGAS evaluation.", - ) - - _llm_model: dict | None = None - _inference_params: dict | None = None - _embed_params: dict | None = None - _secrets: dict[str, SecretRef] = PrivateAttr(default_factory=dict) - _log: logging.Logger = logging.getLogger(__name__) - - def output_spec(self) -> list[MetricOutputSpec]: - """Return outputs emitted by this metric.""" - if isinstance(self.type, MetricType): - return [MetricOutputSpec.continuous_score(self.type.value)] - return [] - - def __init__(self, logger: logging.Logger | None = None, **data): - super().__init__(**data) - if logger: - self._log = logger - - self._configure_models() - - def _configure_models(self) -> None: - """Build provider client configuration from resolved inline model bindings.""" - self._inference_params = {} - self._llm_model = None - self._embed_params = None - self._secrets = {} - inference = getattr(self, "inference", None) - if isinstance(inference, BaseModel): - self._inference_params = inference.model_dump(mode="json", exclude_none=True) - - judge_model = getattr(self, "judge_model", None) - if isinstance(judge_model, Model): - # Determine initial API key: - # - If api_key_secret is configured, resolve secret from env - # - If no api_key_secret, use placeholder immediately (no secret resolution needed) - if judge_model.api_key_secret: - assert judge_model.api_key_env is not None - self._secrets[judge_model.api_key_env] = judge_model.api_key_secret - initial_api_key = judge_model.api_key - else: - initial_api_key = constants.PLACEHOLDER_INFERENCE_API_KEY - - self._llm_model = { - "model": judge_model.name, - "base_url": judge_model.url.replace("/completions", "").replace("/chat", ""), - "api_key": initial_api_key, - } - - embeddings_model = getattr(self, "embeddings_model", None) - if isinstance(embeddings_model, Model): - # Determine initial API key: - # - If api_key_secret is configured, resolve secret from env - # - If no api_key_secret, use placeholder immediately (no secret resolution needed) - if embeddings_model.api_key_secret: - assert embeddings_model.api_key_env is not None - self._secrets[embeddings_model.api_key_env] = embeddings_model.api_key_secret - initial_api_key = embeddings_model.api_key - else: - initial_api_key = constants.PLACEHOLDER_INFERENCE_API_KEY - - self._embed_params = { - "model": embeddings_model.name, - "base_url": embeddings_model.url.replace("/embeddings", ""), - "api_key": initial_api_key, - "truncate": self._inference_params.get("truncate", "NONE"), - } - - async def resolve_models(self, model_resolver: ModelResolver) -> None: - """Resolve RAGAS model references before the metric is used for evaluation.""" - await resolve_model_refs(self, model_resolver) - self._configure_models() - - def model_refs(self) -> dict[str, ModelRef]: - """Return RAGAS model references present on this metric.""" - return collect_model_refs(self) - - async def resolve_secrets(self, secret_resolver: SecretResolver) -> None: - """Resolve API key secrets if configured. Must be called before using the metric. - - This follows the same pattern as LLMJudgeMetric.resolve_secrets(). - - Args: - secret_resolver: Resolver used to look up configured secret references. - """ - # Resolve judge API key (only if api_key_secret is configured) - judge_model = getattr(self, "judge_model", None) - if judge_model is not None: - if not isinstance(judge_model, Model): - raise ValueError( - f"Model reference '{judge_model.root}' has not been resolved. " - "Register it with LocalBackend.model_resolver.register_model() before local execution." - ) - if judge_model.api_key_secret: - secret_name = judge_model.api_key_secret.root - api_key = await secret_resolver.resolve_secret(judge_model.api_key_secret) - if not api_key: - raise ValueError(f"Missing secret '{secret_name}' for API key authentication with LLM judge.") - # Update the model config with resolved API key - if self._llm_model: - self._llm_model["api_key"] = api_key - - # Resolve embeddings API key (only if api_key_secret is configured) - embeddings_model = getattr(self, "embeddings_model", None) - if embeddings_model is not None: - if not isinstance(embeddings_model, Model): - raise ValueError( - f"Model reference '{embeddings_model.root}' has not been resolved. " - "Register it with LocalBackend.model_resolver.register_model() before local execution." - ) - if embeddings_model.api_key_secret: - secret_name = embeddings_model.api_key_secret.root - api_key = await secret_resolver.resolve_secret(embeddings_model.api_key_secret) - if not api_key: - raise ValueError( - f"Missing secret '{secret_name}' for API key authentication with embeddings model." - ) - # Update the model config with resolved API key - if self._embed_params: - self._embed_params["api_key"] = api_key - - def secrets(self) -> dict[str, SecretRef]: - """Return mapping of env var names to secret names. - - This is used by the framework to know which secrets need to be injected. - """ - return self._secrets - - def _ignore_request_failure(self) -> bool: - """Return whether this metric should ignore judge inference-call failures.""" - return getattr(self, "ignore_request_failure", False) - - def _nan_scores_for_metrics(self, metrics: list) -> dict[str, float]: - """Build a NaN score mapping using declared output_spec names.""" - metric_names = [output.name for output in self.output_spec()] - if not metric_names: - for metric in metrics: - metric_name = getattr(metric, "name", None) - if isinstance(metric_name, str) and metric_name: - metric_names.append(metric_name) - - return {metric_name: float("nan") for metric_name in metric_names} - - def _align_scores_to_output_spec(self, scores: dict[str, float]) -> dict[str, float]: - """Map known RAGAS metric keys (e.g. ``nv_accuracy``) to declared output names.""" - declared = [output.name for output in self.output_spec()] - if not declared or not scores: - return scores - - translated_scores = {} - for name, value in scores.items(): - base_name = _strip_ragas_mode_suffix(name) - translated_scores[RAGAS_OUTPUT_NAME_TO_SDK_OUTPUT_NAME.get(base_name, base_name)] = value - aligned = { - name: scores[name] if name in scores else translated_scores[name] - for name in declared - if name in scores or name in translated_scores - } - return aligned if aligned else translated_scores - - def _get_llm_judge(self, client: httpx.AsyncClient | None = None) -> LangchainLLMWrapper | None: - """Get the LLM judge instance based on configuration.""" - if not self._llm_model: - return None - - chat_params: dict[str, Any] = {**self._llm_model} - if self._inference_params: - chat_params.update(self._inference_params) - - # Filter out None values - chat_params = {k: v for k, v in chat_params.items() if v is not None} - - # Lazy load ChatOpenAI and LangchainLLMWrapper - ChatOpenAI = _get_langchain_chat_openai() - LangchainLLMWrapper = get_langchain_llm_wrapper_class() - - llm_judge = ChatOpenAI(**chat_params, http_async_client=client) - return LangchainLLMWrapper(llm_judge) - - def _get_embeddings_client(self): - """Get the RAGAS embeddings client.""" - if not self._embed_params: - return None - - # Lazy load NVIDIAEmbeddings and LangchainEmbeddingsWrapper - NVIDIAEmbeddings = _get_nvidia_embeddings() - LangchainEmbeddingsWrapper = get_langchain_embeddings_wrapper_class() - - embeddings = NVIDIAEmbeddings(**self._embed_params) - return LangchainEmbeddingsWrapper(embeddings) - - def _get_run_config(self): - """Get the RAGAS run configuration.""" - inference_params = {} - inference = getattr(self, "inference", None) - if isinstance(inference, BaseModel): - inference_params = inference.model_dump(exclude_none=True) - - # Lazy load RunConfig - RunConfig = get_run_config_class() - - return RunConfig( - timeout=inference_params.get("request_timeout", DEFAULT_JUDGE_TIMEOUT), - max_retries=inference_params.get("max_retries", DEFAULT_JUDGE_MAX_RETRIES), - max_workers=inference_params.get("max_workers", DEFAULT_JUDGE_MAX_WORKER), - max_wait=RAGAS_MAX_WAIT, - log_tenacity=RAGAS_LOG_TENACITY, - seed=RAGAS_SEED, - ) - - def _run_evaluate(self, dataset: EvaluationDataset, metrics: list) -> dict[str, float]: - """Run evaluation with the given dataset and metrics.""" - run_config = self._get_run_config() - ChatModelCallBackHandler = _get_chat_model_callback_handler_class() - callback_cls = cast(Any, ChatModelCallBackHandler) - cb = callback_cls(self._log) - - # Lazy load evaluate function - evaluate = get_evaluate_function() - - # The evaluate function has a decorator that confuses type checkers. - # Call it and cast the result to bypass decorator type issues. - evaluate_fn = cast(Any, evaluate) - parse_exceptions: tuple[type[BaseException], ...] = (json.JSONDecodeError, ValidationError) - output_parser_exception = _get_output_parser_exception_type() - if output_parser_exception is not None: - parse_exceptions = (*parse_exceptions, output_parser_exception) - - try: - results = evaluate_fn( - dataset=dataset, - metrics=metrics, - run_config=run_config, - callbacks=[cb], - raise_exceptions=True, - ) - except parse_exceptions as error: - self._log.warning( - "RAGAS evaluate failed with parse/output error; returning NaN score", - extra={"error": str(error), "metric_type": self.type}, - ) - return self._nan_scores_for_metrics(metrics) - except (httpx.HTTPError, TimeoutError) as error: - if self._ignore_request_failure(): - self._log.warning( - "RAGAS judge inference failed and is ignored by metric policy; returning NaN score", - extra={"error": str(error), "metric_type": self.type}, - ) - return self._nan_scores_for_metrics(metrics) - raise - except Exception: - raise - - scores: dict[str, float] = {} - for metric_dict in results.scores: - for metric_name, metric_value in metric_dict.items(): - scores[metric_name] = metric_value - - if not scores: - self._log.warning( - "RAGAS evaluation returned no scores; returning NaN score", - extra={"metric_type": self.type}, - ) - return self._nan_scores_for_metrics(metrics) - - invalid_score_names = _invalid_score_names(scores) - if invalid_score_names: - self._log.warning( - "RAGAS evaluation produced invalid scores; returning NaN score", - extra={ - "metric_type": self.type, - "invalid_score_names": sorted(invalid_score_names), - }, - ) - return self._nan_scores_for_metrics(metrics) - - return self._align_scores_to_output_spec(scores) - - def _create_evaluation_dataset(self, item: dict, sample: dict) -> EvaluationDataset: - """Create an EvaluationDataset from the given item and sample.""" - # Lazy load EvaluationDataset class (use different name to avoid shadowing type annotation) - EvaluationDatasetCls = get_evaluation_dataset_class() - - template = self.input_template - response = sample.get("output_text") or sample.get("response") - payload = {} - - # if template is provided, add response to the payload if it's not already present - # otherwise, use the item and add response if it's not already present. For Online evaluation, - # model response supersedes the response in the item. - if template: - payload = render_request(template, context={**item, **sample, "item": item, "sample": sample}) - if response and "response" not in payload: - payload["response"] = response - else: - payload = item.copy() - if response: - payload["response"] = response - - return cast(Any, EvaluationDatasetCls).from_list([payload]) - - async def compute_scores(self, input: MetricInput) -> MetricResult: - """Compute the scores for the metric.""" - return await _run_function_in_plain_loop( - self.compute_scores_async, - input.row.data, - input.candidate.as_sample(), - ) - - async def compute_scores_async(self, item: dict, sample: dict) -> MetricResult: - """Compute the scores for the metric asynchronously.""" - async with httpx.AsyncClient() as client: - data = self._create_evaluation_dataset(item, sample) - llm_judge = self._get_llm_judge(client) - scores = self._metric(data, llm_judge) - return MetricResult( - outputs=[ - MetricOutput(name=metric_name, value=score_value) for metric_name, score_value in scores.items() - ] - ) - - def _metric(self, data: EvaluationDataset, llm_judge: LangchainLLMWrapper | None) -> dict[str, float]: - """ - Compute raw scores for the metric. This method must be implemented by subclasses. - - Args: - data: The evaluation dataset to compute metrics on - llm_judge: The LLM judge to use for metrics that require it. If None, the metric - doesn't use an LLM judge. - """ - raise NotImplementedError(f"{self.__class__.__name__} must implement _metric method") - - -async def _run_function_in_plain_loop(fn, *args, **kwargs): - """ - Run any function inside a dedicated thread with a plain asyncio DefaultEventLoopPolicy (not uvloop). - Args: - fn: The function to execute - *args: Positional arguments to pass to the function - **kwargs: Keyword arguments to pass to the function - """ - - def _call(): - # Make a *fresh* loop for this thread and own its lifecycle - policy = asyncio.DefaultEventLoopPolicy() - loop = policy.new_event_loop() - asyncio.set_event_loop(loop) - - async def _run_and_cleanup(): - try: - # Run the provided function - result = fn(*args, **kwargs) - if inspect.isawaitable(result): - result = await result - - # Cleanup any pending tasks - tasks = [t for t in asyncio.all_tasks(loop) if t is not asyncio.current_task()] - if tasks: - for task in tasks: - task.cancel() - await asyncio.gather(*tasks, return_exceptions=True) - - # Cleanup async generators - await loop.shutdown_asyncgens() - - return result - except Exception as e: - log.error(f"Error during function execution: {str(e)}") - raise - - try: - return loop.run_until_complete(_run_and_cleanup()) - finally: - try: - # Close the loop only if it's still running - if not loop.is_closed(): - loop.close() - except Exception as e: - log.warning(f"Error while closing event loop: {str(e)}") - - return await asyncio.to_thread(_call) - - -@cache -def _get_chat_model_callback_handler_class() -> type[BaseCallbackHandler]: - """Create callback handler class that inherits from BaseCallbackHandler. - - Uses a factory pattern to defer the import of BaseCallbackHandler until first use. - """ - BaseCallbackHandler = _get_base_callback_handler() - - class ChatModelCallBackHandler(BaseCallbackHandler): - """A callback handler that logs chat model interactions using thread-safe context variables.""" - - def __init__(self, logger: logging.Logger | None = None): - super().__init__() - self._logger = logger or get_logger() - # Get the thread-local request log from context - self.request_log = requests_log_var.get([]) - # Store current request data between callbacks - self._current_request = None - - def on_chat_model_start(self, serialized: dict[str, Any], messages: list[list[Any]], **kwargs: Any) -> None: - """Stores request data temporarily until completion or error.""" - # Create a new request entry but don't add it to the log yet - self._current_request = {"request": messages} - - def on_llm_end(self, response, **kwargs) -> None: - """Creates a complete log entry with both request and response data.""" - if self._current_request is None: - self._logger.warning("Received response callback without a matching request") - return - - # Create complete log entry - log_entry = { - **self._current_request, - "response": response, - } - - # Add the complete entry to the log - self.request_log.append(log_entry) - self._current_request = None - - def on_llm_error(self, error: BaseException, **kwargs: Any) -> None: - """Logs error information along with the original request.""" - if self._current_request is None: - self._logger.warning("Received error callback without a matching request") - return - - # Create error log entry - log_entry = { - **self._current_request, - "error": str(error), - "error_type": error.__class__.__name__, - } - - # Add the error entry to the log - self.request_log.append(log_entry) - self._current_request = None - - return ChatModelCallBackHandler - - -def _invalid_score_names(scores: dict[str, float]) -> list[str]: - invalid: list[str] = [] - for metric_name, score in scores.items(): - if isinstance(score, bool): - invalid.append(metric_name) - continue - if not isinstance(score, (int, float)): - invalid.append(metric_name) - continue - if not math.isfinite(float(score)): - invalid.append(metric_name) - return invalid diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/ragas/git_patch.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/ragas/git_patch.py deleted file mode 100644 index 013cdbfc8f..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/ragas/git_patch.py +++ /dev/null @@ -1,36 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -""" -Patch for GitPython to avoid Git dependency in containerized environments. - -This module patches the git.refresh function to be a no-op, preventing -GitPython from trying to find the git executable during import. -This is particularly useful when using ragas which depends on GitPython -but doesn't actually need git functionality in a containerized environment. -""" - -import sys -from types import ModuleType - - -def apply_git_patch(): - """ - Apply patch so that git is not available. - - This must be called before any imports of ragas or other packages - that depend on GitPython. - - If GitPython is already installed and available, this function does nothing - to avoid breaking code that actually uses GitPython. - """ - # If git module is already loaded (real or patched), leave it alone - if "git" in sys.modules: - return - - # Try to import GitPython - try: - import git # noqa: F401 - except ImportError: - # GitPython not available, apply patch - sys.modules["git"] = ModuleType("git") diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/ragas/imports.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/ragas/imports.py deleted file mode 100644 index 05c6830f06..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/ragas/imports.py +++ /dev/null @@ -1,192 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Apply Git patch before any other imports that might use GitPython (like ragas) -from nemo_platform.beta.evaluator.metrics.ragas.git_patch import apply_git_patch - -apply_git_patch() - -# ruff: noqa: E402 -"""Lazy imports for RAGAS library to avoid slow startup times. - -RAGAS and its dependencies (langchain, etc.) take 20-30 seconds to import. -This module provides lazy loading so that the import cost is only paid when -RAGAS metrics are actually used, not when the evaluator service starts. - -Usage: - # Instead of: from ragas import EvaluationDataset - # Use: EvaluationDataset = get_evaluation_dataset_class() -""" -import os -from functools import cache - -# Disable RAGAS telemetry/tracking before importing ragas -os.environ["RAGAS_DO_NOT_TRACK"] = "true" - - -@cache -def _load_ragas(): - """Load all RAGAS modules. Cached so imports only happen once.""" - import ragas - from ragas import EvaluationDataset, RunConfig, evaluate - from ragas.embeddings.base import LangchainEmbeddingsWrapper - from ragas.llms.base import LangchainLLMWrapper - from ragas.metrics import ( - AgentGoalAccuracyWithoutReference, - AgentGoalAccuracyWithReference, - AnswerAccuracy, - AnswerCorrectness, - AnswerSimilarity, - ContextEntityRecall, - ContextPrecision, - ContextRecall, - ContextRelevance, - Faithfulness, - NoiseSensitivity, - ResponseGroundedness, - ResponseRelevancy, - ToolCallAccuracy, - TopicAdherenceScore, - ) - - # Return as a dict for easy access - return { - "ragas": ragas, - "EvaluationDataset": EvaluationDataset, - "RunConfig": RunConfig, - "evaluate": evaluate, - "LangchainEmbeddingsWrapper": LangchainEmbeddingsWrapper, - "LangchainLLMWrapper": LangchainLLMWrapper, - "AgentGoalAccuracyWithoutReference": AgentGoalAccuracyWithoutReference, - "AgentGoalAccuracyWithReference": AgentGoalAccuracyWithReference, - "AnswerAccuracy": AnswerAccuracy, - "AnswerCorrectness": AnswerCorrectness, - "AnswerSimilarity": AnswerSimilarity, - "ContextEntityRecall": ContextEntityRecall, - "ContextPrecision": ContextPrecision, - "ContextRecall": ContextRecall, - "ContextRelevance": ContextRelevance, - "Faithfulness": Faithfulness, - "NoiseSensitivity": NoiseSensitivity, - "ResponseGroundedness": ResponseGroundedness, - "ResponseRelevancy": ResponseRelevancy, - "ToolCallAccuracy": ToolCallAccuracy, - "TopicAdherenceScore": TopicAdherenceScore, - } - - -# ============================================================================= -# Lazy accessor functions -# ============================================================================= - - -def get_evaluation_dataset_class() -> type: - """Get the EvaluationDataset class.""" - return _load_ragas()["EvaluationDataset"] - - -def get_run_config_class() -> type: - """Get the RunConfig class.""" - return _load_ragas()["RunConfig"] - - -def get_evaluate_function(): - """Get the evaluate function.""" - return _load_ragas()["evaluate"] - - -def get_langchain_embeddings_wrapper_class() -> type: - """Get the LangchainEmbeddingsWrapper class.""" - return _load_ragas()["LangchainEmbeddingsWrapper"] - - -def get_langchain_llm_wrapper_class() -> type: - """Get the LangchainLLMWrapper class.""" - return _load_ragas()["LangchainLLMWrapper"] - - -def get_topic_adherence_score_class() -> type: - """Get the TopicAdherenceScore metric class.""" - return _load_ragas()["TopicAdherenceScore"] - - -def get_tool_call_accuracy_class() -> type: - """Get the ToolCallAccuracy metric class.""" - return _load_ragas()["ToolCallAccuracy"] - - -def get_agent_goal_accuracy_with_reference_class() -> type: - """Get the AgentGoalAccuracyWithReference metric class.""" - return _load_ragas()["AgentGoalAccuracyWithReference"] - - -def get_agent_goal_accuracy_without_reference_class() -> type: - """Get the AgentGoalAccuracyWithoutReference metric class.""" - return _load_ragas()["AgentGoalAccuracyWithoutReference"] - - -def get_answer_accuracy_class() -> type: - """Get the AnswerAccuracy metric class.""" - return _load_ragas()["AnswerAccuracy"] - - -def get_context_relevance_class() -> type: - """Get the ContextRelevance metric class.""" - return _load_ragas()["ContextRelevance"] - - -def get_response_groundedness_class() -> type: - """Get the ResponseGroundedness metric class.""" - return _load_ragas()["ResponseGroundedness"] - - -def get_context_recall_class() -> type: - """Get the ContextRecall metric class.""" - return _load_ragas()["ContextRecall"] - - -def get_context_precision_class() -> type: - """Get the ContextPrecision metric class.""" - return _load_ragas()["ContextPrecision"] - - -def get_context_entity_recall_class() -> type: - """Get the ContextEntityRecall metric class.""" - return _load_ragas()["ContextEntityRecall"] - - -def get_response_relevancy_class() -> type: - """Get the ResponseRelevancy metric class.""" - return _load_ragas()["ResponseRelevancy"] - - -def get_faithfulness_class() -> type: - """Get the Faithfulness metric class.""" - return _load_ragas()["Faithfulness"] - - -def get_noise_sensitivity_class() -> type: - """Get the NoiseSensitivity metric class.""" - return _load_ragas()["NoiseSensitivity"] - - -__all__ = [ - "get_evaluation_dataset_class", - "get_run_config_class", - "get_evaluate_function", - "get_langchain_embeddings_wrapper_class", - "get_langchain_llm_wrapper_class", - "get_topic_adherence_score_class", - "get_tool_call_accuracy_class", - "get_agent_goal_accuracy_with_reference_class", - "get_agent_goal_accuracy_without_reference_class", - "get_answer_accuracy_class", - "get_context_relevance_class", - "get_response_groundedness_class", - "get_context_recall_class", - "get_context_precision_class", - "get_context_entity_recall_class", - "get_response_relevancy_class", - "get_faithfulness_class", - "get_noise_sensitivity_class", -] diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/ragas/metrics.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/ragas/metrics.py deleted file mode 100644 index 886ef45012..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/ragas/metrics.py +++ /dev/null @@ -1,183 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -from __future__ import annotations - -import logging -from typing import TYPE_CHECKING - -# Type-only imports for static analysis (not imported at runtime) -if TYPE_CHECKING: - from ragas import EvaluationDataset - from ragas.llms.base import LangchainLLMWrapper - -from nemo_platform.beta.evaluator.enums import MetricType -from nemo_platform.beta.evaluator.metrics.ragas.base import BaseRAGASMetric - -# Lazy imports for RAGAS metric classes - these are getter functions that defer -# the expensive RAGAS/langchain imports (~20-30s) until first use. -from nemo_platform.beta.evaluator.metrics.ragas.imports import ( - get_agent_goal_accuracy_with_reference_class, - get_agent_goal_accuracy_without_reference_class, - get_answer_accuracy_class, - get_context_entity_recall_class, - get_context_precision_class, - get_context_recall_class, - get_context_relevance_class, - get_faithfulness_class, - get_noise_sensitivity_class, - get_response_groundedness_class, - get_response_relevancy_class, - get_tool_call_accuracy_class, - get_topic_adherence_score_class, -) -from nemo_platform.beta.evaluator.values import metrics - -log = logging.getLogger(__name__) - - -# Agentic metrics - - -class TopicAdherenceMetric(metrics.TopicAdherence, BaseRAGASMetric): - """Metric for measuring topic adherence.""" - - def _metric(self, data: EvaluationDataset, llm_judge: LangchainLLMWrapper | None) -> dict[str, float]: - metric_mode = self.metric_mode - TopicAdherenceScore = get_topic_adherence_score_class() - metric = TopicAdherenceScore(llm=llm_judge, mode=metric_mode) - return self._run_evaluate(data, [metric]) - - -class ToolCallAccuracyMetric(metrics.ToolCallAccuracy, BaseRAGASMetric): - """Metric for measuring tool call accuracy.""" - - def _metric(self, data: EvaluationDataset, llm_judge: LangchainLLMWrapper | None) -> dict[str, float]: - ToolCallAccuracy = get_tool_call_accuracy_class() - metric = ToolCallAccuracy() - return self._run_evaluate(data, [metric]) - - -class AgentGoalAccuracyMetric(metrics.AgentGoalAccuracy, BaseRAGASMetric): - """Metric for measuring agent goal accuracy.""" - - def _metric(self, data: EvaluationDataset, llm_judge: LangchainLLMWrapper | None) -> dict[str, float]: - # Choose between with/without reference based on params - if self.use_reference: - metric_class = get_agent_goal_accuracy_with_reference_class() - else: - metric_class = get_agent_goal_accuracy_without_reference_class() - metric = metric_class(llm=llm_judge) - return self._run_evaluate(data, [metric]) - - -# Nvidia metrics - - -class AnswerAccuracyMetric(metrics.AnswerAccuracy, BaseRAGASMetric): - """Metric for measuring answer accuracy.""" - - def _metric(self, data: EvaluationDataset, llm_judge: LangchainLLMWrapper | None) -> dict[str, float]: - AnswerAccuracy = get_answer_accuracy_class() - metric = AnswerAccuracy(llm=llm_judge) - return self._run_evaluate(data, [metric]) - - -class ContextRelevanceMetric(metrics.ContextRelevance, BaseRAGASMetric): - """Metric for measuring context relevance.""" - - def _metric(self, data: EvaluationDataset, llm_judge: LangchainLLMWrapper | None) -> dict[str, float]: - ContextRelevance = get_context_relevance_class() - metric = ContextRelevance(llm=llm_judge) - return self._run_evaluate(data, [metric]) - - -class ResponseGroundednessMetric(metrics.ResponseGroundedness, BaseRAGASMetric): - """Metric for measuring response groundedness.""" - - def _metric(self, data: EvaluationDataset, llm_judge: LangchainLLMWrapper | None) -> dict[str, float]: - ResponseGroundedness = get_response_groundedness_class() - metric = ResponseGroundedness(llm=llm_judge) - return self._run_evaluate(data, [metric]) - - -# RAG Metrics - - -class ContextRecallMetric(metrics.ContextRecall, BaseRAGASMetric): - """Metric for measuring context recall.""" - - def _metric(self, data: EvaluationDataset, llm_judge: LangchainLLMWrapper | None) -> dict[str, float]: - ContextRecall = get_context_recall_class() - metric = ContextRecall(llm=llm_judge) - return self._run_evaluate(data, [metric]) - - -class ContextPrecisionMetric(metrics.ContextPrecision, BaseRAGASMetric): - """Metric for measuring context precision.""" - - def _metric(self, data: EvaluationDataset, llm_judge: LangchainLLMWrapper | None) -> dict[str, float]: - ContextPrecision = get_context_precision_class() - metric = ContextPrecision(llm=llm_judge) - return self._run_evaluate(data, [metric]) - - -class ContextEntityRecallMetric(metrics.ContextEntityRecall, BaseRAGASMetric): - """Metric for measuring context entity recall.""" - - def _metric(self, data: EvaluationDataset, llm_judge: LangchainLLMWrapper | None) -> dict[str, float]: - ContextEntityRecall = get_context_entity_recall_class() - metric = ContextEntityRecall(llm=llm_judge) - return self._run_evaluate(data, [metric]) - - -class ResponseRelevancyMetric(metrics.ResponseRelevancy, BaseRAGASMetric): - """Metric for measuring response relevancy.""" - - def _metric(self, data: EvaluationDataset, llm_judge: LangchainLLMWrapper | None) -> dict[str, float]: - embeddings_client = self._get_embeddings_client() - # Strictness defines number of parallel questions generated. NIM can only generate 1. - # Having a configurable parameter allows computation with non-NIM judges. - ResponseRelevancy = get_response_relevancy_class() - metric = ResponseRelevancy(llm=llm_judge, embeddings=embeddings_client, strictness=self.strictness) - return self._run_evaluate(data, [metric]) - - -class FaithfulnessMetric(metrics.Faithfulness, BaseRAGASMetric): - """Metric for measuring faithfulness.""" - - def _metric(self, data: EvaluationDataset, llm_judge: LangchainLLMWrapper | None) -> dict[str, float]: - Faithfulness = get_faithfulness_class() - metric = Faithfulness(llm=llm_judge) - return self._run_evaluate(data, [metric]) - - -class NoiseSensitivityMetric(metrics.NoiseSensitivity, BaseRAGASMetric): - """Metric for measuring noise sensitivity.""" - - def _metric(self, data: EvaluationDataset, llm_judge: LangchainLLMWrapper | None) -> dict[str, float]: - NoiseSensitivity = get_noise_sensitivity_class() - metric = NoiseSensitivity(llm=llm_judge) - return self._run_evaluate(data, [metric]) - - -# List of all RAGAS metrics -ragas_metrics = [ - AgentGoalAccuracyMetric, - AnswerAccuracyMetric, - ContextEntityRecallMetric, - ContextPrecisionMetric, - ContextRecallMetric, - ContextRelevanceMetric, - FaithfulnessMetric, - NoiseSensitivityMetric, - ResponseGroundednessMetric, - ResponseRelevancyMetric, - ToolCallAccuracyMetric, - TopicAdherenceMetric, -] - -# Map of Metric enum values to class for all RAGAS metrics -RAGAS_METRIC_CLASSES: dict[MetricType, type[BaseRAGASMetric]] = { - metric.model_fields["type"].default: metric for metric in ragas_metrics -} diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/remote.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/remote.py deleted file mode 100644 index 8342df86a5..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/remote.py +++ /dev/null @@ -1,250 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Remote metric runtime implementation.""" - -import logging -import os -from abc import ABC, abstractmethod -from typing import Any, cast - -import httpx -from httpx import Timeout -from jsonpath_ng import parse as jsonpath_parse -from jsonpath_ng.exceptions import JsonPathParserError -from nemo_platform.beta.evaluator.inference import requests_log_var -from nemo_platform.beta.evaluator.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult -from nemo_platform.beta.evaluator.metrics.template_rendering import ( - TemplateSample, - build_template_context, - render_template_or_raise, - template_metric_repr, -) -from nemo_platform.beta.evaluator.resilience.api import run_with_resilience -from nemo_platform.beta.evaluator.resilience.classifier import endpoint_identity -from nemo_platform.beta.evaluator.resolver_protocols import SecretResolver -from nemo_platform.beta.evaluator.values.common import SecretRef -from nemo_platform.beta.evaluator.values.metrics import NemoAgentToolkitRemote, Remote, _RemoteBase -from nemo_platform.beta.evaluator.values.scores import RemoteScore -from pydantic import Field, SecretStr, field_validator - -__all__ = ["RemoteMetric", "NemoAgentToolkitRemoteMetric", "SecretRef"] - -_logger = logging.getLogger(__name__) - - -async def _post_to_remote_endpoint( - url: str, - payload: dict[str, Any], - api_key: str | None = None, - timeout: float = 30.0, - max_retries: int = 0, - log: logging.Logger = _logger, -) -> dict[str, Any]: - """Make a POST request to the remote endpoint.""" - headers = {"Content-Type": "application/json"} - if api_key: - headers["Authorization"] = f"Bearer {api_key}" - - log.debug("Calling remote metric url: %r", url) - - endpoint_key = endpoint_identity(url, model_id="remote-metric", auth_identity=api_key) - max_attempts = max(1, max_retries + 1) - - async with httpx.AsyncClient(timeout=Timeout(timeout)) as client: - - async def _invoke_post() -> dict[str, Any]: - response = await client.post(url, json=payload, headers=headers) - response.raise_for_status() - return response.json() - - try: - return await run_with_resilience(endpoint_key, _invoke_post, max_attempts=max_attempts) - except Exception: - log.exception("Remote metric request failed after %d attempts", max_attempts) - raise - - -class _RemoteMetricBase(_RemoteBase, ABC): - """Shared runtime lifecycle for metrics backed by remote HTTP endpoints.""" - - metric_threshold_score: str | None = Field(default=None) - _api_key: SecretStr | None = None - - def model_post_init(self, __context: Any) -> None: - """Initialize private API key from env when configured.""" - if self.api_key_secret: - env_var_name = self.api_key_secret.root.replace("-", "_") - self._set_api_key(os.getenv(env_var_name)) - return super().model_post_init(__context) - - def _set_api_key(self, api_key: str | None) -> None: - """Store API key as SecretStr private attribute.""" - self._api_key = SecretStr(api_key) if api_key else None - - def _get_api_key(self) -> str | None: - """Read API key value from SecretStr private attribute.""" - return self._api_key.get_secret_value() if self._api_key else None - - def _append_request_log(self, *, payload: dict[str, Any], response: dict[str, Any]) -> None: - """Append the remote request and response payloads to the shared request log.""" - requests_log = requests_log_var.get([]) - requests_log.append({"request": payload, "response": response}) - - async def _post_payload(self, payload: dict[str, Any]) -> dict[str, Any]: - """Send one rendered payload to the remote endpoint and log the result.""" - result_data = await _post_to_remote_endpoint( - url=self.url, - payload=payload, - api_key=self._get_api_key(), - timeout=self.timeout_seconds, - max_retries=self.max_retries, - log=_logger, - ) - self._append_request_log(payload=payload, response=result_data) - return result_data - - async def resolve_secrets(self, secret_resolver: SecretResolver) -> None: - """Resolve API key secret if configured. Must be called before live evaluation.""" - if self.api_key_secret: - secret_name = self.api_key_secret.root - resolved_key = await secret_resolver.resolve_secret(self.api_key_secret) - if resolved_key: - self._set_api_key(resolved_key) - elif not self._get_api_key(): - raise ValueError( - f"Missing secret '{secret_name}' for API key authentication with remote metric server." - ) - - def secrets(self) -> dict[str, SecretRef]: - """Return secret env mappings required by this metric.""" - if self.api_key_secret: - env_var_name = self.api_key_secret.root.replace("-", "_") - return {env_var_name: self.api_key_secret} - return {} - - def _select_metric_score(self, metric_result: MetricResult) -> float: - """Select the default score value for one-row metric results.""" - return float(metric_result.outputs[0].value) - - @abstractmethod - async def compute_scores(self, input: MetricInput) -> MetricResult: - """Compute structured score output for one item/sample pair.""" - ... - - -class RemoteMetric(Remote, _RemoteMetricBase): - """A metric that computes scores via a remote endpoint.""" - - def output_spec(self) -> list[MetricOutputSpec]: - """Return outputs emitted by this metric.""" - return [MetricOutputSpec.continuous_score(score.name) for score in self.scores] - - @field_validator("scores") - @classmethod - def _validate_scores(cls, scores: list[RemoteScore]) -> list[RemoteScore]: - for score in scores: - try: - jsonpath_parse(score.parser.json_path) - except JsonPathParserError as error: - raise ValueError( - f"Score '{score.name}' has invalid JSONPath expression '{score.parser.json_path}': {error}" - ) from error - return scores - - def _select_metric_score(self, metric_result: MetricResult) -> float: - """Select the score value used for single-score consumers.""" - if self.metric_threshold_score: - output_names = [output.name for output in metric_result.outputs] - if self.metric_threshold_score not in output_names: - raise ValueError( - f"Score name '{self.metric_threshold_score}' not found in remote metric response. " - f"Available scores: {output_names}" - ) - return float( - next(output for output in metric_result.outputs if output.name == self.metric_threshold_score).value - ) - - if len(metric_result.outputs) == 1: - return float(metric_result.outputs[0].value) - - raise ValueError( - f"Remote metric returned multiple scores {[output.name for output in metric_result.outputs]}. " - "Please set metric_threshold_score to specify which score to use." - ) - - async def compute_scores(self, input: MetricInput) -> MetricResult: - """Compute structured score output via the remote endpoint.""" - item = input.row.data - sample: TemplateSample = input.candidate - context = build_template_context(item, sample) - rendered_args = render_template_or_raise( - template_name="body", - template=self.body, - context=context, - item=item, - sample=sample, - metric_repr=template_metric_repr(self), - ) - payload = cast(dict[str, Any], rendered_args) if isinstance(rendered_args, dict) else {"args": rendered_args} - result_data = await self._post_payload(payload) - - try: - _logger.debug("Remote metric result received for payload: %r", payload) - outputs: list[MetricOutput] = [] - for score_config in self.scores: - jsonpath_expr = jsonpath_parse(score_config.parser.json_path) - matches = jsonpath_expr.find(result_data) - if not matches: - _logger.warning( - "Could not extract score %r from path: %r, setting to NaN", - score_config.name, - score_config.parser.json_path, - ) - outputs.append(MetricOutput(name=score_config.name, value=float("nan"))) - else: - score_value = matches[0].value - outputs.append(MetricOutput(name=score_config.name, value=float(score_value))) - - return MetricResult(outputs=outputs) - except Exception: - _logger.exception("Error validating remote metric response") - raise - - -class NemoAgentToolkitRemoteMetric(NemoAgentToolkitRemote, _RemoteMetricBase): - """A remote metric that interfaces with NeMo Agent Toolkit evaluators.""" - - _RESULT_SCORE_JSONPATH = jsonpath_parse("$.result.score") - - def output_spec(self) -> list[MetricOutputSpec]: - """Return outputs emitted by this metric.""" - return [MetricOutputSpec.continuous_score(self.evaluator_name)] - - async def compute_scores(self, input: MetricInput) -> MetricResult: - """Compute structured score output via the NeMo Agent Toolkit evaluator endpoint.""" - item = input.row.data - sample: TemplateSample = input.candidate - context = build_template_context(item, sample) - rendered_item = render_template_or_raise( - template_name="body.item", - template="{{ item | tojson }}", - context=context, - item=item, - sample=sample, - metric_repr=template_metric_repr(self), - ) - payload = { - "evaluator_name": self.evaluator_name, - "item": rendered_item, - } - result_data = await self._post_payload(payload) - - matches = self._RESULT_SCORE_JSONPATH.find(result_data) - if not matches: - _logger.warning("Could not extract NeMo Agent Toolkit score from response, setting to NaN") - score = float("nan") - else: - score = float(matches[0].value) - - return MetricResult(outputs=[MetricOutput(name=self.evaluator_name, value=score)]) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/resolution.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/resolution.py deleted file mode 100644 index 05994dcc29..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/resolution.py +++ /dev/null @@ -1,48 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Reusable helpers for metrics that declare resolvable references.""" - -from __future__ import annotations - -from typing import Any, get_args - -from nemo_platform.beta.evaluator.resolver_protocols import ModelResolver -from nemo_platform.beta.evaluator.values.models import ModelRef -from pydantic import BaseModel - - -def _annotation_contains_model_ref(annotation: Any) -> bool: - """Return whether a field annotation allows ``ModelRef``.""" - if annotation is ModelRef: - return True - return any(_annotation_contains_model_ref(arg) for arg in get_args(annotation)) - - -def model_ref_fields(metric: BaseModel) -> tuple[str, ...]: - """Return fields annotated as allowing ``ModelRef`` values.""" - return tuple( - field_name - for field_name, field_info in type(metric).model_fields.items() - if _annotation_contains_model_ref(field_info.annotation) - ) - - -def collect_model_refs(metric: BaseModel, fields: tuple[str, ...] | None = None) -> dict[str, ModelRef]: - """Return model-reference-bearing fields that currently hold ``ModelRef`` values.""" - refs: dict[str, ModelRef] = {} - for field_name in fields if fields is not None else model_ref_fields(metric): - value = getattr(metric, field_name, None) - if isinstance(value, ModelRef): - refs[field_name] = value - return refs - - -async def resolve_model_refs( - metric: BaseModel, - model_resolver: ModelResolver, - fields: tuple[str, ...] | None = None, -) -> None: - """Resolve model-reference-bearing fields in place.""" - for field_name, model_ref in collect_model_refs(metric, fields).items(): - setattr(metric, field_name, await model_resolver.resolve_model(model_ref)) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/rouge.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/rouge.py deleted file mode 100644 index b6973d2274..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/rouge.py +++ /dev/null @@ -1,71 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""ROUGE metric runtime implementation.""" - -from functools import cached_property -from typing import ClassVar, Literal - -from nemo_platform.beta.evaluator.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult -from nemo_platform.beta.evaluator.metrics.template_rendering import ( - TemplateSample, - render_reference_and_candidate, - template_metric_repr, -) -from nemo_platform.beta.evaluator.values.metrics import ROUGE - -__all__ = ["ROUGEMetric", "RougeScoreName"] - -RougeScoreName = Literal["rouge_1_score", "rouge_2_score", "rouge_3_score", "rouge_L_score"] - - -class ROUGEMetric(ROUGE): - """ROUGE metric for overlap-based summarization quality scoring. - - Evaluator-driven runs expose dataset fields through ``item`` and generated - model outputs through ``sample.output_text`` for online execution. - """ - - scores_mapping: ClassVar[dict[RougeScoreName, str]] = { - # Maps the public MetricResult score name to the underlying rouge_scorer key. - "rouge_1_score": "rouge1", - "rouge_2_score": "rouge2", - "rouge_3_score": "rouge3", - "rouge_L_score": "rougeL", - } - - @cached_property - def _scorer(self): - """Lazily initialize the ROUGE scorer to avoid expensive import-time setup.""" - # The RougeScorer loads NLTK's stemmer/tokenizer machinery, so keeping - # this lazy avoids unnecessary startup cost for callers that never use - # ROUGE in a given process. - from rouge_score import rouge_scorer - - return rouge_scorer.RougeScorer(["rouge1", "rouge2", "rouge3", "rougeL"], use_stemmer=True) - - def output_spec(self) -> list[MetricOutputSpec]: - """Return outputs emitted by this metric.""" - return [MetricOutputSpec.continuous_score(score_name) for score_name in self.scores_mapping] - - def _metric(self, item: dict, sample: TemplateSample) -> dict: - """Compute raw ROUGE scores for one item/sample pair.""" - ground_truth, prediction = render_reference_and_candidate( - metric_repr=template_metric_repr(self), - metric_name=self.__class__.__name__, - reference_template=self.reference, - candidate_template=self.candidate, - item=item, - sample=sample, - ) - return self._scorer.score(ground_truth, prediction) - - async def compute_scores(self, input: MetricInput) -> MetricResult: - """Compute structured score output for one item/sample pair.""" - scores = self._metric(input.row.data, input.candidate) - return MetricResult( - outputs=[ - MetricOutput(name=score_name, value=scores[score_key].fmeasure) - for score_name, score_key in self.scores_mapping.items() - ] - ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/string_check.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/string_check.py deleted file mode 100644 index d7a9c916aa..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/string_check.py +++ /dev/null @@ -1,69 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""String-check metric runtime implementation.""" - -from nemo_platform.beta.evaluator.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult -from nemo_platform.beta.evaluator.metrics.template_rendering import ( - TemplateSample, - build_template_context, - render_template_or_raise, - template_metric_repr, -) -from nemo_platform.beta.evaluator.values.metrics import StringCheck, StringCheckOperation - -__all__ = ["StringCheckMetric", "StringCheckOperation"] - - -class StringCheckMetric(StringCheck): - """String-comparison metric with operator-based checks.""" - - def output_spec(self) -> list[MetricOutputSpec]: - """Return outputs emitted by this metric.""" - return [MetricOutputSpec.continuous_score(self.type.value)] - - async def compute_scores(self, input: MetricInput) -> MetricResult: - """Compute the scores for the metric.""" - item = input.row.data - sample: TemplateSample = input.candidate - context = build_template_context(item, sample) - metric_repr = template_metric_repr(self) - left_value = render_template_or_raise( - template_name="left_template", - template=self.left_template, - context=context, - item=item, - sample=sample, - metric_repr=metric_repr, - ) - right_value = render_template_or_raise( - template_name="right_template", - template=self.right_template, - context=context, - item=item, - sample=sample, - metric_repr=metric_repr, - ) - - if not isinstance(left_value, str): - raise TypeError("The left value must be a string.") - if not isinstance(right_value, str): - raise TypeError("The right value must be a string.") - - # Perform the requested string comparison on the rendered operands. - if self.operation in ["equals", "=="]: - score = left_value == right_value - elif self.operation in ["!=", "<>", "not equals"]: - score = left_value != right_value - elif self.operation in ["contains"]: - score = right_value in left_value - elif self.operation in ["not contains"]: - score = right_value not in left_value - elif self.operation in ["startswith"]: - score = left_value.startswith(right_value) - elif self.operation in ["endswith"]: - score = left_value.endswith(right_value) - else: - raise ValueError(f"Unsupported operation: {self.operation}") - - return MetricResult(outputs=[MetricOutput(name=self.type.value, value=1.0 if score else 0.0)]) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/template_rendering.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/template_rendering.py deleted file mode 100644 index e822c54136..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/template_rendering.py +++ /dev/null @@ -1,141 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Free-function helpers for rendering metric templates against row data.""" - -import re -from collections.abc import Mapping -from typing import Any - -from jinja2 import UndefinedError -from nemo_platform.beta.evaluator.metrics.protocol import CandidateOutput -from nemo_platform.beta.evaluator.templates import render_template -from pydantic import BaseModel - -TemplateValue = str | dict[Any, Any] | list[Any] -TemplateSample = dict[str, Any] | CandidateOutput -_DICT_ATTRIBUTE_ERROR_RE = re.compile(r"^'dict object' has no attribute '(?P[^']+)'$") -_UNDEFINED_NAME_ERROR_RE = re.compile(r"^'(?P[^']+)' is undefined$") - - -def sample_template_payload(sample: TemplateSample) -> dict[str, Any]: - """Return a sample-shaped dictionary for template rendering helpers.""" - if isinstance(sample, CandidateOutput): - return sample.as_sample() - return sample - - -def build_template_context(item: dict[str, Any], sample: TemplateSample) -> dict[str, Any]: - """Build the template context shared by item and sample rendering.""" - sample_payload = sample_template_payload(sample) - return {**item, **sample_payload, "item": item, "sample": sample_payload} - - -def template_metric_repr(metric: BaseModel | object) -> str: - """Return a compact repr used in row-level template rendering errors.""" - class_name = metric.__class__.__name__ - if not isinstance(metric, BaseModel): - return class_name - - public_fields = metric.model_dump( - exclude={"type", "description", "labels", "supported_job_types"}, - exclude_none=False, - ) - if not isinstance(public_fields, Mapping): - return class_name - - args = ", ".join(f"{name}={value!r}" for name, value in public_fields.items()) - return f"{class_name}({args})" - - -def extract_missing_template_key(template: TemplateValue, exc: UndefinedError) -> str | None: - """Infer the most specific missing key path from a Jinja undefined error.""" - message = exc.message or str(exc) - missing_leaf: str | None = None - if match := _DICT_ATTRIBUTE_ERROR_RE.fullmatch(message): - missing_leaf = match.group("name") - elif match := _UNDEFINED_NAME_ERROR_RE.fullmatch(message): - missing_leaf = match.group("name") - - return missing_leaf - - -def render_template_or_raise( - *, - template_name: str, - template: TemplateValue, - context: dict[str, Any], - item: dict[str, Any], - sample: TemplateSample, - metric_repr: str, - item_keys_label: str = "item", - sample_keys_label: str = "sample", -) -> object: - """Render one template and raise a specific validation error on missing keys.""" - sample_payload = sample_template_payload(sample) - try: - return render_template(template, context) - except UndefinedError as exc: - missing_key = extract_missing_template_key(template, exc) - base_message = ( - f"{metric_repr} could not render its '{template_name}' template for this row.\n" - f"Available {item_keys_label} keys={sorted(item.keys())}. \n" - f"Available {sample_keys_label} keys={sorted(sample_payload.keys())}.\n" - ) - if missing_key is not None: - detail = f"Dataset item has missing_key='{missing_key}' but the '{template_name}' template references it.\n" - else: - detail = f"jinja_error={str(exc)!r}.\n" - raise ValueError( - base_message + detail + "Ensure that the dataset provides the fields referenced by the templates." - ) from exc - - -def render_default_output_text_candidate_or_raise(*, sample: TemplateSample, metric_name: str) -> object: - """Return the default output-text candidate or raise a clear guidance error.""" - prediction = sample_template_payload(sample).get("output_text") - if prediction is None: - raise ValueError( - f"{metric_name} has missing `candidate` field.\n" - f"For offline evaluation, `candidate=...` field is required when constructing {metric_name}.\n" - "For online evaluation, this usually means the evaluated model produced no output." - ) - return prediction - - -def render_reference_and_candidate( - *, - metric_repr: str, - metric_name: str, - reference_template: str, - candidate_template: str | None, - item: dict[str, Any], - sample: TemplateSample, -) -> tuple[str, str]: - """Render reference and candidate templates, returning validated strings.""" - context = build_template_context(item, sample) - ground_truth = render_template_or_raise( - template_name="reference", - template=reference_template, - context=context, - item=item, - sample=sample, - metric_repr=metric_repr, - ) - if candidate_template: - prediction = render_template_or_raise( - template_name="candidate", - template=candidate_template, - context=context, - item=item, - sample=sample, - metric_repr=metric_repr, - ) - else: - prediction = render_default_output_text_candidate_or_raise(sample=sample, metric_name=metric_name) - - if not isinstance(ground_truth, str): - raise TypeError("The reference must be a string.") - if not isinstance(prediction, str): - raise TypeError("The candidate must be a string.") - return ground_truth, prediction diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/tool_calling.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/tool_calling.py deleted file mode 100644 index 659c82c192..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/tool_calling.py +++ /dev/null @@ -1,172 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Tool-calling metric runtime implementation.""" - -import json -import logging -from collections.abc import Mapping -from typing import ClassVar, cast - -from nemo_platform.beta.evaluator.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult -from nemo_platform.beta.evaluator.metrics.template_rendering import ( - TemplateSample, - build_template_context, - render_template_or_raise, - sample_template_payload, - template_metric_repr, -) -from nemo_platform.beta.evaluator.values.metrics import ToolCalling - -__all__ = ["ToolCallingMetric"] - -_logger = logging.getLogger(__name__) - - -class ToolCallingMetric(ToolCalling): - """Tool-calling accuracy metric for structured function calls. - - A metric that supports checks of tool calling: - - function names - - function names and args - and produces respective scores. - - Important. This metric: - - is case sensitive (for all scores) - - is order insensitive, so parallel multiple tool calls may appear out of order - - expects the ground truth to be formatted in OpenAI-compliant tool calling format - - requires function names with dots (``.``) to be normalized to underscores - (``_``), since dots are not supported in OpenAI function names - - Evaluator-driven runs should usually source ``reference`` from ``item``. - """ - - _score_names: ClassVar[list[str]] = ["function_name_accuracy", "function_name_and_args_accuracy"] - - def output_spec(self) -> list[MetricOutputSpec]: - """Return outputs emitted by this metric.""" - return [MetricOutputSpec.continuous_score(score_name) for score_name in self._score_names] - - def _metric(self, item: dict, sample: TemplateSample) -> dict[str, float]: - """Compute raw tool-calling scores for one item/sample pair.""" - sample_payload = sample_template_payload(sample) - context = build_template_context(item, sample) - ground_truth = render_template_or_raise( - template_name="reference", - template=self.reference, - context=context, - item=item, - sample=sample, - metric_repr=template_metric_repr(self), - ) - if not isinstance(ground_truth, list): - raise TypeError("The reference must render to a list of OpenAI-style tool calls.") - - try: - # TODO: Preserve duplicate tool-call multiplicity. The current - # name-keyed matching keeps existing behavior for this pass. - gt_fn_names: list[str] = [] - gt_fn_args_by_name: dict[str, object] = {} - for gt in ground_truth: - if not isinstance(gt, Mapping): - raise TypeError(f"expected reference item to be a mapping, got {type(gt).__name__}") - - gt_mapping = cast(Mapping[str, object], gt) - function_payload = gt_mapping["function"] - if not isinstance(function_payload, Mapping): - raise TypeError( - f"expected reference function payload to be a mapping, got {type(function_payload).__name__}" - ) - - function_mapping = cast(Mapping[str, object], function_payload) - function_name = function_mapping["name"] - if not isinstance(function_name, str): - raise TypeError( - f"expected reference function name to be a string, got {type(function_name).__name__}" - ) - - gt_fn_names.append(function_name) - gt_fn_args_by_name[function_name] = function_mapping["arguments"] - except (KeyError, TypeError) as e: - raise ValueError( - f"Invalid reference template - expected each item to have function.name and function.arguments: {e}" - ) from e - - # Parse tool calls: check sample (online) first, then item (offline). - response_data = sample_payload.get("response") or item.get("response") - if not response_data: - raise ValueError("No response found in sample or item - tool-calling metric requires model response data") - if not isinstance(response_data, dict): - raise ValueError(f"Invalid response format: expected response dict, got {response_data!r}.") - - choices = response_data.get("choices") - if not isinstance(choices, list) or not choices: - raise ValueError(f"Invalid response format: expected non-empty choices list in response {response_data!r}.") - - first_choice = choices[0] - if not isinstance(first_choice, dict) or "message" not in first_choice: - raise ValueError(f"Invalid response format: expected choices[0].message in response {response_data!r}.") - - message = first_choice["message"] - - if not message.get("tool_calls"): - _logger.info("No tool calls found in %s", sample_payload) - message["tool_calls"] = [] - - tool_calls = message["tool_calls"] - pred_fn_name = [call["function"]["name"] for call in tool_calls if "name" in call.get("function", {})] - - fn_names_match = set(gt_fn_names) == set(pred_fn_name) - fn_name_accuracy_score = 1.0 if fn_names_match else 0.0 - - try: - pred_fn_args_by_name: dict[str, dict] = {} - for pred in tool_calls: - function_payload = pred.get("function", {}) - if "name" not in function_payload: - continue - - args = function_payload.get("arguments") - if args is None: - parsed_args = {} - else: - parsed_args = json.loads(args) - - pred_fn_args_by_name[function_payload["name"]] = parsed_args - - _logger.debug("Comparing %s and %s", gt_fn_args_by_name, pred_fn_args_by_name) - - all_args_match = True - if not fn_names_match: - # If function names do not match, the combined name-and-args - # score must also fail automatically. - fn_name_and_args_accuracy_score = 0.0 - else: - # Compare parsed arguments for each ground-truth function name. - for gt_fn_name, gt_fn_args in gt_fn_args_by_name.items(): - pred_fn_args = pred_fn_args_by_name.get(gt_fn_name) - - if pred_fn_args != gt_fn_args: - all_args_match = False - break - - fn_name_and_args_accuracy_score = 1.0 if all_args_match else 0.0 - except (json.JSONDecodeError, TypeError): - # If the model hallucinated malformed JSON arguments, preserve the - # legacy behavior and report NaN for the args-sensitive score. - _logger.warning("Failed parsing tool calling function args: %s", tool_calls) - fn_name_and_args_accuracy_score = float("nan") - - return { - "function_name_accuracy": fn_name_accuracy_score, - "function_name_and_args_accuracy": fn_name_and_args_accuracy_score, - } - - async def compute_scores(self, input: MetricInput) -> MetricResult: - """Compute the scores for the metric.""" - item = input.row.data - sample = input.candidate - scores = self._metric(item, sample) - return MetricResult( - outputs=[MetricOutput(name=score_name, value=score) for score_name, score in scores.items()] - ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/tunable_rag_defaults.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/tunable_rag_defaults.py deleted file mode 100644 index 3f1e281b89..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/tunable_rag_defaults.py +++ /dev/null @@ -1,91 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Default rubric text and JSON format instructions for tunable RAG evaluation. - -Ported from https://github.com/NVIDIA/NeMo-Agent-Toolkit/blob/main/packages/nvidia_nat_langchain/src/nat/plugins/langchain/eval/tunable_rag_evaluator.py -""" - -from __future__ import annotations - -DEFAULT_SCORING_INSTRUCTIONS = ( - "The coverage score is a measure of how well the generated answer covers the critical aspects mentioned in the " - "expected answer. A low coverage score indicates that the generated answer misses critical aspects of the " - "expected answer. A middle coverage score indicates that the generated answer covers some of the must-haves " - "of the expected answer but lacks other details. A high coverage score indicates that all of the expected " - "aspects are present in the generated answer. The correctness score is a measure of how well the generated " - "answer matches the expected answer. A low correctness score indicates that the generated answer is incorrect " - "or does not match the expected answer. A middle correctness score indicates that the generated answer is " - "correct but lacks some details. A high correctness score indicates that the generated answer is exactly the " - "same as the expected answer. The relevance score is a measure of how well the generated answer is relevant " - "to the question. A low relevance score indicates that the generated answer is not relevant to the question. " - "A middle relevance score indicates that the generated answer is somewhat relevant to the question. A high " - "relevance score indicates that the generated answer is exactly relevant to the question. The reasoning is a " - "1-2 sentence explanation for the scoring." -) - -DEFAULT_SCORE_WEIGHTS: dict[str, float] = { - "coverage": 0.5, - "correctness": 0.3, - "relevance": 0.2, -} - -DEFAULT_SCORING_JSON_SCHEMA = { - "type": "object", - "properties": { - "coverage_score": {"type": "number"}, - "correctness_score": {"type": "number"}, - "relevance_score": {"type": "number"}, - "reasoning": {"type": "string"}, - }, - "required": ["coverage_score", "correctness_score", "relevance_score", "reasoning"], - "additionalProperties": False, -} - -CUSTOM_SCORING_JSON_SCHEMA = { - "type": "object", - "properties": { - "score": {"type": "number"}, - "reasoning": {"type": "string"}, - }, - "required": ["score", "reasoning"], - "additionalProperties": False, -} - - -def build_evaluation_prompt( - *, - judge_llm_prompt: str, - instruction: str, - answer_description: str, - generated_answer: str, - default_scoring: bool, -) -> str: - """Build the judge user prompt (format instructions are passed via structured output).""" - if default_scoring: - return ( - "You are an intelligent assistant that responds strictly in JSON format. " - f"Judge based on the following scoring rubric: {DEFAULT_SCORING_INSTRUCTIONS}" - f"{judge_llm_prompt}\n" - f"Here is the instruction: {instruction}" - f"Here is the description of the expected answer: {answer_description}" - f"Here is the generated answer: {generated_answer}" - ) - return ( - f"You are an intelligent assistant that responds strictly in JSON format. {judge_llm_prompt}\n" - f"Here is the instruction: {instruction}" - f"Here is the description of the expected answer: {answer_description}" - f"Here is the generated answer: {generated_answer}" - ) - - -def normalize_score_weights(weights: dict[str, float] | None) -> tuple[float, float, float]: - """Normalize coverage/correctness/relevance weights to sum to 1.""" - source = weights or DEFAULT_SCORE_WEIGHTS - coverage = float(source.get("coverage", 1 / 3)) - correctness = float(source.get("correctness", 1 / 3)) - relevance = float(source.get("relevance", 1 / 3)) - total = coverage + correctness + relevance - if total <= 0: - return 1 / 3, 1 / 3, 1 / 3 - return coverage / total, correctness / total, relevance / total diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/tunable_rag_evaluator.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/tunable_rag_evaluator.py deleted file mode 100644 index cc7699682b..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/tunable_rag_evaluator.py +++ /dev/null @@ -1,238 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Tunable RAG evaluator metric runtime implementation. - -Ported from https://github.com/NVIDIA/NeMo-Agent-Toolkit/blob/main/packages/nvidia_nat_langchain/src/nat/plugins/langchain/eval/tunable_rag_evaluator.py -""" - -from __future__ import annotations - -import json -import logging -import re -from typing import Any, Literal - -import nemo_platform.beta.evaluator.inference as inference -from nemo_platform.beta.evaluator.inference import InferenceFn -from nemo_platform.beta.evaluator.metrics.hooks import HooksBase -from nemo_platform.beta.evaluator.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult -from nemo_platform.beta.evaluator.metrics.resolution import collect_model_refs, resolve_model_refs -from nemo_platform.beta.evaluator.metrics.tunable_rag_defaults import ( - CUSTOM_SCORING_JSON_SCHEMA, - DEFAULT_SCORING_JSON_SCHEMA, - build_evaluation_prompt, - normalize_score_weights, -) -from nemo_platform.beta.evaluator.resolver_protocols import ModelResolver, SecretResolver -from nemo_platform.beta.evaluator.values.common import SecretRef, SupportedJobTypes -from nemo_platform.beta.evaluator.values.metrics import TunableRagEvaluator -from nemo_platform.beta.evaluator.values.models import Model, ModelRef -from nemo_platform.beta.evaluator.values.params import RunConfig, RunConfigOnline -from openai import AsyncOpenAI -from pydantic import PrivateAttr - -__all__ = ["TunableRagEvaluatorMetric"] - -_logger = logging.getLogger(__name__) - -_JSON_FENCE_RE = re.compile(r"```(?:json)?\s*(.*?)\s*```", re.DOTALL | re.IGNORECASE) - - -class TunableRagEvaluatorMetric(HooksBase, TunableRagEvaluator): - """LLM-judge metric with weighted coverage/correctness/relevance composite scoring.""" - - _api_key: str | None = None - _client: AsyncOpenAI | None = PrivateAttr(default=None) - _inference_fn: InferenceFn | None = None - # Populated from RunConfigOnline.max_retries via apply_evaluation_job_params. - _max_retries: int = PrivateAttr(default=3) - job_type: Literal[SupportedJobTypes.ONLINE, SupportedJobTypes.OFFLINE] = SupportedJobTypes.ONLINE - - @property - def client(self) -> AsyncOpenAI: - if self._client is None: - self._client = inference.new_inference_client(self._require_model(), api_key=self._api_key) - return self._client - - def _require_model(self) -> Model: - if isinstance(self.model, Model): - return self.model - raise ValueError( - f"Model reference '{self.model.root}' has not been resolved. " - "Register it with LocalBackend.model_resolver.register_model() before local execution." - ) - - @property - def inference_fn(self) -> InferenceFn: - return self._inference_fn or inference.make_inference_request - - def apply_evaluation_job_params(self, params: RunConfig) -> None: - """Apply online job params; ``max_retries`` lives on ``RunConfigOnline``, not InferenceParams.""" - self.job_type = SupportedJobTypes.ONLINE if isinstance(params, RunConfigOnline) else SupportedJobTypes.OFFLINE - if isinstance(params, RunConfigOnline): - self._max_retries = params.max_retries - - def model_refs(self) -> dict[str, ModelRef]: - return collect_model_refs(self) - - def secrets(self) -> dict[str, SecretRef]: - if isinstance(self.model, ModelRef): - return {} - if self.model.api_key_secret and self.model.api_key_env: - return {self.model.api_key_env: self.model.api_key_secret} - return {} - - async def resolve_secrets(self, secret_resolver: SecretResolver) -> None: - model = self._require_model() - if model.api_key_secret: - secret_name = model.api_key_secret.root - self._api_key = await secret_resolver.resolve_secret(model.api_key_secret) - if not self._api_key: - raise ValueError(f"Missing secret '{secret_name}' for tunable RAG judge authentication.") - self._client = inference.new_inference_client(model, api_key=self._api_key) - - async def resolve_models(self, model_resolver: ModelResolver) -> None: - await resolve_model_refs(self, model_resolver) - - def output_spec(self) -> list[MetricOutputSpec]: - if self.default_scoring: - return [ - MetricOutputSpec.continuous_score("average_score"), - MetricOutputSpec.continuous_score("coverage_score"), - MetricOutputSpec.continuous_score("correctness_score"), - MetricOutputSpec.continuous_score("relevance_score"), - MetricOutputSpec.label("reasoning"), - ] - return [ - MetricOutputSpec.continuous_score("average_score"), - MetricOutputSpec.label("reasoning"), - ] - - async def compute_scores(self, input: MetricInput) -> MetricResult: - instruction, answer_description, generated_answer = _extract_eval_fields(input) - request = self._build_request(instruction, answer_description, generated_answer) - max_retries = self._max_retries - - try: - response = await self.inference_fn(self._require_model(), request, max_retries, client=self.client) - output_text = inference.process_output(response, hooks=self._postprocess_hooks) - except inference.ClientInferenceError as error: - return self._failed_result(f"Inference failed: {error}") - - if not isinstance(output_text, str) or not output_text.strip(): - return self._failed_result("Judge returned empty output.") - - parsed = _parse_json_object(output_text) - if parsed is None: - return self._failed_result("Error in evaluator from parsing judge LLM response.") - - return self._score_from_parsed(parsed) - - def _build_request(self, instruction: str, answer_description: str, generated_answer: str) -> dict[str, Any]: - prompt = build_evaluation_prompt( - judge_llm_prompt=self.judge_llm_prompt, - instruction=instruction, - answer_description=answer_description, - generated_answer=generated_answer, - default_scoring=self.default_scoring, - ) - schema = DEFAULT_SCORING_JSON_SCHEMA if self.default_scoring else CUSTOM_SCORING_JSON_SCHEMA - request: dict[str, Any] = { - "messages": [ - {"role": "system", "content": "You must respond only in JSON format."}, - {"role": "user", "content": prompt}, - ], - "max_tokens": 1024, - "response_format": { - "type": "json_schema", - "json_schema": { - "name": "tunable_rag_evaluator", - "schema": schema, - "strict": True, - }, - }, - } - if self.inference is not None: - request.update(self.inference.model_dump(exclude_none=True)) - return self._apply_preprocess_hooks(request) - - def _score_from_parsed(self, parsed: dict[str, Any]) -> MetricResult: - if self.default_scoring: - try: - coverage = float(parsed["coverage_score"]) - correctness = float(parsed["correctness_score"]) - relevance = float(parsed["relevance_score"]) - reasoning = str(parsed["reasoning"]) - except (KeyError, TypeError, ValueError): - return self._failed_result("Missing or invalid keys in default scoring judge response.") - - coverage_w, correctness_w, relevance_w = normalize_score_weights(self.default_score_weights) - average = coverage_w * coverage + correctness_w * correctness + relevance_w * relevance - return MetricResult( - outputs=[ - MetricOutput(name="average_score", value=average), - MetricOutput(name="coverage_score", value=coverage), - MetricOutput(name="correctness_score", value=correctness), - MetricOutput(name="relevance_score", value=relevance), - MetricOutput(name="reasoning", value=reasoning), - ] - ) - - try: - average = float(parsed["score"]) - reasoning = str(parsed["reasoning"]) - except (KeyError, TypeError, ValueError): - return self._failed_result("Missing or invalid keys in custom scoring judge response.") - return MetricResult( - outputs=[ - MetricOutput(name="average_score", value=average), - MetricOutput(name="reasoning", value=reasoning), - ] - ) - - def _failed_result(self, reasoning: str) -> MetricResult: - if self.default_scoring: - return MetricResult( - outputs=[ - MetricOutput(name="average_score", value=0.0), - MetricOutput(name="coverage_score", value=0.0), - MetricOutput(name="correctness_score", value=0.0), - MetricOutput(name="relevance_score", value=0.0), - MetricOutput(name="reasoning", value=reasoning), - ] - ) - return MetricResult( - outputs=[ - MetricOutput(name="average_score", value=0.0), - MetricOutput(name="reasoning", value=reasoning), - ] - ) - - -def _extract_eval_fields(metric_input: MetricInput) -> tuple[str, str, str]: - """Pull Fabric agent-eval fields: ``inputs.instruction`` + ``reference.answer``.""" - row = metric_input.row.data - inputs = row.get("inputs") - if not isinstance(inputs, dict): - inputs = row - instruction = str(inputs.get("instruction") or "") - reference = row.get("reference") or {} - if isinstance(reference, dict): - answer_description = str(reference.get("answer") or reference.get("expected") or "") - else: - answer_description = str(reference) - generated_answer = str(metric_input.candidate.output_text or metric_input.candidate.response or "") - return instruction, answer_description, generated_answer - - -def _parse_json_object(text: str) -> dict[str, Any] | None: - stripped = text.strip() - fence_match = _JSON_FENCE_RE.search(stripped) - if fence_match: - stripped = fence_match.group(1).strip() - try: - payload = json.loads(stripped) - except json.JSONDecodeError: - return None - return payload if isinstance(payload, dict) else None diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/types.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/types.py deleted file mode 100644 index 48e79d61ee..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/types.py +++ /dev/null @@ -1,64 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Discriminated unions for SDK metric configuration models.""" - -from typing import Annotated, TypeAlias - -from nemo_platform.beta.evaluator.metrics.bleu import BLEUMetric -from nemo_platform.beta.evaluator.metrics.exact_match import ExactMatchMetric -from nemo_platform.beta.evaluator.metrics.f1 import F1Metric -from nemo_platform.beta.evaluator.metrics.llm_judge import LLMJudgeMetric -from nemo_platform.beta.evaluator.metrics.number_check import NumberCheckMetric -from nemo_platform.beta.evaluator.metrics.ragas.metrics import ( - AgentGoalAccuracyMetric, - AnswerAccuracyMetric, - ContextEntityRecallMetric, - ContextPrecisionMetric, - ContextRecallMetric, - ContextRelevanceMetric, - FaithfulnessMetric, - NoiseSensitivityMetric, - ResponseGroundednessMetric, - ResponseRelevancyMetric, - ToolCallAccuracyMetric, - TopicAdherenceMetric, -) -from nemo_platform.beta.evaluator.metrics.remote import NemoAgentToolkitRemoteMetric, RemoteMetric -from nemo_platform.beta.evaluator.metrics.rouge import ROUGEMetric -from nemo_platform.beta.evaluator.metrics.string_check import StringCheckMetric -from nemo_platform.beta.evaluator.metrics.tool_calling import ToolCallingMetric -from nemo_platform.beta.evaluator.metrics.tunable_rag_evaluator import TunableRagEvaluatorMetric -from pydantic import Field - -MetricVariants: TypeAlias = ( - BLEUMetric - | ExactMatchMetric - | F1Metric - | LLMJudgeMetric - | NumberCheckMetric - | RemoteMetric - | NemoAgentToolkitRemoteMetric - | ROUGEMetric - | StringCheckMetric - | ToolCallingMetric - | TunableRagEvaluatorMetric - | TopicAdherenceMetric - | ToolCallAccuracyMetric - | AgentGoalAccuracyMetric - | AnswerAccuracyMetric - | ContextRelevanceMetric - | ResponseGroundednessMetric - | ContextRecallMetric - | ContextPrecisionMetric - | ContextEntityRecallMetric - | ResponseRelevancyMetric - | FaithfulnessMetric - | NoiseSensitivityMetric -) -"""Raw union of SDK metric configuration models, excluding service-only system metrics.""" - -MetricsUnion: TypeAlias = Annotated[MetricVariants, Field(discriminator="type")] -"""Discriminated union of SDK metric configuration models.""" - -__all__ = ["MetricVariants", "MetricsUnion"] diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/utils.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/utils.py deleted file mode 100644 index 79a7c4b672..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/utils.py +++ /dev/null @@ -1,54 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Shared helpers for runtime metrics.""" - -import re -import string - -from nemo_platform.beta.evaluator.enums import MetricType -from nemo_platform.beta.evaluator.metrics.protocol import Metric - - -def normalize_text(s: str) -> str: - """Normalize free-form text for token/equality-based metric comparisons.""" - if not s: - return "" - s = s.lower() - s = "".join(ch for ch in s if ch not in set(string.punctuation)) - s = re.sub(r"\b(a|an|the)\b", " ", s) - return " ".join(s.split()) - - -def metric_type_name(metric: Metric) -> str: - """Resolve a stable public type name for one runtime metric. - - Args: - metric: Metric object used during execution or optimization. - - Returns: - ``metric.type.value`` for built-in ``MetricType`` members, otherwise - the custom string metric type, otherwise the metric class name. - - This helper exists for generic call sites that operate on the runtime - ``Metric`` protocol and must support the documented ``Metric.type`` shapes - without depending on enum-only APIs: - - - built-in ``MetricType`` members - - plain string custom metric types - - custom string-based enum members, such as ``class MyMetricType(str, Enum)`` - - Examples: - Built-in metrics still commonly expose ``MetricType`` members, so a - BLEU runtime metric resolves to ``"bleu"`` via ``metric.type.value``. - - Custom metrics may expose ``type`` as a plain string such as - ``"my-custom-metric"``, or as a custom string-based enum member; both - are returned as their string identifier. - """ - metric_type = getattr(metric, "type", None) - if isinstance(metric_type, MetricType): - return metric_type.value - if isinstance(metric_type, str): - return metric_type - return metric.__class__.__name__ diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resilience/api.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resilience/api.py deleted file mode 100644 index a92ffd5986..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resilience/api.py +++ /dev/null @@ -1,142 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Public API boundary for evaluator resilience scheduling.""" - -from __future__ import annotations - -import asyncio -import logging -from collections.abc import AsyncIterator, Awaitable, Callable, Sequence -from contextlib import asynccontextmanager -from contextvars import ContextVar -from typing import ParamSpec, TypeVar, cast - -from nemo_platform.beta.evaluator.resilience.config import ResilienceConfig -from nemo_platform.beta.evaluator.resilience.scheduler import ResilienceScheduler - -_T = TypeVar("_T") -_P = ParamSpec("_P") -_logger = logging.getLogger(__name__) - -_current_scheduler: ContextVar[ResilienceScheduler | None] = ContextVar("resilience_scheduler", default=None) -_default_scheduler = ResilienceScheduler(ResilienceConfig()) - - -def _active_scheduler() -> ResilienceScheduler: - scheduler = _current_scheduler.get() - if scheduler is not None: - return scheduler - return _default_scheduler - - -@asynccontextmanager -async def use_resilience_session( - *, - global_limit: int | None = None, - endpoint_max_limit: int | None = None, -) -> AsyncIterator[None]: - """Override active scheduler for the current async context. - - This uses a ContextVar-backed session boundary so concurrent requests/jobs can - each have their own scheduler instance and concurrency cap. - - Note: - Limits are global only within this session. They are not process-wide caps - shared across concurrent evaluator jobs. Service-level coordination remains - out of scope for this process-local V2 design. - """ - base = ResilienceConfig() - scheduler = ResilienceScheduler( - ResilienceConfig( - global_limit=max(1, global_limit) if global_limit is not None else base.global_limit, - endpoint_max_limit=max(1, endpoint_max_limit) - if endpoint_max_limit is not None - else base.endpoint_max_limit, - ) - ) - token = _current_scheduler.set(scheduler) - try: - yield - finally: - summary = await scheduler.summary() - _logger.info("Resilience session summary", extra=summary) - await scheduler.shutdown() - _current_scheduler.reset(token) - - -async def run_with_resilience( - endpoint_key: str, - operation: Callable[_P, Awaitable[_T]], - *args: _P.args, - max_attempts: int, # ty: ignore[invalid-paramspec] - deadline_at: float | None = None, - **kwargs: _P.kwargs, -) -> _T: - """Execute an outbound attempt with scheduler-managed retries/admission.""" - scheduler = _active_scheduler() - effective_attempts = max(1, max_attempts) - return await scheduler.run_with_resilience( - endpoint_key, - operation, - *args, - max_attempts=effective_attempts, - deadline_at=deadline_at, - **kwargs, - ) - - -async def run_indexed_tasks( - indices: Sequence[int], - worker: Callable[[int], Awaitable[_T]], - *, - parallelism: int, -) -> list[_T]: - """Execute index-keyed work with bounded in-flight task dispatch. - - Callers provide item indices and an async worker; this helper schedules up to - `parallelism` tasks at a time, preserving result ordering by input position. - - Notes: - `parallelism` is a hard cap on active row-level tasks for a session. - Scheduler admission still applies inside each task, so endpoint/global - shedding can reduce actual outbound call concurrency below this value. - Keeping this outer cap bounds task memory/CPU overhead while retaining - adaptive network pressure control in the scheduler. - """ - if not indices: - return [] - - results: list[_T | None] = [None] * len(indices) - max_inflight = min(len(indices), max(1, parallelism)) - _logger.debug( - "Resilience indexed task execution started", - extra={"task_count": len(indices), "worker_parallelism": parallelism, "max_inflight_workers": max_inflight}, - ) - next_position = 0 - in_flight: dict[asyncio.Task[_T], int] = {} - - while next_position < len(indices) or in_flight: - try: - while next_position < len(indices) and len(in_flight) < max_inflight: - position = next_position - index = indices[position] - in_flight[asyncio.create_task(worker(index))] = position - next_position += 1 - - done, _ = await asyncio.wait(in_flight.keys(), return_when=asyncio.FIRST_COMPLETED) - for task in done: - position = in_flight.pop(task) - results[position] = task.result() - except Exception: - for task in in_flight: - task.cancel() - if in_flight: - await asyncio.gather(*in_flight.keys(), return_exceptions=True) - raise - - _logger.info( - "Resilience indexed task execution completed", - extra={"task_count": len(indices), "worker_parallelism": parallelism, "max_inflight_workers": max_inflight}, - ) - return cast(list[_T], results) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resilience/classifier.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resilience/classifier.py deleted file mode 100644 index c0a0b91521..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resilience/classifier.py +++ /dev/null @@ -1,142 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Exception classification and endpoint identity helpers for resilience.""" - -from __future__ import annotations - -import hashlib -from datetime import UTC, datetime -from email.utils import parsedate_to_datetime - -import httpx -import openai -from nemo_platform.beta.evaluator.resilience.types import ClassifierResult, FailureClass - -_HARD_OVERLOAD_STATUS_CODES = frozenset({429, 503}) -_TRANSIENT_STATUS_CODES = frozenset({408, 500, 502, 504}) - - -def endpoint_identity(base_url: str, model_id: str | None = None, auth_identity: str | None = None) -> str: - """Build a stable endpoint key for scheduler state and accounting.""" - auth_fingerprint = "" - if auth_identity: - auth_fingerprint = hashlib.blake2b(auth_identity.encode("utf-8"), digest_size=8).hexdigest() - return f"{base_url}|{model_id or '_'}|{auth_fingerprint}" - - -def _status_code_from_exception(exc: Exception) -> int | None: - """Return HTTP status code if present on a known exception type.""" - if isinstance(exc, openai.APIStatusError): - return exc.status_code - if isinstance(exc, httpx.HTTPStatusError): - return exc.response.status_code - return None - - -def _parse_retry_after_value(value: str | None) -> float | None: - """Parse `Retry-After` header values into seconds.""" - if value is None: - return None - raw = value.strip() - try: - parsed = float(raw) - except ValueError: - try: - when = parsedate_to_datetime(raw) - except (TypeError, ValueError): - return None - if when.tzinfo is None: - when = when.replace(tzinfo=UTC) - parsed = (when - datetime.now(tz=UTC)).total_seconds() - if parsed < 0: - return 0.0 - return parsed - - -def _retry_after_from_headers(headers: httpx.Headers | None) -> float | None: - """Extract and parse `Retry-After` from an HTTP header mapping.""" - if not headers: - return None - return _parse_retry_after_value(headers.get("Retry-After") or headers.get("retry-after")) - - -def _retry_after_from_exception(exc: Exception) -> float | None: - """Extract `Retry-After` seconds from supported exception types.""" - if isinstance(exc, httpx.HTTPStatusError): - return _retry_after_from_headers(exc.response.headers) - if isinstance(exc, openai.APIStatusError): - response = getattr(exc, "response", None) - headers = getattr(response, "headers", None) - if isinstance(headers, httpx.Headers): - return _retry_after_from_headers(headers) - if isinstance(headers, dict): - return _parse_retry_after_value(headers.get("Retry-After") or headers.get("retry-after")) - return None - - -def classify_exception(exc: Exception) -> ClassifierResult: - """Classify errors into retry/failure policy buckets.""" - retry_after = _retry_after_from_exception(exc) - status_code = _status_code_from_exception(exc) - error_type = type(exc).__name__ - - if isinstance(exc, (httpx.ReadTimeout, httpx.WriteTimeout, httpx.PoolTimeout, openai.APITimeoutError)): - return ClassifierResult( - failure_class=FailureClass.SOFT_OVERLOAD, - retryable=True, - retry_after_seconds=retry_after, - status_code=status_code, - error_type=error_type, - ) - if isinstance(exc, httpx.ConnectTimeout): - return ClassifierResult( - failure_class=FailureClass.TRANSIENT, - retryable=True, - retry_after_seconds=retry_after, - status_code=status_code, - error_type=error_type, - ) - if isinstance(exc, (httpx.NetworkError, openai.APIConnectionError)): - return ClassifierResult( - failure_class=FailureClass.TRANSIENT, - retryable=True, - retry_after_seconds=retry_after, - status_code=status_code, - error_type=error_type, - ) - - if status_code in _HARD_OVERLOAD_STATUS_CODES: - return ClassifierResult( - failure_class=FailureClass.HARD_OVERLOAD, - retryable=True, - retry_after_seconds=retry_after, - status_code=status_code, - error_type=error_type, - ) - if status_code in _TRANSIENT_STATUS_CODES: - return ClassifierResult( - failure_class=FailureClass.TRANSIENT, - retryable=True, - retry_after_seconds=retry_after, - status_code=status_code, - error_type=error_type, - ) - - # Rate-limit style errors without explicit status code handling above. - if isinstance(exc, (openai.RateLimitError,)): - return ClassifierResult( - failure_class=FailureClass.HARD_OVERLOAD, - retryable=True, - retry_after_seconds=retry_after, - status_code=status_code, - error_type=error_type, - ) - - return ClassifierResult( - failure_class=FailureClass.FATAL, - retryable=False, - retry_after_seconds=retry_after, - status_code=status_code, - error_type=error_type, - ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resilience/config.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resilience/config.py deleted file mode 100644 index c30da27e57..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resilience/config.py +++ /dev/null @@ -1,69 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Internal resilience policy configuration for evaluator outbound calls.""" - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class ResilienceConfig: - """Tuning knobs for retry + admission control. - - These settings are internal to evaluator and are not user-facing API. - - Attributes: - global_limit: Hard cap on concurrent outbound attempts for one resilience - session (job/request context), not a process-wide service cap. - endpoint_initial_limit: Starting per-endpoint concurrency limit. - endpoint_min_limit: Lower bound for per-endpoint adaptive concurrency. - endpoint_max_limit: Upper bound for per-endpoint adaptive concurrency. - success_window: Number of successes needed before additive increase (+1) is applied. - beta_hard_overload: Multiplicative decrease factor for hard overload failures. - beta_soft_overload: Multiplicative decrease factor for soft overload failures (timeouts). - cooldown_seconds_hard: Cooldown duration after hard overload feedback. - cooldown_seconds_soft: Cooldown duration after soft overload feedback. - timeout_soft_overload_escalation_count: Soft overload events needed to escalate to hard behavior. - escalation_window_seconds: Sliding window used for soft-overload escalation counting. - backoff_initial_ms: Initial retry backoff in milliseconds. - backoff_cap_ms: Maximum retry backoff in milliseconds. - pressure_gain_k: Pressure multiplier gain for retry wait inflation. - max_attempts_default: Default maximum attempts if callsites do not override. - task_deadline_seconds_default: Default per-task retry deadline. - global_max_queued: Max queued operations across all endpoints before rejecting work. - endpoint_max_queued: Max queued operations per endpoint before rejecting work. - retry_budget_tokens_per_sec: Retry token refill rate per endpoint. - retry_budget_burst: Retry token burst capacity per endpoint. - shutdown_grace_seconds: Grace period for graceful scheduler shutdown. - endpoint_state_max_entries: Maximum endpoint states retained in memory. - endpoint_state_ttl_seconds: TTL for endpoint states before eviction. - """ - - global_limit: int = 64 - endpoint_initial_limit: int = 4 - endpoint_min_limit: int = 1 - endpoint_max_limit: int = 64 - - success_window: int = 20 - beta_hard_overload: float = 0.5 - beta_soft_overload: float = 0.7 - cooldown_seconds_hard: float = 3.0 - cooldown_seconds_soft: float = 1.5 - timeout_soft_overload_escalation_count: int = 3 - escalation_window_seconds: float = 10.0 - - backoff_initial_ms: float = 250.0 - backoff_cap_ms: float = 15_000.0 - pressure_gain_k: float = 1.0 - max_attempts_default: int = 3 - task_deadline_seconds_default: float = 60.0 - - global_max_queued: int = 8_192 - endpoint_max_queued: int = 1_024 - retry_budget_tokens_per_sec: float = 5.0 - retry_budget_burst: float = 10.0 - - shutdown_grace_seconds: float = 15.0 - - endpoint_state_max_entries: int = 4_096 - endpoint_state_ttl_seconds: float = 300.0 diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resilience/errors.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resilience/errors.py deleted file mode 100644 index 56fb286faf..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resilience/errors.py +++ /dev/null @@ -1,117 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Exception helpers shared by evaluator resilience/task flows.""" - -from collections.abc import Iterator -from typing import TypeVar - -from jinja2.exceptions import UndefinedError as JinjaUndefinedError -from nemo_platform.beta.evaluator.execution.values import EvaluationError - -E = TypeVar("E", bound=BaseException) - - -def iter_leaf_causes(exc: BaseException) -> Iterator[BaseException]: - """Yield all leaf exceptions from an exception-group tree in left-to-right DFS order. - - Recurses into any exception exposing a non-empty tuple ``.exceptions`` - attribute — matches ``BaseExceptionGroup`` on Python >= 3.11 and the - ``exceptiongroup`` backport. Group nesting reflects how tasks were grouped - (e.g. nested ``TaskGroup``s), not causation — leaves are independent, - concurrent failures. - - Example: - Nested group — each branch is fully traversed (children before - siblings) before moving to the next sibling at the parent level:: - - group = ExceptionGroup( - "outer", - [ValueError("a"), - ExceptionGroup("inner", [RuntimeError("b"), KeyError("c")]), - TypeError("d")], - ) - list(iter_leaf_causes(group)) - # [ValueError("a"), RuntimeError("b"), KeyError("c"), TypeError("d")] - """ - stack: list[BaseException] = [exc] - while stack: - current = stack.pop() - exceptions = getattr(current, "exceptions", None) - if not isinstance(exceptions, tuple) or not exceptions: - yield current - continue - if any(not isinstance(child, BaseException) for child in exceptions): - # Defensive: a non-exception child means this node is effectively a leaf. - yield current - continue - # Push reversed so the leftmost child is popped first (preserves DFS left→right). - stack.extend(reversed(exceptions)) - - -def first_failure_cause(exc: BaseException) -> BaseException: - """Return the first leaf failure from an exception/exception-group tree. - - Use case: surface *a* concrete failure in a log line or wrapped message, - without caring which sibling comes first. - """ - return next(iter_leaf_causes(exc), exc) - - -def find_cause(exc: BaseException, cls: type[E]) -> E | None: - """Return the first leaf of type ``cls`` anywhere in the exception tree, or ``None``. - - Unlike :func:`first_failure_cause`, which returns only the first leaf in - traversal order, ``find_cause`` walks every leaf of any ``ExceptionGroup`` - via :func:`iter_leaf_causes`. That means a matching exception is found even - when it is not the leading sibling or sits under a non-leading branch. The - return type is narrowed to ``cls | None`` via ``TypeVar``, avoiding a - duplicate ``isinstance`` check at the call site. Common with - ``asyncio.TaskGroup``, where sibling order depends on task scheduling and - is not deterministic. - - Example: - Nested group — the ``EvaluationError`` is buried under a non-leading - branch, so ``first_failure_cause`` misses it:: - - group = ExceptionGroup("outer", [ - RuntimeError("sibling"), - ExceptionGroup("inner", [ - RuntimeError("noise"), - EvaluationError(index=5, message="template error"), - ]), - ]) - first_failure_cause(group) # RuntimeError("sibling") - find_cause(group, EvaluationError) # EvaluationError(index=5, ...) - """ - for leaf in iter_leaf_causes(exc): - if isinstance(leaf, cls): - return leaf - return None - - -def normalize_evaluation_failure( - exc: BaseException, - *, - prefix: str = "Metric evaluation has failed", -) -> RuntimeError: - """Convert queue/task execution failures into the public evaluator error shape.""" - root = first_failure_cause(exc) - if isinstance(root, JinjaUndefinedError): - return RuntimeError(f"{prefix} due to templating error: {str(root)}") - if isinstance(exc, JinjaUndefinedError): - return RuntimeError(f"{prefix} due to templating error: {str(exc)}") - return RuntimeError(f"{prefix} with error: {str(root) or root.__class__.__name__}") - - -def get_evaluation_error(exc: BaseException) -> EvaluationError | RuntimeError: - """Classify a pipeline failure into an ``EvaluationError`` or normalized ``RuntimeError``. - - Returns the first ``EvaluationError`` found anywhere in the exception tree - (via :func:`find_cause`) if present; otherwise falls back to - :func:`normalize_evaluation_failure`. Callers decide how to re-raise. - """ - evaluation_error = find_cause(exc, EvaluationError) - if evaluation_error is not None: - return evaluation_error - return normalize_evaluation_failure(exc) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resilience/policy.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resilience/policy.py deleted file mode 100644 index 8bf6df7e4b..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resilience/policy.py +++ /dev/null @@ -1,50 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Policy math for resilience retry waits and endpoint adaptation.""" - -from __future__ import annotations - -import math -import random - -from nemo_platform.beta.evaluator.resilience.config import ResilienceConfig -from nemo_platform.beta.evaluator.resilience.types import FailureClass, RetryContext - - -def pressure(limit: int, max_limit: int) -> float: - """Compute normalized pressure (0..1) from endpoint capacity state.""" - if max_limit <= 0: - return 1.0 - return max(0.0, min(1.0, 1.0 - (limit / max_limit))) - - -def retry_wait_seconds(context: RetryContext, config: ResilienceConfig) -> float: - """Compute retry delay with jitter, pressure scaling, and overload floors.""" - exp = min( - config.backoff_cap_ms / 1000.0, (config.backoff_initial_ms / 1000.0) * (2 ** max(0, context.attempt_number - 1)) - ) - jittered = random.uniform(0.0, exp) - pressure_mult = 1.0 + (config.pressure_gain_k * context.pressure) - computed = jittered * pressure_mult - server_floor = context.retry_after_seconds or 0.0 - return max(server_floor, context.cooldown_remaining_seconds, computed) - - -def additive_increase(limit: int, max_limit: int) -> int: - """Increase endpoint limit by one within configured bounds.""" - return min(max_limit, limit + 1) - - -def multiplicative_decrease(limit: int, min_limit: int, beta: float) -> int: - """Reduce endpoint limit by multiplicative factor within bounds.""" - return max(min_limit, math.ceil(limit * beta)) - - -def cooldown_for_failure_class(failure_class: FailureClass, config: ResilienceConfig) -> float: - """Return cooldown period in seconds for a failure class.""" - if failure_class == FailureClass.HARD_OVERLOAD: - return config.cooldown_seconds_hard - if failure_class == FailureClass.SOFT_OVERLOAD: - return config.cooldown_seconds_soft - return 0.0 diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resilience/scheduler.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resilience/scheduler.py deleted file mode 100644 index d3b3b70fe5..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resilience/scheduler.py +++ /dev/null @@ -1,459 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Process-local resilience scheduler for admission control and retries.""" - -from __future__ import annotations - -import asyncio -import logging -from collections.abc import Awaitable, Callable -from dataclasses import dataclass -from typing import Deque, ParamSpec, TypeVar - -import anyio -from nemo_platform.beta.evaluator.resilience.classifier import classify_exception -from nemo_platform.beta.evaluator.resilience.config import ResilienceConfig -from nemo_platform.beta.evaluator.resilience.policy import ( - additive_increase, - cooldown_for_failure_class, - multiplicative_decrease, - pressure, - retry_wait_seconds, -) -from nemo_platform.beta.evaluator.resilience.types import ( - ClassifierResult, - Clock, - EndpointState, - FailureClass, - OperationCounters, - RetryContext, - SystemClock, -) - -_logger = logging.getLogger(__name__) -_T = TypeVar("_T") -_P = ParamSpec("_P") - - -class ResilienceError(RuntimeError): - """Base class for resilience runtime failures.""" - - def __init__(self, message: str, *, endpoint_key: str, attempt: int, cause: BaseException | None = None) -> None: - super().__init__(message) - self.endpoint_key = endpoint_key - self.attempt = attempt - self.cause = cause - - -class ResilienceQueueFullError(ResilienceError): - """Raised when scheduler queue bounds are exceeded.""" - - def __init__(self, message: str, *, endpoint_key: str, reason: str, attempt: int = 0) -> None: - super().__init__(message, endpoint_key=endpoint_key, attempt=attempt) - self.reason = reason - - -class ResilienceDeadlineExceededError(ResilienceError): - """Raised when retry would violate deadline.""" - - -class ResilienceMaxAttemptsExceededError(ResilienceError): - """Raised when retry attempts are exhausted.""" - - -class ResilienceCancelledError(ResilienceError): - """Raised when operation is cancelled during scheduler flow.""" - - -@dataclass -class _Controller: - state: EndpointState - limiter: anyio.CapacityLimiter - lock: asyncio.Lock - - -class ResilienceScheduler: - """Central process-local scheduler for retries and adaptive admission. - - A scheduler instance is intended to be session-scoped (for example one job or - one live-eval request). Its global limiter enforces bounds within that session, - not across the entire evaluator process. - """ - - def __init__(self, config: ResilienceConfig, *, clock: Clock | None = None) -> None: - self._config = config - self._clock = clock or SystemClock() - self._global_limiter = anyio.CapacityLimiter(config.global_limit) - self._controllers: dict[str, _Controller] = {} - self._lock = asyncio.Lock() - self._global_queued = 0 - self._shutdown = False - self._metrics = OperationCounters() - - def now(self) -> float: - """Return scheduler monotonic time.""" - return self._clock.monotonic() - - async def shutdown(self) -> None: - """Mark scheduler as shutting down.""" - async with self._lock: - self._shutdown = True - - async def run_with_resilience( - self, - endpoint_key: str, - operation: Callable[_P, Awaitable[_T]], - *args: _P.args, - max_attempts: int, # ty: ignore[invalid-paramspec] - deadline_at: float | None, - **kwargs: _P.kwargs, - ) -> _T: - """Execute operation with adaptive scheduling and retry policy.""" - last_failure_class: FailureClass | None = None - _logger.debug( - "Resilience operation started", - extra={"endpoint_key": endpoint_key, "max_attempts": max_attempts, "deadline_at": deadline_at}, - ) - controller = await self._get_controller(endpoint_key) - async with controller.lock: - controller.state.counters.operations_started += 1 - async with self._lock: - self._metrics.operations_started += 1 - for attempt in range(1, max_attempts + 1): - try: - result = await self._run_once(endpoint_key, operation, *args, attempt=attempt, **kwargs) - except ResilienceCancelledError: - raise - except Exception as exc: - classified = classify_exception(exc) - last_failure_class = classified.failure_class - controller = await self._get_controller(endpoint_key) - await self._record_failure(controller, classified) - - if not classified.retryable: - raise - if attempt >= max_attempts: - raise ResilienceMaxAttemptsExceededError( - f"Retry attempts exhausted: {attempt} out of {max_attempts}", - endpoint_key=endpoint_key, - attempt=attempt, - cause=exc, - ) from exc - - now = self.now() - cooldown_remaining = max(0.0, controller.state.cooldown_until - now) - wait_seconds = retry_wait_seconds( - RetryContext( - attempt_number=attempt, - retry_after_seconds=classified.retry_after_seconds, - pressure=pressure(controller.state.limit, controller.state.max_limit), - cooldown_remaining_seconds=cooldown_remaining, - ), - self._config, - ) - if deadline_at is not None and now + wait_seconds > deadline_at: - raise ResilienceDeadlineExceededError( - "Retry deadline exceeded", - endpoint_key=endpoint_key, - attempt=attempt, - cause=exc, - ) from exc - _logger.info( - "Resilience retry scheduled", - extra={ - "endpoint_key": endpoint_key, - "attempt": attempt, - "failure_class": classified.failure_class.value, - "wait_seconds": wait_seconds, - "retry_after_seconds": classified.retry_after_seconds, - "pressure": pressure(controller.state.limit, controller.state.max_limit), - "cooldown_remaining_seconds": cooldown_remaining, - "endpoint_limit": controller.state.limit, - }, - ) - async with controller.lock, self._lock: - self._metrics.retries_scheduled += 1 - controller.state.counters.retries_scheduled += 1 - await asyncio.sleep(wait_seconds) - else: - await self.record_success(endpoint_key) - controller = await self._get_controller(endpoint_key) - _logger.debug( - "Resilience operation completed", - extra={ - "endpoint_key": endpoint_key, - "attempts_used": attempt, - "last_failure_class": last_failure_class.value if last_failure_class else None, - "endpoint_limit": controller.state.limit, - "endpoint_max_inflight_seen": controller.state.max_inflight_seen, - }, - ) - async with self._lock: - self._metrics.operations_completed += 1 - async with controller.lock: - controller.state.counters.operations_completed += 1 - return result - - raise RuntimeError("Unreachable: retry loop exhausted without terminal outcome") - - async def _run_once( - self, - endpoint_key: str, - operation: Callable[_P, Awaitable[_T]], - *args: _P.args, - attempt: int, # ty: ignore[invalid-paramspec] - **kwargs: _P.kwargs, - ) -> _T: - controller = await self._get_controller(endpoint_key) - await self._enqueue_or_raise(controller.state, endpoint_key) - - endpoint_acquired = False - global_acquired = False - dispatched = False - started = self.now() - try: - await controller.limiter.acquire() - endpoint_acquired = True - await self._global_limiter.acquire() - global_acquired = True - await self._on_dispatch(controller.state, attempt=attempt) - dispatched = True - return await operation(*args, **kwargs) - except asyncio.CancelledError as exc: - async with controller.lock, self._lock: - self._metrics.cancellations += 1 - controller.state.counters.cancellations += 1 - raise ResilienceCancelledError( - "Operation cancelled", - endpoint_key=endpoint_key, - attempt=attempt, - cause=exc, - ) from exc - finally: - elapsed = max(0.0, self.now() - started) - await self._on_complete(controller.state, dispatched=dispatched, elapsed_seconds=elapsed) - if global_acquired: - self._global_limiter.release() - if endpoint_acquired: - controller.limiter.release() - - async def _get_controller(self, endpoint_key: str) -> _Controller: - """Get or create endpoint controller state and refresh its last-seen time.""" - now = self.now() - async with self._lock: - self._evict_stale_endpoints(now) - controller = self._controllers.get(endpoint_key) - if controller is None: - state = EndpointState( - key=endpoint_key, - limit=max( - self._config.endpoint_min_limit, - min(self._config.endpoint_initial_limit, self._config.endpoint_max_limit), - ), - min_limit=self._config.endpoint_min_limit, - max_limit=self._config.endpoint_max_limit, - retry_budget_tokens=self._config.retry_budget_burst, - retry_budget_last_refill=now, - last_seen=now, - ) - controller = _Controller(state=state, limiter=anyio.CapacityLimiter(state.limit), lock=asyncio.Lock()) - self._controllers[endpoint_key] = controller - controller.state.last_seen = now - return controller - - def _evict_stale_endpoints(self, now: float) -> None: - """Evict endpoint state by TTL first, then by least-recently-seen capacity pressure.""" - stale_keys = [ - key - for key, controller in self._controllers.items() - if now - controller.state.last_seen > self._config.endpoint_state_ttl_seconds - ] - for key in stale_keys: - self._controllers.pop(key, None) - if len(self._controllers) <= self._config.endpoint_state_max_entries: - return - victims = sorted(self._controllers.items(), key=lambda item: item[1].state.last_seen)[ - : len(self._controllers) - self._config.endpoint_state_max_entries - ] - for key, _ in victims: - self._controllers.pop(key, None) - - async def _enqueue_or_raise(self, state: EndpointState, endpoint_key: str) -> None: - """Account queued work or raise typed queue/shutdown errors before dispatch.""" - async with self._lock: - if self._shutdown: - raise ResilienceCancelledError("Scheduler is shutting down", endpoint_key=endpoint_key, attempt=0) - if self._global_queued >= self._config.global_max_queued: - raise ResilienceQueueFullError( - f"Global queue is full: {self._global_queued} out of {self._config.global_max_queued}", - endpoint_key=endpoint_key, - reason="global_queue_full", - ) - if state.queued >= self._config.endpoint_max_queued: - raise ResilienceQueueFullError( - f"Endpoint queue is full for {endpoint_key}: {state.queued} out of {self._config.endpoint_max_queued}", - endpoint_key=endpoint_key, - reason="endpoint_queue_full", - ) - self._global_queued += 1 - state.queued += 1 - - async def _on_dispatch(self, state: EndpointState, *, attempt: int) -> None: - """Move one queued item to inflight and apply retry-budget checks.""" - now = self.now() - async with self._lock: - self._refill_retry_budget(state, now) - if attempt > 1: - if state.retry_budget_tokens < 1.0: - raise ResilienceQueueFullError( - "Retry budget exhausted", - endpoint_key=state.key, - reason="retry_budget_exhausted", - ) - state.retry_budget_tokens -= 1.0 - self._global_queued = max(0, self._global_queued - 1) - state.queued = max(0, state.queued - 1) - state.inflight += 1 - state.max_inflight_seen = max(state.max_inflight_seen, state.inflight) - - async def _on_complete(self, state: EndpointState, *, dispatched: bool, elapsed_seconds: float) -> None: - """Reconcile queued/inflight counters and update latency EWMA on completion.""" - async with self._lock: - if not dispatched: - self._global_queued = max(0, self._global_queued - 1) - state.queued = max(0, state.queued - 1) - else: - state.inflight = max(0, state.inflight - 1) - if state.latency_ewma is None: - state.latency_ewma = elapsed_seconds - else: - state.latency_ewma = (0.2 * elapsed_seconds) + (0.8 * state.latency_ewma) - - async def _record_failure(self, controller: _Controller, classified: ClassifierResult) -> None: - """Apply failure feedback to endpoint state (AIMD and timeout escalation).""" - failure_class = classified.failure_class - now = self.now() - async with controller.lock: - state = controller.state - state.success_streak = 0 - state.saw_failure = True - previous_limit = state.limit - if failure_class == FailureClass.HARD_OVERLOAD: - state.overload_hard_count += 1 - state.limit = multiplicative_decrease(state.limit, state.min_limit, self._config.beta_hard_overload) - state.cooldown_until = max( - state.cooldown_until, now + cooldown_for_failure_class(failure_class, self._config) - ) - elif failure_class == FailureClass.SOFT_OVERLOAD: - state.overload_soft_count += 1 - state.soft_overload_events.append(now) - state.limit = multiplicative_decrease(state.limit, state.min_limit, self._config.beta_soft_overload) - state.cooldown_until = max( - state.cooldown_until, now + cooldown_for_failure_class(failure_class, self._config) - ) - self._trim_events(state.soft_overload_events, now, self._config.escalation_window_seconds) - if len(state.soft_overload_events) >= self._config.timeout_soft_overload_escalation_count: - state.limit = multiplicative_decrease(state.limit, state.min_limit, self._config.beta_hard_overload) - state.cooldown_until = max(state.cooldown_until, now + self._config.cooldown_seconds_hard) - state.overload_hard_count += 1 - if failure_class in {FailureClass.HARD_OVERLOAD, FailureClass.SOFT_OVERLOAD}: - controller.limiter.total_tokens = state.limit - if state.limit != previous_limit: - state.counters.limit_decreases += 1 - async with self._lock: - self._metrics.limit_decreases += 1 - _logger.info( - "Resilience endpoint limit decreased", - extra={ - "endpoint_key": state.key, - "failure_class": failure_class.value, - "previous_limit": previous_limit, - "new_limit": state.limit, - "cooldown_until": state.cooldown_until, - "overload_soft_count": state.overload_soft_count, - "overload_hard_count": state.overload_hard_count, - "cause_status_code": classified.status_code, - "cause_error_type": classified.error_type, - }, - ) - - async def record_success(self, endpoint_key: str) -> None: - """Record success feedback for callers that need explicit signaling.""" - controller = await self._get_controller(endpoint_key) - now = self.now() - async with controller.lock: - state = controller.state - state.success_streak += 1 - if now < state.cooldown_until: - return - if state.success_streak >= self._config.success_window: - state.success_streak = 0 - previous_limit = state.limit - # Fast-start before the first failure: double capacity each window. - # After any failure, revert to conservative +1 additive increase. - if state.saw_failure: - state.limit = additive_increase(state.limit, state.max_limit) - else: - state.limit = min(state.max_limit, max(state.min_limit, state.limit * 2)) - controller.limiter.total_tokens = state.limit - if state.limit != previous_limit: - state.counters.limit_increases += 1 - async with self._lock: - self._metrics.limit_increases += 1 - _logger.info( - "Resilience endpoint limit increased", - extra={ - "endpoint_key": state.key, - "previous_limit": previous_limit, - "new_limit": state.limit, - "success_window": self._config.success_window, - }, - ) - - def _refill_retry_budget(self, state: EndpointState, now: float) -> None: - """Refill endpoint retry tokens using elapsed monotonic time.""" - elapsed = max(0.0, now - state.retry_budget_last_refill) - state.retry_budget_last_refill = now - state.retry_budget_tokens = min( - self._config.retry_budget_burst, - state.retry_budget_tokens + elapsed * self._config.retry_budget_tokens_per_sec, - ) - - @staticmethod - def _trim_events(events: Deque[float], now: float, window_seconds: float) -> None: - """Drop soft-overload timestamps older than the escalation window.""" - cutoff = now - max(0.0, window_seconds) - while events and events[0] < cutoff: - events.popleft() - - async def summary(self) -> dict[str, object]: - """Return aggregate session metrics for resilience diagnostics.""" - async with self._lock: - max_inflight_seen = max((c.state.max_inflight_seen for c in self._controllers.values()), default=0) - per_endpoint: dict[str, dict[str, int | float]] = {} - for endpoint_key, controller in self._controllers.items(): - state = controller.state - per_endpoint[endpoint_key] = { - "operations_started": state.counters.operations_started, - "operations_completed": state.counters.operations_completed, - "retries_scheduled": state.counters.retries_scheduled, - "limit_decreases": state.counters.limit_decreases, - "limit_increases": state.counters.limit_increases, - "cancellations": state.counters.cancellations, - "overload_hard_count": state.overload_hard_count, - "overload_soft_count": state.overload_soft_count, - "current_limit": state.limit, - "max_inflight_seen": state.max_inflight_seen, - } - return { - "operations_started": self._metrics.operations_started, - "operations_completed": self._metrics.operations_completed, - "retries_scheduled": self._metrics.retries_scheduled, - "limit_decreases": self._metrics.limit_decreases, - "limit_increases": self._metrics.limit_increases, - "cancellations": self._metrics.cancellations, - "endpoints_tracked": len(self._controllers), - "max_endpoint_inflight_seen": max_inflight_seen, - "per_endpoint": per_endpoint, - } diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resilience/types.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resilience/types.py deleted file mode 100644 index 3d5b0576dd..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resilience/types.py +++ /dev/null @@ -1,93 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Types used by the evaluator resilience control plane.""" - -from __future__ import annotations - -import time -from collections import deque -from dataclasses import dataclass, field -from enum import StrEnum -from typing import Deque, Protocol - - -class FailureClass(StrEnum): - """Failure taxonomy for retry + adaptation decisions.""" - - HARD_OVERLOAD = "hard_overload" - SOFT_OVERLOAD = "soft_overload" - TRANSIENT = "transient_non_overload" - FATAL = "fatal" - - -class Clock(Protocol): - """Monotonic clock abstraction for deterministic tests.""" - - def monotonic(self) -> float: - """Return monotonic time in seconds.""" - ... - - -class SystemClock: - """System monotonic clock implementation.""" - - def monotonic(self) -> float: - return time.monotonic() - - -@dataclass -class OperationCounters: - """Common operation counters used for scheduler and endpoint metrics.""" - - operations_started: int = 0 - operations_completed: int = 0 - retries_scheduled: int = 0 - limit_decreases: int = 0 - limit_increases: int = 0 - cancellations: int = 0 - - -@dataclass -class EndpointState: - """Mutable per-endpoint resilience state.""" - - key: str - limit: int - min_limit: int - max_limit: int - cooldown_until: float = 0.0 - success_streak: int = 0 - overload_hard_count: int = 0 - overload_soft_count: int = 0 - latency_ewma: float | None = None - retry_budget_tokens: float = 0.0 - retry_budget_last_refill: float = 0.0 - last_seen: float = 0.0 - soft_overload_events: Deque[float] = field(default_factory=deque) - queued: int = 0 - inflight: int = 0 - max_inflight_seen: int = 0 - saw_failure: bool = False - counters: OperationCounters = field(default_factory=OperationCounters) - - -@dataclass(frozen=True) -class ClassifierResult: - """Classification output for one raised exception.""" - - failure_class: FailureClass - retryable: bool - retry_after_seconds: float | None - status_code: int | None = None - error_type: str | None = None - - -@dataclass(frozen=True) -class RetryContext: - """Inputs for retry wait calculation.""" - - attempt_number: int - retry_after_seconds: float | None - pressure: float - cooldown_remaining_seconds: float diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resolver_protocols.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resolver_protocols.py deleted file mode 100644 index 3ca00c3bb8..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resolver_protocols.py +++ /dev/null @@ -1,29 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Resolver protocols for evaluator SDK references.""" - -from __future__ import annotations - -from typing import Protocol, runtime_checkable - -from nemo_platform.beta.evaluator.values.common import SecretRef -from nemo_platform.beta.evaluator.values.models import Model, ModelRef - - -@runtime_checkable -class SecretResolver(Protocol): - """Resolve evaluator secret references to secret values.""" - - async def resolve_secret(self, secret_ref: SecretRef) -> str | None: - """Return the secret value for ``secret_ref`` when available.""" - ... - - -@runtime_checkable -class ModelResolver(Protocol): - """Resolve evaluator model references to concrete SDK model bindings.""" - - async def resolve_model(self, model_ref: ModelRef) -> Model: - """Return the concrete model binding for ``model_ref``.""" - ... diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resolvers.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resolvers.py deleted file mode 100644 index f8b9433e83..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/resolvers.py +++ /dev/null @@ -1,69 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Local resolver implementations for evaluator SDK refs.""" - -from __future__ import annotations - -import os - -from nemo_platform.beta.evaluator.values.common import SecretRef -from nemo_platform.beta.evaluator.values.models import Model, ModelRef - - -def _candidate_env_names(secret_name: str) -> list[str]: - """Generate environment variable names that may contain one secret.""" - names = [secret_name, secret_name.upper()] - normalized = secret_name.replace("-", "_").replace("/", "_") - names.extend([normalized, normalized.upper()]) - if normalized and normalized[0].isdigit(): - prefixed = f"_{normalized}" - names.extend([prefixed, prefixed.upper()]) - return list(dict.fromkeys(names)) - - -class LocalSecretResolver: - """Resolve secrets from local environment variables.""" - - async def resolve_secret(self, secret_ref: SecretRef) -> str | None: - """Resolve one secret value from environment variables.""" - for candidate in _candidate_env_names(secret_ref.root): - value = os.getenv(candidate) - if value: - return value - return None - - -class LocalModelResolver: - """Resolve model references from an in-process registry.""" - - def __init__(self) -> None: - """Create a resolver with an empty local model registry.""" - self._models: dict[str, Model] = {} - - def register_model(self, model_ref: ModelRef, model: Model, *, replace: bool = False) -> None: - """Register a local model binding for a model reference.""" - if not replace and model_ref.root in self._models: - raise ValueError(f"Model reference '{model_ref.root}' is already registered.") - self._models[model_ref.root] = model - - def get_model(self, model_ref: ModelRef) -> Model: - """Return the registered model binding for a model reference.""" - try: - return self._models[model_ref.root] - except KeyError as exc: - raise ValueError( - f"Model reference '{model_ref.root}' is not registered. " - "Register it with LocalBackend.model_resolver.register_model() before local execution." - ) from exc - - def unregister_model(self, model_ref: ModelRef) -> Model: - """Remove and return a registered local model binding.""" - try: - return self._models.pop(model_ref.root) - except KeyError as exc: - raise ValueError(f"Model reference '{model_ref.root}' is not registered.") from exc - - async def resolve_model(self, model_ref: ModelRef) -> Model: - """Resolve one model reference from the local registry.""" - return self.get_model(model_ref) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/structured_output.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/structured_output.py deleted file mode 100644 index b9565c39e4..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/structured_output.py +++ /dev/null @@ -1,184 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -import json -from enum import Enum - -from jsonschema.exceptions import SchemaError -from jsonschema.validators import validator_for -from pydantic import BaseModel, Field, field_validator - -from nemo_platform.beta.evaluator.enums import ModelFormat -from nemo_platform.beta.evaluator.inference import InferenceFn, PreprocessRequest, deep_merge -from nemo_platform.beta.evaluator.values import Model - - -class StructuredOutputMode(str, Enum): - OPENAI_RESPONSE_FORMAT = "openai_response_format" - ROOT_GUIDED_JSON = "root_guided_json" - NVEXT_GUIDED_JSON = "nvext_guided_json" - UNSUPPORTED = "unsupported" - - -class StructuredOutput(BaseModel): - name: str | None = None - json_schema: dict = Field(alias="schema") - strict: bool = False - - @field_validator("json_schema") - @classmethod - def validate_json_schema(cls, value: dict): - validator = validator_for(value) - validator.check_schema(value) - return value - - -class InferenceStructuredOutput(PreprocessRequest): - """Format structured output request parameters based on provider mode.""" - - def __init__(self, mode: StructuredOutputMode, structured_output: dict): - if not structured_output: - raise ValueError("structured_output cannot be empty") - try: - output = StructuredOutput(**structured_output) - self._json_schema = output.json_schema - self._strict = output.strict - self.mode = mode - self.inference_param = self._build_inference_param(mode) - except SchemaError as e: - raise ValueError("structured output contains invalid JSON schema") from e - - @property - def json_schema(self) -> dict: - return self._json_schema - - def set_mode(self, mode: StructuredOutputMode) -> None: - self.mode = mode - self.inference_param = self._build_inference_param(mode) - - def _build_inference_param(self, mode: StructuredOutputMode) -> dict: - if mode == StructuredOutputMode.OPENAI_RESPONSE_FORMAT: - return { - "response_format": { - "type": "json_schema", - "json_schema": { - "name": "structured_output", - "schema": self._json_schema, - "strict": self._strict, - }, - } - } - if mode == StructuredOutputMode.ROOT_GUIDED_JSON: - return {"extra_body": {"guided_json": self._json_schema}} - if mode == StructuredOutputMode.NVEXT_GUIDED_JSON: - return {"extra_body": {"nvext": {"guided_json": self._json_schema}}} - if mode == StructuredOutputMode.UNSUPPORTED: - return {} - raise ValueError(f"Unsupported structured output mode: {mode}") - - def _apply_fallback_instruction(self, request: dict) -> dict: - schema_str = json.dumps(self._json_schema, separators=(",", ":")) - instruction = f"Return ONLY valid JSON and ensure it matches this JSON schema exactly: {schema_str}" - if request.get("messages"): - msg = request["messages"][0] - if msg.get("role") == "system": - request["messages"][0]["content"] = f"{instruction} {msg['content']}" - else: - request["messages"].insert(0, {"role": "system", "content": instruction}) - elif request.get("prompt"): - request["prompt"] = f"{instruction} {request['prompt']}" - return request - - def preprocess(self, request: dict, id: str | None = None) -> dict: - _ = id # Required by preprocess hook interface. - if self.mode == StructuredOutputMode.UNSUPPORTED: - return self._apply_fallback_instruction(request) - # Use merge instead of update to avoid overwriting nested dicts - return deep_merge(request, self.inference_param) - - -def default_structured_output_mode(format: str) -> StructuredOutputMode: - if format == ModelFormat.OPEN_AI: - return StructuredOutputMode.OPENAI_RESPONSE_FORMAT - if format == ModelFormat.NVIDIA_NIM: - # Backward-compatible default before preflight detection overrides this. - return StructuredOutputMode.NVEXT_GUIDED_JSON - raise ValueError(f"Unsupported structured output format: {format}") - - -def _looks_like_unsupported_guided_json_error(message: str) -> bool: - lowered = message.lower() - signatures = ( - "guided_json is unsupported", - "unexpected keyword argument 'guided_json'", - "unexpected keyword argument 'nvext'", - "extra_forbidden", - "extra inputs are not permitted", - ) - if any(sig in lowered for sig in signatures): - return "guided_json" in lowered or "nvext" in lowered or "extra_body" in lowered - return False - - -def _extract_chat_content(response: dict) -> str | None: - choices = response.get("choices") - if not isinstance(choices, list) or not choices: - return None - msg = choices[0].get("message", {}) - content = msg.get("content") - return content if isinstance(content, str) else None - - -def _is_probe_valid_json(content: str, probe_schema: dict) -> bool: - try: - obj = json.loads(content) - except (TypeError, ValueError): - return False - if not isinstance(obj, dict): - return False - try: - validator = validator_for(probe_schema) - validator.check_schema(probe_schema) - validator(probe_schema).validate(obj) - return True - except Exception: - return False - - -async def detect_structured_output_mode( - *, - format: str, - model: Model, - inference_fn: InferenceFn, - api_key: str | None, - probe_schema: dict, -) -> StructuredOutputMode: - """Detect working structured output mode for the given model/format.""" - if format == ModelFormat.OPEN_AI: - return StructuredOutputMode.OPENAI_RESPONSE_FORMAT - if format != ModelFormat.NVIDIA_NIM: - return StructuredOutputMode.UNSUPPORTED - - probe_message = "Return ONLY a JSON object that matches the provided schema exactly. No prose or code fences." - base_request = { - "messages": [{"role": "user", "content": probe_message}], - "temperature": 0, - "max_tokens": 128, - } - candidates: list[tuple[StructuredOutputMode, dict]] = [ - (StructuredOutputMode.ROOT_GUIDED_JSON, {"extra_body": {"guided_json": probe_schema}}), - (StructuredOutputMode.NVEXT_GUIDED_JSON, {"extra_body": {"nvext": {"guided_json": probe_schema}}}), - ] - for mode, structured_param in candidates: - try: - response = await inference_fn(model, {**base_request, **structured_param}, 1, api_key=api_key) - content = _extract_chat_content(response) - if content and _is_probe_valid_json(content, probe_schema): - return mode - except Exception as e: - if _looks_like_unsupported_guided_json_error(str(e)): - continue - # Probe failures should not abort evaluation startup. If no mode works, - # caller will fall back to prompt-level strict JSON instruction. - continue - return StructuredOutputMode.UNSUPPORTED diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/templates.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/templates.py deleted file mode 100644 index 2bf5b8b452..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/templates.py +++ /dev/null @@ -1,113 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Template rendering helpers for evaluator SDK runtime.""" - -import json -from typing import Any - -from jinja2 import StrictUndefined -from jinja2.sandbox import SandboxedEnvironment - -env = SandboxedEnvironment(undefined=StrictUndefined) - -# Preserve native Python values for bare expressions like `{{ item.score }}`. -# Full template rendering always returns strings, which breaks structured payloads. -_EXPR_PREFIX = "{{" -_EXPR_SUFFIX = "}}" - - -def _is_single_expression_template(template: str) -> bool: - """Check whether a template is exactly one Jinja expression. - - Args: - template: Raw template string. - - Returns: - ``True`` when the string is a single ``{{ ... }}`` expression with no - statement blocks; otherwise ``False``. - """ - stripped = template.strip() - return ( - stripped.startswith(_EXPR_PREFIX) - and stripped.endswith(_EXPR_SUFFIX) - and stripped.count(_EXPR_PREFIX) == 1 - and stripped.count(_EXPR_SUFFIX) == 1 - and "{%" not in stripped - and "%}" not in stripped - ) - - -def _identifier_kwargs(context: dict) -> dict: - """Filter context keys that can be passed to ``compile_expression`` kwargs. - - Args: - context: Template rendering context. - - Returns: - Dictionary containing only string keys that are valid Python identifiers. - """ - # compile_expression() only accepts keyword arguments, so keys like - # `foo-bar` need to stay accessible through `item`/`sample` instead. - return {k: v for k, v in context.items() if isinstance(k, str) and k.isidentifier()} - - -def render_template(template: str | dict | list, context: dict) -> Any: - """Render strings, dicts, or lists using sandboxed Jinja evaluation. - - For bare expression templates (for example ``{{ item.payload }}``), the - function uses ``compile_expression`` to preserve native value types instead - of forcing string output. - - Args: - template: Template payload to render. - context: Variables available to the Jinja runtime. - - Returns: - Rendered value preserving dict/list structure and native expression types. - - Raises: - jinja2.UndefinedError: If the template references a missing variable. - json.JSONDecodeError: If ``tojson`` output is not valid JSON. - """ - if isinstance(template, dict): - return {k: render_template(v, context) for k, v in template.items()} - if isinstance(template, list): - return [render_template(v, context) for v in template] - if isinstance(template, str): - if _is_single_expression_template(template): - # Use the expression compiler here so `{{ some_dict }}` returns a dict - # instead of a stringified representation. - expr = template.strip()[len(_EXPR_PREFIX) : -len(_EXPR_SUFFIX)].strip() - compiled = env.compile_expression(expr, undefined_to_none=False) - result = compiled(**_identifier_kwargs(context)) - if isinstance(result, StrictUndefined): - str(result) - if "tojson" in template and isinstance(result, str): - return json.loads(result) - return result - - rendered_str = env.from_string(template).render(context) - if "tojson" in template: - return json.loads(rendered_str) - return rendered_str - return template - - -def render_request(template: str | dict, context: dict) -> dict: - """Render a request payload and normalize string output into prompt dicts. - - Args: - template: String or dictionary request template. - context: Variables available to the Jinja runtime. - - Returns: - Dictionary request payload. String templates are wrapped as - ``{"prompt": rendered_text}``. - """ - request = render_template(template, context=context) - if isinstance(request, str): - request = {"prompt": request} - if isinstance(request, list): - raise TypeError("Request template must not produce a list output.") - return request diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/__init__.py deleted file mode 100644 index 241043f551..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/__init__.py +++ /dev/null @@ -1,369 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Public value types for evaluator SDK runtime. - -The public interface resolves lazily (PEP 562), for the same reason the package root does: -every ``from nemo_platform.beta.evaluator.values.X import ...`` runs this barrel first, so -eagerly re-exporting all 97 names dragged ``.datasets``/``.results`` (pyarrow, numpy) and -``.metrics``/``.scores`` (jsonschema, jinja2) into ``agent_eval``, which uses none of them. -Measured: 485 modules and +57 MB RSS for ``import agent_eval.runtimes.harbor_runtime`` before, -300 modules and pydantic alone after. - -Add a new re-export to ``_LAZY_ATTRS``, the ``TYPE_CHECKING`` block and ``__all__`` — never as a -module-level import. ``tests/test_lazy_public_api.py`` locks the boundary in. -""" - -# ruff: noqa: I001 - the vendored SDK mirror uses different import-order settings. - -from importlib import import_module as _import_module -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from nemo_platform.beta.evaluator.values.agents import ( - Agent, - AgentBase, - GenericAgent, - NatAgentConfig, - NemoAgentToolkitAgent, - ) - from nemo_platform.beta.evaluator.values.atif import ( - FinalMetrics, - Metrics, - Observation, - ObservationResult, - Step, - ToolCall, - Trajectory, - ) - from nemo_platform.beta.evaluator.values.common import SecretRef, SupportedJobTypes - from nemo_platform.beta.evaluator.values.dataset_schemas import ( - FieldMapping, - InputSchema, - ) - from nemo_platform.beta.evaluator.values.datasets import DatasetInput, DatasetRows - from nemo_platform.beta.evaluator.values.evidence import ( - CandidateEvidence, - CommandResult, - EvidenceDescriptor, - FilesystemDiff, - FilesystemEntry, - LocalFilesystemEvidence, - LogHandle, - TraceHandle, - WellKnownEvidenceKey, - parse_atif, - ) - from nemo_platform.beta.evaluator.values.metrics import ( - BLEU, - F1, - ROUGE, - AgentGoalAccuracy, - AnswerAccuracy, - ContextEntityRecall, - ContextPrecision, - ContextRecall, - ContextRelevance, - ExactMatch, - Faithfulness, - LLMJudge, - MetricBase, - NemoAgentToolkitRemote, - NoiseSensitivity, - NumberCheck, - Remote, - ResponseGroundedness, - ResponseRelevancy, - StringCheck, - ToolCallAccuracy, - ToolCalling, - TopicAdherence, - TunableRagEvaluator, - ) - from nemo_platform.beta.evaluator.values.models import Model, ModelRef, ReasoningParams - from nemo_platform.beta.evaluator.values.multi_metric_results import BenchmarkEvaluationResult - from nemo_platform.beta.evaluator.values.params import ( - InferenceParams, - RunConfig, - RunConfigOnline, - RunConfigOnlineModel, - ) - from nemo_platform.beta.evaluator.values.protocol import ( - BooleanValue, - CandidateOutput, - ContinuousScore, - DatasetRow, - DiscreteScore, - Label, - MetricDescriptor, - MetricDiagnostic, - MetricInput, - MetricOutput, - MetricOutputSpec, - MetricResult, - MetricTypeName, - ) - from nemo_platform.beta.evaluator.values.results import ( - AggregatedMetricResult, - AggregateFieldName, - AggregateRangeScore, - AggregateRubricScore, - AggregateScore, - AggregateScoreBase, - DefaultAggregateFieldName, - EvaluationResult, - Histogram, - HistogramBin, - MetricScore, - Percentiles, - RowScore, - RubricScoreStat, - RubricScoreValue, - SampleResult, - ScoreStats, - ) - from nemo_platform.beta.evaluator.values.scores import ( - JSONScoreParser, - RangeScore, - RegexScoreParser, - RemoteScore, - Rubric, - RubricScore, - Score, - score_discriminator, - ) - - -# Re-exported name -> the submodule that defines it, relative to this package. Relative on -# purpose: the vendoring tool mirrors this file into nemo_platform.beta.evaluator by rewriting -# module paths, and a relative name has nothing to rewrite, so the mirror is correct by -# construction. Mirrors the TYPE_CHECKING block above, in the same order. -_LAZY_ATTRS: dict[str, str] = { - "Agent": ".agents", - "AgentBase": ".agents", - "GenericAgent": ".agents", - "NatAgentConfig": ".agents", - "NemoAgentToolkitAgent": ".agents", - "FinalMetrics": ".atif", - "Metrics": ".atif", - "Observation": ".atif", - "ObservationResult": ".atif", - "Step": ".atif", - "ToolCall": ".atif", - "Trajectory": ".atif", - "SecretRef": ".common", - "SupportedJobTypes": ".common", - "FieldMapping": ".dataset_schemas", - "InputSchema": ".dataset_schemas", - "DatasetInput": ".datasets", - "DatasetRows": ".datasets", - "CandidateEvidence": ".evidence", - "CommandResult": ".evidence", - "EvidenceDescriptor": ".evidence", - "FilesystemDiff": ".evidence", - "FilesystemEntry": ".evidence", - "LocalFilesystemEvidence": ".evidence", - "LogHandle": ".evidence", - "TraceHandle": ".evidence", - "WellKnownEvidenceKey": ".evidence", - "parse_atif": ".evidence", - "BLEU": ".metrics", - "F1": ".metrics", - "ROUGE": ".metrics", - "AgentGoalAccuracy": ".metrics", - "AnswerAccuracy": ".metrics", - "ContextEntityRecall": ".metrics", - "ContextPrecision": ".metrics", - "ContextRecall": ".metrics", - "ContextRelevance": ".metrics", - "ExactMatch": ".metrics", - "Faithfulness": ".metrics", - "LLMJudge": ".metrics", - "MetricBase": ".metrics", - "NemoAgentToolkitRemote": ".metrics", - "NoiseSensitivity": ".metrics", - "NumberCheck": ".metrics", - "Remote": ".metrics", - "ResponseGroundedness": ".metrics", - "ResponseRelevancy": ".metrics", - "StringCheck": ".metrics", - "ToolCallAccuracy": ".metrics", - "ToolCalling": ".metrics", - "TopicAdherence": ".metrics", - "TunableRagEvaluator": ".metrics", - "Model": ".models", - "ModelRef": ".models", - "ReasoningParams": ".models", - "BenchmarkEvaluationResult": ".multi_metric_results", - "InferenceParams": ".params", - "RunConfig": ".params", - "RunConfigOnline": ".params", - "RunConfigOnlineModel": ".params", - "BooleanValue": ".protocol", - "CandidateOutput": ".protocol", - "ContinuousScore": ".protocol", - "DatasetRow": ".protocol", - "DiscreteScore": ".protocol", - "Label": ".protocol", - "MetricDescriptor": ".protocol", - "MetricDiagnostic": ".protocol", - "MetricInput": ".protocol", - "MetricOutput": ".protocol", - "MetricOutputSpec": ".protocol", - "MetricResult": ".protocol", - "MetricTypeName": ".protocol", - "AggregatedMetricResult": ".results", - "AggregateFieldName": ".results", - "AggregateRangeScore": ".results", - "AggregateRubricScore": ".results", - "AggregateScore": ".results", - "AggregateScoreBase": ".results", - "DefaultAggregateFieldName": ".results", - "EvaluationResult": ".results", - "Histogram": ".results", - "HistogramBin": ".results", - "MetricScore": ".results", - "Percentiles": ".results", - "RowScore": ".results", - "RubricScoreStat": ".results", - "RubricScoreValue": ".results", - "SampleResult": ".results", - "ScoreStats": ".results", - "JSONScoreParser": ".scores", - "RangeScore": ".scores", - "RegexScoreParser": ".scores", - "RemoteScore": ".scores", - "Rubric": ".scores", - "RubricScore": ".scores", - "Score": ".scores", - "score_discriminator": ".scores", -} - -__all__ = [ - "Agent", - "AgentBase", - "GenericAgent", - "NatAgentConfig", - "NemoAgentToolkitAgent", - "AggregateFieldName", - "AggregatedMetricResult", - "AggregateRangeScore", - "AggregateRubricScore", - "AggregateScore", - "AggregateScoreBase", - "BenchmarkEvaluationResult", - "BooleanValue", - "CandidateEvidence", - "CandidateOutput", - "CommandResult", - "ContinuousScore", - "FilesystemDiff", - "FilesystemEntry", - "FinalMetrics", - "LogHandle", - "Metrics", - "Observation", - "ObservationResult", - "Step", - "ToolCall", - "Trajectory", - "TraceHandle", - "WellKnownEvidenceKey", - "parse_atif", - "DatasetRow", - "DatasetRows", - "DefaultAggregateFieldName", - "DiscreteScore", - "RunConfig", - "RunConfigOnline", - "RunConfigOnlineModel", - "FieldMapping", - "Histogram", - "HistogramBin", - "InferenceParams", - "JSONScoreParser", - "Label", - "LocalFilesystemEvidence", - "MetricDescriptor", - "MetricDiagnostic", - "MetricInput", - "MetricOutput", - "MetricOutputSpec", - "MetricResult", - "MetricTypeName", - "MetricScore", - "Model", - "ModelRef", - "DatasetInput", - "EvaluationResult", - "EvidenceDescriptor", - "Percentiles", - "RangeScore", - "ReasoningParams", - "InputSchema", - "RegexScoreParser", - "RemoteScore", - "RowScore", - "Rubric", - "RubricScore", - "RubricScoreStat", - "RubricScoreValue", - "SampleResult", - "Score", - "ScoreStats", - "SecretRef", - "SupportedJobTypes", - "score_discriminator", - # Metrics - "AgentGoalAccuracy", - "AnswerAccuracy", - "BLEU", - "ContextEntityRecall", - "ContextPrecision", - "ContextRecall", - "ContextRelevance", - "ExactMatch", - "F1", - "Faithfulness", - "LLMJudge", - "MetricBase", - "NemoAgentToolkitRemote", - "NoiseSensitivity", - "NumberCheck", - "Remote", - "ResponseGroundedness", - "ResponseRelevancy", - "ROUGE", - "StringCheck", - "ToolCallAccuracy", - "ToolCalling", - "TopicAdherence", - "TunableRagEvaluator", -] - - -def __getattr__(name: str) -> object: - """Import the submodule that defines ``name`` on first access (PEP 562). - - An *unknown* name raises ``AttributeError``, which is required: ``from pkg import sub`` only - falls back to importing a submodule when attribute lookup raises ``AttributeError``. - - A *known* name whose submodule fails to import propagates that ``ImportError`` unchanged, so - the real cause is not hidden behind a bogus "no attribute". The consequence is that - ``hasattr`` raises rather than returning ``False`` when a name's dependencies are missing; - catch ``ImportError`` around the access, or test membership against ``__all__``. - """ - submodule = _LAZY_ATTRS.get(name) - if submodule is None: - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") - value = getattr(_import_module(submodule, __name__), name) - globals()[name] = value # cache, so later lookups skip __getattr__ entirely - return value - - -def __dir__() -> list[str]: - # The declared surface plus any submodule the caller has already imported. Machinery is - # imported under a leading underscore so the filter keeps it out of autocomplete without a - # denylist; ``TYPE_CHECKING`` is the one exception, unaliased so type checkers recognise it. - public = {name for name in globals() if not name.startswith("_")} - {"TYPE_CHECKING"} - return sorted(set(__all__) | public) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/agents.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/agents.py deleted file mode 100644 index 298f7a6844..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/agents.py +++ /dev/null @@ -1,138 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Agent-related value types.""" - -# ruff: noqa: I001 - the vendored SDK mirror uses different import-order settings. - -from __future__ import annotations - -import os -from functools import cached_property -from typing import Any, Annotated, Literal, TypeAlias - -from pydantic import BaseModel, ConfigDict, Field - -from nemo_platform.beta.evaluator.enums import AgentFormat -from nemo_platform.beta.evaluator.values.common import SecretRef - - -def _require_format_in_json_schema(schema: dict[str, Any]) -> None: - """Require the discriminator in serialized agent payloads.""" - required = schema.setdefault("required", []) - if "format" not in required: - required.append("format") - - -# How matched data-frame values are combined into one final output when a -# target is streamed as JSON SSE: -# - "last": keep only the final matched value. Correct for endpoints that -# emit a complete response snapshot per frame. -# - "concat": join matched string values in arrival order. Correct for -# token-delta endpoints (e.g. NAT /generate/full emits one token -# per frame), where the last frame is only the final token. -StreamAggregation: TypeAlias = Literal["last", "concat"] - - -class NatAgentConfig(BaseModel): - """NeMo Agent Toolkit request and stream handling configuration.""" - - model_config = ConfigDict(extra="forbid") - - endpoint: str = Field( - default="/generate/full", - description="Relative path below agent.url, or an absolute NAT endpoint URL.", - ) - request_mode: Literal["input_message", "passthrough"] = Field( - default="input_message", - description="Derive the legacy input_message payload or send the rendered request unchanged.", - ) - query_params: dict[str, str] = Field( - default_factory=lambda: {"filter_steps": "none"}, - description="Query parameters sent to the NAT endpoint.", - ) - response_path: str = Field( - default="$.value", - description="JSONPath applied to each data-channel payload to extract its emitted value.", - ) - response_aggregation: StreamAggregation = Field( - default="concat", - description=( - "How to combine matched data-frame values into the final output. NAT /generate/full " - "emits token-level deltas, so 'concat' reconstructs the complete response; 'last' keeps " - "only the final matched value (for endpoints that emit a full snapshot per frame)." - ), - ) - - -class AgentBase(BaseModel): - """Fields shared by every inference agent target.""" - - # TODO: Much of this is duplicated between agent and model. Once we have aligned on model defination. - # the duplication can be removed by defining EndPoint class and reusing it across both model and agent. - model_config = ConfigDict(extra="forbid") - - url: str = Field(description="Base URL of the agent endpoint.") - name: str = Field(description="Agent name / identifier.") - api_key_secret: SecretRef | None = Field( - default=None, - description="API key secret reference for the agent. Format: workspace/secret_name or secret_name within the job workspace.", - ) - - @cached_property - def api_key_env(self) -> str | None: - if self.api_key_secret: - env_name = self.api_key_secret.root - if env_name[0].isdigit(): - env_name = f"_{env_name}" - return env_name.replace("-", "_").replace("/", "_") - return None - - @cached_property - def api_key(self) -> str | None: - api_key_env = self.api_key_env - return os.getenv(api_key_env) if api_key_env is not None else None - - -class GenericAgent(AgentBase): - """Configurable HTTP agent with optional JSON SSE response handling.""" - - model_config = ConfigDict(json_schema_extra=_require_format_in_json_schema) - - format: Literal[AgentFormat.GENERIC] = AgentFormat.GENERIC - body: dict[str, Any] = Field(description="Jinja template for the request payload.") - response_path: str = Field(description="JSONPath expression used to extract the response value.") - trajectory_path: str | None = Field( - default=None, - description="Optional JSONPath expression used to extract trajectory data.", - ) - stream: bool = Field( - default=False, - description="Read JSON SSE data frames instead of a single JSON response body.", - ) - response_aggregation: StreamAggregation = Field( - default="last", - description=( - "How to combine matched data-frame values when 'stream' is true. 'last' keeps the final " - "matched value (endpoints that emit a full snapshot per frame); 'concat' joins matched " - "string values in arrival order (token-delta endpoints)." - ), - ) - - -class NemoAgentToolkitAgent(AgentBase): - """NeMo Agent Toolkit target normalized to the shared streaming transport.""" - - model_config = ConfigDict(json_schema_extra=_require_format_in_json_schema) - - format: Literal[AgentFormat.NEMO_AGENT_TOOLKIT] = AgentFormat.NEMO_AGENT_TOOLKIT - nat: NatAgentConfig | None = Field( - default=None, - description="Optional NAT endpoint and stream configuration; defaults preserve /generate/full behavior.", - ) - - -Agent: TypeAlias = Annotated[ - GenericAgent | NemoAgentToolkitAgent, - Field(discriminator="format"), -] diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/atif.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/atif.py deleted file mode 100644 index 5db6667586..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/atif.py +++ /dev/null @@ -1,135 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Permissive ATIF read models for the evaluator SDK. - -The evaluator ingests traces that producers emit in the Agent Trajectory -Interchange Format (ATIF; RFC 0001, schema_version ``ATIF-v1.x``). Selected -fields are derived from Harbor's reference models at commit -``aaf0561340fd2f03257ec3084732f98537a2d2b1``. They are intentionally not a -byte-for-byte vendoring: this SDK keeps ``extra="ignore"``, makes newly consumed -fields optional where possible, and omits producer-side cross-field validators, -multimodal content models, and embedded subagent trajectories. - -Validation here means "this payload carries the ATIF fields evaluator metrics -read", not full RFC conformance. The producer's original dictionary remains the -authoritative persistence representation; these models provide typed read access. -""" - -# ruff: noqa: I001 - the vendored SDK mirror uses different import-order settings. - -from typing import Any, Literal - -from pydantic import BaseModel, ConfigDict, Field, field_validator - - -class ToolCall(BaseModel): - """A single tool/function invocation within an agent step.""" - - model_config = ConfigDict(extra="ignore") - - tool_call_id: str | None = Field(default=None, description="Producer-assigned tool call id, if any.") - function_name: str = Field(description="Name of the invoked tool/function.") - arguments: dict[str, Any] | None = Field(default=None, description="Arguments passed to the tool.") - extra: dict[str, Any] | None = Field(default=None, description="Custom tool-call metadata.") - - -class ObservationResult(BaseModel): - """One tool or environment result attached to an observation.""" - - model_config = ConfigDict(extra="ignore") - - source_call_id: str | None = Field(default=None, description="Related ToolCall.tool_call_id, if any.") - content: Any | None = Field(default=None, description="Tool result content retained for evidence readers.") - extra: dict[str, Any] | None = Field(default=None, description="Custom result-level metadata.") - - -class Observation(BaseModel): - """Environment feedback following tool calls or other actions.""" - - model_config = ConfigDict(extra="ignore") - - results: list[ObservationResult] = Field(default_factory=list, description="Results produced by the action.") - - -class Metrics(BaseModel): - """Per-step token metrics.""" - - model_config = ConfigDict(extra="ignore") - - prompt_tokens: int | None = None - completion_tokens: int | None = None - cached_tokens: int | None = None - cost_usd: float | None = None - prompt_token_ids: list[int] | None = None - completion_token_ids: list[int] | None = None - logprobs: list[float] | None = None - extra: dict[str, Any] | None = None - - -class FinalMetrics(BaseModel): - """Trajectory-level aggregate token metrics.""" - - model_config = ConfigDict(extra="ignore") - - total_prompt_tokens: int | None = None - total_completion_tokens: int | None = None - total_cached_tokens: int | None = None - total_cost_usd: float | None = None - total_steps: int | None = Field(default=None, ge=0) - extra: dict[str, Any] | None = None - - -class Agent(BaseModel): - """ATIF protocol producer recorded in a trajectory, not an inference target.""" - - model_config = ConfigDict(extra="ignore") - - name: str | None = None - version: str | None = None - model_name: str | None = None - tool_definitions: list[dict[str, Any]] | None = None - extra: dict[str, Any] | None = None - - -class Step(BaseModel): - """One step in an agent trajectory.""" - - model_config = ConfigDict(extra="ignore") - - step_id: int | None = Field(default=None, ge=1, description="Producer-assigned ordinal step index.") - timestamp: str | None = Field(default=None, description="Producer-reported ISO 8601 timestamp.") - source: Literal["system", "user", "agent"] = Field(description="Who produced this step.") - model_name: str | None = Field(default=None, description="Model used for this step, if reported.") - reasoning_effort: str | float | None = Field(default=None, description="Reported reasoning effort.") - message: str = Field(default="", description="Step text content.") - reasoning_content: str | None = Field(default=None, description="Explicit reasoning content, if exposed.") - tool_calls: list[ToolCall] | None = Field(default=None, description="Tool calls issued in this step.") - observation: Observation | None = Field(default=None, description="Environment feedback for this step.") - metrics: Metrics | None = Field(default=None, description="Per-step token metrics, if reported.") - is_copied_context: bool | None = Field(default=None, description="Whether the step was copied as context.") - llm_call_count: int | None = Field(default=None, ge=0, description="LLM calls represented by this step.") - extra: dict[str, Any] | None = Field(default=None, description="Custom step-level metadata.") - - -class Trajectory(BaseModel): - """An ATIF trajectory read view over the fields evaluator metrics consume.""" - - model_config = ConfigDict(extra="ignore") - - schema_version: str = Field(description="ATIF schema version, e.g. 'ATIF-v1.7'.") - session_id: str | None = Field(default=None, description="Identifier for the logical agent run.") - trajectory_id: str | None = Field(default=None, description="Identifier for this trajectory document.") - agent: Agent | None = Field(default=None, description="Producer-recorded agent configuration.") - steps: list[Step] = Field(min_length=1, description="Ordered trajectory steps.") - notes: str | None = Field(default=None, description="Producer notes about the trajectory.") - final_metrics: FinalMetrics | None = Field(default=None, description="Aggregate token metrics, if reported.") - continued_trajectory_ref: str | None = Field(default=None, description="Reference to a continuation trace.") - extra: dict[str, Any] | None = Field(default=None, description="Custom trajectory-level metadata.") - - @field_validator("schema_version") - @classmethod - def _looks_like_atif(cls, value: str) -> str: - # Cheap sanity gate so arbitrary JSON isn't silently accepted as a trace. - if not value.startswith("ATIF-"): - raise ValueError(f"unexpected trace schema_version {value!r}; expected an 'ATIF-*' version") - return value diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/common.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/common.py deleted file mode 100644 index 85ab3cad86..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/common.py +++ /dev/null @@ -1,25 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Common value types used throughout evaluator SDK runtime.""" - -from enum import Enum - -from pydantic import Field, RootModel - - -class SupportedJobTypes(str, Enum): - ONLINE = "online" - OFFLINE = "offline" - - -class SecretRef(RootModel): - root: str = Field( - description="Reference to a platform secret or local environment variable. Format: 'secret_name' (uses request workspace) or 'workspace/secret_name' (explicit workspace).", - pattern=r"^[A-Za-z0-9_-]+(/[A-Za-z0-9_-]+)?$", - examples=[ - "my-secret", - "my-workspace/my-secret", - "NVIDIA_API_KEY", - ], - ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/dataset_schemas.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/dataset_schemas.py deleted file mode 100644 index e79577bb98..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/dataset_schemas.py +++ /dev/null @@ -1,112 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Public Pydantic models for canonical evaluator schema and dataset column mapping.""" - -from __future__ import annotations - -from typing import Annotated, Self - -from pydantic import BaseModel, ConfigDict, Field, model_validator - -_KNOWN_BINDING_FIELDS = ( - "input", - "output", - "context", - "reference", - "trajectory", - "messages", - "tool_calls", - "tools", -) - -_FIELD_MAPPING_PATH_PATTERN = r"^[^\[\]]*$" -_FieldMappingPath = Annotated[str, Field(pattern=_FIELD_MAPPING_PATH_PATTERN, min_length=1)] - - -class InputSchema(BaseModel): - model_config = ConfigDict(serialize_by_alias=True) - - schema_: dict = Field( - alias="schema", - description=( - "Canonical evaluator input schema expressed as JSON Schema. " - "This describes the normalized template context required by the metric, " - "not the raw dataset row shape." - ), - ) - - @model_validator(mode="after") - def validate_schema(self) -> Self: - from nemo_platform.beta.evaluator.dataset_schemas.common import validate_json_schema - - validate_json_schema(self.schema_) - return self - - -class _FieldMappingBase(BaseModel): - model_config = ConfigDict(extra="forbid") - - input: _FieldMappingPath | None = Field( - default=None, description="Binding for the canonical 'input' evaluator field." - ) - output: _FieldMappingPath | None = Field( - default=None, description="Binding for the canonical 'output' evaluator field." - ) - context: _FieldMappingPath | None = Field( - default=None, description="Binding for the canonical 'context' evaluator field." - ) - reference: _FieldMappingPath | None = Field( - default=None, - description="Binding for the canonical 'reference' evaluator field.", - ) - trajectory: _FieldMappingPath | None = Field( - default=None, - description="Binding for the canonical 'trajectory' evaluator field.", - ) - messages: _FieldMappingPath | None = Field( - default=None, - description="Binding for the canonical 'messages' evaluator field.", - ) - tool_calls: _FieldMappingPath | None = Field( - default=None, - description="Binding for the canonical 'tool_calls' evaluator field.", - ) - tools: _FieldMappingPath | None = Field( - default=None, description="Binding for the canonical 'tools' evaluator field." - ) - custom: dict[str, _FieldMappingPath] = Field( - default_factory=dict, - description="Additional evaluator field bindings keyed by canonical field name.", - ) - - @model_validator(mode="after") - def validate_custom_keys(self) -> Self: - duplicates = sorted(set(self.custom).intersection(_KNOWN_BINDING_FIELDS)) - if duplicates: - raise ValueError(f"custom binding keys overlap with reserved evaluator fields: {duplicates}") - return self - - def mapping(self) -> dict[str, str]: - result = {name: value for name in _KNOWN_BINDING_FIELDS if (value := getattr(self, name)) is not None} - result.update(self.custom) - return result - - -class FieldMapping(_FieldMappingBase): - """Maps canonical evaluator fields to raw dataset column paths. - Example: {'input': 'question', 'output': 'answer', 'reference': 'gold', 'trajectory': 'steps'} - """ - - model_config = ConfigDict(extra="forbid") - - @model_validator(mode="after") - def validate_supported_dataset_paths(self) -> Self: - unsupported = sorted( - canonical_name - for canonical_name, dataset_path in self.mapping().items() - if "[" in dataset_path or "]" in dataset_path - ) - if unsupported: - raise ValueError(f"array path segments are not supported for column mappings: {unsupported}") - return self diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/datasets.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/datasets.py deleted file mode 100644 index ae636c330d..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/datasets.py +++ /dev/null @@ -1,26 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Dataset-related value types for evaluator SDK runtime.""" - -from typing import Any, TypeAlias - -import pyarrow as pa -from pydantic import BaseModel, ConfigDict, Field - - -class DatasetRows(BaseModel): - """Inline dataset definition with embedded rows. - - Use this for quick evaluations without persisting the dataset first. - """ - - model_config = ConfigDict(extra="forbid") - - rows: list[dict[str, Any]] = Field( - min_length=1, - description="Array of data rows. Each row can be any valid JSON value (object, string, array, etc.).", - ) - - -DatasetInput: TypeAlias = list[dict[str, Any]] | DatasetRows | pa.Table diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/evidence.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/evidence.py deleted file mode 100644 index 9f263e88ec..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/evidence.py +++ /dev/null @@ -1,495 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Evidence value types shared by protocol metrics and agent evaluations.""" - -from __future__ import annotations - -import asyncio -import difflib -import hashlib -import json -import os -import shutil -import signal -import tempfile -from collections.abc import Mapping -from pathlib import Path -from typing import Any, Literal -from urllib.parse import urlparse - -from pydantic import BaseModel, ConfigDict, Field, JsonValue, PrivateAttr, model_validator - -from nemo_platform.beta.evaluator.values.atif import FinalMetrics, Step, ToolCall, Trajectory - -# Standard evidence keys shared by inference, persistence, and agent evaluation. -EVIDENCE_INITIAL_STATE = "initial_state" -EVIDENCE_TRACE = "trace" -EVIDENCE_LOGS = "logs" -EVIDENCE_FINAL_STATE = "final_state" -EVIDENCE_VERIFIER_LOGS = "verifier_logs" -EVIDENCE_RAW_STREAM = "raw_stream" -EVIDENCE_STREAM_EVENTS = "stream_events" -EVIDENCE_REQUEST_PAYLOAD = "request_payload" -EVIDENCE_REQUEST_HEADERS = "request_headers" -EVIDENCE_HTTP_METADATA = "http_metadata" -EVIDENCE_TRANSLATION_ERROR = "translation_error" - -EVIDENCE_FORMAT_ATIF = "atif" -EVIDENCE_FORMAT_JSON = "json" -EVIDENCE_FORMAT_TEXT = "text" - -# Well-known evidence keys used by the core agent-eval artifact contract. -WellKnownEvidenceKey = Literal["initial_state", "trace", "logs", "final_state", "verifier_logs"] - - -class FilesystemEntry(BaseModel): - """One path that differs between two filesystem snapshots.""" - - model_config = ConfigDict(extra="forbid") - - path: str - change_type: Literal["added", "modified", "deleted"] - - -class FilesystemDiff(BaseModel): - """Set of paths that changed between two filesystem snapshots.""" - - model_config = ConfigDict(extra="forbid") - - entries: list[FilesystemEntry] = Field(default_factory=list) - - def changed( - self, - *, - prefix: str | None = None, - kinds: set[str] | None = None, - ) -> list[FilesystemEntry]: - """Return entries optionally filtered by path prefix and change kind.""" - return [ - entry - for entry in self.entries - if (prefix is None or entry.path.startswith(prefix)) and (kinds is None or entry.change_type in kinds) - ] - - -class CommandResult(BaseModel): - """Outcome of running a verifier command against filesystem evidence.""" - - model_config = ConfigDict(extra="forbid") - - exit_code: int - stdout: str = "" - stderr: str = "" - timed_out: bool = False - - @property - def ok(self) -> bool: - """Whether the command exited 0 without timing out.""" - return self.exit_code == 0 and not self.timed_out - - -class LocalFilesystemEvidence: - """Constrained local filesystem handle for metric evidence access.""" - - def __init__(self, root: str | Path) -> None: - self._root = Path(root).expanduser().resolve() - - @property - def root(self) -> Path: - """Resolved root path for this local evidence handle.""" - return self._root - - def path(self, relative_path: str | Path = ".") -> Path: - """Return a path under the evidence root, rejecting traversal outside it.""" - relative = Path(relative_path) - candidate = relative.resolve() if relative.is_absolute() else (self._root / relative).resolve() - if not self._within_root(candidate): - raise ValueError(f"evidence path {relative_path!r} resolves outside evidence root") - return candidate - - def _within_root(self, path: Path) -> bool: - """Whether ``path`` (after resolving symlinks) stays inside the evidence root.""" - resolved = path.resolve() - return resolved == self._root or self._root in resolved.parents - - async def exists(self, relative_path: str | Path = ".") -> bool: - """Return whether a path exists under the evidence root.""" - path = self.path(relative_path) - return await asyncio.to_thread(path.exists) - - async def read_text(self, relative_path: str | Path, *, encoding: str = "utf-8") -> str: - """Read a text file under the evidence root.""" - path = self.path(relative_path) - return await asyncio.to_thread(path.read_text, encoding=encoding) - - async def iter_paths(self, relative_path: str | Path = ".", *, recursive: bool = False) -> list[str]: - """List entries (files and directories) rooted at ``relative_path``.""" - base = self.path(relative_path) - return await asyncio.to_thread(self._iter_paths_sync, base, recursive) - - def _iter_paths_sync(self, base: Path, recursive: bool) -> list[str]: - if base.is_file(): - return [base.relative_to(self._root).as_posix()] - iterator = base.rglob("*") if recursive else base.iterdir() - return sorted(path.relative_to(self._root).as_posix() for path in iterator) - - async def read_bytes(self, relative_path: str | Path) -> bytes: - """Read a binary file under the evidence root.""" - path = self.path(relative_path) - return await asyncio.to_thread(path.read_bytes) - - async def list_files(self, pattern: str = "**/*") -> list[str]: - """List relative posix paths of files (not directories) matching ``pattern``.""" - return await asyncio.to_thread(self._list_sync, pattern) - - def _list_sync(self, pattern: str) -> list[str]: - return sorted( - path.relative_to(self._root).as_posix() - for path in self._root.glob(pattern) - if path.is_file() and self._within_root(path) - ) - - async def diff(self, other: LocalFilesystemEvidence) -> FilesystemDiff: - """Diff this snapshot (before) against ``other`` (after) by file content hash.""" - return await asyncio.to_thread(self._diff_sync, other) - - def _diff_sync(self, other: LocalFilesystemEvidence) -> FilesystemDiff: - before = self._hashes() - after = other._hashes() - entries = [FilesystemEntry(path=path, change_type="added") for path in sorted(after.keys() - before.keys())] - entries += [FilesystemEntry(path=path, change_type="deleted") for path in sorted(before.keys() - after.keys())] - entries += [ - FilesystemEntry(path=path, change_type="modified") - for path in sorted(before.keys() & after.keys()) - if before[path] != after[path] - ] - return FilesystemDiff(entries=entries) - - async def unified_diff( - self, - other: LocalFilesystemEvidence, - relative_path: str | Path, - *, - context: int = 3, - ) -> str: - """Unified diff of one path between this snapshot (before) and ``other`` (after); ``""`` if identical or binary.""" - return await asyncio.to_thread(self._unified_diff_sync, other, relative_path, context) - - def _unified_diff_sync(self, other: LocalFilesystemEvidence, relative_path: str | Path, context: int) -> str: - before_path, after_path = self.path(relative_path), other.path(relative_path) - before = before_path.read_bytes() if before_path.is_file() else b"" - after = after_path.read_bytes() if after_path.is_file() else b"" - if before == after: - return "" - try: - before_lines = before.decode("utf-8").splitlines(keepends=True) - after_lines = after.decode("utf-8").splitlines(keepends=True) - except UnicodeDecodeError: - return "" # binary content: no textual patch - rel = Path(relative_path).as_posix() - return "".join( - difflib.unified_diff(before_lines, after_lines, fromfile=f"a/{rel}", tofile=f"b/{rel}", n=context) - ) - - def _hashes(self) -> dict[str, str]: - hashes: dict[str, str] = {} - for path in self._safe_files(): - hashes[path.relative_to(self._root).as_posix()] = hashlib.sha256(path.read_bytes()).hexdigest() - return hashes - - def _safe_files(self) -> list[Path]: - """Regular files under the root, skipping symlinks whose target escapes it.""" - files: list[Path] = [] - for dirpath, _dirnames, filenames in os.walk(self._root, followlinks=False): - for name in filenames: - full = Path(dirpath) / name - if full.is_symlink() and not self._within_root(full): - continue - files.append(full) - return files - - async def run_verifier( - self, - command: list[str], - *, - cwd: str = ".", - overlay_files: Mapping[str, str] | None = None, - timeout_s: float | None = None, - ) -> CommandResult: - """Run ``command`` (no shell) against a throwaway copy of the evidence; not a sandbox (host privileges). - - ``overlay_files`` is a ``{relative_path: contents}`` map of trusted files written *over* the copy - after it is made, before the command runs. This is how a grader supplies held-out artifacts (a - canonical test suite, a reference implementation) that must not live in — and cannot be edited - through — the agent's own workspace. Overlay paths that escape the copy are rejected. - """ - sandbox = Path(tempfile.mkdtemp(prefix="evidence-verify-")).resolve() - try: - workdir = (sandbox / cwd).resolve() - if workdir != sandbox and sandbox not in workdir.parents: - raise ValueError(f"verifier cwd {cwd!r} resolves outside evidence overlay") - # symlinks=True copies links as-is (no host deref); the ignore hook drops links whose - # target escapes the evidence root so the verifier can't read or write through them. - await asyncio.to_thread( - shutil.copytree, - self._root, - sandbox, - dirs_exist_ok=True, - symlinks=True, - ignore=self._ignore_escaping_symlinks, - ) - if overlay_files: - await asyncio.to_thread(_write_overlay_files, sandbox, overlay_files) - return await self._exec(command, workdir, timeout_s) - finally: - await asyncio.to_thread(shutil.rmtree, sandbox, True) - - def _ignore_escaping_symlinks(self, directory: str, names: list[str]) -> set[str]: - """copytree ignore hook: drop absolute symlinks and links whose target escapes the evidence root.""" - ignored: set[str] = set() - for name in names: - full = Path(directory) / name - if not full.is_symlink(): - continue - if os.path.isabs(os.readlink(full)) or not self._within_root(full): - ignored.add(name) - return ignored - - @staticmethod - async def _exec(command: list[str], cwd: Path, timeout_s: float | None) -> CommandResult: - # start_new_session: child leads its own process group so a timeout can reap the whole tree. - process = await asyncio.create_subprocess_exec( - *command, - cwd=str(cwd), - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - start_new_session=True, - ) - try: - stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=timeout_s) - except TimeoutError: - # wait_for leaves the tree running; kill the whole process group. - try: - os.killpg(os.getpgid(process.pid), signal.SIGKILL) - except ProcessLookupError: - pass - await process.wait() - return CommandResult(exit_code=-1, timed_out=True) - return CommandResult( - exit_code=process.returncode if process.returncode is not None else -1, - stdout=stdout.decode(errors="replace"), - stderr=stderr.decode(errors="replace"), - ) - - -def _write_overlay_files(root: Path, files: Mapping[str, str]) -> None: - """Write ``{relative_path: contents}`` into ``root``, rejecting paths that escape it.""" - resolved_root = root.resolve() - for rel_path, contents in files.items(): - target = (resolved_root / str(rel_path)).resolve() - if target != resolved_root and resolved_root not in target.parents: - raise ValueError(f"overlay file path escapes the verifier copy: {rel_path!r}") - target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(contents, encoding="utf-8") - - -class EvidenceDescriptor(BaseModel): - """Descriptor for a candidate trace, source, or artifact.""" - - # ``anyOf`` mirrors the ``_requires_ref_or_data`` validator into the OpenAPI schema, so a payload - # with neither ``ref`` nor ``data`` is rejected by the contract, not just at runtime. - model_config = ConfigDict( - extra="forbid", - json_schema_extra={"anyOf": [{"required": ["ref"]}, {"required": ["data"]}]}, - ) - - kind: str = Field(description="Evidence type, e.g. 'filesystem', 'trace', 'log_bundle', or 'review'.") - ref: str | None = Field( - default=None, - description="Reference to externally stored evidence (e.g. a local path or storage ref).", - ) - format: str | None = Field( - default=None, - description="Parser hint for the evidence payload, e.g. 'atif' for normalized traces.", - ) - data: JsonValue | None = Field( - default=None, - description="Small inline evidence payload; at least one of ref or data must be set.", - ) - metadata: dict[str, Any] = Field( - default_factory=dict, - description="Free-form metadata associated with the evidence descriptor.", - ) - - @model_validator(mode="after") - def _requires_ref_or_data(self) -> EvidenceDescriptor: - if self.ref is None and self.data is None: - raise ValueError("evidence descriptor requires ref or data") - return self - - -# ATIF ingest: producers emit conformant ATIF (see values/atif.py, RFC 0001); we validate on read, not normalize. -def parse_atif(payload: Any) -> Trajectory: - """Validate a payload as a canonical ATIF :class:`Trajectory` (raises ``ValidationError`` if non-conformant).""" - return payload if isinstance(payload, Trajectory) else Trajectory.model_validate(payload) - - -class TraceHandle: - """Lazily validated read handle exposing a trace descriptor as an ATIF :class:`Trajectory`.""" - - def __init__(self, descriptor: EvidenceDescriptor) -> None: - self._descriptor = descriptor - self._trajectory: Trajectory | None = None - - async def trace(self) -> Trajectory: - """Return the ATIF trajectory, reading and validating on first access.""" - if self._trajectory is None: - payload = await asyncio.to_thread(self._load_payload) - self._trajectory = parse_atif(payload) - return self._trajectory - - def _load_payload(self) -> Any: - descriptor = self._descriptor - if descriptor.data is not None: - return descriptor.data - if descriptor.ref is None: - raise ValueError("trace evidence descriptor requires ref or data") - return json.loads(_local_filesystem_ref(descriptor.ref).read_text(encoding="utf-8")) - - async def steps(self) -> list[Step]: - """Return the ATIF steps in order.""" - return (await self.trace()).steps - - async def tool_calls(self) -> list[ToolCall]: - """Return all tool calls flattened across agent steps, in order.""" - calls: list[ToolCall] = [] - for step in await self.steps(): - calls.extend(step.tool_calls or []) - return calls - - async def token_usage(self) -> FinalMetrics: - """Return aggregate token usage (trajectory ``final_metrics``, else summed per step).""" - trajectory = await self.trace() - if trajectory.final_metrics is not None: - return trajectory.final_metrics - prompt = sum((step.metrics.prompt_tokens or 0) for step in trajectory.steps if step.metrics is not None) - completion = sum((step.metrics.completion_tokens or 0) for step in trajectory.steps if step.metrics is not None) - return FinalMetrics(total_prompt_tokens=prompt or None, total_completion_tokens=completion or None) - - -class LogHandle: - """Read handle over a log-bundle directory.""" - - def __init__(self, root: str | Path) -> None: - self._fs = LocalFilesystemEvidence(root) - - async def list_files(self) -> list[str]: - """Return relative paths of log files in the bundle.""" - return await self._fs.list_files("**/*") - - async def read_text(self, name: str) -> str: - """Read one log file's full text.""" - return await self._fs.read_text(name) - - async def tail(self, name: str, lines: int = 50) -> str: - """Return the last ``lines`` lines of a log file.""" - text = await self._fs.read_text(name) - return "\n".join(text.splitlines()[-lines:]) - - -class CandidateEvidence(BaseModel): - """Named evidence descriptors attached to an AgentEvalTrial.""" - - model_config = ConfigDict(extra="forbid") - - descriptors: dict[str, EvidenceDescriptor] = Field( - default_factory=dict, - description="Evidence descriptors keyed by name (e.g. 'final_state', 'trace', 'logs').", - ) - metadata: dict[str, Any] = Field( - default_factory=dict, - description="Free-form metadata associated with the evidence collection.", - ) - _filesystem_cache: dict[str, LocalFilesystemEvidence] = PrivateAttr(default_factory=dict) - _trace_cache: dict[str, TraceHandle] = PrivateAttr(default_factory=dict) - _log_cache: dict[str, LogHandle] = PrivateAttr(default_factory=dict) - - @model_validator(mode="before") - @classmethod - def _coerce_descriptor_mapping(cls, value: Any) -> Any: - if isinstance(value, cls): - return value - if isinstance(value, dict) and "descriptors" not in value and "metadata" not in value: - return {"descriptors": value} - return value - - def names(self, *, kind: str | None = None) -> list[str]: - """Return evidence names, optionally filtered by descriptor kind.""" - if kind is None: - return list(self.descriptors) - return [name for name, descriptor in self.descriptors.items() if descriptor.kind == kind] - - def get(self, name: str) -> EvidenceDescriptor | None: - """Return a descriptor by name without materializing evidence.""" - return self.descriptors.get(name) - - def require(self, name: str, *, kind: str | None = None) -> EvidenceDescriptor: - """Return a descriptor by name, raising when it is missing or has the wrong kind.""" - descriptor = self.get(name) - if descriptor is None: - raise KeyError(f"missing evidence descriptor {name!r}") - if kind is not None and descriptor.kind != kind: - raise ValueError(f"evidence descriptor {name!r} has kind {descriptor.kind!r}, expected {kind!r}") - return descriptor - - async def filesystem(self, name: str) -> LocalFilesystemEvidence: - """Return a cached local filesystem handle for a named filesystem descriptor.""" - cached = self._filesystem_cache.get(name) - if cached is not None: - return cached - - descriptor = self.require(name, kind="filesystem") - if descriptor.ref is None: - raise ValueError(f"filesystem evidence descriptor {name!r} requires a local ref") - - root = _local_filesystem_ref(descriptor.ref) - handle = LocalFilesystemEvidence(root) - self._filesystem_cache[name] = handle - return handle - - async def trace(self, name: str = "trace") -> TraceHandle: - """Return a cached trace handle for a named trace descriptor (read lazily on first access).""" - cached = self._trace_cache.get(name) - if cached is not None: - return cached - handle = TraceHandle(self.require(name, kind="trace")) - self._trace_cache[name] = handle - return handle - - async def logs(self, name: str = "logs") -> LogHandle: - """Return a cached log-bundle handle for a named logs descriptor.""" - cached = self._log_cache.get(name) - if cached is not None: - return cached - descriptor = self.require(name, kind="logs") - if descriptor.ref is None: - raise ValueError(f"logs evidence descriptor {name!r} requires a local ref") - handle = LogHandle(_local_filesystem_ref(descriptor.ref)) - self._log_cache[name] = handle - return handle - - -def _local_filesystem_ref(ref: str) -> Path: - """Resolve a local ref (POSIX path, ``file://`` URI, or Windows drive path) to a Path; reject network/cloud URIs.""" - parsed = urlparse(ref) - # A single-letter scheme is a Windows drive letter (e.g. "C:\\dir"), not a URI scheme. - if len(parsed.scheme) == 1 and parsed.scheme.isalpha(): - return Path(ref) - if parsed.scheme in {"http", "https", "s3", "gs"}: - raise ValueError("CandidateEvidence.filesystem only supports local filesystem refs") - if parsed.scheme == "file": - return Path(parsed.path) - if parsed.scheme: - raise ValueError(f"CandidateEvidence.filesystem does not support {parsed.scheme!r} refs") - return Path(ref) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/llm_judge_defaults.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/llm_judge_defaults.py deleted file mode 100644 index 6508e124ba..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/llm_judge_defaults.py +++ /dev/null @@ -1,80 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Default prompt templates for LLM judge metric values.""" - -from typing import Any - -from nemo_platform.beta.evaluator.values.common import SupportedJobTypes -from nemo_platform.beta.evaluator.values.models import Model, ModelRef - -DEFAULT_PROMPT_TEMPLATE = "{{item}}" -LLM_JUDGE_SCORES_CONTEXT_KEY = "scores" -DEFAULT_JUDGE_SYSTEM_PROMPT_TEMPLATE = """You are an expert evaluator for answers to user queries. Your task is to assess responses to user queries based on {{ scores.keys() | join(", ") }} -{% if scores | length > 1 %}Scores:{% endif %} -{%- for score_name, score in scores.items() %} -{{ score_name }}{%- if "minimum" in score %} with a score range from {{ score.minimum }} to {{ score.maximum }}{%- endif %}{% if score.description %}: {{score.description}}{% endif %} -{%- if "rubric" in score %} -{%- for rubric in score.rubric %} -* {{ rubric.label }}{% if rubric.description %}: {{rubric.description}}{% endif %} -{%- endfor -%} -{%- endif -%} -{%- endfor -%} -""" -DEFAULT_JUDGE_PROMPT_TEMPLATE_WITH_TARGET_MODEL = "{{sample.output_text}}" - - -def is_chat_inference(url: str) -> bool: - """Check if the URL is for chat inference (vs completions).""" - return "/v1/completions" not in url - - -def default_judge_prompt_template_chat(job_type: SupportedJobTypes = SupportedJobTypes.ONLINE) -> dict: - prompt = ( - DEFAULT_JUDGE_PROMPT_TEMPLATE_WITH_TARGET_MODEL - if job_type == SupportedJobTypes.ONLINE - else DEFAULT_PROMPT_TEMPLATE - ) - return { - "messages": [ - {"role": "system", "content": DEFAULT_JUDGE_SYSTEM_PROMPT_TEMPLATE}, - {"role": "user", "content": prompt}, - ] - } - - -def default_judge_prompt_template_completions(job_type: SupportedJobTypes = SupportedJobTypes.ONLINE) -> dict: - prompt = ( - DEFAULT_JUDGE_PROMPT_TEMPLATE_WITH_TARGET_MODEL - if job_type == SupportedJobTypes.ONLINE - else DEFAULT_PROMPT_TEMPLATE - ) - return {"prompt": f"{DEFAULT_JUDGE_SYSTEM_PROMPT_TEMPLATE}\n{prompt}"} - - -def _model_uses_chat_prompt_default(model: Model | ModelRef | dict[str, Any]) -> bool: - """Return whether a model-like value should default to chat prompt format.""" - if isinstance(model, ModelRef): - return True - if isinstance(model, Model): - return is_chat_inference(model.url) - - url = model.get("url") - if isinstance(url, str): - return is_chat_inference(url) - - root = model.get("root") - if isinstance(root, str): - return True - - raise ValueError("model.url or ModelRef.root is required to infer the default prompt template") - - -def default_judge_prompt_template_for_model( - model: Model | ModelRef | dict[str, Any], - job_type: SupportedJobTypes = SupportedJobTypes.ONLINE, -) -> dict: - """Return the default judge prompt template for a model-like value.""" - if _model_uses_chat_prompt_default(model): - return default_judge_prompt_template_chat(job_type) - return default_judge_prompt_template_completions(job_type) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/metrics.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/metrics.py deleted file mode 100644 index 827051e0d0..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/metrics.py +++ /dev/null @@ -1,588 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Metric types for evaluator SDK. - -These types contain all metric configuration fields but do not require -workspace/name (they do not inherit from EntityBase). They can be used -directly for inline metric definitions in API requests. -""" - -from __future__ import annotations - -from collections.abc import Iterable -from typing import Annotated, Any, ClassVar, Literal - -from pydantic import BaseModel, ConfigDict, Field, model_validator -from typing_extensions import Self - -from nemo_platform.beta.evaluator.dataset_schemas.common import empty_object_schema -from nemo_platform.beta.evaluator.dataset_schemas.compatibility import merge_metric_required_schemas -from nemo_platform.beta.evaluator.dataset_schemas.templates import infer_required_schema_from_template -from nemo_platform.beta.evaluator.enums import MetricType -from nemo_platform.beta.evaluator.values.common import SecretRef, SupportedJobTypes -from nemo_platform.beta.evaluator.values.dataset_schemas import InputSchema -from nemo_platform.beta.evaluator.values.llm_judge_defaults import ( - LLM_JUDGE_SCORES_CONTEXT_KEY, - default_judge_prompt_template_for_model, -) -from nemo_platform.beta.evaluator.values.llm_judge_defaults import ( - default_judge_prompt_template_chat as default_judge_prompt_template_chat, -) -from nemo_platform.beta.evaluator.values.llm_judge_defaults import ( - default_judge_prompt_template_completions as default_judge_prompt_template_completions, -) -from nemo_platform.beta.evaluator.values.llm_judge_defaults import ( - is_chat_inference as is_chat_inference, -) -from nemo_platform.beta.evaluator.values.models import Model, ModelRef, ReasoningParams -from nemo_platform.beta.evaluator.values.params import InferenceParams -from nemo_platform.beta.evaluator.values.protocol import MetricTypeName -from nemo_platform.beta.evaluator.values.scores import RemoteScore, Score - -# ============================================================================= -# Prompt Template Constants and Helpers -# ============================================================================= - -# TODO: Align optional_fields with template path semantics. -# Keep support for dataset-relative nested paths (for example "reference.text") -# and runtime sample paths (for example "sample.output_text"), while avoiding -# dependence on the "item." alias form (normalize "item.foo" -> "foo"). -OptionalFieldName = Annotated[str, Field(min_length=1)] - - -def _input_schema_from_template( - template: str | dict | list, - *, - ignored_roots: set[str] | None = None, - optional_fields: set[str] | None = None, -) -> InputSchema: - return InputSchema( - schema=infer_required_schema_from_template( - template, - ignored_roots=ignored_roots, - optional_fields=optional_fields, - ) - ) - - -def _input_schema_from_templates(templates: Iterable[str | dict | list]) -> InputSchema: - schemas = [infer_required_schema_from_template(template) for template in templates] - if not schemas: - return InputSchema(schema=empty_object_schema()) - - merged_schema = merge_metric_required_schemas((f"template_{index}", schema) for index, schema in enumerate(schemas)) - return InputSchema(schema=merged_schema) - - -# ============================================================================= -# Base Metric Type -# ============================================================================= - - -class MetricBase(BaseModel): - """Base class for inline metrics. - - Contains common fields shared by all metric types. - """ - - __entity_type__: ClassVar[str] = "metric" - - type: MetricTypeName = Field(description="The type of metric. Used as a discriminator for the metric type.") - description: str | None = Field(default=None, description="Human-readable description of the metric.") - labels: dict[str, str] = Field( - default_factory=dict, description="Labels are key-value pairs that can be used for grouping and filtering." - ) - supported_job_types: list[Literal[SupportedJobTypes.ONLINE, SupportedJobTypes.OFFLINE]] = Field( - default=[SupportedJobTypes.ONLINE, SupportedJobTypes.OFFLINE], - description="A metric can evaluate model outputs for online evaluations or pre-generated outputs for offline evaluations.", - ) - - def model_post_init(self, __context: Any) -> None: - """Mark ``type`` as set so it is included when exclude_unset=True.""" - self.__pydantic_fields_set__.add("type") - - def input_schema(self) -> InputSchema: - """Return the canonical evaluator input schema required by this metric.""" - return InputSchema(schema=empty_object_schema()) - - -# ============================================================================= -# Metric Types -# ============================================================================= - - -class BLEU(MetricBase): - """BLEU metric configuration.""" - - type: Literal[MetricType.BLEU] = MetricType.BLEU - references: list[str] = Field( - description="The templates for the ground truth references to calculate BLEU metric with." - ) - candidate: str | None = Field( - default=None, - description="The template for the candidate to calculate BLEU metric on. If not provided, the output text from the model is used.", - ) - - def input_schema(self) -> InputSchema: - templates: list[str] = [*self.references] - if self.candidate is not None: - templates.append(self.candidate) - return _input_schema_from_templates(templates) - - -class ExactMatch(MetricBase): - """Exact Match metric configuration.""" - - type: Literal[MetricType.EXACT_MATCH] = MetricType.EXACT_MATCH - reference: str = Field( - description="The template for the ground truth reference to calculate the exact match metric with.", - ) - candidate: str | None = Field( - default=None, - description="The template for the candidate to evaluate the exact match metric on. If not provided, the output text from the model is used.", - ) - - def input_schema(self) -> InputSchema: - templates = [self.reference] - if self.candidate is not None: - templates.append(self.candidate) - return _input_schema_from_templates(templates) - - -class F1(MetricBase): - """F1 metric configuration.""" - - type: Literal[MetricType.F1] = MetricType.F1 - reference: str = Field(description="The template for the ground truth reference to calculate the F1 metric with.") - candidate: str | None = Field( - default=None, - description="The template for the candidate to evaluate the F1 metric on. If not provided, the output text from the model is used.", - ) - - def input_schema(self) -> InputSchema: - templates = [self.reference] - if self.candidate is not None: - templates.append(self.candidate) - return _input_schema_from_templates(templates) - - -class LLMJudge(MetricBase): - """LLM-as-a-Judge metric configuration.""" - - type: Literal[MetricType.LLM_JUDGE] = MetricType.LLM_JUDGE - model: Model | ModelRef = Field( - description="The judge model to use for the metric.", - examples=[ - { - "endpoint": "https://api.openai.com/v1", - "name": "gpt-4o", - "api_key_secret": "secret/my_openai_api_key", - "format": "openai", - } - ], - ) - scores: list[Score] = Field( - description="Definitions of scores that will be extracted from the judge's output.", min_length=1 - ) - prompt_template: str | dict | None = Field( - default=None, - description="The prompt template for the judge. Can be either a simple string or a structured object (e.g., OpenAI messages format). Use Jinja template variables like {{sample.output_text}} to use the model output within the template or {{item.xxx}} to reference input columns from the dataset.", - examples=[ - {"type": "string", "content": "You are an expert judge evaluating the correctness of AI responses."}, - { - "type": "object", - "content": { - "messages": [ - { - "role": "system", - "content": "You are an expert judge evaluating the correctness of AI responses.", - }, - { - "role": "user", - "content": "Question: {{item.prompt}}\nAnswer: {{sample.output_text}}\nReference: {{item.reference}}\nRate the correctness from 1-5.", - }, - ] - }, - }, - ], - ) - optional_fields: list[OptionalFieldName] = Field( - default_factory=list, - description=( - "Prompt template fields that should remain in the inferred input schema but not be required. " - "Use this for fields like 'reference' when the metric can still run without them." - ), - examples=[["reference"]], - ) - structured_output: dict | None = Field( - default=None, - description="JSON schema to apply structured output for the judge model evaluation. Structured output is derived from scores when omitted. Use this option if there are custom requirements for the output of the judge.", - ) - inference: InferenceParams | None = Field(default=None, description="Inference parameters for the judge model.") - system_prompt: str | None = Field( - default=None, - description="Initial instructions that define the judge model's role and behavior for the conversation. " - "This is prepended to the messages as a system message.", - ) - reasoning: ReasoningParams | None = Field( - default=None, - description="Custom settings that control the judge model's reasoning behavior. " - "For reasoning models (e.g., Nemotron), use `end_token` to strip reasoning traces from the output.", - ) - ignore_request_failure: bool = Field( - default=False, - description="If True, request failures will be ignored and the result will be marked as NaN. " - "If False (default), request failures will raise an exception.", - ) - - @model_validator(mode="after") - def unique_scores(self) -> Self: - if not self.scores: - return self - - scores = {score.name for score in self.scores} - if len(scores) != len(self.scores): - raise ValueError("score names must be unique") - return self - - @model_validator(mode="after") - def reject_reserved_prompt_template_keys(self) -> Self: - """Fail fast on misplaced evaluator controls in `prompt_template`.""" - if self.prompt_template is None or not isinstance(self.prompt_template, dict): - return self - - reserved_keys = {"system_prompt", "reasoning"} - found_keys = sorted(reserved_keys.intersection(self.prompt_template.keys())) - if found_keys: - keys_str = ", ".join(found_keys) - raise ValueError( - f"prompt_template cannot include {keys_str}. " - "Use top-level fields 'system_prompt' and 'reasoning' instead." - ) - return self - - def input_schema(self) -> InputSchema: - job_type = getattr(self, "job_type", SupportedJobTypes.ONLINE) - prompt_template = ( - self.prompt_template - if self.prompt_template is not None - else default_judge_prompt_template_for_model(self.model, job_type) - ) - return _input_schema_from_template( - prompt_template, - ignored_roots={LLM_JUDGE_SCORES_CONTEXT_KEY}, - optional_fields=set(self.optional_fields), - ) - - -NumberCheckOperation = Literal[ - "equals", - "==", - "!=", - "<>", - "not equals", - ">=", - "gte", - "greater than or equal", - ">", - "gt", - "greater than", - "<=", - "lte", - "less than or equal", - "<", - "lt", - "less than", - "absolute difference", -] - - -class NumberCheck(MetricBase): - """Number check metric configuration.""" - - type: Literal[MetricType.NUMBER_CHECK] = MetricType.NUMBER_CHECK - operation: NumberCheckOperation = Field(description="The operation to compute for the metric.") - left_template: str = Field( - description="The template to use for rendering the left value of the operator to compute the metric.", - examples=["{{item.dataset_column_name}}"], - ) - right_template: str = Field( - description="The template to use for rendering the right value of the operator to compute the metric.", - examples=["{{sample.output_text}}"], - ) - epsilon: int | float | None = Field( - default=None, description="Specify the tolerance for the absolute difference of values." - ) - - @model_validator(mode="after") - def absolute_difference(self) -> Self: - if self.operation == "absolute difference": - if self.epsilon is None: - raise ValueError(f"epsilon value is required with operation {self.operation}") - elif self.epsilon: - raise ValueError(f"epsilon value can only be used with absolute difference operation: {self.operation}") - return self - - def input_schema(self) -> InputSchema: - return _input_schema_from_templates([self.left_template, self.right_template]) - - -class _RemoteBase(MetricBase): - url: str = Field(description="The URL of the remote endpoint.") - api_key_secret: SecretRef | None = Field( - default=None, - description="Optional secret reference of an API key for authentication. Format: workspace/secret_name or secret_name within the job workspace.", - ) - timeout_seconds: float = Field(default=30.0, description="Request timeout in seconds.") - max_retries: int = Field(default=3, description="Maximum number of retry attempts.") - - -class Remote(_RemoteBase): - """Remote metric configuration.""" - - type: Literal[MetricType.REMOTE] = MetricType.REMOTE - body: dict[str, Any] = Field(description="Jinja template for request payload") - scores: list[RemoteScore] = Field(description="List of scores to extract from the remote response") - - def input_schema(self) -> InputSchema: - # NAT evaluators currently receive the entire dataset row as an opaque `item` - # payload, and this metric config does not carry a machine-readable schema for - # what that evaluator expects. Until we can discover or declare the NAT input - # contract (for example from evaluator metadata), treat the accepted input as an - # unconstrained object rather than over-specifying required fields here. - return InputSchema(schema=empty_object_schema()) - - -class NemoAgentToolkitRemote(_RemoteBase): - """NeMo Agent Toolkit Remote metric configuration.""" - - type: Literal[MetricType.NEMO_AGENT_TOOLKIT_REMOTE] = MetricType.NEMO_AGENT_TOOLKIT_REMOTE - evaluator_name: str = Field(description="The name of the evaluator (also used as the score name).") - - def input_schema(self) -> InputSchema: - # NAT evaluators currently receive the entire dataset row as an opaque `item` - # payload, and this metric config does not carry a machine-readable schema for - # what that evaluator expects. Until we can discover or declare the NAT input - # contract (for example from evaluator metadata), treat the accepted input as an - # unconstrained object rather than over-specifying required fields here. - return InputSchema(schema=empty_object_schema()) - - -class ROUGE(MetricBase): - """ROUGE metric configuration.""" - - type: Literal[MetricType.ROUGE] = MetricType.ROUGE - reference: str = Field(description="The template for the ground truth reference to evaluate the ROUGE metric with.") - candidate: str | None = Field( - default=None, - description="The template for the candidate to evaluate the ROUGE metric on. If not provided, the output text from the model is used.", - ) - - def input_schema(self) -> InputSchema: - templates = [self.reference] - if self.candidate is not None: - templates.append(self.candidate) - return _input_schema_from_templates(templates) - - -StringCheckOperation = Literal[ - "equals", - "==", - "!=", - "<>", - "not equals", - "contains", - "not contains", - "startswith", - "endswith", -] - - -class StringCheck(MetricBase): - """String check metric configuration.""" - - type: Literal[MetricType.STRING_CHECK] = MetricType.STRING_CHECK - operation: StringCheckOperation = Field(description="The operation to compute for the metric.") - left_template: str = Field( - description="The template to use for rendering the left value of the operator to compute the metric.", - examples=["{{item.dataset_column_name}}"], - ) - right_template: str = Field( - description="The template to use for rendering the right value of the operator to compute the metric.", - examples=["{{sample.output_text | trim}}"], - ) - - def input_schema(self) -> InputSchema: - return _input_schema_from_templates([self.left_template, self.right_template]) - - -class ToolCalling(MetricBase): - """Tool Calling metric configuration.""" - - model_config = ConfigDict(validate_assignment=True) - - type: Literal[MetricType.TOOL_CALLING] = MetricType.TOOL_CALLING - reference: str = Field(description="The template for the ground truth reference to evaluate tool calling accuracy.") - - def input_schema(self) -> InputSchema: - return _input_schema_from_template(self.reference) - - -# ============================================================================= -# Inline RAGAS Metric Types -# ============================================================================= - - -class _RAGASJudgeConfig(BaseModel): - """Configuration for the LLM judge used by RAGAS metrics.""" - - judge_model: Model | ModelRef = Field(description="The LLM model to use as judge.") - inference: InferenceParams = Field( - default_factory=InferenceParams, description="Inference parameters for the judge." - ) - ignore_request_failure: bool = Field( - default=False, - description="If True, request failures to the judge model are ignored and the metric result " - "is marked as NaN. Parse/output formatting failures are always converted to NaN.", - ) - - -class _RAGASEmbeddingsConfig(BaseModel): - """Configuration for embeddings used by RAGAS metrics.""" - - embeddings_model: Model | ModelRef = Field(description="The embeddings model to use.") - - -class _RAGASBase(MetricBase): - """Base class for inline RAGAS metrics.""" - - input_template: dict[str, Any] | None = Field( - default=None, - description="Optional Jinja template for rendering the input payload for RAGAS evaluation.", - ) - - def input_schema(self) -> InputSchema: - if self.input_template is None: - return InputSchema(schema=empty_object_schema()) - return _input_schema_from_template(self.input_template) - - -class TopicAdherence(_RAGASBase, _RAGASJudgeConfig): - """RAGAS metric for measuring topic adherence.""" - - type: Literal[MetricType.TOPIC_ADHERENCE] = MetricType.TOPIC_ADHERENCE - metric_mode: Literal["f1", "precision", "recall"] = Field( - default="f1", description="The mode for computing topic adherence score." - ) - - -class ToolCallAccuracy(_RAGASBase): - """RAGAS metric for measuring tool call accuracy.""" - - type: Literal[MetricType.TOOL_CALL_ACCURACY] = MetricType.TOOL_CALL_ACCURACY - - -class AgentGoalAccuracy(_RAGASBase, _RAGASJudgeConfig): - """RAGAS metric for measuring agent goal accuracy.""" - - type: Literal[MetricType.AGENT_GOAL_ACCURACY] = MetricType.AGENT_GOAL_ACCURACY - use_reference: bool = Field(default=True, description="Whether to use reference for goal accuracy evaluation.") - - -class AnswerAccuracy(_RAGASBase, _RAGASJudgeConfig): - """RAGAS metric for measuring answer accuracy.""" - - type: Literal[MetricType.ANSWER_ACCURACY] = MetricType.ANSWER_ACCURACY - - -class ContextRelevance(_RAGASBase, _RAGASJudgeConfig): - """RAGAS metric for measuring context relevance.""" - - type: Literal[MetricType.CONTEXT_RELEVANCE] = MetricType.CONTEXT_RELEVANCE - - -class ResponseGroundedness(_RAGASBase, _RAGASJudgeConfig): - """RAGAS metric for measuring response groundedness.""" - - type: Literal[MetricType.RESPONSE_GROUNDEDNESS] = MetricType.RESPONSE_GROUNDEDNESS - - -class ContextRecall(_RAGASBase, _RAGASJudgeConfig): - """RAGAS metric for measuring context recall.""" - - type: Literal[MetricType.CONTEXT_RECALL] = MetricType.CONTEXT_RECALL - - -class ContextPrecision(_RAGASBase, _RAGASJudgeConfig): - """RAGAS metric for measuring context precision.""" - - type: Literal[MetricType.CONTEXT_PRECISION] = MetricType.CONTEXT_PRECISION - - -class ContextEntityRecall(_RAGASBase, _RAGASJudgeConfig): - """RAGAS metric for measuring context entity recall.""" - - type: Literal[MetricType.CONTEXT_ENTITY_RECALL] = MetricType.CONTEXT_ENTITY_RECALL - - -class ResponseRelevancy(_RAGASBase, _RAGASJudgeConfig, _RAGASEmbeddingsConfig): - """RAGAS metric for measuring response relevancy.""" - - type: Literal[MetricType.RESPONSE_RELEVANCY] = MetricType.RESPONSE_RELEVANCY - strictness: int = Field( - default=1, - description="Number of parallel questions generated. NIM can only generate 1.", - ) - - -class Faithfulness(_RAGASBase, _RAGASJudgeConfig): - """RAGAS metric for measuring faithfulness.""" - - type: Literal[MetricType.FAITHFULNESS] = MetricType.FAITHFULNESS - - -class NoiseSensitivity(_RAGASBase, _RAGASJudgeConfig): - """RAGAS metric for measuring noise sensitivity.""" - - type: Literal[MetricType.NOISE_SENSITIVITY] = MetricType.NOISE_SENSITIVITY - - -class TunableRagEvaluator(MetricBase): - """Tunable RAG evaluator with customizable judge prompt and weighted sub-scores.""" - - type: Literal[MetricType.TUNABLE_RAG_EVALUATOR] = MetricType.TUNABLE_RAG_EVALUATOR - model: Model | ModelRef = Field(description="Judge model used to score generated answers.") - judge_llm_prompt: str = Field( - default="", - description="Optional custom judge rubric. Ignored when default_scoring is true except as extra context.", - ) - default_scoring: bool = Field( - default=True, - description="Use built-in coverage/correctness/relevance rubric and weighted composite.", - ) - default_score_weights: dict[str, float] = Field( - default_factory=lambda: {"coverage": 0.5, "correctness": 0.3, "relevance": 0.2}, - description="Weights for coverage/correctness/relevance when default_scoring is true.", - ) - inference: InferenceParams | None = Field( - default=None, - description="Optional inference parameters for the judge model.", - ) - - def input_schema(self) -> InputSchema: - return InputSchema( - schema={ - "type": "object", - "properties": { - "inputs": { - "type": "object", - "properties": {"instruction": {"type": "string"}}, - }, - "reference": { - "type": "object", - "properties": {"answer": {"type": "string"}}, - }, - }, - } - ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/models.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/models.py deleted file mode 100644 index b080548a00..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/models.py +++ /dev/null @@ -1,179 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Model-related value types.""" - -from __future__ import annotations - -import logging -import os -from functools import cached_property -from typing import Annotated, Literal - -from pydantic import BaseModel, ConfigDict, Field, RootModel, field_validator -from pydantic.config import JsonDict - -from nemo_platform.beta.evaluator.enums import ModelFormat -from nemo_platform.beta.evaluator.values.common import SecretRef - -logger = logging.getLogger(__name__) - -_AUTH_HEADER_PATTERNS: tuple[str, ...] = ( - "auth", - "token", - "key", - "secret", - "credential", - "bearer", - "cookie", - "set-cookie", -) -# Keep this aligned with nmp.common.entities.constants.NAME_PATTERN without adding an SDK dependency on nmp_common. -# This pydantic-core-compatible form avoids lookarounds while preserving the naming syntax: -# start with lowercase alpha, require at least one more valid character, allow single hyphens between non-hyphen chars, -# and do not end in hyphen. -_ENTITY_NAME_SEGMENT = r"[a-z](?:[a-z0-9@.+_]|-[a-z0-9@.+_]){1,62}" -_QUALIFIED_MODEL_REF_PATTERN = rf"^{_ENTITY_NAME_SEGMENT}/{_ENTITY_NAME_SEGMENT}$" - - -_ModelRefRoot = Annotated[ - str, - Field( - pattern=_QUALIFIED_MODEL_REF_PATTERN, - description="Reference to a model (format: workspace/name).", - examples=[ - "workspace/model_name", - ], - ), -] - - -class ReasoningParams(BaseModel): - """Custom settings that control the model's reasoning behavior.""" - - end_token: str | None = Field( - default=None, - description="Configure the end token to trim reasoning context based on the model's reasoning API. Example for Nemotron models: ''", - ) - include_if_not_finished: bool | None = Field( - default=None, - description="Configure whether to include reasoning context if the model has not finished reasoning.", - ) - effort: str | None = Field( - default=None, - description="Option for OpenAI models to specify low, medium, or high reasoning effort.", - ) - - -class ModelRef(RootModel[_ModelRefRoot]): - """Reference to a model that can be resolved by an evaluator backend.""" - - -def _strip_internal_fields(schema: JsonDict) -> None: - """Remove internal-only fields from the JSON schema so they don't appear in the OpenAPI spec.""" - props = schema.get("properties") - if isinstance(props, dict): - props.pop("default_headers", None) - props.pop("host_url", None) - - -def normalize_header_name(header_name: str) -> str: - """Normalize a transport header name for policy checks.""" - return header_name.strip().lower().replace("_", "-") - - -def is_auth_header_name(header_name: str) -> bool: - """Return whether a header name appears to carry authentication material.""" - normalized_header_name = normalize_header_name(header_name) - return any(pattern in normalized_header_name for pattern in _AUTH_HEADER_PATTERNS) - - -def filter_auth_headers(headers: dict[str, str] | None) -> dict[str, str] | None: - """Return only non-auth headers from a header mapping.""" - if headers is None: - return None - - filtered_headers: dict[str, str] = {} - for header_name, header_value in headers.items(): - if is_auth_header_name(header_name): - logger.debug( - f"Filtered header {header_name} because it was recognized as an auth header", - ) - continue - filtered_headers[header_name] = header_value - - return filtered_headers or None - - -class Model(BaseModel): - """Model definition for use without persisting to the Models API.""" - - model_config = ConfigDict(extra="forbid", json_schema_extra=_strip_internal_fields) - - url: str = Field(description="URL of the model.") - name: str = Field(description="Name of the model.") - default_headers: dict[str, str] | None = Field( - default=None, - exclude=True, - description="Runtime-only non-auth headers automatically applied to requests made with this model. " - "Authentication must be configured via model.api_key_secret; auth headers such as Authorization will be rejected.", - ) - host_url: str | None = Field( - default=None, - description="Direct NIM endpoint URL (http://host:port). Populated when resolved from a ModelRef. " - "Used by EvalFactory containers that reject path-based URLs (e.g., Haystack NvidiaDocumentEmbedder).", - ) - api_key_secret: SecretRef | None = Field( - default=None, - description="API key secret reference for the model. Format: workspace/secret_name or secret_name within the job workspace.", - ) - format: Literal[ModelFormat.NVIDIA_NIM, ModelFormat.OPEN_AI, ModelFormat.LLAMA_STACK] = Field( - default=ModelFormat.NVIDIA_NIM, description="API format of the model." - ) - - @field_validator("default_headers") - @classmethod - def validate_default_headers(cls, value: dict[str, str] | None) -> dict[str, str] | None: - """Reject auth-style headers and direct users to api_key_secret for credentials.""" - if value is None: - return None - - for header_name in value: - if is_auth_header_name(header_name): - raise ValueError( - f"Header {header_name} in model.default_headers is rejected because model.default_headers cannot include authentication headers (Authorization, X-API-Key, etc.). " - f"Header names are not allowed to contain the following substrings: {', '.join(_AUTH_HEADER_PATTERNS)}. " - f"Configure model auth via model.api_key_secret instead." - ) - return value - - @cached_property - def api_key_env(self) -> str | None: - if self.api_key_secret: - env_name = self.api_key_secret.root - if env_name[0].isdigit(): - env_name = f"_{env_name}" # prefix with valid character for environment variable - return env_name.replace("-", "_").replace("/", "_") - - @cached_property - def api_key(self) -> str | None: - if self.api_key_secret: - api_key_env = self.api_key_env - assert api_key_env is not None - return os.getenv(api_key_env) or os.getenv(api_key_env.upper()) - - def with_default_headers(self, headers: dict[str, str] | None) -> "Model": - """Return a copy of the model with merged runtime default headers.""" - if not headers: - return self - - return self.model_copy( - update={ - "default_headers": { - **(self.default_headers or {}), - **headers, - } - }, - # flat dict, no deep copy is needed - deep=False, - ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/multi_metric_results.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/multi_metric_results.py deleted file mode 100644 index 7fd92bd2f4..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/multi_metric_results.py +++ /dev/null @@ -1,386 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Helpers for assembling multi-metric evaluator results.""" - -from __future__ import annotations - -import json -from typing import Any - -from pydantic import BaseModel - -from nemo_platform.beta.evaluator.values.protocol import MetricDiagnostic, MetricOutput -from nemo_platform.beta.evaluator.values.results import ( - AggregatedMetricResult, - AggregateFieldName, - EvaluationResult, - ResultView, - RowScore, - diagnostics_records, - flatten_dict, - format_error_details, - format_table, - row_error_text, - row_status, - serialize_value, - summary_aggregate_record, - summary_header, - summary_row_base_record, -) - - -def _filter_aggregate_fields( - aggregate_scores: AggregatedMetricResult, aggregate_fields: tuple[AggregateFieldName, ...] | None -) -> AggregatedMetricResult: - """Trim aggregate score payloads to the requested field subset. - - Args: - aggregate_scores: Full aggregate metric output for one or more metrics. - aggregate_fields: Optional field names to retain on each score object. - - Returns: - The original aggregate scores when no filtering is requested, otherwise - a copy that keeps only the selected fields. - """ - if not aggregate_fields: - return aggregate_scores - - filtered_scores = [score.with_fields(frozenset(aggregate_fields)) for score in aggregate_scores.scores] - return AggregatedMetricResult(scores=filtered_scores) - - -def _extract_metric_outputs(row_score: RowScore, expected_key: str) -> list[MetricOutput]: - """Resolve the output list for one metric from a row result. - - This exists because local and remote execution paths may return row scores - either already keyed by the final metric key or as a single unnamed metric - payload that still needs to be attached to that key. - - Args: - row_score: Row-level result payload to inspect. - expected_key: Metric key the caller expects to find on the row. - - Returns: - The output list for the requested metric key, or an empty list when the - row has no metric output because evaluation failed. - - Raises: - ValueError: If the row contains multiple metric entries and none match - the requested key. - """ - if expected_key in row_score.metrics: - return row_score.metrics[expected_key] - if not row_score.metrics: - return [] - if len(row_score.metrics) == 1: - return next(iter(row_score.metrics.values())) - raise ValueError(f"Unable to resolve row metric outputs for key {expected_key!r}") - - -def _extract_metric_error(row_score: RowScore, expected_key: str) -> str | None: - """Resolve one metric error message from a row result.""" - metric_errors = row_score.metric_errors - if metric_errors and expected_key in metric_errors: - return metric_errors[expected_key] - if metric_errors: - if len(metric_errors) == 1: - return next(iter(metric_errors.values())) - raise ValueError(f"Unable to resolve row metric error for key {expected_key!r}") - return row_score.error - - -def _extract_metric_diagnostics(row_score: RowScore, expected_key: str) -> list[MetricDiagnostic] | None: - """Return diagnostics for ``expected_key``, or ``None`` if absent.""" - if not row_score.metric_diagnostics: - return None - return row_score.metric_diagnostics.get(expected_key) - - -def _row_identity(row_score: RowScore, fallback_index: int) -> int: - """Resolve the stable row identity used when combining metric results. - - Args: - row_score: Row-level result payload. - fallback_index: Positional fallback when no explicit row identity is present. - - Returns: - Stable row identity for result alignment. - """ - return row_score.row_index if row_score.row_index is not None else fallback_index - - -def namespace_result( - metric_key: str, - result: EvaluationResult, - aggregate_fields: tuple[AggregateFieldName, ...] | None, -) -> EvaluationResult: - """Rewrite a single-metric result to use a stable `v4` metric namespace. - - Args: - metric_key: Public metric key assigned by the evaluator. - result: Raw single-metric evaluation result from one backend. - aggregate_fields: Optional aggregate field subset to keep. - - Returns: - A result whose row metrics and aggregate score names are prefixed with - the evaluator-assigned metric key. - """ - row_scores = [ - RowScore( - row_index=row_score.row_index, - item=row_score.item, - sample=row_score.sample, - metrics={metric_key: _extract_metric_outputs(row_score, metric_key)}, - requests=row_score.requests, - metric_errors={metric_key: error} if (error := _extract_metric_error(row_score, metric_key)) else None, - metric_diagnostics=( - {metric_key: diagnostics} - if (diagnostics := _extract_metric_diagnostics(row_score, metric_key)) - else None - ), - ) - for row_score in result.row_scores - ] - aggregate_scores = AggregatedMetricResult( - scores=[ - score.model_copy(update={"name": f"{metric_key}.{score.name}"}) for score in result.aggregate_scores.scores - ] - ) - aggregate_scores = _filter_aggregate_fields(aggregate_scores, aggregate_fields) - return EvaluationResult(row_scores=row_scores, aggregate_scores=aggregate_scores) - - -def collapse_results( - results_by_key: dict[str, EvaluationResult], - aggregate_fields: tuple[AggregateFieldName, ...] | None, -) -> BenchmarkEvaluationResult: - """Merge multiple single-metric results into one multi-metric view. - - Args: - results_by_key: Namespaced single-metric results keyed by metric name. - aggregate_fields: Optional aggregate field subset to keep. - - Returns: - A combined multi-metric result with merged row scores, merged aggregate - scores, and the original per-metric mapping. - - Raises: - ValueError: If the provided metric results do not all contain the same - number of rows. - """ - if not results_by_key: - empty = AggregatedMetricResult(scores=[]) - return BenchmarkEvaluationResult(row_scores=[], aggregate_scores=empty, per_metric={}) - - ordered_keys = list(results_by_key.keys()) - row_count = len(results_by_key[ordered_keys[0]].row_scores) - baseline_identities = [ - _row_identity(row_score, index) for index, row_score in enumerate(results_by_key[ordered_keys[0]].row_scores) - ] - for metric_key in ordered_keys[1:]: - result = results_by_key[metric_key] - if len(result.row_scores) != row_count: - raise ValueError(f"Cannot combine metric results with different row counts: {metric_key}") - current_identities = [_row_identity(row_score, index) for index, row_score in enumerate(result.row_scores)] - if current_identities != baseline_identities: - raise ValueError(f"Cannot combine metric results with different row identities: {metric_key}") - - combined_rows: list[RowScore] = [] - for index in range(row_count): - first_row = results_by_key[ordered_keys[0]].row_scores[index] - metrics: dict[str, list[MetricOutput]] = {} - requests: list[dict[str, Any]] = [] - metric_errors: dict[str, str] = {} - metric_diagnostics: dict[str, list[MetricDiagnostic]] = {} - for metric_key in ordered_keys: - row_score = results_by_key[metric_key].row_scores[index] - metrics[metric_key] = _extract_metric_outputs(row_score, metric_key) - requests.extend(row_score.requests) - if row_score.metric_errors: - metric_errors.update(row_score.metric_errors) - elif row_score.error: - metric_errors[metric_key] = row_score.error - if diagnostics := _extract_metric_diagnostics(row_score, metric_key): - metric_diagnostics[metric_key] = diagnostics - combined_rows.append( - RowScore( - row_index=_row_identity(first_row, index), - item=first_row.item, - sample=first_row.sample, - metrics=metrics, - requests=requests, - metric_errors=metric_errors or None, - metric_diagnostics=metric_diagnostics or None, - ) - ) - - aggregate_scores = AggregatedMetricResult( - scores=[score for result in results_by_key.values() for score in result.aggregate_scores.scores] - ) - aggregate_scores = _filter_aggregate_fields(aggregate_scores, aggregate_fields) - return BenchmarkEvaluationResult( - row_scores=combined_rows, - aggregate_scores=aggregate_scores, - per_metric=results_by_key, - ) - - -class BenchmarkEvaluationResult(BaseModel): - """Unified benchmark evaluation result.""" - - row_scores: list[RowScore] - aggregate_scores: AggregatedMetricResult - per_metric: dict[str, EvaluationResult] - - def metric_result(self, metric_key: str) -> EvaluationResult: - """Return the original single-metric result for one metric key. - - Args: - metric_key: Metric key to retrieve from the combined result. - - Returns: - The single-metric result stored for that key. - """ - return self.per_metric[metric_key] - - def to_records(self, view: ResultView = "rows") -> list[dict[str, Any]]: - """Convert the result into flat dictionaries for export or inspection. - - Args: - view: Which logical result view to flatten, either `"rows"` or - `"aggregate"`. - - Returns: - A list of flat dictionaries suitable for tabular rendering. - - Raises: - ValueError: If `view` is not one of the supported options. - """ - if view == "rows": - records: list[dict[str, Any]] = [] - for index, row_score in enumerate(self.row_scores): - record: dict[str, Any] = { - "row_index": row_score.row_index if row_score.row_index is not None else index, - "status": row_status(row_score), - } - flatten_dict("item", serialize_value(row_score.item), record) - flatten_dict("sample", serialize_value(row_score.sample), record) - if error_text := row_error_text(row_score): - record["error"] = error_text - for metric_key, metric_scores in row_score.metrics.items(): - for output in metric_scores: - record[f"output.{metric_key}.{output.name}"] = serialize_value(output.value) - for column, diagnostics_json in diagnostics_records(row_score).items(): - record[column] = diagnostics_json - records.append(record) - return records - - if view == "aggregate": - records = [] - for score in self.aggregate_scores.scores: - record = {} - for key, value in score.model_dump(mode="json").items(): - if key == "percentiles" and isinstance(value, dict): - flatten_dict("percentiles", value, record) - elif key == "histogram" and value is not None: - record[key] = json.dumps(value, sort_keys=True) - else: - record[key] = value - records.append(record) - return records - - raise ValueError(f"Unsupported view {view!r}. Expected 'rows' or 'aggregate'.") - - def to_table(self, view: ResultView = "rows"): - """Convert the result into a `pyarrow.Table`. - - Args: - view: Which logical result view to materialize. - - Returns: - A PyArrow table built from the flattened result records. - """ - import pyarrow as pa - - return pa.Table.from_pylist(self.to_records(view=view)) - - def to_pandas(self, view: ResultView = "rows"): - """Convert the result into a pandas `DataFrame`. - - Args: - view: Which logical result view to materialize. - - Returns: - A pandas dataframe built from the flattened result records. - """ - import pandas as pd - - return pd.DataFrame.from_records(self.to_records(view=view)) - - def format_summary(self, max_rows: int = 10, *, max_error_rows: int | None = None) -> str: - """Build a human-readable text summary of the result. - - Args: - max_rows: Maximum number of row records to include in the preview. - max_error_rows: Maximum number of failed rows included in the full - error-details section. Defaults to ``max_rows``. - - Returns: - A formatted multiline string summary. - """ - if max_error_rows is None: - max_error_rows = max_rows - aggregate_records = [summary_aggregate_record(score) for score in self.aggregate_scores.scores] - parts = [ - summary_header("BenchmarkEvaluationResult", self.row_scores, len(self.aggregate_scores.scores)), - "", - "Aggregate scores", - format_table(aggregate_records), - ] - for metric_key, metric_result in self.per_metric.items(): - preview_records = [] - for index, row_score in enumerate(metric_result.row_scores[:max_rows]): - record = summary_row_base_record(row_score, index) - for metric_scores in row_score.metrics.values(): - for score in metric_scores: - record[f"score.{score.name}"] = serialize_value(score.value) - preview_records.append(record) - if not preview_records: - continue - parts.extend( - [ - "", - f"Row preview for metric '{metric_key}' (first {len(preview_records)} of {len(metric_result.row_scores)})", - format_table(preview_records), - ] - ) - parts.extend( - format_error_details( - self.row_scores, - max_error_rows=max_error_rows, - label_metric_errors=True, - ) - ) - return "\n".join(parts) - - def print_summary(self, max_rows: int = 10, *, max_error_rows: int | None = None) -> None: - """Print the formatted summary to standard output. - - Args: - max_rows: Maximum number of row records to include in the preview. - max_error_rows: Maximum number of failed rows included in the full - error-details section. Defaults to ``max_rows``. - - Returns: - None. - """ - print(self.format_summary(max_rows=max_rows, max_error_rows=max_error_rows)) - - def __str__(self) -> str: - """Return the default compact summary representation. - - Returns: - Summary string with up to five preview rows. - """ - return self.format_summary(max_rows=5) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/params.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/params.py deleted file mode 100644 index 0a6e9af392..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/params.py +++ /dev/null @@ -1,109 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Shared execution parameter types for evaluator SDK and service runtimes.""" - -from typing import Self - -from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator - -from nemo_platform.beta.evaluator.values.models import ReasoningParams - - -class InferenceParams(BaseModel): - """ - Parameters for model inference. Extra fields can be supplied for additional options applied to the inference request directly. Fields not supported by the model may cause inference errors during evaluation. - """ - - model_config = ConfigDict(extra="allow") - - temperature: float | None = Field( - default=None, - ge=0, - le=2, - description="Float value between 0 and 1. temp of 0 indicates greedy decoding, " - "where the token with highest prob is chosen. Temperature can't be set to 0.0 currently", - ) - max_tokens: int | None = Field(default=None, ge=1, description="Max tokens to generate") - max_completion_tokens: int | None = Field(default=None, ge=1, description="Max tokens to generate") - top_p: float | None = Field( - default=None, - ge=0, - le=1, - description="Float value between 0 and 1; limits to the top tokens within a certain " - "probability. top_p=0 means the model will only consider the single most likely " - "token for the next prediction", - ) - stop: list[str] | None = Field(default=None) - - @model_validator(mode="after") - def check_max_tokens(self) -> Self: - if self.max_tokens and self.max_completion_tokens: - raise ValueError( - "max_tokens and max_completion_tokens cannot both be configured. " - "Choose the appropriate tokens parameter for the model." - ) - return self - - -class RunConfig(BaseModel): - """Job parameters.""" - - model_config = ConfigDict(extra="forbid") - - parallelism: int = Field( - default=8, - ge=1, - description="Parallelism to be used for the evaluation job. " - "Typically, this represents the maximum number of concurrent requests made to the model.", - ) - limit_samples: int | None = Field( - default=None, - ge=1, - description="Limit number of evaluation samples, taking the first `limit` samples from the dataset.", - ) - - -class RunConfigOnline(RunConfig): - """Job parameters for online evaluation.""" - - ignore_request_failure: bool = Field( - default=False, - description="If True, request failures will be ignored and the result will be marked as NaN. " - "If False (default), request failures will raise an exception.", - ) - request_timeout: int | None = Field( - default=None, description="The timeout to be used for requests made to the model." - ) - max_retries: int = Field(default=3, ge=0, description="Maximum number of retries for failed requests.") - - -class RunConfigOnlineModel(RunConfigOnline): - """Job parameters for model online evaluation.""" - - inference: InferenceParams | None = Field( - default=None, - description="Custom settings that control the model's text generation behavior.", - ) - system_prompt: str | None = Field( - default=None, - description="Initial instructions that define the model's role and behavior for the conversation.", - ) - reasoning: ReasoningParams | None = Field( - default=None, description="Custom settings that control the model's reasoning behavior." - ) - structured_output: dict | None = Field( - default=None, - description="JSON schema to apply structured output for the model.", - ) - - @field_validator("inference", mode="before") - @classmethod - def coerce_inference_params(cls, value: object) -> object: - """Normalize equivalent Pydantic inference models before SDK validation runs.""" - # Compatibility shim for service-side inference models. This converts other - # Pydantic models into plain data and lets SDK InferenceParams validation - # decide whether the payload is compatible. - if isinstance(value, BaseModel) and not isinstance(value, InferenceParams): - return value.model_dump(mode="python", exclude_none=True) - return value diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/protocol.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/protocol.py deleted file mode 100644 index 4d834c96f9..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/protocol.py +++ /dev/null @@ -1,214 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Protocol-adjacent value types for evaluator SDK metrics.""" - -from __future__ import annotations - -import math -from typing import Annotated, Any - -from pydantic import BaseModel, ConfigDict, Field, RootModel, StringConstraints, field_serializer, field_validator - -from nemo_platform.beta.evaluator.values.evidence import CandidateEvidence - -MetricTypeName = Annotated[str, StringConstraints(min_length=1)] - - -class DatasetRow(BaseModel): - """Original dataset row plus optional stable row identity.""" - - model_config = ConfigDict(extra="forbid") - - row_index: int | None = None - data: dict[str, Any] - - -class CandidateOutput(BaseModel): - """Candidate or prediction output being scored for one dataset row.""" - - model_config = ConfigDict(extra="forbid") - - output_text: str | None = None - response: Any | None = None - trajectory: Any | None = None - evidence: CandidateEvidence | None = Field( - default=None, - description="Named evidence captured for the candidate (final state, traces, logs, ...) for agent eval.", - ) - metadata: dict[str, Any] = Field(default_factory=dict) - - def as_sample(self) -> dict[str, Any]: - """Return a sample-shaped payload for template rendering helpers.""" - sample = dict(self.metadata) - if self.output_text is not None: - sample["output_text"] = self.output_text - if self.response is not None: - sample["response"] = self.response - if self.trajectory is not None: - sample["trajectory"] = self.trajectory - if self.evidence is not None: - sample["evidence"] = self.evidence - return sample - - -class MetricInput(BaseModel): - """Complete per-row scoring input passed to a metric.""" - - model_config = ConfigDict(extra="forbid") - - row: DatasetRow - candidate: CandidateOutput - - -class ContinuousScore(RootModel[float]): - """Continuous numeric metric value.""" - - -class DiscreteScore(RootModel[int]): - """Discrete numeric metric value.""" - - -class Label(RootModel[str]): - """String label metric value.""" - - -class BooleanValue(RootModel[bool]): - """Boolean metric value.""" - - -class MetricOutputSpec(BaseModel): - """Schema for one named value emitted by a metric.""" - - model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) - - name: str - description: str | None = None - value_schema: type[BaseModel] - - @field_validator("name") - @classmethod - def _name_must_not_be_empty(cls, value: str) -> str: - if not value: - raise ValueError("metric output name must not be empty") - return value - - @staticmethod - def continuous_score(name: str, description: str | None = None) -> "MetricOutputSpec": - return MetricOutputSpec(name=name, description=description, value_schema=ContinuousScore) - - @staticmethod - def discrete_score(name: str, description: str | None = None) -> "MetricOutputSpec": - return MetricOutputSpec(name=name, description=description, value_schema=DiscreteScore) - - @staticmethod - def label(name: str, description: str | None = None) -> "MetricOutputSpec": - return MetricOutputSpec(name=name, description=description, value_schema=Label) - - @staticmethod - def boolean(name: str, description: str | None = None) -> "MetricOutputSpec": - return MetricOutputSpec(name=name, description=description, value_schema=BooleanValue) - - @staticmethod - def model(name: str, value_schema: type[BaseModel], description: str | None = None) -> "MetricOutputSpec": - return MetricOutputSpec(name=name, description=description, value_schema=value_schema) - - def coerce_value(self, value: Any) -> BaseModel: - """Validate and coerce a raw output value to this spec's declared schema.""" - return self.value_schema.model_validate(value) - - def coerce_output(self, output: "MetricOutput") -> BaseModel: - """Validate and coerce a named metric output against this spec.""" - if output.name != self.name: - raise ValueError(f"Expected metric output {self.name!r}, got {output.name!r}") - return self.coerce_value(output.value) - - def value_json_schema(self) -> dict[str, Any]: - return self.value_schema.model_json_schema() - - -class MetricDescriptor(BaseModel): - """Metadata describing a metric implementation and its declared outputs.""" - - model_config = ConfigDict(extra="forbid") - - type: MetricTypeName - outputs: list[MetricOutputSpec] = Field(min_length=1) - - @field_validator("outputs") - @classmethod - def _output_names_must_be_unique(cls, value: list[MetricOutputSpec]) -> list[MetricOutputSpec]: - names = [output.name for output in value] - duplicates = sorted({name for name in names if names.count(name) > 1}) - if duplicates: - raise ValueError(f"duplicate metric output names: {duplicates}") - return value - - -class MetricOutput(BaseModel): - """One named value emitted by a metric.""" - - model_config = ConfigDict(extra="forbid") - - name: str - value: Any - - @field_serializer("value") - def serialize_nan(self, value: Any) -> Any: - if isinstance(value, float) and math.isnan(value): - return "NaN" - return value - - -class MetricDiagnostic(BaseModel): - """One diagnostic finding explaining how a metric score was derived. - - ``message`` is the human-readable entry point. Metric-specific structured - context (expected/actual values, diffs, check breakdowns, judge rationales, - etc.) belongs in ``details``. - """ - - model_config = ConfigDict(extra="forbid") - - message: str = Field(description="Human-readable diagnostic message.") - details: dict[str, Any] | None = Field( - default=None, - description="Optional metric-specific structured context for debugging and inspection.", - ) - - -class MetricResult(BaseModel): - """Structured row-level metric result.""" - - model_config = ConfigDict(extra="forbid") - - outputs: list[MetricOutput] - diagnostics: list[MetricDiagnostic] = Field( - default_factory=list, - description=( - "Optional diagnostic findings that explain how the result was derived. For debugging and inspection only." - ), - ) - - -def validate_metric_result(result: MetricResult, outputs: list[MetricOutputSpec]) -> MetricResult: - """Validate a metric result against its declared outputs.""" - returned_names = [output.name for output in result.outputs] - duplicates = sorted({name for name in returned_names if returned_names.count(name) > 1}) - if duplicates: - raise ValueError(f"Duplicate metric output names: {duplicates}") - - outputs_by_name = {output.name: output for output in outputs} - declared_names = [output.name for output in outputs] - declared = set(declared_names) - returned = set(returned_names) - missing = [name for name in declared_names if name not in returned] - undeclared = [name for name in returned_names if name not in declared] - - if missing: - raise ValueError(f"Missing declared metric outputs: {missing}") - if undeclared: - raise ValueError(f"Undeclared metric outputs: {undeclared}") - for output in result.outputs: - outputs_by_name[output.name].coerce_output(output) - return result diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py deleted file mode 100644 index 05220d061d..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py +++ /dev/null @@ -1,878 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Result types for evaluator SDK runtime.""" - -from __future__ import annotations - -import json -import math -from collections.abc import Mapping -from difflib import get_close_matches -from typing import TYPE_CHECKING, Any, Literal, Self - -from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator, model_serializer - -if TYPE_CHECKING: - import pyarrow as pa - -from nemo_platform.beta.evaluator.values.protocol import MetricDiagnostic, MetricOutput, MetricResult - -ResultView = Literal["rows", "aggregate"] -AggregateFieldName = Literal[ - # Base statistics - "nan_count", - "sum", - "mean", - "min", - "max", - "median", - "std_dev", - "variance", - "sample_std_dev", - "sample_variance", - # Range-specific fields - "score_type", - "percentiles", - "histogram", - # Rubric-specific fields - "rubric_distribution", - "mode_category", -] -DefaultAggregateFieldName = Literal["nan_count", "sum", "mean", "min", "max"] - - -def flatten_dict(prefix: str, value: Any, output: dict[str, Any]) -> None: - """Flatten nested dictionaries into dot-delimited key/value pairs. - - Args: - prefix: Current key path prefix. - value: Value to flatten (possibly nested dictionary). - output: Mutable destination mapping populated in place. - - Returns: - ``None``. ``output`` is mutated with flattened entries. - """ - # Flatten nested payloads into dot-separated columns so row views round-trip - # cleanly into tables and data frames. - if isinstance(value, dict): - for key, nested_value in value.items(): - nested_prefix = f"{prefix}.{key}" if prefix else str(key) - flatten_dict(nested_prefix, nested_value, output) - return - - output[prefix] = value - - -def serialize_value(value: Any) -> Any: - """Recursively convert SDK values into JSON-serializable primitives. - - Args: - value: Value to serialize. - - Returns: - JSON-compatible representation of ``value``. - """ - if isinstance(value, BaseModel): - return value.model_dump(mode="json") - if isinstance(value, list): - return [serialize_value(item) for item in value] - if isinstance(value, dict): - return {key: serialize_value(item) for key, item in value.items()} - return value - - -def _truncate(value: Any, max_length: int = 40) -> str: - """Render values as bounded-width strings for text-table output. - - Args: - value: Value to render as text. - max_length: Maximum output length before ellipsis truncation. - - Returns: - Truncated or original string representation. - """ - text = str(value) - if len(text) <= max_length: - return text - return f"{text[: max_length - 3]}..." - - -def format_table(records: list[dict[str, Any]]) -> str: - """Render flat records as an aligned ASCII table. - - The formatter derives a stable column order from first appearance, computes - per-column widths, and truncates long cell values for compact output. - - Args: - records: Flat dictionaries to print. - - Returns: - Multi-line table string. - """ - if not records: - return "(no rows)" - - seen: set[str] = set() - columns: list[str] = [] - for record in records: - for key in record: - if key not in seen: - seen.add(key) - columns.append(key) - - widths: dict[str, int] = {} - for column in columns: - widths[column] = len(column) - for record in records: - widths[column] = max(widths[column], len(_truncate(record.get(column, "")))) - - header = " | ".join(column.ljust(widths[column]) for column in columns) - divider = "-+-".join("-" * widths[column] for column in columns) - body = [ - " | ".join(_truncate(record.get(column, "")).ljust(widths[column]) for column in columns) for record in records - ] - return "\n".join([header, divider, *body]) - - -class RubricScoreValue(BaseModel): - """Rubric-based score definition for grading criteria.""" - - label: str = Field(description="The label to use for the level of the rubric grading criteria.") - description: str | None = Field( - default=None, - description="Describe the semantic meaning of each criteria for the given rubric.", - ) - value: float | int = Field(description="The score value to assign for the criteria.") - - -class RubricScoreStat(RubricScoreValue): - """Rubric score with count statistics.""" - - count: int = Field(default=0, description="The number of samples evaluated with the rubric level.") - - -class ScoreStats(BaseModel): - """Stats for a score. Fields that are NaN are serialized as the string "NaN" in the API response.""" - - count: int | None = Field( - default=None, - description="The number of values used for computing the score.", - ) - sum: float | None = Field( - default=None, - description="The sum of all values used for computing the score.", - ) - sum_squared: float | None = Field( - default=None, - description="The sum of the square of all values used for computing the score.", - ) - min: float | None = Field( - default=None, - description="The minimum of all values used for computing the score.", - ) - max: float | None = Field( - default=None, - description="The maximum of all values used for computing the score.", - ) - mean: float | None = Field( - default=None, - description="The mean of all values used for computing the score.", - ) - variance: float | None = Field( - default=None, - description="""The population variance, (note: not the sample variance).""", - ) - stddev: float | None = Field( - default=None, - description="""The population standard deviation, (note: not the sample standard deviation).""", - ) - sample_variance: float | None = Field( - default=None, - description="The sample (Bessel-corrected, n-1) variance. None when fewer than two values.", - ) - sample_stddev: float | None = Field( - default=None, - description="The sample (Bessel-corrected, n-1) standard deviation, estimating the spread of the " - "process the values were drawn from. None when fewer than two values (undefined, not zero).", - ) - stderr: float | None = Field(default=None, description="The standard error.") - nan_count: int | None = Field( - default=None, - description="The number of values that are not a number (NaN) and are excluded from the score stats calculations.", - ) - rubric_distribution: list[RubricScoreStat] | None = Field( - default=None, description="The distribution of the rubric grading criteria for the score." - ) - - @field_serializer( - "sum", "sum_squared", "min", "max", "mean", "variance", "stddev", "sample_variance", "sample_stddev", "stderr" - ) - def serialize_nan(self, v: float | None) -> float | str | None: - """Serialize NaN stats as string values for JSON compatibility. - - Args: - v: Float statistic value or ``None``. - - Returns: - ``"NaN"`` for NaN floats, otherwise the original value. - """ - if isinstance(v, float) and math.isnan(v): - return "NaN" - return v - - -class MetricScore(BaseModel): - """ - A computed score for the metric - """ - - name: str - value: float - stats: ScoreStats | None = Field( - default=None, - description="Computed score statistics for the score.", - ) - - @field_validator("value", mode="before") - @classmethod - def convert_value(cls, v): - """ - If incoming object is string with value "nan", it is converted to float nan. - """ - if isinstance(v, str): - if v.strip().lower() == "nan": - return float("nan") - raise ValueError("The only string value allowed for value is NaN") - return v - - @field_serializer("value") - def serialize_nan(self, v): - """ - JSON serializers do not consistently support float NaN values. - Serialize them as the string "NaN" so results remain portable. - """ - if isinstance(v, float) and math.isnan(v): - return "NaN" - return v - - -class Percentiles(BaseModel): - """Percentile distribution of scores.""" - - model_config = ConfigDict(extra="forbid") - p10: float | int = Field(description="10th percentile.") - p20: float | int = Field(description="20th percentile.") - p30: float | int = Field(description="30th percentile.") - p40: float | int = Field(description="40th percentile.") - p50: float | int = Field(description="50th percentile (median).") - p60: float | int = Field(description="60th percentile.") - p70: float | int = Field(description="70th percentile.") - p80: float | int = Field(description="80th percentile.") - p90: float | int = Field(description="90th percentile.") - p100: float | int = Field(description="100th percentile.") - - -class HistogramBin(BaseModel): - """A single bin in a histogram.""" - - model_config = ConfigDict(extra="forbid") - lower_bound: float | int = Field(description="Lower bound of the bin (inclusive).") - upper_bound: float | int = Field(description="Upper bound of the bin (exclusive for all but last bin).") - count: int = Field(description="Number of values in this bin.") - - -class Histogram(BaseModel): - """Histogram of score distribution.""" - - model_config = ConfigDict(extra="forbid") - bins: list[HistogramBin] = Field(description="Histogram bins.") - - -class AggregateScoreBase(BaseModel): - """Base statistics shared by all aggregated score types. - - This base class is used by both the app layer aggregation and API response schemas. - """ - - model_config = ConfigDict(extra="forbid") - name: str = Field(description="Name of the score.") - count: int | None = Field( - default=None, - description="Number of samples evaluated (excluding NaN). Omitted when the sample size is unknown " - "— e.g. a figure imported from a backend that reports statistics without the n behind them. " - "Distinct from 0, which asserts that nothing was evaluated. (``None`` on the model; the result " - "routes serialize with exclude_none, so the field is absent from the response rather than null.)", - ) - nan_count: int = Field(description="Number of samples that produced NaN scores.") - sum: float | None = Field(default=None, description="Sum of all score values.") - mean: float | None = Field(default=None, description="Mean score value.") - min: float | None = Field(default=None, description="Minimum score value.") - max: float | None = Field(default=None, description="Maximum score value.") - median: float | None = Field( - default=None, - description="Median score value. Equal to percentiles.p50 when a percentile distribution is " - "also present; carried separately because a backend may report a median without one.", - ) - std_dev: float | None = Field( - default=None, - description="Population standard deviation of the scores (divides by n). Describes the spread of " - "the values actually evaluated. See sample_std_dev to estimate the spread of the wider process.", - ) - variance: float | None = Field( - default=None, - description="Population variance of the scores (divides by n). See sample_variance.", - ) - sample_std_dev: float | None = Field( - default=None, - description="Sample standard deviation of the scores (Bessel-corrected, divides by n-1). Estimates " - "the spread of the process the values were drawn from — the right choice when repeated trials " - "sample a stochastic system. Omitted when fewer than two values (undefined, not zero).", - ) - sample_variance: float | None = Field( - default=None, - description="Sample variance of the scores (Bessel-corrected, divides by n-1). Omitted when fewer " - "than two values.", - ) - - -class AggregateRangeScore(AggregateScoreBase): - """Aggregated statistics for a range-type score with percentiles and histogram.""" - - score_type: Literal["range"] = Field(default="range", description="Type of score.") - percentiles: Percentiles | None = Field(default=None, description="Percentile distribution of scores.") - histogram: Histogram | None = Field(default=None, description="Histogram of score distribution.") - - _include_fields: frozenset[str] | None = None - - def with_fields(self, fields: frozenset[AggregateFieldName]) -> Self: - """Return a copy configured to serialize only the specified fields.""" - copy = self.model_copy() - object.__setattr__(copy, "_include_fields", {*fields, "name", "count"}) - return copy - - @model_serializer(mode="wrap") - def _serialize(self, handler): - data = handler(self) - if self._include_fields is not None: - # Always include required fields (name, count), plus requested fields - fields_to_include = self._include_fields | {"name", "count"} - return {k: v for k, v in data.items() if k in fields_to_include} - return data - - -class AggregateRubricScore(AggregateScoreBase): - """Aggregated statistics for a rubric-type score with category distribution.""" - - score_type: Literal["rubric"] = Field(default="rubric", description="Type of score.") - rubric_distribution: list[RubricScoreStat] = Field(description="Distribution of rubric categories.") - mode_category: str | None = Field(default=None, description="Most frequent rubric category.") - - _include_fields: frozenset[str] | None = None - - def with_fields(self, fields: frozenset[AggregateFieldName]) -> Self: - """Return a copy configured to serialize only the specified fields.""" - copy = self.model_copy() - object.__setattr__(copy, "_include_fields", {*fields, "name", "count"}) - return copy - - @model_serializer(mode="wrap") - def _serialize(self, handler): - data = handler(self) - if self._include_fields is not None: - # Always include required fields (name, count), plus requested fields - fields_to_include = self._include_fields | {"name", "count"} - return {k: v for k, v in data.items() if k in fields_to_include} - return data - - -class AggregateScalarScore(AggregateScoreBase): - """A single pre-computed value with no underlying distribution available. - - For figures a backend reports as one number (e.g. an environment's own ``pass@1`` or Elo) rather - than a set of per-sample values the SDK could aggregate itself. ``value`` carries the number; - ``mean``/``min``/``max`` are optional and normally unset, since there is no sample to describe — - a producer may still supply them, but readers key off ``score_type`` and read ``value``. Distinct from - :class:`AggregateRangeScore` so a reader can tell "this is the whole story" from "this summarizes - ``count`` samples", instead of seeing a range score with a suspicious ``count`` of 1. - """ - - score_type: Literal["scalar"] = Field(default="scalar", description="Type of score.") - value: float = Field(description="The reported value.") - - _include_fields: frozenset[str] | None = None - - def with_fields(self, fields: frozenset[AggregateFieldName]) -> Self: - """Return a copy configured to serialize only the specified fields.""" - copy = self.model_copy() - object.__setattr__(copy, "_include_fields", {*fields, "name", "count"}) - return copy - - @model_serializer(mode="wrap") - def _serialize(self, handler): - data = handler(self) - if self._include_fields is not None: - # Always include required fields (name, count, value), plus requested fields - fields_to_include = self._include_fields | {"name", "count", "value"} - return {k: v for k, v in data.items() if k in fields_to_include} - return data - - -AggregateScore = AggregateRangeScore | AggregateRubricScore | AggregateScalarScore - - -#: How many names a lookup-miss message lists when nothing resembles what was asked for. A judgement -#: call rather than a measured optimum -- enough to show the naming convention, few enough to stay -#: readable, since a run with several metrics times pass@k can carry dozens. Only this fallback is -#: truncated; a near-miss is surfaced by similarity, so finding the name you meant never depends on -#: where it happens to fall alphabetically. -_MISS_NAME_LIMIT = 10 - - -class AggregatedMetricResult(BaseModel): - """Result of aggregating metric scores with full statistics.""" - - model_config = ConfigDict(extra="forbid") - scores: list[AggregateScore] = Field(description="The list of aggregated scores.") - - @property - def scores_by_name(self) -> Mapping[str, AggregateScore]: - """Aggregates keyed by :attr:`AggregateScoreBase.name`, for ``in``, ``.get()``, and iteration. - - Reach for this when a score's absence is a legitimate outcome ("did this metric run?"); use - :meth:`score` when it isn't. Names are expected unique, but runner-contributed extras are - appended as-is, so a collision is possible: the first wins, matching the ``next(...)`` scans - this replaces. - """ - by_name: dict[str, AggregateScore] = {} - for score in self.scores: - by_name.setdefault(score.name, score) - return by_name - - def score(self, name: str) -> AggregateScore: - """Return the aggregate named ``name``, raising :class:`KeyError` if there isn't one. - - Raises rather than returning ``None`` because an unknown name is nearly always a typo or a - metric that didn't run. Both are bugs worth surfacing at the lookup, where the name is in - hand, instead of as an ``AttributeError`` on ``.mean`` further downstream. When absence is a - real possibility, use ``scores_by_name.get(...)``. - """ - # A direct scan rather than a lookup into `scores_by_name`: building the whole mapping to - # return one element allocates a dict per call, and the score list is short enough that the - # scan wins outright. - for score in self.scores: - if score.name == name: - return score - raise KeyError(self._unknown_score_message(name)) - - def _unknown_score_message(self, name: str) -> str: - """Explain a lookup miss, leading with near-misses when the name looks like a typo. - - "Close" is :func:`difflib.get_close_matches`: SequenceMatcher (Ratcliff/Obershelp) similarity - of at least 0.6, best three first. That is a subsequence-overlap ratio, not an edit distance. - - Listing every name alphabetically would be simpler, and is the better answer if the list is - left whole. Truncating one is what breaks it -- the name a caller meant is not reliably in - the first :data:`_MISS_NAME_LIMIT`, since a typo'd ``view.solved`` sits behind a page of - ``gym_reward.*`` in a run carrying pass@1..8 for two metrics. - """ - # Deduplicated: names are expected unique, but runner-contributed extras are appended as-is, - # and a repeat would otherwise be suggested twice, listed twice, and counted twice in the - # "N other aggregates" tally -- making a collision look like two distinct near-misses. - available = sorted({score.name for score in self.scores}) - if not available: - return f"no aggregate score named {name!r}: this result has no aggregates at all" - close = get_close_matches(name, available, n=3) - if close: - suggestions = ", ".join(repr(match) for match in close) - message = f"no aggregate score named {name!r}; did you mean {suggestions}?" - # Say how many others there are, so a wrong guess isn't a dead end: without this the - # caller can't tell whether the suggestions are the whole set or three of forty. - others = len(available) - len(close) - if others == 0: - return message - noun = "aggregate" if others == 1 else "aggregates" - return f"{message} ({others} other {noun} in this result)" - shown = ", ".join(repr(score_name) for score_name in available[:_MISS_NAME_LIMIT]) - remainder = len(available) - _MISS_NAME_LIMIT - if remainder > 0: - shown = f"{shown}, ... ({remainder} more)" - return f"no aggregate score named {name!r}; available: {shown}" - - -class RowScore(BaseModel): - """Normalized row-level score payload for metric/benchmark job results.""" - - model_config = ConfigDict(extra="allow") - - row_index: int | None = Field(default=None, description="Stable row position used for result alignment.", ge=0) - item: dict[str, Any] = Field(description="Input item metadata for the evaluated row.") - sample: dict[str, Any] = Field(description="Sample output payload for the evaluated row.") - metrics: dict[str, list[MetricOutput]] = Field(description="Metric-level row outputs by metric key.") - requests: list[dict[str, Any]] = Field(description="Request details captured during evaluation.") - metric_errors: dict[str, str] | None = Field( - default=None, - description="Full row-level error text keyed by metric for summary rendering.", - ) - metric_diagnostics: dict[str, list[MetricDiagnostic]] | None = Field( - default=None, - description="Optional row-level diagnostic findings keyed by metric used for debugging.", - ) - - @property - def error(self) -> str | None: - """Derived row-level summary error text.""" - if self.metric_errors: - return "; ".join(f"{metric_key}: {message}" for metric_key, message in self.metric_errors.items()) - return None - - -class SampleResult(BaseModel): - """Result of evaluating a single sample.""" - - model_config = ConfigDict(extra="forbid") - - index: int = Field(description="Index of the sample in the input dataset.") - result: MetricResult | None = Field(default=None, description="Metric result if evaluation succeeded.") - error: str | None = Field(default=None, description="Error message if evaluation failed.") - - @property - def is_success(self) -> bool: - """Return whether this sample completed without an error payload. - - Returns: - ``True`` when ``result`` is present. - """ - return self.result is not None - - @classmethod - def success(cls, index: int, result: MetricResult) -> "SampleResult": - """Create a successful ``SampleResult``. - - Args: - index: Sample index in the source dataset. - result: Metric result for that sample. - - Returns: - Successful sample result object. - """ - return cls(index=index, result=result) - - @classmethod - def failure(cls, index: int, exc: Exception) -> "SampleResult": - """Create a failed ``SampleResult`` from an exception. - - Args: - index: Sample index in the source dataset. - exc: Exception raised during sample evaluation. - - Returns: - Failed sample result object with normalized error text. - """ - return cls(index=index, error=str(exc) or exc.__class__.__name__) - - -def row_error_text(row_score: RowScore) -> str | None: - """Return the human-facing row error text.""" - return row_score.error - - -def diagnostics_records(row_score: RowScore) -> dict[str, str]: - """Return JSON-encoded diagnostic columns for a row, keyed by ``diagnostics.``. - - Diagnostics are rendered as compact JSON strings so tabular exports stay - flat regardless of the (metric-defined) diagnostic shape. Returns an empty - mapping when the row carries no diagnostics. - """ - if not row_score.metric_diagnostics: - return {} - - return { - f"diagnostics.{metric_key}": json.dumps(serialize_value(diagnostics), sort_keys=True) - for metric_key, diagnostics in row_score.metric_diagnostics.items() - } - - -def _row_has_scores(row_score: RowScore) -> bool: - """Return whether the row contains any metric score values.""" - return any(metric_scores for metric_scores in row_score.metrics.values()) - - -def _row_has_errors(row_score: RowScore) -> bool: - """Return whether the row contains any error payload.""" - return bool(row_error_text(row_score)) - - -def row_status(row_score: RowScore) -> str: - """Return a compact row status used by summary and export views.""" - if _row_has_errors(row_score): - return "error" - return "ok" - - -def _summary_status_counts(row_scores: list[RowScore]) -> dict[str, int]: - """Count summary statuses for inclusion in summary headers.""" - counts = {"ok": 0, "error": 0} - for row_score in row_scores: - counts[row_status(row_score)] += 1 - return counts - - -def summary_header(name: str, row_scores: list[RowScore], aggregate_count: int) -> str: - """Build the compact summary header line for result objects.""" - counts = _summary_status_counts(row_scores) - parts = [f"{name}(rows={len(row_scores)}, aggregate_scores={aggregate_count}"] - if counts["ok"]: - parts.append(f", ok={counts['ok']}") - if counts["error"]: - parts.append(f", error={counts['error']}") - parts.append(")") - return "".join(parts) - - -def _row_display_index(row_score: RowScore, index: int) -> int: - """Resolve the row index used in preview tables and error sections.""" - return row_score.row_index if row_score.row_index is not None else index - - -def _flatten_summary_dict(prefix: str, value: Any, output: dict[str, Any]) -> None: - """Flatten only summary-friendly scalar values into a preview record.""" - if isinstance(value, dict): - for key, nested_value in value.items(): - nested_prefix = f"{prefix}.{key}" if prefix else str(key) - _flatten_summary_dict(nested_prefix, nested_value, output) - return - if isinstance(value, (str, int, float, bool)) or value is None: - output[prefix] = value - - -def summary_aggregate_record(score: AggregateScore) -> dict[str, Any]: - """Project an aggregate score into a concise summary row.""" - record: dict[str, Any] = { - "name": score.name, - "count": score.count, - "nan_count": score.nan_count, - "mean": score.mean, - "min": score.min, - "max": score.max, - } - score_type = getattr(score, "score_type", None) - if score_type is not None: - record["score_type"] = score_type - percentiles = getattr(score, "percentiles", None) - if percentiles is not None and getattr(percentiles, "p50", None) is not None: - record["p50"] = percentiles.p50 - mode_category = getattr(score, "mode_category", None) - if mode_category is not None: - record["mode_category"] = mode_category - return record - - -def summary_row_base_record(row_score: RowScore, index: int) -> dict[str, Any]: - """Build the shared non-score columns for summary row previews.""" - record: dict[str, Any] = { - "row_index": _row_display_index(row_score, index), - "status": row_status(row_score), - } - _flatten_summary_dict("item", serialize_value(row_score.item), record) - sample = serialize_value(row_score.sample) - if isinstance(sample, dict): - sample = {key: value for key, value in sample.items() if key != "response"} - _flatten_summary_dict("sample", sample, record) - if error_text := row_error_text(row_score): - record["error"] = error_text - return record - - -def format_error_details( - row_scores: list[RowScore], - *, - max_error_rows: int | None, - label_metric_errors: bool, -) -> list[str]: - """Render a detailed full-error section for failed rows.""" - failed_rows = [(index, row_score) for index, row_score in enumerate(row_scores) if _row_has_errors(row_score)] - if not failed_rows: - return [] - - shown_limit = len(failed_rows) if max_error_rows is None else max(0, max_error_rows) - shown_rows = failed_rows[:shown_limit] - parts = [ - "", - f"Error details ({len(shown_rows)} of {len(failed_rows)} failed rows)", - ] - - for index, row_score in shown_rows: - display_index = _row_display_index(row_score, index) - parts.extend(["", f"[row {display_index}]"]) - - if row_score.metric_errors: - for metric_key, message in row_score.metric_errors.items(): - if label_metric_errors: - parts.append(f"{metric_key}: {message}") - else: - parts.append(message) - else: - if error_text := row_error_text(row_score): - parts.append(error_text) - - for column, diagnostics_json in diagnostics_records(row_score).items(): - parts.append(f"{column}: {diagnostics_json}") - - if len(shown_rows) < len(failed_rows): - parts.extend(["", f"... {len(failed_rows) - len(shown_rows)} more failed rows omitted"]) - - return parts - - -class EvaluationResult(BaseModel): - """Result object returned by SDK offline evaluation.""" - - row_scores: list[RowScore] = Field(description="Row-level scores.") - aggregate_scores: AggregatedMetricResult = Field(description="Aggregate score statistics.") - - def to_records(self, view: ResultView = "rows") -> list[dict[str, Any]]: - """Convert evaluation output into flat dictionaries. - - For ``view="rows"``, nested ``item`` and ``sample`` payloads are - flattened with dotted keys. For ``view="aggregate"``, percentile fields - are flattened while histograms are kept as JSON strings to preserve - tabular shape. - - Args: - view: Output projection, either ``"rows"`` or ``"aggregate"``. - - Returns: - Flat record dictionaries for downstream table/dataframe conversion. - - Raises: - ValueError: If ``view`` is unsupported. - """ - if view == "rows": - records: list[dict[str, Any]] = [] - for index, row_score in enumerate(self.row_scores): - record: dict[str, Any] = { - "row_index": row_score.row_index if row_score.row_index is not None else index, - "status": row_status(row_score), - } - flatten_dict("item", serialize_value(row_score.item), record) - flatten_dict("sample", serialize_value(row_score.sample), record) - if error_text := row_error_text(row_score): - record["error"] = error_text - for metric_scores in row_score.metrics.values(): - for output in metric_scores: - record[f"output.{output.name}"] = serialize_value(output.value) - for column, diagnostics_json in diagnostics_records(row_score).items(): - record[column] = diagnostics_json - records.append(record) - return records - - if view == "aggregate": - records = [] - for score in self.aggregate_scores.scores: - record = {} - for key, value in score.model_dump(mode="json").items(): - if key == "percentiles" and isinstance(value, dict): - flatten_dict("percentiles", value, record) - elif key == "histogram" and value is not None: - # Histograms stay as JSON strings so aggregate views remain - # tabular instead of expanding variable-width nested columns. - record[key] = json.dumps(value, sort_keys=True) - else: - record[key] = value - records.append(record) - return records - - raise ValueError(f"Unsupported view {view!r}. Expected 'rows' or 'aggregate'.") - - def to_table(self, view: ResultView = "rows") -> pa.Table: - """Convert records into a ``pyarrow.Table``. - - Args: - view: Output projection, either ``"rows"`` or ``"aggregate"``. - - Returns: - Table built from ``to_records(view=view)``. - """ - # Imported here rather than at module scope: pyarrow (plus its numpy tail) costs ~31 MB - # RSS and 223 modules, this is its only runtime use in the module, and the module is on - # the agent_eval import path, which never calls this method. The return annotation is a - # string already (`from __future__ import annotations`), so it needs no import. - import pyarrow as pa - - return pa.Table.from_pylist(self.to_records(view=view)) - - def to_pandas(self, view: ResultView = "rows"): - """Convert records into a pandas ``DataFrame``. - - Args: - view: Output projection, either ``"rows"`` or ``"aggregate"``. - - Returns: - DataFrame built from ``to_records(view=view)``. - """ - import pandas as pd - - return pd.DataFrame.from_records(self.to_records(view=view)) - - def format_summary(self, max_rows: int = 10, *, max_error_rows: int | None = None) -> str: - """Render a human-readable summary with aggregates and row preview. - - Args: - max_rows: Maximum number of row-level records included in preview. - max_error_rows: Maximum number of failed rows included in the full - error-details section. Defaults to ``max_rows``. - - Returns: - Multi-line summary string suitable for terminal/notebook display. - """ - if max_error_rows is None: - max_error_rows = max_rows - aggregate_records = [summary_aggregate_record(score) for score in self.aggregate_scores.scores] - preview_records = [] - for index, row_score in enumerate(self.row_scores[:max_rows]): - record = summary_row_base_record(row_score, index) - for metric_scores in row_score.metrics.values(): - for output in metric_scores: - record[f"output.{output.name}"] = serialize_value(output.value) - preview_records.append(record) - parts = [ - summary_header("EvaluationResult", self.row_scores, len(self.aggregate_scores.scores)), - "", - "Aggregate scores", - format_table(aggregate_records), - ] - if preview_records: - parts.extend( - [ - "", - f"Row preview (first {len(preview_records)} of {len(self.row_scores)})", - format_table(preview_records), - ] - ) - parts.extend( - format_error_details( - self.row_scores, - max_error_rows=max_error_rows, - label_metric_errors=False, - ) - ) - return "\n".join(parts) - - def print_summary(self, max_rows: int = 10, *, max_error_rows: int | None = None) -> None: - """Print ``format_summary`` output. - - Args: - max_rows: Maximum number of row-level records included in preview. - max_error_rows: Maximum number of failed rows included in the full - error-details section. Defaults to ``max_rows``. - """ - print(self.format_summary(max_rows=max_rows, max_error_rows=max_error_rows)) - - def __str__(self) -> str: - """Return the default compact summary representation. - - Returns: - Summary string with up to five preview rows. - """ - return self.format_summary(max_rows=5) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/scores.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/scores.py deleted file mode 100644 index 7c48e29be7..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/scores.py +++ /dev/null @@ -1,300 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Score configuration types for metric definitions.""" - -from __future__ import annotations - -import json -import logging -import re -from abc import ABC, abstractmethod -from typing import Annotated, Any, Literal, Self - -import jsonschema -from jsonschema import validators -from pydantic import BaseModel, ConfigDict, Discriminator, Field, Tag, model_validator - -from nemo_platform.beta.evaluator.values.results import MetricScore, RubricScoreStat, ScoreStats - -_logger = logging.getLogger(__name__) - - -class JSONScoreParser(BaseModel): - """Parse a score from JSON structured content.""" - - type: Literal["json"] = "json" - json_path: str = Field( - description="The JSON path to parse the score from the judge response when using structured output." - ) - - -class RegexScoreParser(BaseModel): - """Parse a score from content in any format using regular expression.""" - - type: Literal["regex"] = "regex" - pattern: str = Field(description="The regular expression to parse the score from the judge response.") - method: Literal["search", "match"] = Field( - default="match", - description="The regex method to use: 'search' looks anywhere in the string, 'match' starts from the beginning.", - ) - - @model_validator(mode="after") - def valid_regex(self) -> Self: - if self.type != "regex": - # model_validator will run when resolving parser JSONScoreParser | RegexScoreParser - # if it's RegexScoreParser, continue with validation. - return self - - try: - re.compile(self.pattern) - except re.error as e: - raise ValueError(f"invalid regex for score parser: {e}") - return self - - -class _Score(BaseModel): - model_config = ConfigDict(extra="forbid") - name: str = Field( - pattern=r"^[a-z0-9_]+$", - description="The name of the score. Only lowercase letters, numbers, and underscores allowed.", - ) - description: str | None = Field(default=None, description="Human-readable description of the score.") - parser: JSONScoreParser | RegexScoreParser = Field( - default_factory=lambda data: JSONScoreParser(json_path=data["name"]), - description="The method to parse the score. When used with llm-judge metric, and no parser is set, JSONScoreParser is the default parser inferred from the score parameters.", - ) - - -class Rubric(BaseModel): - model_config = ConfigDict(extra="forbid") - label: str = Field( - description='The label to use for the level of the rubric grading criteria. (e.g., "helpful", "not_helpful", "positive")' - ) - description: str | None = Field( - default=None, - description="Describe the semantic meaning of each criteria for the given rubric. If no judge template is set, the input description for labels are included in the generated judge prompt.", - ) - value: float | int = Field( - description="The score value to assign for the criteria used for aggregation and ranking." - ) - - -class RubricScore(_Score): - """Score definition for a rubric with optional parser. If no parser is set, JSONScoreParser is the default parser inferred from the score parameters""" - - model_config = ConfigDict(extra="forbid") - rubric: list[Rubric] = Field(min_length=2, description="The rubric for the score.") - - -class RangeScore(_Score): - """Score definition for a range of values with optional parser. If no parser is set, JSONScoreParser is the default parser inferred from the score parameters""" - - model_config = ConfigDict(extra="forbid") - minimum: float | int = Field(description="Minimum value for the score range. Must be less than maximum.") - maximum: float | int = Field(description="Maximum value for the score range. Must be greater than minimum.") - - @model_validator(mode="after") - def valid_range(self) -> Self: - if self.minimum >= self.maximum: - raise ValueError(f"minimum must be less than maximum: {self.minimum} < {self.maximum}") - return self - - -class RemoteScore(_Score): - """Score configuration for remote metrics. - - Unlike RangeScore, minimum and maximum are optional (default to None = no bounds). - This avoids JSON serialization issues with infinity values. - """ - - minimum: float | int | None = Field( - default=None, - description="Minimum value for the score range. Defaults to None (no lower bound).", - ) - maximum: float | int | None = Field( - default=None, - description="Maximum value for the score range. Defaults to None (no upper bound).", - ) - parser: JSONScoreParser = Field( - default_factory=lambda data: JSONScoreParser(json_path=data["name"]), - description="The method to parse the score. Only JSON parsing is supported for remote metrics.", - ) - - @model_validator(mode="after") - def valid_range(self) -> Self: - """Validate that minimum < maximum when both are configured.""" - if self.minimum is not None and self.maximum is not None: - if self.minimum >= self.maximum: - raise ValueError(f"minimum must be less than maximum: {self.minimum} >= {self.maximum}") - return self - - -def score_discriminator(data: dict[str, Any] | RubricScore | RangeScore) -> Literal["rubric", "range"]: - if "rubric" in data or isinstance(data, RubricScore): - return "rubric" - return "range" - - -Score = Annotated[ - (Annotated[RubricScore, Tag("rubric")] | Annotated[RangeScore, Tag("range")]), Discriminator(score_discriminator) -] - - -class ScoreParser(ABC): - """Parse model output text into a normalized ``MetricScore``.""" - - class Params(BaseModel): - pass - - rubric_mapping: dict[str, Rubric] | None = None - - def __init__(self, score: Score): - self.score = score - if isinstance(score, RubricScore): - self.rubric_mapping = {rubric.label: rubric for rubric in score.rubric} - - @abstractmethod - def parse(self, text: str | None) -> MetricScore: - """Parse the provided text and extract a single score.""" - raise NotImplementedError - - def _get_rubric_score(self, label: str) -> MetricScore: - """Map rubric label to score value and build rubric distribution stats.""" - assert isinstance(self.score, RubricScore) - assert self.rubric_mapping is not None - rubric_distribution = [ - RubricScoreStat( - label=rubric.label, description=rubric.description, value=rubric.value, count=int(label == rubric.label) - ) - for rubric in self.score.rubric - ] - rubric = self.rubric_mapping.get(label) - score_value = float("nan") if rubric is None else rubric.value - return MetricScore( - name=self.score.name, value=score_value, stats=ScoreStats(rubric_distribution=rubric_distribution) - ) - - -class ScoreParserRegex(ScoreParser): - """Parse score values from free-form text using a regex capture group.""" - - parser_type: Literal["regex"] = "regex" - pattern: re.Pattern - method: Literal["search", "match"] - - def __init__(self, score: Score): - super().__init__(score) - if not isinstance(score.parser, RegexScoreParser): - raise ValueError(f"incompatible score parser to initialize ScoreParserRegex: {type(score.parser)}") - try: - self.pattern = re.compile(score.parser.pattern) - except re.error as e: - raise ValueError(f"invalid regex pattern for score parser with LLM-as-a-Judge: {score.parser.pattern} {e}") - self.method = score.parser.method - - def parse(self, text: str | None) -> MetricScore: - if not text: - return MetricScore(name=self.score.name, value=float("nan")) - - match = self.pattern.search(text) if self.method == "search" else self.pattern.match(text) - if match is None: - return MetricScore(name=self.score.name, value=float("nan")) - - groups = match.groups() - if len(groups) == 0: - return MetricScore(name=self.score.name, value=float("nan")) - - try: - if self.rubric_mapping: - return self._get_rubric_score(groups[0]) - return MetricScore(name=self.score.name, value=float(groups[0])) - except ValueError: - # This is expected when models drift from requested output format. - _logger.info("Failed to parse score from text: %s.", text) - return MetricScore(name=self.score.name, value=float("nan")) - - -class ScoreParserJSON(ScoreParser): - """Parse score values from JSON output using ``json_path`` key lookup.""" - - parser_type: Literal["json"] = "json" - json_path: str - - # Optional when structured output is defined. - structured_output: dict | None = None - json_schema: dict | None = None - json_validator: jsonschema.Validator | None = None - - def __init__(self, score: Score, structured_output: dict | None = None): - super().__init__(score) - if not isinstance(score.parser, JSONScoreParser): - raise ValueError(f"incompatible score parser to initialize ScoreParserJSON: {type(score.parser)}") - - self.json_path = score.parser.json_path - - if structured_output: - self._validate_structured_output(score, structured_output) - self.structured_output = structured_output - self.json_schema = structured_output["schema"] - - def _validate_structured_output(self, score: Score, structured_output: dict) -> None: - json_schema = structured_output.get("schema") - if not json_schema: - raise ValueError("missing schema for structured output") - - # Validate schema itself and verify parser expectations for json_path. - validator = validators.validator_for(json_schema) - validator.check_schema(json_schema) - - schema_type = json_schema.get("type", "") - if schema_type != "object": - raise ValueError(f"schema must be type 'object' for JSON score parser: {schema_type}") - - assert score.parser is not None - assert isinstance(score.parser, JSONScoreParser) - assert score.parser.json_path is not None - - schema_defined_json_path = json_schema.get("properties", {}).get(score.parser.json_path) - if not schema_defined_json_path: - raise ValueError( - f"schema must have {score.parser.json_path} defined as an object property for JSON score parser: {json_schema}" - ) - json_type = schema_defined_json_path.get("type") - if isinstance(score, RubricScore): - if json_type != "string": - raise ValueError( - f"expected string type in schema for property '{score.parser.json_path}' when used with score rubric and JSON score parser: {json_type}" - ) - elif json_type not in ["number", "integer", "boolean"]: - raise ValueError( - f"schema property {score.parser.json_path} must be type number, integer, or boolean for JSON score parser: {schema_defined_json_path}" - ) - - def parse(self, text: str | None) -> MetricScore: - if not isinstance(text, str): - return MetricScore(name=self.score.name, value=float("nan")) - try: - obj = json.loads(text) - except json.JSONDecodeError: - return MetricScore(name=self.score.name, value=float("nan")) - - if not isinstance(obj, dict): - # handles if model returns just a number, for example - _logger.warning("Expected JSON object, got %s. Returning NaN score", type(obj)) - return MetricScore(name=self.score.name, value=float("nan")) - - score = obj.get(self.json_path) - if score is None: - return MetricScore(name=self.score.name, value=float("nan")) - - if self.rubric_mapping: - return self._get_rubric_score(score) - - if isinstance(score, str): - return MetricScore(name=self.score.name, value=float("nan")) - - if isinstance(score, bool): - score = 1.0 if score else 0.0 - - return MetricScore(name=self.score.name, value=score) diff --git a/sdk/python/nemo-platform/src/nemo_platform/cli/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/cli/__init__.py index 19c8081361..645edbe614 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/cli/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/cli/__init__.py @@ -1,4 +1,6 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""NeMo CLI - Command-line interface for NeMo Platform.""" +from nemo_platform._alias import alias_package as _alias_package + +_alias_package("nemo_platform_ext.cli", globals()) diff --git a/sdk/python/nemo-platform/src/nemo_platform/cli/app.py b/sdk/python/nemo-platform/src/nemo_platform/cli/app.py deleted file mode 100644 index 1b1c852c4a..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/cli/app.py +++ /dev/null @@ -1,345 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""NeMo CLI - Command-line interface for NeMo Platform.""" - -from __future__ import annotations - -import logging -from importlib.metadata import EntryPoint -from typing import TYPE_CHECKING, Annotated, cast - -import typer - -from nemo_platform.cli.commands.api import API_TOP_LEVEL_ENTRIES -from nemo_platform.cli.commands.manifest_registry import TOP_LEVEL_ENTRIES -from nemo_platform.cli.core.help_formatter import HELP_OPTION_NAMES -from nemo_platform.cli.core.lazy_load import ( - ManifestBackedNmpGroup, - attach_lazy_entries, -) -from nemo_platform.cli.core.logging import configure_logging -from nemo_platform.cli.core.types import ListOutputFormat, TimestampFormat -from nemo_platform.cli.manifest import ( - TopLevelEntry, - build_top_level_entries, -) -from nemo_platform.config.types import OutputFormat as ConfigOutputFormat - -if TYPE_CHECKING: - from nemo_platform_plugin.cli import NemoCLI - from nemo_platform_plugin.function import NemoFunction - from nemo_platform_plugin.job import NemoJob - - from nemo_platform.config.models import ConfigParams - -logger = logging.getLogger(__name__) - -_SKIP_AUTH_CHECK_SUBCOMMANDS = frozenset( - {"agent", "auth", "config", "setup", "quickstart", "cluster-info", "skills", "docs", "services", "plugins"} -) -# Create the main CLI app with custom help formatting -app = typer.Typer( - name="nemo", - no_args_is_help=True, - add_completion=True, - pretty_exceptions_enable=False, - rich_markup_mode=None, - cls=ManifestBackedNmpGroup, - context_settings=dict(help_option_names=list(HELP_OPTION_NAMES)), -) - - -def _build_top_level_lazy_entries() -> tuple[TopLevelEntry, ...]: - plugin_entry_points = _installed_plugin_command_entry_points() - # Plugin `nemo.cli` entry points own their command name (e.g. safe-synthesizer). - # Drop generated API top-level groups with the same name so run-local/runtime stay available. - api_entries = tuple(entry for entry in API_TOP_LEVEL_ENTRIES if entry.name not in plugin_entry_points) - return build_top_level_entries( - (*TOP_LEVEL_ENTRIES, *api_entries), - plugin_entry_points, - include_hidden=True, - ) - - -def _installed_plugin_command_entry_points() -> dict[str, EntryPoint]: - """Return installed plugin CLI entry points without importing plugin code.""" - try: - from nemo_platform_plugin.discovery import discover_entry_points - except ImportError: - return {} - try: - return discover_entry_points("nemo.cli") - except Exception: # noqa: BLE001 - logger.warning("Failed to discover CLI plugin entry points", exc_info=True) - return {} - - -def _add_plugin_job_commands( - plugin_app: typer.Typer, - plugin_jobs: dict[str, type[NemoJob]], - *, - cli: NemoCLI | None = None, -) -> None: - # TODO: nemo-platform-plugin is temporarily optional while it is being published - # to the nightly PyPI feed. Once available, it should become an - # unconditional dependency and these guards can be removed. - try: - from nemo_platform_plugin.commands import add_job_commands - except ImportError: - logger.warning( - "nemo_platform_plugin.commands unavailable; skipping plugin job command injection", exc_info=True - ) - return - - add_job_commands(plugin_app, plugin_jobs, cli=cli) - - -def _discover_plugin_job_entry_points() -> dict[str, EntryPoint] | None: - # TODO: nemo-platform-plugin is temporarily optional while it is being published - # to the nightly PyPI feed. Once available, it should become an - # unconditional dependency and these guards can be removed. - try: - from nemo_platform_plugin.discovery import discover_entry_points - except ImportError: - return None - - return discover_entry_points("nemo.jobs") - - -def _add_plugin_function_commands( - plugin_app: typer.Typer, - plugin_functions: dict[str, type[NemoFunction]], - *, - cli: NemoCLI | None = None, -) -> None: - # Same nemo-platform-plugin optionality guard as the jobs path. When the - # package isn't installed we silently skip — the plugin's bare CLI - # surface (whatever it shipped via `nemo.cli`) still loads. - try: - from nemo_platform_plugin.commands import add_function_commands - except ImportError: - logger.warning( - "nemo_platform_plugin.commands unavailable; skipping plugin function command injection", - exc_info=True, - ) - return - - add_function_commands(plugin_app, plugin_functions, cli=cli) - - -def _discover_plugin_function_entry_points() -> dict[str, EntryPoint] | None: - try: - from nemo_platform_plugin.discovery import discover_entry_points - except ImportError: - return None - - return discover_entry_points("nemo.functions") - - -# Global options -@app.callback(options_metavar="[GLOBAL OPTIONS]") -def main( - ctx: typer.Context, - version: Annotated[ - bool | None, - typer.Option( - "--version", - "-V", - help="Show version information and exit.", - callback=_version_callback, - is_eager=True, - rich_help_panel="Help", - ), - ] = None, - context_name: Annotated[ - str | None, - typer.Option( - "--context", - "-c", - help="The name of the context to use. Overrides the current context in the config file.", - rich_help_panel="Global Options", - hidden=True, - ), - ] = None, - base_url: Annotated[ - str | None, - typer.Option( - "--base-url", - help="Base URL for the NeMo Platform API", - rich_help_panel="Global Options", - ), - ] = None, - output_format: Annotated[ - ListOutputFormat | None, - typer.Option( - "--output-format", - "--output", - "-f", - help="Output format for how results are printed.", - rich_help_panel="Global Options", - ), - ] = None, - no_truncate: Annotated[ - bool | None, - typer.Option( - "--no-truncate", - help="Don't truncate long values in table/markdown/csv output", - rich_help_panel="Global Options", - ), - ] = None, - timestamp_format: Annotated[ - TimestampFormat | None, - typer.Option( - help="Timestamp format for table/markdown/csv output", - rich_help_panel="Global Options", - ), - ] = None, - verbose: Annotated[ - bool | None, - typer.Option( - "--verbose", - "-v", - help="Enable verbose messaging. This only impacts logs that are visible, it doesn't change any data outputs.", - rich_help_panel="Global Options", - ), - ] = None, - agent_mode: Annotated[ - bool | None, - typer.Option( - "--agent-mode", - "-A", - help="Enable agent-friendly output mode with extra context for coding agents.", - rich_help_panel="Global Options", - ), - ] = None, - no_auto_refresh: Annotated[ - bool, - typer.Option( - "--no-auto-refresh", - help="Disable automatic token refresh when token is about to expire.", - rich_help_panel="Global Options", - hidden=True, - ), - ] = False, - no_telemetry: Annotated[ - bool, - typer.Option( - "--no-telemetry", - help="Disable anonymous usage telemetry for this invocation.", - rich_help_panel="Global Options", - ), - ] = False, -) -> None: - """ - Command-line interface for NeMo Platform. - - :books: Documentation: https://docs.nvidia.com/nemo-platform - - [green]Getting started:[/] - - Browse documentation with [cyan]`nemo docs --list`[/] - - Run local platform services with [cyan]`nemo services run --help`[/] - - Read the Kubernetes deployment guide with [cyan]`nemo docs set-up/helm/install`[/] - - [green]Examples:[/] - nemo workspaces list --output-format markdown - nemo workspaces get default -f json - - [green]Exit codes:[/] - - 0: Success - - 1: Local or unexpected error - - 2: Command usage error - - 3: Remote/API error - """ - # Lazy imports for performance (avoid loading pydantic_settings for --help) - from nemo_platform.cli.core.context import CLIContext - from nemo_platform.quickstart import QuickstartConfig - - # Configure logging (always call to silence httpx in non-verbose mode) - configure_logging(1 if verbose else 0) - - if ctx.obj is None: - ctx.obj = CLIContext() - - # Resolve agent mode: explicit flag > env var > default False - import os - - if agent_mode is None: - env_val = os.environ.get("NMP_AGENT_MODE", "").lower() - agent_mode = env_val in ("1", "true", "yes") - ctx.obj.agent_mode = agent_mode - - # Capture command name + agent mode for the command_invoked telemetry event, wire - # the per-invocation opt-out, and print the first-run notice. Best effort inside. - from nemo_platform.cli.telemetry import runtime as telemetry_runtime - - telemetry_runtime.on_callback(ctx, no_telemetry=no_telemetry) - - # Build ConfigParams from CLI args - overrides: ConfigParams = {} - if context_name is not None: - overrides["current_context"] = context_name - if base_url is not None: - overrides["base_url"] = base_url - if output_format is not None: - overrides["output_format"] = cast(ConfigOutputFormat, output_format) - elif agent_mode: - overrides["output_format"] = "markdown" - if timestamp_format is not None: - overrides["timestamp_format"] = timestamp_format - if no_truncate is not None: - overrides["truncate"] = not no_truncate - - # Update CLIContext overrides - ctx.obj.overrides.update(overrides) - ctx.obj.verbosity = 1 if verbose else 0 - ctx.obj.quickstart_config = QuickstartConfig.load() - - # Non-quickstart contexts always require auth. Some quickstart contexts require auth (opt-in) - context_requires_auth = ctx.obj.quickstart_config is None or ctx.obj.quickstart_config.auth_enabled - command_requires_auth = ctx.invoked_subcommand not in _SKIP_AUTH_CHECK_SUBCOMMANDS - auto_refresh_enabled = not no_auto_refresh - - if context_requires_auth and command_requires_auth and auto_refresh_enabled: - from nemo_platform.cli.commands.auth import AuthError, ensure_valid_token - - try: - token_valid = ensure_valid_token(ctx.obj.get_sdk_context()) - if not token_valid: - typer.echo( - "Error: Your access token has expired and could not be refreshed.\n" - "Hint: Run 'nemo auth login' to re-authenticate.", - err=True, - ) - raise typer.Exit(code=1) - - # reset the context, so we load the refreshed auth info - ctx.obj.reset_sdk_context() - except typer.Exit: - raise # Re-raise Exit to stop execution - except AuthError as e: - typer.echo(f"Error: {e}", err=True) - raise typer.Exit(code=1) - except Exception as e: - typer.echo(f"Warning: Failed to check/refresh token: {e}", err=True) - - -attach_lazy_entries(main, _build_top_level_lazy_entries()) - - -def _version_callback(value: bool) -> None: - """Print version information and exit.""" - if value: - import nemo_platform - - typer.echo(f"nemo version {nemo_platform.__version__}") - raise typer.Exit() - - -def cli() -> None: - """Main entry point for the CLI.""" - app() - - -if __name__ == "__main__": - cli() diff --git a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/__init__.py deleted file mode 100644 index 1275d78dff..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - diff --git a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/__init__.py deleted file mode 100644 index 6588131429..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/__init__.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# NOTE: This file is auto-generated -from __future__ import annotations - -from nemo_platform.cli.manifest import TopLevelEntry - -API_TOP_LEVEL_ENTRIES = ( - TopLevelEntry( - import_path=f"{__package__}.adapters:app", - name="adapters", - help="Manage adapters.", - panel="Core plugins", - kind="group", - hidden=True, - ), - TopLevelEntry( - import_path=f"{__package__}.experiments:app", - name="experiments", - help="Manage experiments.", - panel="Functional plugins", - kind="group", - hidden=False, - ), - TopLevelEntry( - import_path=f"{__package__}.files:app", - name="files", - help="Manage files.", - panel="Core plugins", - kind="group", - hidden=False, - ), - TopLevelEntry( - import_path=f"{__package__}.guardrail:app", - name="guardrail", - help="Manage guardrails.", - panel="Functional plugins", - kind="group", - hidden=False, - ), - TopLevelEntry( - import_path=f"{__package__}.iam:app", - name="iam", - help="IAM operations.", - panel="Core plugins", - kind="group", - hidden=True, - ), - TopLevelEntry( - import_path=f"{__package__}.inference:app", - name="inference", - help="Inference operations.", - panel="Core plugins", - kind="group", - hidden=False, - ), - TopLevelEntry( - import_path=f"{__package__}.intake:app", - name="intake", - help="Intake operations.", - panel="Functional plugins", - kind="group", - hidden=False, - ), - TopLevelEntry( - import_path=f"{__package__}.jobs:app", - name="jobs", - help="Manage jobs.", - panel="Core plugins", - kind="group", - hidden=False, - ), - TopLevelEntry( - import_path=f"{__package__}.models:app", - name="models", - help="Manage models.", - panel="Core plugins", - kind="group", - hidden=False, - ), - TopLevelEntry( - import_path=f"{__package__}.projects:app", - name="projects", - help="Manage projects.", - panel="Core plugins", - kind="group", - hidden=True, - ), - TopLevelEntry( - import_path=f"{__package__}.secrets:app", - name="secrets", - help="Manage secrets.", - panel="Core plugins", - kind="group", - hidden=False, - ), - TopLevelEntry( - import_path=f"{__package__}.workspaces:app", - name="workspaces", - help="Manage workspaces.", - panel="Core plugins", - kind="group", - hidden=False, - ), -) diff --git a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/adapters.py b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/adapters.py deleted file mode 100644 index 7e8cf86935..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/adapters.py +++ /dev/null @@ -1,393 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# NOTE: This file is auto-generated -from __future__ import annotations - -from typing import Annotated, Literal - -import typer - -from nemo_platform.cli.core.api import build_kwargs, merge_filter_dict -from nemo_platform.cli.core.code_generator import handle_code_generation -from nemo_platform.cli.core.context import CLIContext -from nemo_platform.cli.core.errors import handle_errors -from nemo_platform.cli.core.formatters import ( - Column, - check_output_columns_with_format, - format_output, - validate_stream_output_format, -) -from nemo_platform.cli.core.help_formatter import collect_warnings, create_typer_app -from nemo_platform.cli.core.pagination import PaginationType, fetch_all_pages, warn_if_more_pages -from nemo_platform.cli.core.stdin_utils import read_data_input_with_flags, read_payload, validate_required_fields -from nemo_platform.cli.core.types import ( - EntityOutputFormatOption, - ListOutputFormatOption, - NoTruncateOption, - OutputColumnsOption, - StreamOutputOption, -) - -app = create_typer_app(name="adapters", help="Manage adapters") - - -@app.command("create") -@collect_warnings -@handle_errors -def create_adapters( - ctx: typer.Context, - name: Annotated[ - str | None, - typer.Argument( - help="Name of the adapter. Name must be unique in the workspace. Name must start with a lowercase letter, be 2-63 characters, and use lowercase letters, digits, hyphens, and dots (no consecutive hyphens, cannot end with a hyphen). (required)" - ), - ] = None, - workspace: Annotated[str | None, typer.Option("--workspace")] = None, - fileset: Annotated[ - str | None, - typer.Option( - "--fileset", - help="Location where adapter files are stored - expected format {workspace}/{fileset_name} (required)", - ), - ] = None, - finetuning_type: Annotated[ - Literal[ - "lora_merged", - "all_weights", - "last_layer", - "top_layers", - "gradual_unfreezing", - "bias_only", - "attention_only", - "lora", - "qlora", - "adalora", - "dora", - "lora_plus", - "prompt_tuning", - "prefix_tuning", - "p_tuning", - "p_tuning_v2", - "soft_prompt", - "ppo", - "dpo", - "cdpo", - "ipo", - "orpo", - "kto", - "rrhf", - "grpo", - ] - | None, - typer.Option("--finetuning-type", help="Finetuning types. (required)"), - ] = None, - model: Annotated[ - str | None, - typer.Option( - "--model", - help="Base model entity. Use `{workspace}/{model_name}` to reference a model in any workspace, or a single `{model_name}` resolved in the path workspace. A single name (2-63 characters) or 'workspace/model*name' where each segment is a valid name (lowercase, digits, hyphens, and temporarily @ . + *; no leading/trailing or consecutive hyphens). If one slash, both sides must be non-empty. (required)", - ), - ] = None, - description: Annotated[ - str | None, typer.Option("--description", help="Optional description of the adapter") - ] = None, - enabled: Annotated[ - bool | None, - typer.Option("--enabled", help="Whether to make this adapter available for inference post training"), - ] = None, - lora_config: Annotated[ - str | None, typer.Option("--lora-config", help="Lora configuration specifics (JSON string)") - ] = None, - input_file: Annotated[ - str | None, - typer.Option("--input-file", help="Path to JSON file (use '-' for stdin)", rich_help_panel="Input Options"), - ] = None, - input_data: Annotated[ - str | None, - typer.Option("--input-data", help="Input data for the request (JSON or YAML)", rich_help_panel="Input Options"), - ] = None, - output_format: EntityOutputFormatOption = None, -) -> None: - """Create an adapter under a base model specified by the "model" field in the body. - - [bold red]Required fields:[/] fileset, finetuning_type, model, name - - [green]Examples:[/] - nemo adapters create --input-file config.json - nemo adapters create --input-data '{"fileset": "value", "finetuning_type": "value", "model": "value", "name": "value"}' - echo '{"json": "data"}' | nemo adapters create --input-file - - nemo adapters create --