diff --git a/.gitignore b/.gitignore index 96eac96c6e3..0c3bd69c06d 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ venv* .venv* .nemo/ .env* +!**/.env.example !**/env.py !packages/data_designer/**/environment.py !packages/nemo_data_designer/**/environment.py diff --git a/plugins/README.md b/plugins/README.md index 0e23f0fafa0..9452a437b92 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -61,6 +61,7 @@ The package name is the `name` field in the plugin's `pyproject.toml`, not the d | `nemo-agents/` | `nemo-agents-plugin` | | `nemo-anonymizer/` | `nemo-anonymizer-plugin` | | `nemo-data-designer/` | `nemo-data-designer-plugin` | +| `nemo-eval-author/` | `nemo-eval-author-plugin` | | `nemo-evaluator/` | `nemo-evaluator-plugin` | | `nemo-guardrails/` | `nemo-guardrails-plugin` | | `nemo-switchyard/` | `nemo-switchyard` | diff --git a/plugins/nemo-eval-author/.env.example b/plugins/nemo-eval-author/.env.example new file mode 100644 index 00000000000..d45305de9a1 --- /dev/null +++ b/plugins/nemo-eval-author/.env.example @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Example environment for standalone Eval Author runs. +# Copy to `.env` next to this file (or export in your shell): +# +# cp .env.example .env +# +# AUTHOR_* is Eval Author's credential contract. The EXPERIMENTALIST_* interplay +# below is transitional and goes away once Eval Author no longer reuses any +# Experimentalist code: +# +# - an unset AUTHOR_* falls back to its EXPERIMENTALIST_* equivalent, so insight +# mode works from one Experimentalist profile .env +# - a set AUTHOR_* is copied into any unset EXPERIMENTALIST_* slot, because the +# Experimentalist helpers Eval Author still borrows read only EXPERIMENTALIST_* +# +# TODO(eval-author-standalone): remove both, at which point AUTHOR_* becomes required +# here rather than optional. Set AUTHOR_* explicitly now to avoid that break. +# +# TODO(cli): wire `nemo eval-author` to load this `.env` the way Experimentalist +# loads the profile-dir `.env`. + +# Required for LLM-backed Eval Author / TraceAnalyzer (NOOA CompletionClient). +AUTHOR_API_BASE=https://inference-api.nvidia.com/v1 +AUTHOR_API_KEY= + +# Optional model overrides (defaults shown). Eval Author itself uses the smart and fast +# tiers; the mid tier is listed only because it is bridged to Experimentalist helpers, +# and it goes away with the bridge. +# AUTHOR_SMART_MODEL_NAME=openai/openai/openai/gpt-5.5 +# AUTHOR_MID_MODEL_NAME=openai/gcp/google/gemini-3.5-flash +# AUTHOR_FAST_MODEL_NAME=openai/openai/openai/gpt-5-mini + +# Optional: NVIDIA Inference Gateway virtual key. When AUTHOR_API_KEY is unset and +# AUTHOR_API_BASE is the gateway over HTTPS, standalone runners fall back to +# INFERENCE_API_KEY the same way Experimentalist does today. The HTTPS requirement +# is deliberate: the key is never forwarded to a plain-http base. +# INFERENCE_API_KEY= + +# Optional: NeMo Platform URL for Insight / Fileset / agent-code access. +# Leave unset to use the active `nemo auth` context, or set for a local stack. +# NMP_BASE_URL=http://localhost:8080 diff --git a/plugins/nemo-eval-author/README.md b/plugins/nemo-eval-author/README.md new file mode 100644 index 00000000000..7f22176cac3 --- /dev/null +++ b/plugins/nemo-eval-author/README.md @@ -0,0 +1,66 @@ +# NeMo Eval Author Plugin + +Library-only plugin that owns the Eval Author agent (`eval_author/`). + +## Direction of travel + +**Eval Author is meant to become standalone, with nothing imported from +Experimentalist.** Prefer duplicating a helper over sharing one, even when sharing +looks tidier. + +Right now the two packages depend on each other: + +| Arrow | Status | Why | +| --- | --- | --- | +| Experimentalist → Eval Author | permanent | insight mode imports `EvalAuthor` and `EvalAuthorConfig` at module scope | +| Eval Author → Experimentalist | temporary | still borrows evaluator/Harbor, staging, trace, tools, cache, backend | + +[`tests/test_plugin_boundary.py`](tests/test_plugin_boundary.py) pins the second list +so it can only shrink, and names what each remaining import is still for. `uv` +resolves the current cycle; install both packages with: + +```bash +uv sync --group experimentalist +``` + +## Public API + +```python +from nemo_eval_author_plugin.eval_author.agent import EvalAuthor, build_eval_author_agent +from nemo_eval_author_plugin.eval_author.models import EvalAuthorConfig, EvalAuthorResult +from nemo_eval_author_plugin.eval_author.run import run_eval_author + +# Still borrowed from Experimentalist, and on the way out. Treat these as Eval Author's +# own types once they move; do not build new code on the Experimentalist paths. +from nemo_experimentalist_plugin.experimentalist.components.evaluator import Dataset +from nemo_experimentalist_plugin.experimentalist.components.evaluator.models import DatasetRef +from nemo_experimentalist_plugin.experimentalist.components.dataset_staging import stage_task_template +from nemo_experimentalist_plugin.experimentalist.components.trace_analyzer import TraceAnalyzer +from nemo_experimentalist_plugin.experimentalist.components.trace_explorer import TraceExplorer +``` + +## Credentials (standalone) + +Copy [`.env.example`](.env.example) to `.env` and set `AUTHOR_API_KEY` (and optionally model names / `NMP_BASE_URL`): + +```bash +cp plugins/nemo-eval-author/.env.example plugins/nemo-eval-author/.env +``` + +`AUTHOR_*` is Eval Author's credential contract, and `model_config` imports nothing +from Experimentalist. When the API base is the NVIDIA Inference Gateway over HTTPS, +`INFERENCE_API_KEY` is also accepted. + +Two pieces of that module are transitional and disappear with the last +Experimentalist import, both tagged `TODO(eval-author-standalone)`: + +- unset `AUTHOR_*` variables fall back to `EXPERIMENTALIST_*`, so insight mode works + from a single Experimentalist profile `.env`. Setting `AUTHOR_*` explicitly today + avoids the break when the fallback is removed. +- importing `nemo_eval_author_plugin._env_bridge` copies `AUTHOR_*` into unset + `EXPERIMENTALIST_*` slots, so the Experimentalist helpers Eval Author still + borrows see credentials during a standalone run. `eval_author.agent` imports it + ahead of any Experimentalist agent, because those agents read the environment when + their class body executes. + +A `nemo eval-author` CLI that auto-loads this `.env` is not wired yet. diff --git a/plugins/nemo-eval-author/pyproject.toml b/plugins/nemo-eval-author/pyproject.toml new file mode 100644 index 00000000000..94e72320b0b --- /dev/null +++ b/plugins/nemo-eval-author/pyproject.toml @@ -0,0 +1,26 @@ +[project] +name = "nemo-eval-author-plugin" +version = "0.1.0" +description = "Eval Author agent for NeMo Platform (hard-depends on Experimentalist for evaluator/trace helpers)." +requires-python = ">=3.12,<3.14" +dependencies = [ + "pydantic>=2", + "harbor>=0.16", + "nooa", + "nemo-experimentalist-plugin", + "nemo-insights-plugin", + "nemo-platform", + "tomlkit>=0.13.3", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/nemo_eval_author_plugin"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +pythonpath = ["src"] +testpaths = ["tests"] diff --git a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/_env_bridge.py b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/_env_bridge.py new file mode 100644 index 00000000000..c4df059e32b --- /dev/null +++ b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/_env_bridge.py @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Import this module for its side effect: bridging ``AUTHOR_*`` credentials. + +Transitional. This module exists only because Eval Author still reuses Experimentalist +agents, and it should be deleted along with the last ``nemo_experimentalist_plugin`` import +in this package. ``tests/test_plugin_boundary.py`` tracks what is left to remove. + +``TraceAnalyzer`` and the other Experimentalist agents Eval Author reuses build their LLM +client in the class body, so they read ``EXPERIMENTALIST_*`` the moment their module is +first imported. A module that imports them therefore has to bridge *before* that import, +which no function call inside the module can do. + +Importing this module is how Eval Author expresses that ordering. ``import +nemo_eval_author_plugin._env_bridge`` sorts ahead of every ``from ...`` line in the same +isort section, so the ordering is maintained by the linter rather than by an ``E402`` +waiver and a comment asking future readers not to reshuffle the imports. +""" + +from nemo_eval_author_plugin.model_config import bridge_author_env_to_experimentalist + +bridge_author_env_to_experimentalist() diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/README.md b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/README.md similarity index 90% rename from plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/README.md rename to plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/README.md index 09acb7f7061..59a33c09f57 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/README.md +++ b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/README.md @@ -5,13 +5,15 @@ SPDX-License-Identifier: Apache-2.0 # Eval Author -The top-level `nemo_experimentalist_plugin.eval_author` package is the canonical Eval Author +The top-level `nemo_eval_author_plugin.eval_author` package is the canonical Eval Author implementation. It turns an Experimentalist Insight and its production trace refs into evaluator dataset changes, creating or augmenting regression signals that capture the failure mode before optimization begins. -Experimentalist is a consumer of this package: its insight mode imports and -runs the top-level Eval Author before beginning optimization. +This package hard-depends on Experimentalist for evaluator, staging, and trace +helpers. Experimentalist insight mode imports and runs this Eval Author before +beginning optimization (both packages are installed via the `experimentalist` +uv group). ## Current Files @@ -89,8 +91,8 @@ directly: import asyncio from pathlib import Path -from nemo_experimentalist_plugin.eval_author.models import EvalAuthorConfig -from nemo_experimentalist_plugin.eval_author.run import run_eval_author +from nemo_eval_author_plugin.eval_author.models import EvalAuthorConfig +from nemo_eval_author_plugin.eval_author.run import run_eval_author from nemo_experimentalist_plugin.experimentalist.components.evaluator.models import DatasetRef diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/REFERENCE.md b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/REFERENCE.md similarity index 100% rename from plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/REFERENCE.md rename to plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/REFERENCE.md diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/agent.py b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/agent.py similarity index 97% rename from plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/agent.py rename to plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/agent.py index 33fe71da8c4..780a372c10c 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/agent.py +++ b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/agent.py @@ -12,8 +12,12 @@ from pathlib import Path from typing import Any -from nemo_experimentalist_plugin.eval_author.materialization import InsightSuite -from nemo_experimentalist_plugin.eval_author.models import EvalAuthorConfig, EvalAuthorResult +# Populates EXPERIMENTALIST_* from AUTHOR_*, which the Experimentalist agent imports below +# read when their class bodies execute. Must stay ahead of them; isort keeps it there. +import nemo_eval_author_plugin._env_bridge # noqa: F401 +from nemo_eval_author_plugin.eval_author.materialization import InsightSuite +from nemo_eval_author_plugin.eval_author.models import EvalAuthorConfig, EvalAuthorResult +from nemo_eval_author_plugin.model_config import get_fast_model, get_smart_model from nemo_experimentalist_plugin.experimentalist.components import cache from nemo_experimentalist_plugin.experimentalist.components.evaluator import ( Dataset, @@ -22,7 +26,6 @@ TrialResult, ) from nemo_experimentalist_plugin.experimentalist.components.evaluator.models import ResourceRef -from nemo_experimentalist_plugin.experimentalist.components.model_config import get_fast_model, get_smart_model from nemo_experimentalist_plugin.experimentalist.components.tools import GuardedShellTools from nemo_experimentalist_plugin.experimentalist.components.trace_analyzer import ( Diagnostic, diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/config.yaml b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/config.yaml similarity index 100% rename from plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/config.yaml rename to plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/config.yaml diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/materialization.py b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/materialization.py similarity index 100% rename from plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/materialization.py rename to plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/materialization.py diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/models.py b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/models.py similarity index 100% rename from plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/models.py rename to plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/models.py diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/run.py b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/run.py similarity index 95% rename from plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/run.py rename to plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/run.py index aad8158bb5c..b0ea41dddf8 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/run.py +++ b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/run.py @@ -7,8 +7,8 @@ from pathlib import Path from typing import Literal, Protocol, cast +from nemo_eval_author_plugin.eval_author.models import EvalAuthorConfig, EvalAuthorResult from nemo_experimentalist_plugin.client import make_client -from nemo_experimentalist_plugin.eval_author.models import EvalAuthorConfig, EvalAuthorResult from nemo_experimentalist_plugin.experimentalist.components.dataset_staging import stage_task_template from nemo_experimentalist_plugin.experimentalist.components.evaluator import Dataset, Task from nemo_experimentalist_plugin.experimentalist.components.evaluator.base import EvaluatorType @@ -113,7 +113,7 @@ async def run_eval_author( def build_eval_author_agent(*, experiment_dir: Path, config: EvalAuthorConfig) -> _EvalAuthorAgent: """Build the LLM-backed Eval Author agent lazily.""" - from nemo_experimentalist_plugin.eval_author.agent import build_eval_author_agent as _build_eval_author_agent + from nemo_eval_author_plugin.eval_author.agent import build_eval_author_agent as _build_eval_author_agent return _build_eval_author_agent(experiment_dir=experiment_dir, config=config) diff --git a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/model_config.py b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/model_config.py new file mode 100644 index 00000000000..584a4bcd7fd --- /dev/null +++ b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/model_config.py @@ -0,0 +1,203 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""LLM clients for Eval Author. + +This module deliberately imports nothing from Experimentalist. Eval Author is meant to +become a standalone plugin, so its credential handling duplicates Experimentalist's rather +than sharing it. See the plugin README for the wider split, and +``tests/test_plugin_boundary.py`` for the check that holds this module independent. + +``AUTHOR_*`` is the real contract. The ``EXPERIMENTALIST_*`` fallback and +:func:`bridge_author_env_to_experimentalist` are both transitional, and exist only while +Eval Author still reuses Experimentalist agents. + +TODO(eval-author-standalone): drop the ``EXPERIMENTALIST_*`` fallback and the bridge once +this plugin imports nothing from ``nemo_experimentalist_plugin``. Concretely: delete +``_BRIDGED_ENV_PAIRS`` and :func:`bridge_author_env_to_experimentalist`, delete +``_env_bridge.py`` and its import in ``eval_author/agent.py``, and narrow every ``_env`` / +``_env_or_default`` call here to its ``AUTHOR_*`` name. That is a breaking change for +anyone running insight mode from an Experimentalist-only profile ``.env``, so it needs a +release note. ``rg 'eval-author-standalone'`` finds the other sites. +""" + +import functools +import logging +import os +from urllib.parse import urlsplit + +from nooa.unifiedllm import CompletionClient + +logger = logging.getLogger(__name__) + +# Duplicated from `nemo_experimentalist_plugin.cli` on purpose: sharing it would mean +# importing Experimentalist from a module that is otherwise already standalone. +_GATEWAY_HOST = "inference-api.nvidia.com" + +_SMART_MODEL_DEFAULT = "openai/openai/openai/gpt-5.5" +_FAST_MODEL_DEFAULT = "openai/openai/openai/gpt-5-mini" + +# Transitional; see TODO(eval-author-standalone) above. Every pair targets an Experimentalist +# helper rather than a client Eval Author owns, which is why the mid tier appears here +# despite Eval Author having no mid-tier client of its own. +_BRIDGED_ENV_PAIRS = ( + ("AUTHOR_API_BASE", "EXPERIMENTALIST_API_BASE"), + ("AUTHOR_API_KEY", "EXPERIMENTALIST_API_KEY"), + ("AUTHOR_SMART_MODEL_NAME", "EXPERIMENTALIST_SMART_MODEL_NAME"), + ("AUTHOR_MID_MODEL_NAME", "EXPERIMENTALIST_MID_MODEL_NAME"), + ("AUTHOR_FAST_MODEL_NAME", "EXPERIMENTALIST_FAST_MODEL_NAME"), +) + + +def _env(*names: str) -> str | None: + """Return the first non-empty value among ``names``, or ``None`` when all are empty.""" + for name in names: + value = os.environ.get(name, "").strip() + if value: + return value + return None + + +def _env_or_default(*names: str, default: str) -> str: + """Return the first non-empty value among ``names``, falling back to ``default``.""" + return _env(*names) or default + + +def _api_base() -> str: + base = _env("AUTHOR_API_BASE", "EXPERIMENTALIST_API_BASE") + if not base: + raise ValueError("AUTHOR_API_BASE (or EXPERIMENTALIST_API_BASE) must be set") + return base + + +def _is_gateway_base(value: str) -> bool: + """Report whether ``value`` is the NVIDIA Inference Gateway reached over HTTPS. + + Mirrors ``nemo_experimentalist_plugin.cli._is_gateway_base``. The scheme check is not + cosmetic: a host-only match would forward ``INFERENCE_API_KEY`` to a plain-HTTP base. + """ + try: + parsed = urlsplit(value) + except ValueError: + return False + return parsed.scheme == "https" and parsed.hostname == _GATEWAY_HOST + + +def _api_key() -> str: + """Resolve the LLM API key for Eval Author. + + Preference order: + 1. ``AUTHOR_API_KEY`` + 2. ``EXPERIMENTALIST_API_KEY`` + 3. ``INFERENCE_API_KEY``, but only when the resolved base is the HTTPS NVIDIA + Inference Gateway, matching how the Experimentalist CLI forwards that key. + + Returns: + str: the resolved API key. + + Raises: + ValueError: if no key is available for the resolved base. + + """ + key = _env("AUTHOR_API_KEY", "EXPERIMENTALIST_API_KEY") + if key: + return key + if _is_gateway_base(_api_base()): + inference = _env("INFERENCE_API_KEY") + if inference: + return inference + raise ValueError( + "AUTHOR_API_KEY (or EXPERIMENTALIST_API_KEY) must be set; " + "when using the NVIDIA Inference Gateway, INFERENCE_API_KEY is also accepted" + ) + + +@functools.cache +def _completion_client(name: str, api_base: str, api_key: str) -> CompletionClient: + """Return Eval Author's client for an already-resolved model name, base URL, and key. + + Those three arguments are the cache key, so any tier resolving to the same model gets the + same client. The tier getters below are cached as well, but that alone only promises one + client per tier: two tiers configured with the same model would each build their own, and + every ``CompletionClient`` opens its own pool of keepalive connections. The base URL and + key belong in the key too, since the same model reached through a different endpoint, or + with different credentials, is a different client and must not be reused. + + Experimentalist caches its clients separately, so an insight-mode run holds two clients + for what is usually the same model. Sharing one factory across the plugins would fix + that, but only by importing Experimentalist into the one Eval Author module that has no + such import (see the module docstring). A duplicate connection pool is the cheaper cost. + + Args: + name: Fully qualified model name, as ``CompletionClient`` expects it. + api_base: Resolved API base URL. + api_key: Resolved API key. + + Returns: + CompletionClient: the client for this name, base URL, and key. + + """ + return CompletionClient(name, api_base=api_base, api_key=api_key) + + +@functools.cache +def get_smart_model() -> CompletionClient: + """Return the cached smart (high-capability) LLM client configured from environment variables. + + Returns: + CompletionClient: the singleton smart-model client. + + Raises: + ValueError: if Eval Author API credentials are unset or empty. + + """ + name = _env_or_default( + "AUTHOR_SMART_MODEL_NAME", + "EXPERIMENTALIST_SMART_MODEL_NAME", + default=_SMART_MODEL_DEFAULT, + ) + return _completion_client(name, _api_base(), _api_key()) + + +@functools.cache +def get_fast_model() -> CompletionClient: + """Return the cached fast (low-latency) LLM client configured from environment variables. + + Returns: + CompletionClient: the singleton fast-model client. + + Raises: + ValueError: if Eval Author API credentials are unset or empty. + + """ + name = _env_or_default( + "AUTHOR_FAST_MODEL_NAME", + "EXPERIMENTALIST_FAST_MODEL_NAME", + default=_FAST_MODEL_DEFAULT, + ) + return _completion_client(name, _api_base(), _api_key()) + + +def bridge_author_env_to_experimentalist() -> list[str]: + """Copy ``AUTHOR_*`` into unset ``EXPERIMENTALIST_*`` slots. + + Eval Author owns ``AUTHOR_*`` for its own clients, but the Experimentalist helpers it + reuses (``TraceAnalyzer`` and the summarizers it pulls in) bind their LLM in the class + body from ``EXPERIMENTALIST_*``. Bridging lets a standalone ``AUTHOR_*``-only run + supply those helpers without the caller duplicating credentials. Values already present + in ``EXPERIMENTALIST_*`` always win, so insight-mode runs are left alone. + + Returns: + list[str]: the ``EXPERIMENTALIST_*`` names this call populated, in declaration + order. Empty once everything is in place, which makes repeat calls no-ops. + + """ + applied: list[str] = [] + for src, dst in _BRIDGED_ENV_PAIRS: + value = os.environ.get(src, "").strip() + if value and not os.environ.get(dst, "").strip(): + os.environ[dst] = value + applied.append(dst) + if applied: + logger.info("Bridged Eval Author credentials into %s", ", ".join(applied)) + return applied diff --git a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/py.typed b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/py.typed new file mode 100644 index 00000000000..e69de29bb2d diff --git a/plugins/nemo-eval-author/tests/conftest.py b/plugins/nemo-eval-author/tests/conftest.py new file mode 100644 index 00000000000..391f6fa2cc8 --- /dev/null +++ b/plugins/nemo-eval-author/tests/conftest.py @@ -0,0 +1,37 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Top-level conftest: runs before any test module is imported. + +Eval Author and Experimentalist agents call get_smart_model() at class-definition +time. Placeholder credentials keep collection working when real keys are absent. +setdefault does not overwrite CI or developer-provided values. + +Both stand-in values contain "placeholder" on purpose: real-model canaries such as +test_eval_author_repair_e2e cannot use "is it set?" to decide whether to skip, because +these assignments make every credential look present. Keep the substring if you change +the values. +""" + +import os + +import litellm +import pytest + +# An unroutable HTTPS host: if a real key is present but the base is not, an +# unmocked call fails to connect rather than sending the key over plaintext. +os.environ.setdefault("AUTHOR_API_BASE", "https://placeholder.invalid") +os.environ.setdefault("AUTHOR_API_KEY", "placeholder-for-import") +os.environ.setdefault("EXPERIMENTALIST_API_BASE", "https://placeholder.invalid") +os.environ.setdefault("EXPERIMENTALIST_API_KEY", "placeholder-for-import") + +litellm.drop_params = True + + +@pytest.fixture(autouse=True) +def _restore_environ(): + """Undo environment changes that monkeypatch cannot restore.""" + snapshot = os.environ.copy() + yield + os.environ.clear() + os.environ.update(snapshot) diff --git a/plugins/nemo-experimentalist/tests/test_eval_author_agent.py b/plugins/nemo-eval-author/tests/test_eval_author_agent.py similarity index 95% rename from plugins/nemo-experimentalist/tests/test_eval_author_agent.py rename to plugins/nemo-eval-author/tests/test_eval_author_agent.py index 943b9775bbc..e6177c7318f 100644 --- a/plugins/nemo-experimentalist/tests/test_eval_author_agent.py +++ b/plugins/nemo-eval-author/tests/test_eval_author_agent.py @@ -12,16 +12,15 @@ from typing import Any, cast import pytest -from nemo_experimentalist_plugin.eval_author import agent as eval_author_module -from nemo_experimentalist_plugin.eval_author.agent import EvalAuthor -from nemo_experimentalist_plugin.eval_author.models import EvalAuthorConfig +from nemo_eval_author_plugin.eval_author import agent as eval_author_module +from nemo_eval_author_plugin.eval_author.agent import EvalAuthor +from nemo_eval_author_plugin.eval_author.models import EvalAuthorConfig from nemo_experimentalist_plugin.experimentalist.components.evaluator import ( Dataset, DatasetValidationError, Task, TrialResult, ) -from nemo_experimentalist_plugin.experimentalist.components.loop import EvolutionaryOptimizerConfig from nemo_experimentalist_plugin.experimentalist.components.trace_analyzer import ( Diagnostic, TraceAnalyzerConfig, @@ -679,21 +678,10 @@ async def test_run_raises_after_validation_repair_budget_is_exhausted( assert insight_dataset.validate_calls == 2 -def test_evolutionary_optimizer_uses_top_level_eval_author_config() -> None: - config = EvolutionaryOptimizerConfig().eval_author - - assert type(config) is EvalAuthorConfig - assert config.max_validation_repair_attempts == 5 +def test_eval_author_config_defaults_and_bounds_validation_repair_attempts() -> None: + # Experimentalist's loop config asserts this default too, but from the other side of + # the plugin boundary; owning it here is what keeps the default a plugin contract. + assert EvalAuthorConfig().max_validation_repair_attempts == 5 assert EvalAuthorConfig(max_validation_repair_attempts=10).max_validation_repair_attempts == 10 with pytest.raises(ValueError, match="less than or equal to 10"): EvalAuthorConfig(max_validation_repair_attempts=11) - - -def test_evolutionary_optimizer_tolerates_unknown_eval_author_config() -> None: - config = EvolutionaryOptimizerConfig.model_validate({"eval_author": {"bogus": 1}}) - assert type(config.eval_author) is EvalAuthorConfig - - -def test_evolutionary_optimizer_rejects_legacy_curator_config() -> None: - with pytest.raises(ValueError, match="'curator' was renamed to 'eval_author'"): - EvolutionaryOptimizerConfig.model_validate({"curator": {"max_traces": 1}}) diff --git a/plugins/nemo-experimentalist/tests/test_eval_author_materialization.py b/plugins/nemo-eval-author/tests/test_eval_author_materialization.py similarity index 98% rename from plugins/nemo-experimentalist/tests/test_eval_author_materialization.py rename to plugins/nemo-eval-author/tests/test_eval_author_materialization.py index da415afd0ba..a110fcfcf01 100644 --- a/plugins/nemo-experimentalist/tests/test_eval_author_materialization.py +++ b/plugins/nemo-eval-author/tests/test_eval_author_materialization.py @@ -10,8 +10,8 @@ from pathlib import Path import pytest -from nemo_experimentalist_plugin.eval_author import materialization as materialization_module -from nemo_experimentalist_plugin.eval_author.materialization import InsightSuite +from nemo_eval_author_plugin.eval_author import materialization as materialization_module +from nemo_eval_author_plugin.eval_author.materialization import InsightSuite from nemo_experimentalist_plugin.experimentalist.components.evaluator import Task from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor import HarborDataset diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_eval_author_repair_e2e.py b/plugins/nemo-eval-author/tests/test_eval_author_repair_e2e.py similarity index 88% rename from plugins/nemo-experimentalist/tests/experimentalist/test_eval_author_repair_e2e.py rename to plugins/nemo-eval-author/tests/test_eval_author_repair_e2e.py index 6dd3a23b71a..3dfbf4cfb08 100644 --- a/plugins/nemo-experimentalist/tests/experimentalist/test_eval_author_repair_e2e.py +++ b/plugins/nemo-eval-author/tests/test_eval_author_repair_e2e.py @@ -9,8 +9,9 @@ from pathlib import Path import pytest -from nemo_experimentalist_plugin.eval_author.agent import EvalAuthor -from nemo_experimentalist_plugin.eval_author.models import EvalAuthorConfig +from nemo_eval_author_plugin.eval_author.agent import EvalAuthor +from nemo_eval_author_plugin.eval_author.models import EvalAuthorConfig +from nemo_eval_author_plugin.model_config import get_fast_model from nemo_experimentalist_plugin.experimentalist.components.evaluator import ( DatasetValidationError, local_path_from_uri, @@ -20,13 +21,24 @@ HarborEvaluator, HarborEvaluatorConfig, ) -from nemo_experimentalist_plugin.experimentalist.components.model_config import get_fast_model from nemo_experimentalist_plugin.experimentalist.components.trace_analyzer import Diagnostic from nemo_insights_plugin.entities import Insight -_HAS_EXPERIMENTALIST_LLM = bool( - os.environ.get("EXPERIMENTALIST_API_BASE") and os.environ.get("EXPERIMENTALIST_API_KEY") -) + +def _has_real_llm_credentials() -> bool: + """Report whether real credentials are present, rather than the conftest stand-ins. + + conftest fills placeholder credentials so agent classes can be imported at collection + time, which means a plain "is it set?" check always passes. These canaries call a live + model, so they have to look past the stand-ins. + """ + base = os.environ.get("AUTHOR_API_BASE") or os.environ.get("EXPERIMENTALIST_API_BASE") or "" + key = os.environ.get("AUTHOR_API_KEY") or os.environ.get("EXPERIMENTALIST_API_KEY") or "" + return bool(base and key) and "placeholder" not in base and "placeholder" not in key + + +_HAS_LLM = _has_real_llm_credentials() +_CREDENTIALS_HINT = "AUTHOR_API_BASE and AUTHOR_API_KEY (or their EXPERIMENTALIST_* equivalents)" _RUN_EVAL_AUTHOR_REPAIR_E2E = os.environ.get("RUN_EVAL_AUTHOR_REPAIR_E2E") == "1" _RUN_EVAL_AUTHOR_HARBOR_E2E = os.environ.get("RUN_EVAL_AUTHOR_HARBOR_E2E") == "1" _MALFORMED_VERIFIER = """\ @@ -191,10 +203,8 @@ async def run( @pytest.mark.skipif( - not (_RUN_EVAL_AUTHOR_REPAIR_E2E and _HAS_EXPERIMENTALIST_LLM), - reason=( - "Set RUN_EVAL_AUTHOR_REPAIR_E2E=1 with EXPERIMENTALIST_API_BASE and EXPERIMENTALIST_API_KEY to run the Eval Author repair canary." - ), + not (_RUN_EVAL_AUTHOR_REPAIR_E2E and _HAS_LLM), + reason=f"Set RUN_EVAL_AUTHOR_REPAIR_E2E=1 with {_CREDENTIALS_HINT} to run the Eval Author repair canary.", ) async def test_gpt5_mini_repairs_malformed_harbor_verifiers( tmp_path: Path, @@ -215,7 +225,9 @@ async def test_gpt5_mini_repairs_malformed_harbor_verifiers( assert "SyntaxError: expected 'except' or 'finally' block" in validation_feedback llm = get_fast_model() - monkeypatch.delenv("EXPERIMENTALIST_API_KEY") + # Construction must not need EXPERIMENTALIST_API_KEY: Eval Author owns AUTHOR_* and only + # bridges into the Experimentalist slots, which are already resolved by this point. + monkeypatch.delenv("EXPERIMENTALIST_API_KEY", raising=False) eval_author = EvalAuthor( experiment_dir=tmp_path, config=EvalAuthorConfig(), @@ -259,11 +271,8 @@ async def test_gpt5_mini_repairs_malformed_harbor_verifiers( @pytest.mark.skipif( - not (_RUN_EVAL_AUTHOR_HARBOR_E2E and _HAS_EXPERIMENTALIST_LLM), - reason=( - "Set RUN_EVAL_AUTHOR_HARBOR_E2E=1 with EXPERIMENTALIST_API_BASE and EXPERIMENTALIST_API_KEY " - "to run the live Harbor metric canary." - ), + not (_RUN_EVAL_AUTHOR_HARBOR_E2E and _HAS_LLM), + reason=f"Set RUN_EVAL_AUTHOR_HARBOR_E2E=1 with {_CREDENTIALS_HINT} to run the live Harbor metric canary.", ) async def test_eval_author_metric_scores_known_failing_harbor_baseline_low( tmp_path: Path, @@ -279,7 +288,8 @@ async def test_eval_author_metric_scores_known_failing_harbor_baseline_low( llm = get_fast_model() llm.config["temperature"] = 0.0 - monkeypatch.delenv("EXPERIMENTALIST_API_KEY") + # See the repair canary above: construction must not need EXPERIMENTALIST_API_KEY. + monkeypatch.delenv("EXPERIMENTALIST_API_KEY", raising=False) eval_author = EvalAuthor( experiment_dir=tmp_path, config=EvalAuthorConfig(), diff --git a/plugins/nemo-experimentalist/tests/test_eval_author_run.py b/plugins/nemo-eval-author/tests/test_eval_author_run.py similarity index 98% rename from plugins/nemo-experimentalist/tests/test_eval_author_run.py rename to plugins/nemo-eval-author/tests/test_eval_author_run.py index cc5625765fd..6b31bf9be52 100644 --- a/plugins/nemo-experimentalist/tests/test_eval_author_run.py +++ b/plugins/nemo-eval-author/tests/test_eval_author_run.py @@ -6,8 +6,8 @@ from typing import Any, Literal import pytest -from nemo_experimentalist_plugin.eval_author import run as eval_author_run -from nemo_experimentalist_plugin.eval_author.models import EvalAuthorConfig, EvalAuthorResult +from nemo_eval_author_plugin.eval_author import run as eval_author_run +from nemo_eval_author_plugin.eval_author.models import EvalAuthorConfig, EvalAuthorResult from nemo_experimentalist_plugin.experimentalist.components.evaluator.models import Dataset, DatasetRef, Task from nemo_insights_plugin.entities import Insight diff --git a/plugins/nemo-eval-author/tests/test_model_config.py b/plugins/nemo-eval-author/tests/test_model_config.py new file mode 100644 index 00000000000..55cfb96732e --- /dev/null +++ b/plugins/nemo-eval-author/tests/test_model_config.py @@ -0,0 +1,233 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Credential and model-client resolution for Eval Author.""" + +import os +import subprocess +import sys +from collections.abc import Iterator + +import pytest +from nemo_eval_author_plugin import model_config + + +@pytest.fixture(autouse=True) +def _clear_model_cache() -> Iterator[None]: + """Drop every cached client so each test resolves credentials from scratch.""" + + def clear() -> None: + model_config.get_smart_model.cache_clear() + model_config.get_fast_model.cache_clear() + model_config._completion_client.cache_clear() + + clear() + yield + clear() + + +def test_api_base_prefers_eval_author_over_experimentalist( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("AUTHOR_API_BASE", "https://eval-author.example/v1") + monkeypatch.setenv("EXPERIMENTALIST_API_BASE", "https://experimentalist.example/v1") + monkeypatch.setenv("AUTHOR_API_KEY", "eval-key") + + assert model_config._api_base() == "https://eval-author.example/v1" + + +def test_api_base_falls_back_to_experimentalist( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("AUTHOR_API_BASE", raising=False) + monkeypatch.setenv("EXPERIMENTALIST_API_BASE", "https://experimentalist.example/v1") + monkeypatch.setenv("EXPERIMENTALIST_API_KEY", "exp-key") + + assert model_config._api_base() == "https://experimentalist.example/v1" + + +def test_api_base_requires_configuration(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("AUTHOR_API_BASE", raising=False) + monkeypatch.delenv("EXPERIMENTALIST_API_BASE", raising=False) + + with pytest.raises(ValueError, match="AUTHOR_API_BASE"): + model_config._api_base() + + +def test_whitespace_only_values_count_as_unset(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("AUTHOR_API_BASE", " ") + monkeypatch.setenv("EXPERIMENTALIST_API_BASE", " https://experimentalist.example/v1 ") + + assert model_config._api_base() == "https://experimentalist.example/v1" + + +def test_api_key_accepts_inference_key_on_gateway( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("AUTHOR_API_BASE", "https://inference-api.nvidia.com/v1") + monkeypatch.delenv("AUTHOR_API_KEY", raising=False) + monkeypatch.delenv("EXPERIMENTALIST_API_KEY", raising=False) + monkeypatch.setenv("INFERENCE_API_KEY", "sk-gateway") + + assert model_config._api_key() == "sk-gateway" + + +def test_api_key_refuses_to_forward_inference_key_over_plain_http( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("AUTHOR_API_BASE", "http://inference-api.nvidia.com/v1") + monkeypatch.delenv("AUTHOR_API_KEY", raising=False) + monkeypatch.delenv("EXPERIMENTALIST_API_KEY", raising=False) + monkeypatch.setenv("INFERENCE_API_KEY", "sk-gateway") + + with pytest.raises(ValueError, match="AUTHOR_API_KEY"): + model_config._api_key() + + +def test_api_key_refuses_to_forward_inference_key_to_a_lookalike_host( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("AUTHOR_API_BASE", "https://inference-api.nvidia.com.attacker.example/v1") + monkeypatch.delenv("AUTHOR_API_KEY", raising=False) + monkeypatch.delenv("EXPERIMENTALIST_API_KEY", raising=False) + monkeypatch.setenv("INFERENCE_API_KEY", "sk-gateway") + + with pytest.raises(ValueError, match="AUTHOR_API_KEY"): + model_config._api_key() + + +def test_api_key_requires_credentials_when_missing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("AUTHOR_API_BASE", "https://custom.example/v1") + monkeypatch.delenv("AUTHOR_API_KEY", raising=False) + monkeypatch.delenv("EXPERIMENTALIST_API_KEY", raising=False) + monkeypatch.delenv("INFERENCE_API_KEY", raising=False) + + with pytest.raises(ValueError, match="AUTHOR_API_KEY"): + model_config._api_key() + + +@pytest.mark.parametrize("tier", ["SMART", "FAST"]) +def test_model_name_prefers_author_then_experimentalist_then_default( + monkeypatch: pytest.MonkeyPatch, + tier: str, +) -> None: + getter = {"SMART": model_config.get_smart_model, "FAST": model_config.get_fast_model}[tier] + default = {"SMART": model_config._SMART_MODEL_DEFAULT, "FAST": model_config._FAST_MODEL_DEFAULT}[tier] + author_var = f"AUTHOR_{tier}_MODEL_NAME" + experimentalist_var = f"EXPERIMENTALIST_{tier}_MODEL_NAME" + + monkeypatch.setenv("AUTHOR_API_BASE", "https://eval-author.example/v1") + monkeypatch.setenv("AUTHOR_API_KEY", "eval-key") + + monkeypatch.setenv(author_var, "vendor/from-author") + monkeypatch.setenv(experimentalist_var, "vendor/from-experimentalist") + assert getter().model == "vendor/from-author" + + getter.cache_clear() + monkeypatch.delenv(author_var) + assert getter().model == "vendor/from-experimentalist" + + getter.cache_clear() + monkeypatch.delenv(experimentalist_var) + assert getter().model == default + + +def test_tiers_pointing_at_one_model_share_a_client(monkeypatch: pytest.MonkeyPatch) -> None: + # The @functools.cache on each getter only promises one client per tier. Keying the + # factory on the resolved name, base URL, and key is what collapses identical tiers. + monkeypatch.setenv("AUTHOR_API_BASE", "https://eval-author.example/v1") + monkeypatch.setenv("AUTHOR_API_KEY", "eval-key") + monkeypatch.setenv("AUTHOR_SMART_MODEL_NAME", "vendor/one-model") + monkeypatch.setenv("AUTHOR_FAST_MODEL_NAME", "vendor/one-model") + + assert model_config.get_smart_model() is model_config.get_fast_model() + + +def test_differing_tiers_get_separate_clients(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("AUTHOR_API_BASE", "https://eval-author.example/v1") + monkeypatch.setenv("AUTHOR_API_KEY", "eval-key") + monkeypatch.setenv("AUTHOR_SMART_MODEL_NAME", "vendor/smart") + monkeypatch.setenv("AUTHOR_FAST_MODEL_NAME", "vendor/fast") + + assert model_config.get_smart_model() is not model_config.get_fast_model() + + +def test_bridge_author_env_fills_unset_experimentalist_slots( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("AUTHOR_API_BASE", "https://eval-author.example/v1") + monkeypatch.setenv("AUTHOR_API_KEY", "eval-key") + monkeypatch.delenv("EXPERIMENTALIST_API_BASE", raising=False) + monkeypatch.delenv("EXPERIMENTALIST_API_KEY", raising=False) + + applied = model_config.bridge_author_env_to_experimentalist() + + assert applied == ["EXPERIMENTALIST_API_BASE", "EXPERIMENTALIST_API_KEY"] + assert os.environ["EXPERIMENTALIST_API_BASE"] == "https://eval-author.example/v1" + assert os.environ["EXPERIMENTALIST_API_KEY"] == "eval-key" + + +def test_bridge_author_env_covers_every_declared_pair( + monkeypatch: pytest.MonkeyPatch, +) -> None: + for src, dst in model_config._BRIDGED_ENV_PAIRS: + monkeypatch.setenv(src, f"value-from-{src}") + monkeypatch.delenv(dst, raising=False) + + applied = model_config.bridge_author_env_to_experimentalist() + + assert applied == [dst for _, dst in model_config._BRIDGED_ENV_PAIRS] + for src, dst in model_config._BRIDGED_ENV_PAIRS: + assert os.environ[dst] == f"value-from-{src}" + + +def test_bridge_author_env_does_not_overwrite_existing_experimentalist( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("AUTHOR_API_BASE", "https://eval-author.example/v1") + monkeypatch.setenv("EXPERIMENTALIST_API_BASE", "https://experimentalist.example/v1") + + applied = model_config.bridge_author_env_to_experimentalist() + + assert applied == [] + assert os.environ["EXPERIMENTALIST_API_BASE"] == "https://experimentalist.example/v1" + + +def test_bridge_author_env_is_a_no_op_once_applied( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("AUTHOR_API_BASE", "https://eval-author.example/v1") + monkeypatch.delenv("EXPERIMENTALIST_API_BASE", raising=False) + + assert model_config.bridge_author_env_to_experimentalist() == ["EXPERIMENTALIST_API_BASE"] + assert model_config.bridge_author_env_to_experimentalist() == [] + + +def test_agent_module_imports_under_author_credentials_alone() -> None: + """Guard the import ordering that lets an AUTHOR_*-only run build Experimentalist agents. + + ``TraceAnalyzer`` and friends read ``EXPERIMENTALIST_*`` while their class body runs, so + ``agent.py`` has to import ``_env_bridge`` before importing them. That ordering cannot be + checked in-process: this test suite's conftest already populates ``EXPERIMENTALIST_*``, + and the modules are imported once per session. Hence the subprocess with a pruned + environment -- it fails with ``ValueError`` if the bridge stops running early enough. + """ + env = { + key: value + for key, value in os.environ.items() + if not key.startswith(("AUTHOR_", "EXPERIMENTALIST_", "INFERENCE_API_KEY")) + } + env["AUTHOR_API_BASE"] = "https://placeholder.invalid" + env["AUTHOR_API_KEY"] = "placeholder-for-import" + + result = subprocess.run( + [sys.executable, "-c", "import nemo_eval_author_plugin.eval_author.agent"], + env=env, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr diff --git a/plugins/nemo-eval-author/tests/test_plugin_boundary.py b/plugins/nemo-eval-author/tests/test_plugin_boundary.py new file mode 100644 index 00000000000..4c851ce06ae --- /dev/null +++ b/plugins/nemo-eval-author/tests/test_plugin_boundary.py @@ -0,0 +1,100 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Ratchet on Eval Author's remaining dependency on Experimentalist. + +Eval Author is meant to end up standalone, with Experimentalist depending on it and not the +other way around. Until then this test pins the modules Eval Author still reaches for, so +the coupling can only shrink. Prefer duplicating a helper over adding a row here. + +Emptying the allowlist is what unblocks the TODO(eval-author-standalone) cleanups: the +``EXPERIMENTALIST_*`` credential fallback and the ``_env_bridge`` side-effect module both +exist only to serve the imports listed below. +""" + +import ast +from pathlib import Path + +_SRC = Path(__file__).resolve().parents[1] / "src" / "nemo_eval_author_plugin" + +# Each entry is the module named in the import statement, so `from ...components import +# cache` is recorded as `...components`. Each one is debt: deleting a row is the goal, and +# adding a row needs a deliberate argument for why duplicating the helper is worse. What +# Eval Author still borrows, by row: +# +# client -> make_client, the platform client factory +# ...components -> the cache module, for run artifacts +# ...dataset_staging -> stage_task_template +# ...evaluator -> Dataset / Task / TrialResult / DatasetValidationError +# ...evaluator.base -> EvaluatorType +# ...evaluator.factory-> DatasetFactory +# ...evaluator.harbor -> HarborDataset +# ...evaluator.models -> DatasetRef, ResourceRef, Task, local_path_from_uri +# ...tools -> GuardedShellTools +# ...trace_analyzer -> TraceAnalyzer, TraceAnalyzerConfig, Diagnostic +# ...trace_explorer -> TraceExplorer and its view models +# experimentalist_backend -> make_experimentalist_backend +_ALLOWED_EXPERIMENTALIST_IMPORTS = { + "nemo_experimentalist_plugin.client", + "nemo_experimentalist_plugin.experimentalist.components", + "nemo_experimentalist_plugin.experimentalist.components.dataset_staging", + "nemo_experimentalist_plugin.experimentalist.components.evaluator", + "nemo_experimentalist_plugin.experimentalist.components.evaluator.base", + "nemo_experimentalist_plugin.experimentalist.components.evaluator.factory", + "nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor", + "nemo_experimentalist_plugin.experimentalist.components.evaluator.models", + "nemo_experimentalist_plugin.experimentalist.components.tools", + "nemo_experimentalist_plugin.experimentalist.components.trace_analyzer", + "nemo_experimentalist_plugin.experimentalist.components.trace_explorer", + "nemo_experimentalist_plugin.experimentalist.experimentalist_backend", +} + + +def _imported_experimentalist_modules() -> dict[str, set[str]]: + """Map each Experimentalist module imported under ``src`` to the files importing it.""" + found: dict[str, set[str]] = {} + for path in sorted(_SRC.rglob("*.py")): + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + names = [alias.name for alias in node.names] + elif isinstance(node, ast.ImportFrom): + # level > 0 is a relative import, which cannot reach another distribution. + names = [node.module] if node.module and not node.level else [] + else: + continue + for name in names: + if name.split(".")[0] == "nemo_experimentalist_plugin": + found.setdefault(name, set()).add(path.name) + return found + + +def test_experimentalist_imports_only_shrink() -> None: + found = _imported_experimentalist_modules() + + added = set(found) - _ALLOWED_EXPERIMENTALIST_IMPORTS + assert not added, ( + "New Experimentalist imports in the Eval Author plugin: " + + ", ".join(f"{name} (in {', '.join(sorted(found[name]))})" for name in sorted(added)) + + ". Eval Author is moving toward standalone, so duplicate the helper instead of " + "importing it. If the import is genuinely unavoidable, add it to " + "_ALLOWED_EXPERIMENTALIST_IMPORTS with a note explaining why." + ) + + removed = _ALLOWED_EXPERIMENTALIST_IMPORTS - set(found) + assert not removed, ( + "These Experimentalist imports are gone, which is the point: " + + ", ".join(sorted(removed)) + + ". Drop them from _ALLOWED_EXPERIMENTALIST_IMPORTS so the list keeps meaning something." + ) + + +def test_model_config_is_already_standalone() -> None: + """Credential resolution is the one part of the plugin that owes Experimentalist nothing. + + It was briefly refactored to share Experimentalist's client cache, which read as a + cleanup but moved the plugin away from standalone. Keep it independent. + """ + importers = {file for files in _imported_experimentalist_modules().values() for file in files} + + assert "model_config.py" not in importers diff --git a/plugins/nemo-experimentalist/AGENTS.md b/plugins/nemo-experimentalist/AGENTS.md index 8cbeb1fbd2f..f3a5d43939a 100644 --- a/plugins/nemo-experimentalist/AGENTS.md +++ b/plugins/nemo-experimentalist/AGENTS.md @@ -13,6 +13,41 @@ Inherited from the NeMo Platform monorepo that now hosts this plugin: ## Active migrations +### 2026-07-28: Eval Author extracted to its own plugin, heading for standalone + +`plugins/nemo-eval-author/` (`nemo-eval-author-plugin`) owns the Eval Author agent package +(`eval_author/`) plus its own `AUTHOR_*` `model_config`. + +**The target is one arrow: Experimentalist → Eval Author.** Eval Author is meant to stop +depending on Experimentalist entirely, even where that means duplicating code. Today the +arrow points both ways: + +- Experimentalist → Eval Author is permanent. Insight mode imports `EvalAuthor` in + `components/loop.py` and `EvalAuthorConfig` in `resolve.py`, both at module scope, so + the dependency is declared in `pyproject.toml`. +- Eval Author → Experimentalist is temporary. It still borrows the evaluator/Harbor + abstractions, dataset staging, trace analyzer/explorer, `GuardedShellTools`, the cache, + and the backend factory. + +`plugins/nemo-eval-author/tests/test_plugin_boundary.py` pins that second list so it can +only shrink. **When you are tempted to share a helper between the two plugins, duplicate it +into Eval Author instead.** Sharing reads like a cleanup and is a regression here; the +boundary test will reject it, which is the intended answer, not an obstacle to route +around. `uv` resolves the current cycle fine — install both with +`uv sync --group experimentalist`. + +Two transitional mechanisms exist only because of the second arrow. Both should be deleted +with the last `nemo_experimentalist_plugin` import, and both are tagged +`TODO(eval-author-standalone)` — `rg 'eval-author-standalone'` lists every site: + +- `nemo_eval_author_plugin/_env_bridge.py` copies `AUTHOR_*` into unset `EXPERIMENTALIST_*` + slots and is imported for that side effect. `eval_author/agent.py` imports it ahead of + every Experimentalist agent, because those agents build their LLM in the class body and + so read the environment at import time. isort keeps a plain `import` ahead of `from` + imports in the same section, so do not convert it to a `from` import. +- `AUTHOR_*` falls back to `EXPERIMENTALIST_*` in `model_config`. `AUTHOR_*` is the real + contract; the fallback only keeps one credential set working in insight mode. + ### 2026-07-24: Optimizer renamed to Experimentalist Ahead of the move into the `nemo-platform` monorepo, the plugin was renamed from @@ -50,7 +85,7 @@ nesting under `nemo agents` needs a Platform-side change first. The Curator agent was renamed directly to Eval Author in ASE-643. This is a breaking rename with no compatibility aliases or migration layer: -- `nemo_experimentalist_plugin.curator` → `nemo_experimentalist_plugin.eval_author` +- `nemo_experimentalist_plugin.curator` → `nemo_eval_author_plugin.eval_author` (was `nemo_experimentalist_plugin.eval_author` before extraction) - `Curator`, `CuratorConfig`, and `CuratorResult` → `EvalAuthor`, `EvalAuthorConfig`, and `EvalAuthorResult` - `run_curator(...)` and `build_curator_agent(...)` → diff --git a/plugins/nemo-experimentalist/pyproject.toml b/plugins/nemo-experimentalist/pyproject.toml index f0f081c4002..e0add47d4f3 100644 --- a/plugins/nemo-experimentalist/pyproject.toml +++ b/plugins/nemo-experimentalist/pyproject.toml @@ -11,6 +11,7 @@ dependencies = [ "protobuf>=6.0.0", "nooa", "pyyaml>=6.0.3", + "nemo-eval-author-plugin", "nemo-insights-plugin", "nemo-platform", "nemo-platform-plugin", diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/analyzer.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/analyzer.py index a65faea12f8..335441e571e 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/analyzer.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/analyzer.py @@ -17,6 +17,12 @@ Task, TrialResult, ) +from nemo_experimentalist_plugin.experimentalist.components.trace_analyzer import ( # noqa: F401 + Diagnostic, + TraceAnalyzer, + TraceAnalyzerConfig, +) +from nemo_experimentalist_plugin.experimentalist.components.trace_explorer import TraceExplorer # noqa: F401 from nemo_platform import AsyncNeMoPlatform from nooa import Agent, CodeActStrategy, strategy from nooa.agentdoc import doc, spec @@ -31,8 +37,6 @@ from .model_config import get_fast_model, get_smart_model from .rationalizer import Rationale, Rationalizer, RationalizerConfig # noqa: F401 from .tools import GuardedShellTools -from .trace_analyzer import Diagnostic, TraceAnalyzer, TraceAnalyzerConfig # noqa: F401 -from .trace_explorer import TraceExplorer # noqa: F401 from .util import load_framework_skills diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/coder.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/coder.py index 928359f2d5f..352e0376851 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/coder.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/coder.py @@ -14,6 +14,14 @@ import httpx from nemo_experimentalist_plugin.entities import Candidate +from nemo_experimentalist_plugin.experimentalist.components.evaluator import ( + Dataset, + EvaluationResult, + Evaluator, + EvaluatorConfig, + Task, + TrialResult, +) from nooa import Agent, CodeActStrategy, strategy from nooa.agentdoc import doc, spec from nooa.agents import TokenBudgetSummarizer @@ -25,7 +33,6 @@ from pydantic import BaseModel, Field from .cards import Optimize -from .evaluator import Dataset, EvaluationResult, Evaluator, EvaluatorConfig, Task, TrialResult from .model_config import get_fast_model, get_mid_model, get_smart_model from .tools import GuardedShellTools from .util import load_framework_skills diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/loop.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/loop.py index 80649146f99..0dade050434 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/loop.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/loop.py @@ -18,8 +18,8 @@ from pathlib import Path from typing import Any, Literal, cast, get_args +from nemo_eval_author_plugin.eval_author.agent import EvalAuthor from nemo_experimentalist_plugin.entities import Candidate, ExperimentRun -from nemo_experimentalist_plugin.eval_author.agent import EvalAuthor from nemo_experimentalist_plugin.experimentalist.components.analyzer import AgentAnalyzer, AnalyzerConfig from nemo_experimentalist_plugin.experimentalist.components.coder import Coder, CoderConfig from nemo_experimentalist_plugin.experimentalist.components.dataset_staging import stage_eval_author_inputs diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/rationale.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/rationale.py new file mode 100644 index 00000000000..831961e67dd --- /dev/null +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/rationale.py @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Task-level rationale models shared by Rationalizer and TraceAnalyzer.""" + +from pydantic import BaseModel, Field + + +class RationaleStep(BaseModel): + """One reasoning step in a task rationale.""" + + thought: str + action: str + observation: str = "" + + +class Rationale(BaseModel): + """Task-level context produced by the Rationalizer before trace analysis.""" + + task_name: str + steps: list[RationaleStep] = Field(default_factory=list) diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/rationalizer.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/rationalizer.py index 7756ffc216c..6be50d9423b 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/rationalizer.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/rationalizer.py @@ -22,23 +22,11 @@ from . import cache from .model_config import get_fast_model, get_smart_model +from .rationale import Rationale, RationaleStep from .tools import GuardedShellTools from .util import load_framework_skills - -class RationaleStep(BaseModel): - """One reasoning step in a task rationale.""" - - thought: str - action: str - observation: str = "" - - -class Rationale(BaseModel): - """Task-level context produced by the Rationalizer before trace analysis.""" - - task_name: str - steps: list[RationaleStep] = Field(default_factory=list) +__all__ = ["Rationale", "RationaleStep", "Rationalizer", "RationalizerConfig"] class RationalizerConfig(BaseModel): diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/trace_analyzer.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/trace_analyzer.py index 319d0b4dfb1..e46ae16ffe1 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/trace_analyzer.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/trace_analyzer.py @@ -11,8 +11,11 @@ from typing import Any, Literal from urllib.parse import unquote, urlparse +from nemo_experimentalist_plugin.experimentalist.components import cache from nemo_experimentalist_plugin.experimentalist.components.evaluator import MetricResult, Task, TrialResult from nemo_experimentalist_plugin.experimentalist.components.evaluator.models import DependencyRuntime +from nemo_experimentalist_plugin.experimentalist.components.model_config import get_fast_model, get_smart_model +from nemo_experimentalist_plugin.experimentalist.components.tools import GuardedShellTools, WorkspaceTool from nemo_insights_plugin.entities import Insight from nemo_platform import AsyncNeMoPlatform from nooa import Agent, CodeActStrategy, strategy @@ -24,10 +27,7 @@ from nooa.tools import Match, TodoManager from pydantic import BaseModel, Field -from . import cache -from .model_config import get_fast_model, get_smart_model -from .rationalizer import Rationale -from .tools import GuardedShellTools, WorkspaceTool +from .rationale import Rationale from .trace_explorer import SearchResult, SessionData, SessionSummary, TraceExplorer, TurnInfo from .util import load_framework_skills diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/trace_scorer.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/trace_scorer.py index 7e73161f8e9..30d0c8ae00b 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/trace_scorer.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/trace_scorer.py @@ -6,6 +6,7 @@ from typing import Any from nemo_experimentalist_plugin.experimentalist.components.evaluator import Dataset, TrialResult +from nemo_experimentalist_plugin.experimentalist.components.trace_explorer import TraceExplorer # noqa: F401 from nemo_platform import AsyncNeMoPlatform from nooa import Agent, CodeActStrategy, strategy from nooa.agentdoc import doc @@ -14,7 +15,6 @@ from .goal_tree import GoalNode from .model_config import get_mid_model -from .trace_explorer import TraceExplorer # noqa: F401 logger = logging.getLogger(__name__) diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/resolve.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/resolve.py index 639ab509b7d..82be6af0b4b 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/resolve.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/resolve.py @@ -22,7 +22,7 @@ from typing import Any, Literal import yaml -from nemo_experimentalist_plugin.eval_author.models import EvalAuthorConfig +from nemo_eval_author_plugin.eval_author.models import EvalAuthorConfig from nemo_experimentalist_plugin.experimentalist.components.evaluator.models import DatasetRef from nemo_experimentalist_plugin.experimentalist.components.repository import looks_like_git from nemo_experimentalist_plugin.profile import AgentProfile diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_dataset_staging.py b/plugins/nemo-experimentalist/tests/experimentalist/test_dataset_staging.py index eb813d2c6a0..f0462534513 100644 --- a/plugins/nemo-experimentalist/tests/experimentalist/test_dataset_staging.py +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_dataset_staging.py @@ -4,17 +4,13 @@ from pathlib import Path from types import SimpleNamespace from typing import cast -from unittest.mock import AsyncMock import pytest -from nemo_experimentalist_plugin.experimentalist.components import loop as loop_module from nemo_experimentalist_plugin.experimentalist.components.dataset_staging import ( stage_eval_author_inputs, stage_task_template, ) from nemo_experimentalist_plugin.experimentalist.components.evaluator.models import DatasetRef -from nemo_experimentalist_plugin.experimentalist.components.loop import EvolutionaryOptimizer -from nemo_experimentalist_plugin.resolve import EvolutionaryOptimizerConfig from nemo_platform import AsyncNeMoPlatform @@ -157,93 +153,3 @@ async def download(self, *, remote_path: str, local_path: str, workspace: str) - "workspace": "workspace-a", }, ] - - -@pytest.mark.asyncio -async def test_insight_run_stages_inputs_before_eval_author( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - source = tmp_path / "source" - train = source / "train" - validation = source / "validation" - template = source / "task-template" - _write_tree(train, "train source") - _write_tree(validation, "validation source") - _write_tree(template, "template source") - experiment = tmp_path / "experiment" - captured: dict[str, Path] = {} - - class RecordingDatasetFactory: - def build_dataset(self, evaluator_type: str, ref: DatasetRef) -> SimpleNamespace: - return SimpleNamespace(ref=ref) - - def build_task_template(self, evaluator_type: str, ref: DatasetRef) -> SimpleNamespace: - return SimpleNamespace(uri=ref.uri) - - class MutatingEvalAuthor: - def __init__(self, **kwargs: object) -> None: - pass - - async def run( - self, - *, - task_template: SimpleNamespace, - train_dataset: SimpleNamespace, - validation_dataset: SimpleNamespace, - **kwargs: object, - ) -> None: - captured["train"] = Path(train_dataset.ref.uri) - captured["validation"] = Path(validation_dataset.ref.uri) - captured["template"] = Path(task_template.uri) - (captured["train"] / "task-1" / "tests" / "test.sh").write_text("curated", encoding="utf-8") - (captured["template"].parent / "generated-task").mkdir() - raise RuntimeError("stop after eval_author") - - monkeypatch.setattr( - loop_module, - "EvaluatorFactory", - lambda: SimpleNamespace(build_evaluator=lambda *args, **kwargs: object()), - ) - monkeypatch.setattr(loop_module, "DatasetFactory", RecordingDatasetFactory) - monkeypatch.setattr(loop_module, "EvalAuthor", MutatingEvalAuthor) - monkeypatch.setattr( - EvolutionaryOptimizer, - "_init_structure", - lambda self: (experiment / "agents", experiment / "analysis", experiment / "results"), - ) - - backend = SimpleNamespace( - client=object(), - get_insight=AsyncMock(return_value=SimpleNamespace(agent=str(tmp_path / "agent"))), - get_agent_code=AsyncMock(), - ) - config = EvolutionaryOptimizerConfig() - optimizer = object.__new__(EvolutionaryOptimizer) - optimizer.working_dir = experiment - optimizer.config = config - optimizer.shell = SimpleNamespace(close=AsyncMock()) - deps = SimpleNamespace( - backend=backend, - workspace="default", - config=config, - evaluator_type="harbor", - train_dataset=DatasetRef(uri=str(train)), - validation_dataset=DatasetRef(uri=str(validation)), - task_template=DatasetRef(uri=str(template)), - insight="insight-1", - agent=tmp_path / "agent", - agent_spec=None, - ) - - with pytest.raises(RuntimeError, match="stop after eval_author"): - await optimizer.run(deps) - - optimizer.shell.close.assert_awaited_once() - assert captured == { - "train": experiment / "dataset" / "train", - "validation": experiment / "dataset" / "validation", - "template": experiment / "dataset" / "task-template", - } - assert (train / "task-1" / "tests" / "test.sh").read_text(encoding="utf-8") == "train source" - assert not (template.parent / "generated-task").exists() diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_dataset_staging_loop.py b/plugins/nemo-experimentalist/tests/experimentalist/test_dataset_staging_loop.py new file mode 100644 index 00000000000..a03990f009b --- /dev/null +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_dataset_staging_loop.py @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Loop integration: Experimentalist stages Eval Author inputs before calling Eval Author.""" + +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest +from nemo_experimentalist_plugin.experimentalist.components import loop as loop_module +from nemo_experimentalist_plugin.experimentalist.components.evaluator.models import DatasetRef +from nemo_experimentalist_plugin.experimentalist.components.loop import EvolutionaryOptimizer +from nemo_experimentalist_plugin.resolve import EvolutionaryOptimizerConfig + + +def _write_tree(root: Path, content: str) -> None: + tests = root / "task-1" / "tests" + tests.mkdir(parents=True) + (tests / "test.sh").write_text(content, encoding="utf-8") + + +@pytest.mark.asyncio +async def test_insight_run_stages_inputs_before_eval_author( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + source = tmp_path / "source" + train = source / "train" + validation = source / "validation" + template = source / "task-template" + _write_tree(train, "train source") + _write_tree(validation, "validation source") + _write_tree(template, "template source") + experiment = tmp_path / "experiment" + captured: dict[str, Path] = {} + + class RecordingDatasetFactory: + def build_dataset(self, evaluator_type: str, ref: DatasetRef) -> SimpleNamespace: + return SimpleNamespace(ref=ref) + + def build_task_template(self, evaluator_type: str, ref: DatasetRef) -> SimpleNamespace: + return SimpleNamespace(uri=ref.uri) + + class MutatingEvalAuthor: + def __init__(self, **kwargs: object) -> None: + pass + + async def run( + self, + *, + task_template: SimpleNamespace, + train_dataset: SimpleNamespace, + validation_dataset: SimpleNamespace, + **kwargs: object, + ) -> None: + captured["train"] = Path(train_dataset.ref.uri) + captured["validation"] = Path(validation_dataset.ref.uri) + captured["template"] = Path(task_template.uri) + (captured["train"] / "task-1" / "tests" / "test.sh").write_text("curated", encoding="utf-8") + (captured["template"].parent / "generated-task").mkdir() + raise RuntimeError("stop after eval_author") + + monkeypatch.setattr( + loop_module, + "EvaluatorFactory", + lambda: SimpleNamespace(build_evaluator=lambda *args, **kwargs: object()), + ) + monkeypatch.setattr(loop_module, "DatasetFactory", RecordingDatasetFactory) + monkeypatch.setattr(loop_module, "EvalAuthor", MutatingEvalAuthor) + monkeypatch.setattr( + EvolutionaryOptimizer, + "_init_structure", + lambda self: (experiment / "agents", experiment / "analysis", experiment / "results"), + ) + + backend = SimpleNamespace( + client=object(), + get_insight=AsyncMock(return_value=SimpleNamespace(agent=str(tmp_path / "agent"))), + get_agent_code=AsyncMock(), + ) + config = EvolutionaryOptimizerConfig() + optimizer = object.__new__(EvolutionaryOptimizer) + optimizer.working_dir = experiment + optimizer.config = config + optimizer.shell = SimpleNamespace(close=AsyncMock()) + deps = SimpleNamespace( + backend=backend, + workspace="default", + config=config, + evaluator_type="harbor", + train_dataset=DatasetRef(uri=str(train)), + validation_dataset=DatasetRef(uri=str(validation)), + task_template=DatasetRef(uri=str(template)), + insight="insight-1", + agent=tmp_path / "agent", + agent_spec=None, + ) + + with pytest.raises(RuntimeError, match="stop after eval_author"): + await optimizer.run(deps) + + optimizer.shell.close.assert_awaited_once() + assert captured == { + "train": experiment / "dataset" / "train", + "validation": experiment / "dataset" / "validation", + "template": experiment / "dataset" / "task-template", + } + assert (train / "task-1" / "tests" / "test.sh").read_text(encoding="utf-8") == "train source" + assert not (template.parent / "generated-task").exists() diff --git a/plugins/nemo-experimentalist/tests/test_trace_explorer_from_ref.py b/plugins/nemo-experimentalist/tests/experimentalist/test_trace_explorer_from_ref.py similarity index 100% rename from plugins/nemo-experimentalist/tests/test_trace_explorer_from_ref.py rename to plugins/nemo-experimentalist/tests/experimentalist/test_trace_explorer_from_ref.py diff --git a/plugins/nemo-experimentalist/tests/test_eval_author_config.py b/plugins/nemo-experimentalist/tests/test_eval_author_config.py new file mode 100644 index 00000000000..23eb45b8aa1 --- /dev/null +++ b/plugins/nemo-experimentalist/tests/test_eval_author_config.py @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Experimentalist config wiring for the nested Eval Author settings block.""" + +import pytest +from nemo_eval_author_plugin.eval_author.models import EvalAuthorConfig +from nemo_experimentalist_plugin.experimentalist.components.loop import EvolutionaryOptimizerConfig + + +def test_evolutionary_optimizer_uses_top_level_eval_author_config() -> None: + config = EvolutionaryOptimizerConfig().eval_author + + assert type(config) is EvalAuthorConfig + assert config.max_validation_repair_attempts == 5 + + +def test_evolutionary_optimizer_tolerates_unknown_eval_author_config() -> None: + config = EvolutionaryOptimizerConfig.model_validate({"eval_author": {"bogus": 1}}) + assert type(config.eval_author) is EvalAuthorConfig + + +def test_evolutionary_optimizer_rejects_legacy_curator_config() -> None: + with pytest.raises(ValueError, match="'curator' was renamed to 'eval_author'"): + EvolutionaryOptimizerConfig.model_validate({"curator": {"max_traces": 1}}) diff --git a/pyproject.toml b/pyproject.toml index 99cd6f423ac..46e588b5e99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,8 +68,14 @@ version = "0.0.0" [dependency-groups] # Optional Insights analyst plugin. insights = ["nemo-insights-plugin"] -# Optional Experimentalist agent-optimization plugin. -experimentalist = ["nemo-experimentalist-plugin"] +# Optional Experimentalist agent-optimization plugin. Experimentalist depends on Eval Author +# because insight mode imports it at module scope; Eval Author still depends back on +# Experimentalist for evaluator/trace helpers, but that arrow is being removed (see +# plugins/nemo-eval-author/README.md). +experimentalist = [ + "nemo-experimentalist-plugin", + "nemo-eval-author-plugin", +] dev = [ "pre-commit>=3.8.0", "pydantic-settings>=2.6.1", @@ -365,6 +371,7 @@ nmp-testing = { workspace = true } nmp-build-tools = { workspace = true } nemo-platform-plugin = { workspace = true } nemo-insights-plugin = { workspace = true } +nemo-eval-author-plugin = { workspace = true } nemo-experimentalist-plugin = { workspace = true } # Not published to PyPI yet; pinned to an immutable public tag. nooa = { git = "https://github.com/NVIDIA-NeMo/labs-OO-Agents.git", tag = "v0.0.6" } @@ -434,6 +441,7 @@ members = [ "plugins/nemo-agents", "plugins/nemo-deployments", "plugins/nemo-insights", + "plugins/nemo-eval-author", "plugins/nemo-experimentalist", "plugins/nemo-agents/examples/calculator-agent", "plugins/nemo-agents/examples/email-phishing-analyzer", @@ -557,6 +565,8 @@ extra-paths = [ "plugins/nemo-evaluator/src", # The optional Insights analyst is not part of the default environment. "plugins/nemo-insights/src", + # Same for the optional Eval Author plugin. + "plugins/nemo-eval-author/src", # Same for the optional Experimentalist; its tests also import the benchmark runner. "plugins/nemo-experimentalist/src", # Agentic-use tests place both tests/agentic-use and tests/agentic-use/shared @@ -632,9 +642,12 @@ exclude = [ # nooa agents declare their LLM-generated methods as annotated signatures with an # ellipsis body; the framework fills them in at runtime. ty reads that as a missing -# return, so the rule is off for the Experimentalist's agent classes only. +# return, so the rule is off for the Experimentalist and Eval Author agent classes only. [[tool.ty.overrides]] -include = ["plugins/nemo-experimentalist/src/**"] +include = [ + "plugins/nemo-experimentalist/src/**", + "plugins/nemo-eval-author/src/**", +] [tool.ty.overrides.rules] empty-body = "ignore" diff --git a/tests/discovery_exclusions.py b/tests/discovery_exclusions.py index 0648577791e..7e2e609b769 100644 --- a/tests/discovery_exclusions.py +++ b/tests/discovery_exclusions.py @@ -17,5 +17,11 @@ if find_spec("nemo_experimentalist_plugin") is None: TEST_DISCOVERY_EXCLUSIONS[Path("plugins/nemo-experimentalist/tests")] = ( - "The optional experimentalist dependency group is 3.12-only, so it is absent from a 3.11 test environment." + "The optional experimentalist dependency group is not installed in the root test environment." + ) + +if find_spec("nemo_eval_author_plugin") is None: + TEST_DISCOVERY_EXCLUSIONS[Path("plugins/nemo-eval-author/tests")] = ( + "Eval Author ships in the optional experimentalist dependency group, which is not installed " + "in the root test environment." ) diff --git a/uv.lock b/uv.lock index ff076b0d3ff..eb6d0c48fa3 100644 --- a/uv.lock +++ b/uv.lock @@ -30,6 +30,7 @@ members = [ "nemo-customizer-plugin", "nemo-data-designer-plugin", "nemo-deployments-plugin", + "nemo-eval-author-plugin", "nemo-evaluator-plugin", "nemo-evaluator-sdk", "nemo-experimentalist-plugin", @@ -4029,6 +4030,31 @@ dev = [ { name = "pytest-asyncio", specifier = ">=0.25.3" }, ] +[[package]] +name = "nemo-eval-author-plugin" +version = "0.1.0" +source = { editable = "plugins/nemo-eval-author" } +dependencies = [ + { name = "harbor", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nemo-experimentalist-plugin", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nemo-insights-plugin", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nemo-platform", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nooa", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pydantic", extra = ["email"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "tomlkit", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, +] + +[package.metadata] +requires-dist = [ + { name = "harbor", specifier = ">=0.16" }, + { name = "nemo-experimentalist-plugin", editable = "plugins/nemo-experimentalist" }, + { name = "nemo-insights-plugin", editable = "plugins/nemo-insights" }, + { name = "nemo-platform", editable = "packages/nemo_platform" }, + { name = "nooa", git = "https://github.com/NVIDIA-NeMo/labs-OO-Agents.git?tag=v0.0.6" }, + { name = "pydantic", specifier = ">=2" }, + { name = "tomlkit", specifier = ">=0.13.3" }, +] + [[package]] name = "nemo-evaluator-plugin" version = "0.0.0" @@ -4139,6 +4165,7 @@ source = { editable = "plugins/nemo-experimentalist" } dependencies = [ { name = "harbor", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "httpx", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nemo-eval-author-plugin", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nemo-insights-plugin", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nemo-platform", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nemo-platform-plugin", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, @@ -4154,6 +4181,7 @@ dependencies = [ requires-dist = [ { name = "harbor", specifier = ">=0.16" }, { name = "httpx" }, + { name = "nemo-eval-author-plugin", editable = "plugins/nemo-eval-author" }, { name = "nemo-insights-plugin", editable = "plugins/nemo-insights" }, { name = "nemo-platform", editable = "packages/nemo_platform" }, { name = "nemo-platform-plugin", editable = "packages/nemo_platform_plugin" }, @@ -6131,6 +6159,7 @@ enabled-plugins = [ { name = "nemo-unsloth-plugin", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] experimentalist = [ + { name = "nemo-eval-author-plugin", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nemo-experimentalist-plugin", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] functional-services = [ @@ -6337,7 +6366,10 @@ enabled-plugins = [ { name = "nemo-switchyard", editable = "plugins/nemo-switchyard" }, { name = "nemo-unsloth-plugin", editable = "plugins/nemo-unsloth" }, ] -experimentalist = [{ name = "nemo-experimentalist-plugin", editable = "plugins/nemo-experimentalist" }] +experimentalist = [ + { name = "nemo-eval-author-plugin", editable = "plugins/nemo-eval-author" }, + { name = "nemo-experimentalist-plugin", editable = "plugins/nemo-experimentalist" }, +] functional-services = [ { name = "nemo-agents-plugin", editable = "plugins/nemo-agents" }, { name = "nemo-anonymizer-plugin", editable = "plugins/nemo-anonymizer" },