feat: extract Eval Author into nemo-eval-author-plugin

.gitignore

@@ -18,6 +18,7 @@ venv*
 .venv*
 .nemo/
 .env*
+!**/.env.example
 !**/env.py
 !packages/data_designer/**/environment.py
 !packages/nemo_data_designer/**/environment.py

plugins/README.md

@@ -61,6 +61,7 @@ The package name is the `name` field in the plugin's `pyproject.toml`, not the d
 | `nemo-agents/` | `nemo-agents-plugin` |
 | `nemo-anonymizer/` | `nemo-anonymizer-plugin` |
 | `nemo-data-designer/` | `nemo-data-designer-plugin` |
+| `nemo-eval-author/` | `nemo-eval-author-plugin` |
 | `nemo-evaluator/` | `nemo-evaluator-plugin` |
 | `nemo-guardrails/` | `nemo-guardrails-plugin` |
 | `nemo-switchyard/` | `nemo-switchyard` |

plugins/nemo-eval-author/.env.example

@@ -0,0 +1,43 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Example environment for standalone Eval Author runs.
+# Copy to `.env` next to this file (or export in your shell):
+#
+#   cp .env.example .env
+#
+# AUTHOR_* is Eval Author's credential contract. The EXPERIMENTALIST_* interplay
+# below is transitional and goes away once Eval Author no longer reuses any
+# Experimentalist code:
+#
+#   - an unset AUTHOR_* falls back to its EXPERIMENTALIST_* equivalent, so insight
+#     mode works from one Experimentalist profile .env
+#   - a set AUTHOR_* is copied into any unset EXPERIMENTALIST_* slot, because the
+#     Experimentalist helpers Eval Author still borrows read only EXPERIMENTALIST_*
+#
+# TODO(eval-author-standalone): remove both, at which point AUTHOR_* becomes required
+# here rather than optional. Set AUTHOR_* explicitly now to avoid that break.
+#
+# TODO(cli): wire `nemo eval-author` to load this `.env` the way Experimentalist
+# loads the profile-dir `.env`.
+
+# Required for LLM-backed Eval Author / TraceAnalyzer (NOOA CompletionClient).
+AUTHOR_API_BASE=https://inference-api.nvidia.com/v1
+AUTHOR_API_KEY=
+
+# Optional model overrides (defaults shown). Eval Author itself uses the smart and fast
+# tiers; the mid tier is listed only because it is bridged to Experimentalist helpers,
+# and it goes away with the bridge.
+# AUTHOR_SMART_MODEL_NAME=openai/openai/openai/gpt-5.5
+# AUTHOR_MID_MODEL_NAME=openai/gcp/google/gemini-3.5-flash
+# AUTHOR_FAST_MODEL_NAME=openai/openai/openai/gpt-5-mini
+
+# Optional: NVIDIA Inference Gateway virtual key. When AUTHOR_API_KEY is unset and
+# AUTHOR_API_BASE is the gateway over HTTPS, standalone runners fall back to
+# INFERENCE_API_KEY the same way Experimentalist does today. The HTTPS requirement
+# is deliberate: the key is never forwarded to a plain-http base.
+# INFERENCE_API_KEY=
+
+# Optional: NeMo Platform URL for Insight / Fileset / agent-code access.
+# Leave unset to use the active `nemo auth` context, or set for a local stack.
+# NMP_BASE_URL=http://localhost:8080

plugins/nemo-eval-author/README.md

@@ -0,0 +1,66 @@
+# NeMo Eval Author Plugin
+
+Library-only plugin that owns the Eval Author agent (`eval_author/`).
+
+## Direction of travel
+
+**Eval Author is meant to become standalone, with nothing imported from
+Experimentalist.** Prefer duplicating a helper over sharing one, even when sharing
+looks tidier.
+
+Right now the two packages depend on each other:
+
+| Arrow | Status | Why |
+| --- | --- | --- |
+| Experimentalist → Eval Author | permanent | insight mode imports `EvalAuthor` and `EvalAuthorConfig` at module scope |
+| Eval Author → Experimentalist | temporary | still borrows evaluator/Harbor, staging, trace, tools, cache, backend |
+
+[`tests/test_plugin_boundary.py`](tests/test_plugin_boundary.py) pins the second list
+so it can only shrink, and names what each remaining import is still for. `uv`
+resolves the current cycle; install both packages with:
+
+```bash
+uv sync --group experimentalist
+```
+
+## Public API
+
+```python
+from nemo_eval_author_plugin.eval_author.agent import EvalAuthor, build_eval_author_agent
+from nemo_eval_author_plugin.eval_author.models import EvalAuthorConfig, EvalAuthorResult
+from nemo_eval_author_plugin.eval_author.run import run_eval_author
+
+# Still borrowed from Experimentalist, and on the way out. Treat these as Eval Author's
+# own types once they move; do not build new code on the Experimentalist paths.
+from nemo_experimentalist_plugin.experimentalist.components.evaluator import Dataset
+from nemo_experimentalist_plugin.experimentalist.components.evaluator.models import DatasetRef
+from nemo_experimentalist_plugin.experimentalist.components.dataset_staging import stage_task_template
+from nemo_experimentalist_plugin.experimentalist.components.trace_analyzer import TraceAnalyzer
+from nemo_experimentalist_plugin.experimentalist.components.trace_explorer import TraceExplorer
+```
+
+## Credentials (standalone)
+
+Copy [`.env.example`](.env.example) to `.env` and set `AUTHOR_API_KEY` (and optionally model names / `NMP_BASE_URL`):
+
+```bash
+cp plugins/nemo-eval-author/.env.example plugins/nemo-eval-author/.env
+```
+
+`AUTHOR_*` is Eval Author's credential contract, and `model_config` imports nothing
+from Experimentalist. When the API base is the NVIDIA Inference Gateway over HTTPS,
+`INFERENCE_API_KEY` is also accepted.
+
+Two pieces of that module are transitional and disappear with the last
+Experimentalist import, both tagged `TODO(eval-author-standalone)`:
+
+- unset `AUTHOR_*` variables fall back to `EXPERIMENTALIST_*`, so insight mode works
+  from a single Experimentalist profile `.env`. Setting `AUTHOR_*` explicitly today
+  avoids the break when the fallback is removed.
+- importing `nemo_eval_author_plugin._env_bridge` copies `AUTHOR_*` into unset
+  `EXPERIMENTALIST_*` slots, so the Experimentalist helpers Eval Author still
+  borrows see credentials during a standalone run. `eval_author.agent` imports it
+  ahead of any Experimentalist agent, because those agents read the environment when
+  their class body executes.
+
+A `nemo eval-author` CLI that auto-loads this `.env` is not wired yet.

plugins/nemo-eval-author/pyproject.toml

@@ -0,0 +1,26 @@
+[project]
+name = "nemo-eval-author-plugin"
+version = "0.1.0"
+description = "Eval Author agent for NeMo Platform (hard-depends on Experimentalist for evaluator/trace helpers)."
+requires-python = ">=3.12,<3.14"
+dependencies = [
+    "pydantic>=2",
+    "harbor>=0.16",
+    "nooa",
+    "nemo-experimentalist-plugin",
+    "nemo-insights-plugin",
+    "nemo-platform",
+    "tomlkit>=0.13.3",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/nemo_eval_author_plugin"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+pythonpath = ["src"]
+testpaths = ["tests"]

plugins/nemo-eval-author/src/nemo_eval_author_plugin/_env_bridge.py

@@ -0,0 +1,23 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Import this module for its side effect: bridging ``AUTHOR_*`` credentials.
+
+Transitional. This module exists only because Eval Author still reuses Experimentalist
+agents, and it should be deleted along with the last ``nemo_experimentalist_plugin`` import
+in this package. ``tests/test_plugin_boundary.py`` tracks what is left to remove.
+
+``TraceAnalyzer`` and the other Experimentalist agents Eval Author reuses build their LLM
+client in the class body, so they read ``EXPERIMENTALIST_*`` the moment their module is
+first imported. A module that imports them therefore has to bridge *before* that import,
+which no function call inside the module can do.
+
+Importing this module is how Eval Author expresses that ordering. ``import
+nemo_eval_author_plugin._env_bridge`` sorts ahead of every ``from ...`` line in the same
+isort section, so the ordering is maintained by the linter rather than by an ``E402``
+waiver and a comment asking future readers not to reshuffle the imports.
+"""
+
+from nemo_eval_author_plugin.model_config import bridge_author_env_to_experimentalist
+
+bridge_author_env_to_experimentalist()

plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/README.md → plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/README.md

@@ -5,13 +5,15 @@ SPDX-License-Identifier: Apache-2.0
 
 # Eval Author
 
-The top-level `nemo_experimentalist_plugin.eval_author` package is the canonical Eval Author
+The top-level `nemo_eval_author_plugin.eval_author` package is the canonical Eval Author
 implementation. It turns an Experimentalist Insight and its production trace refs into
 evaluator dataset changes, creating or augmenting regression signals that
 capture the failure mode before optimization begins.
 
-Experimentalist is a consumer of this package: its insight mode imports and
-runs the top-level Eval Author before beginning optimization.
+This package hard-depends on Experimentalist for evaluator, staging, and trace
+helpers. Experimentalist insight mode imports and runs this Eval Author before
+beginning optimization (both packages are installed via the `experimentalist`
+uv group).
 
 ## Current Files
 
@@ -89,8 +91,8 @@ directly:
 import asyncio
 from pathlib import Path
 
-from nemo_experimentalist_plugin.eval_author.models import EvalAuthorConfig
-from nemo_experimentalist_plugin.eval_author.run import run_eval_author
+from nemo_eval_author_plugin.eval_author.models import EvalAuthorConfig
+from nemo_eval_author_plugin.eval_author.run import run_eval_author
 from nemo_experimentalist_plugin.experimentalist.components.evaluator.models import DatasetRef
 
 

plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/agent.py → plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/agent.py

@@ -12,8 +12,12 @@
 from pathlib import Path
 from typing import Any
 
-from nemo_experimentalist_plugin.eval_author.materialization import InsightSuite
-from nemo_experimentalist_plugin.eval_author.models import EvalAuthorConfig, EvalAuthorResult
+# Populates EXPERIMENTALIST_* from AUTHOR_*, which the Experimentalist agent imports below
+# read when their class bodies execute. Must stay ahead of them; isort keeps it there.
+import nemo_eval_author_plugin._env_bridge  # noqa: F401
+from nemo_eval_author_plugin.eval_author.materialization import InsightSuite
+from nemo_eval_author_plugin.eval_author.models import EvalAuthorConfig, EvalAuthorResult
+from nemo_eval_author_plugin.model_config import get_fast_model, get_smart_model
 from nemo_experimentalist_plugin.experimentalist.components import cache
 from nemo_experimentalist_plugin.experimentalist.components.evaluator import (
     Dataset,
@@ -22,7 +26,6 @@
     TrialResult,
 )
 from nemo_experimentalist_plugin.experimentalist.components.evaluator.models import ResourceRef
-from nemo_experimentalist_plugin.experimentalist.components.model_config import get_fast_model, get_smart_model
 from nemo_experimentalist_plugin.experimentalist.components.tools import GuardedShellTools
 from nemo_experimentalist_plugin.experimentalist.components.trace_analyzer import (
     Diagnostic,

plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/eval_author/run.py → plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/run.py

@@ -7,8 +7,8 @@
 from pathlib import Path
 from typing import Literal, Protocol, cast
 
+from nemo_eval_author_plugin.eval_author.models import EvalAuthorConfig, EvalAuthorResult
 from nemo_experimentalist_plugin.client import make_client
-from nemo_experimentalist_plugin.eval_author.models import EvalAuthorConfig, EvalAuthorResult
 from nemo_experimentalist_plugin.experimentalist.components.dataset_staging import stage_task_template
 from nemo_experimentalist_plugin.experimentalist.components.evaluator import Dataset, Task
 from nemo_experimentalist_plugin.experimentalist.components.evaluator.base import EvaluatorType
@@ -113,7 +113,7 @@ async def run_eval_author(
 
 def build_eval_author_agent(*, experiment_dir: Path, config: EvalAuthorConfig) -> _EvalAuthorAgent:
     """Build the LLM-backed Eval Author agent lazily."""
-    from nemo_experimentalist_plugin.eval_author.agent import build_eval_author_agent as _build_eval_author_agent
+    from nemo_eval_author_plugin.eval_author.agent import build_eval_author_agent as _build_eval_author_agent
 
     return _build_eval_author_agent(experiment_dir=experiment_dir, config=config)