feat: extract Eval Author into nemo-eval-author-plugin

plugins/README.md

@@ -61,6 +61,7 @@ The package name is the `name` field in the plugin's `pyproject.toml`, not the d
 | `nemo-agents/` | `nemo-agents-plugin` |
 | `nemo-anonymizer/` | `nemo-anonymizer-plugin` |
 | `nemo-data-designer/` | `nemo-data-designer-plugin` |
+| `nemo-eval-author/` | `nemo-eval-author-plugin` |
 | `nemo-evaluator/` | `nemo-evaluator-plugin` |
 | `nemo-guardrails/` | `nemo-guardrails-plugin` |
 | `nemo-switchyard/` | `nemo-switchyard` |

plugins/nemo-eval-author/README.md

@@ -0,0 +1,36 @@
+# NeMo Eval Author Plugin
+
+Library-only plugin that owns the Eval Author agent, Harbor evaluator stack, trace analysis helpers, and dataset staging used to author evaluation suites from Insights.
+
+Experimentalist depends on this plugin the same way it depends on `nemo-insights-plugin`.
+
+## Public API
+
+```python
+from nemo_eval_author_plugin.eval_author.agent import EvalAuthor, build_eval_author_agent
+from nemo_eval_author_plugin.eval_author.models import EvalAuthorConfig, EvalAuthorResult
+from nemo_eval_author_plugin.eval_author.run import run_eval_author
+from nemo_eval_author_plugin.evaluator import Dataset, DatasetRef, Evaluator
+from nemo_eval_author_plugin.evaluator.factory import DatasetFactory, EvaluatorFactory
+from nemo_eval_author_plugin.dataset_staging import stage_eval_author_inputs, stage_task_template
+from nemo_eval_author_plugin.trace_analyzer import TraceAnalyzer, TraceAnalyzerConfig, Diagnostic
+from nemo_eval_author_plugin.trace_explorer import TraceExplorer
+```
+
+## Install
+
+From the repository root:
+
+```bash
+uv sync --group experimentalist
+```
+
+## TODO(shared-module)
+
+The following modules are exact copies of Experimentalist helpers and should eventually live in a shared package:
+
+- `tools.py`
+- `model_config.py`
+- `cache.py`
+- `client.py`
+- `repository.py` (agent clone helpers used by `backend.py`)

plugins/nemo-eval-author/pyproject.toml

@@ -0,0 +1,30 @@
+[project]
+name = "nemo-eval-author-plugin"
+version = "0.1.0"
+description = "Eval Author agent and Harbor evaluator stack for NeMo Platform."
+requires-python = ">=3.11,<3.14"
+dependencies = [
+    "pydantic>=2",
+    "httpx",
+    "harbor>=0.16 ; python_full_version >= '3.12'",
+    "opentelemetry-proto>=1.42.1",
+    "protobuf>=6.0.0",
+    "nooa ; python_full_version >= '3.12'",
+    "pyyaml>=6.0.3",
+    "nemo-insights-plugin",
+    "nemo-platform",
+    "nemo-platform-plugin",
+    "tomlkit>=0.13.3",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/nemo_eval_author_plugin"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+pythonpath = ["src"]
+testpaths = ["tests"]

plugins/nemo-eval-author/src/nemo_eval_author_plugin/backend.py

@@ -0,0 +1,122 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Minimal backend for standalone Eval Author runs.
+
+Replicates the ``get_insight`` and ``get_agent_code`` behavior from
+``LocalExperimentalistBackend`` so ``run_eval_author`` does not depend on
+Experimentalist.
+
+TODO(shared-module): unify agent materialization with Experimentalist backend.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import shutil
+import subprocess
+from pathlib import Path
+from typing import Any, Literal, TypeVar, cast
+
+import httpx
+from nemo_eval_author_plugin.repository import (
+    AgentSource,
+    clone_agent_repo,
+    looks_like_git,
+    split_agent_spec,
+    split_git_ref,
+)
+from nemo_insights_plugin.entities import Insight
+from nemo_platform import AsyncNeMoPlatform
+
+_ModelT = TypeVar("_ModelT")
+
+_AGENT_COPY_EXCLUDE_NAMES = {
+    ".git",
+    ".venv",
+    "artifacts",
+    "dataset",
+    "eval-and-optimize",
+    "scratch",
+}
+
+
+def _ignore_agent_copy(directory: str, contents: list[str]) -> set[str]:
+    del directory
+    return {name for name in contents if name in _AGENT_COPY_EXCLUDE_NAMES}
+
+
+def _load_entity(cls: type[_ModelT], path: Path) -> _ModelT:
+    """Deserialize *path* as JSON into *cls*, restoring the private ``_id`` field."""
+    data = json.loads(path.read_text())
+    entity_id = data.get("id", "")
+    obj = cls.model_validate(data)
+    if entity_id:
+        cast(Any, obj)._id = entity_id
+    return obj
+
+
+class EvalAuthorBackend:
+    """Insight and agent-code materialization for Eval Author runs."""
+
+    def __init__(self, *, client: AsyncNeMoPlatform | None, path: Path) -> None:
+        self.client = client
+        self.path = path
+
+    async def get_insight(self, *, workspace: str, insight_id: str) -> Insight:
+        p = Path(insight_id)
+        if p.exists():
+            return _load_entity(Insight, p)
+        if self.client is None:
+            raise ValueError(
+                f"Insight {insight_id!r} is not an existing local file and no platform "
+                "client is available to fetch it from the platform."
+            )
+        try:
+            return await self.client.insights.insights.get(workspace=workspace, insight_id=insight_id)
+        except httpx.HTTPStatusError as exc:
+            if exc.response.status_code == 404:
+                raise ValueError(f"Insight not found on the platform: {insight_id!r}") from exc
+            raise ValueError(f"Failed to fetch insight {insight_id!r} from the platform: {exc}") from exc
+
+    async def get_agent_code(
+        self, *, workspace: str, agent: str | Path, dest: Path, clone_depth: int | None = None
+    ) -> AgentSource | None:
+        del workspace
+        if looks_like_git(str(agent)):
+            return await self._clone_git_agent(str(agent), dest, clone_depth=clone_depth)
+        src = Path(agent)
+        if not src.exists():
+            raise FileNotFoundError(f"Local agent path not found: {src}")
+        if dest.resolve() != src.resolve():
+            if dest.exists():
+                shutil.rmtree(dest)
+            shutil.copytree(src, dest, ignore=_ignore_agent_copy)
+        return None
+
+    async def _clone_git_agent(self, agent: str, dest: Path, *, clone_depth: int | None = None) -> AgentSource:
+        try:
+            return await asyncio.to_thread(clone_agent_repo, agent, dest, clone_depth=clone_depth)
+        except subprocess.CalledProcessError:
+            remote, _ = split_git_ref(split_agent_spec(agent)[0])
+            from nemo_eval_author_plugin.repository import _redact_url
+
+            raise ValueError(f"failed to fetch --agent {_redact_url(remote)!r}") from None
+
+
+def make_eval_author_backend(
+    *,
+    client: AsyncNeMoPlatform | None,
+    experiments_output: str,
+    mode: Literal["local", "remote"],
+) -> EvalAuthorBackend:
+    """Select the Eval Author backend for *mode*.
+
+    Both local and remote modes use the same insight/agent materialization
+    implementation; remote mode requires a platform client for platform-backed
+    insight ids.
+    """
+    if client is None and mode == "remote":
+        raise ValueError("remote Eval Author backend requires a platform client")
+    return EvalAuthorBackend(client=client, path=Path(experiments_output))

plugins/nemo-eval-author/src/nemo_eval_author_plugin/cache.py

@@ -0,0 +1,121 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# TODO(shared-module): exact copy of experimentalist components/cache.py; unify into a shared package.
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+from pathlib import Path
+from typing import TypeVar
+from uuid import uuid4
+
+from pydantic import BaseModel, ValidationError
+
+_logger = logging.getLogger(__name__)
+
+# PEP 695 syntax would read better, but the workspace lints against Python 3.11.
+T = TypeVar("T", bound=BaseModel)
+
+
+def task_hash(task_name: str) -> str:
+    """Return a namespaced sha256 hex digest for a task key.
+
+    Args:
+        task_name: the task identifier to hash.
+
+    Returns:
+        str: a ``task-<hex>`` prefixed digest string.
+
+    """
+    digest = hashlib.sha256(task_name.encode()).hexdigest()
+    return f"task-{digest}"
+
+
+def agent_hash(agent_id: str) -> str:
+    """Return a namespaced sha256 hex digest for an agent key.
+
+    Args:
+        agent_id: the agent identifier to hash.
+
+    Returns:
+        str: an ``agent-<hex>`` prefixed digest string.
+
+    """
+    digest = hashlib.sha256(agent_id.encode()).hexdigest()
+    return f"agent-{digest}"
+
+
+def trace_hash(trace_path: str | Path) -> str:
+    """Return a namespaced sha256 hex digest of a trace file's contents.
+
+    Args:
+        trace_path: path to the trace file to hash.
+
+    Returns:
+        str: a ``trace-<hex>`` prefixed digest string.
+
+    """
+    h = hashlib.sha256()
+    with open(trace_path, "rb") as f:
+        for chunk in iter(lambda: f.read(1 << 20), b""):
+            h.update(chunk)
+    digest = h.hexdigest()
+    return f"trace-{digest}"
+
+
+def _cache_path(workspace: Path, key: str) -> Path:
+    return workspace / "eval-and-optimize" / "cache" / f"{key}.json"
+
+
+def load(workspace: Path, key: str, model: type[T]) -> T | None:
+    """Return a previously-stored model instance for this key, or None.
+
+    Args:
+        workspace: root workspace directory containing the cache.
+        key: cache key returned by one of the ``*_hash`` functions.
+        model: Pydantic model class used to deserialise the stored payload.
+
+    Returns:
+        T | None: the deserialised model instance, or None if the entry is
+            missing, unreadable, or fails validation.
+
+    """
+    path = _cache_path(workspace, key)
+    if not path.exists():
+        return None
+    try:
+        payload = json.loads(path.read_text())
+    except (OSError, ValueError) as exc:
+        _logger.warning(f"Cache read failed at {path}: {exc} -- ignoring")
+        return None
+    try:
+        return model.model_validate(payload)
+    except ValidationError as exc:
+        _logger.warning(f"Cache payload at {path} does not validate against {model.__name__}: {exc} -- ignoring")
+        return None
+
+
+def store(workspace: Path, key: str, value: BaseModel) -> None:
+    """Persist a model instance under the given key, overwriting any prior entry.
+
+    Args:
+        workspace: root workspace directory containing the cache.
+        key: cache key returned by one of the ``*_hash`` functions.
+        value: Pydantic model instance to serialise and store.
+
+    """
+    path = _cache_path(workspace, key)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    payload = value.model_dump(mode="json")
+    tmp = path.parent / f"{path.name}.{uuid4().hex}.tmp"
+    try:
+        tmp.write_text(json.dumps(payload, indent=2))
+        tmp.replace(path)
+    except OSError as exc:
+        _logger.warning(f"Cache write failed for key {key}: {exc} -- skipping")
+        try:
+            tmp.unlink(missing_ok=True)
+        except OSError:
+            pass

plugins/nemo-eval-author/src/nemo_eval_author_plugin/client.py

@@ -0,0 +1,51 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# TODO(shared-module): exact copy of experimentalist client.py; unify into a shared package.
+
+"""Shared NeMo Platform SDK client construction for Insight consumers.
+
+Auth lives in the active ``nemo auth login`` context in
+``~/.config/nmp/config.yaml``. The SDK only wires up that context (and the
+transparent OIDC token refresh that comes with it) when it runs its config
+bootstrap. Passing ``base_url`` *alone* puts the SDK in "direct mode", which
+skips the bootstrap and injects **no** auth headers — fine for an
+unauthenticated local ``nemo services run``, but it 401s against a remote
+deployment. To authenticate against a remote URL we must trigger the bootstrap
+(by also passing ``config_path``) so the explicit ``base_url`` is combined with
+the context's credentials.
+
+Every Platform Insight consumer takes ``base_url`` from its workflow context,
+so this helper is the one place that branch lives.
+"""
+
+from urllib.parse import urlparse
+
+from nemo_platform import AsyncNeMoPlatform
+from nemo_platform.config.config import Config
+
+# Loopback hosts are served by an unauthenticated local platform; attaching
+# (and refreshing) OAuth tokens there is both unnecessary and a failure mode
+# when the cached token is stale and OIDC discovery against localhost fails.
+LOOPBACK_HOSTS = frozenset({"localhost", "127.0.0.1", "::1", "0.0.0.0"})
+
+
+def make_client(base_url: str | None) -> AsyncNeMoPlatform:
+    """Construct an :class:`AsyncNeMoPlatform` honoring an optional ``base_url``.
+
+    - No ``base_url``: use the active nmp context for both URL and auth.
+    - Loopback ``base_url``: direct mode (local platform is unauthenticated).
+    - Remote ``base_url`` with an nmp config present: combine the URL with the
+      context's auth so the SDK injects and refreshes a Bearer token.
+    - Remote ``base_url`` without an nmp config: direct mode (no credentials to
+      use; the request will surface a clear auth error).
+    """
+    if not base_url:
+        return AsyncNeMoPlatform()
+
+    host = (urlparse(base_url).hostname or "").lower()
+    config_path = Config.get_default_config_path()
+    if host in LOOPBACK_HOSTS or not config_path.exists():
+        return AsyncNeMoPlatform(base_url=base_url)
+
+    return AsyncNeMoPlatform(base_url=base_url, config_path=config_path)

plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/dataset_staging.py → plugins/nemo-eval-author/src/nemo_eval_author_plugin/dataset_staging.py

@@ -8,7 +8,7 @@
 from pathlib import Path
 from urllib.parse import urlparse
 
-from nemo_experimentalist_plugin.experimentalist.components.evaluator.models import DatasetRef, local_path_from_uri
+from nemo_eval_author_plugin.evaluator.models import DatasetRef, local_path_from_uri
 from nemo_platform import AsyncNeMoPlatform