diff --git a/plugins/nemo-eval-author/README.md b/plugins/nemo-eval-author/README.md index 6e105239c5..d3e249e986 100644 --- a/plugins/nemo-eval-author/README.md +++ b/plugins/nemo-eval-author/README.md @@ -54,7 +54,6 @@ For non-interactive and isolated environments, `NEMO_DEFAULT_MODEL` and `workspace/model-name` and refer to Model Entities on the target Platform. A `nemo agents eval-author` CLI is registered under `nemo.cli.agents` and -mounted by the agents plugin. Verb scaffolding is in place -(`discover`, `audit`, `propose`, `run`, `doctor`); bodies are still -placeholders until ASE-673–678 land. The library runner already uses the +mounted by the agents plugin. `discover` is implemented; `audit`, `propose`, +`run`, and `doctor` remain placeholders. The library runner already uses the configured Platform model pair. diff --git a/plugins/nemo-eval-author/pyproject.toml b/plugins/nemo-eval-author/pyproject.toml index 33212fcb09..1ba44422c9 100644 --- a/plugins/nemo-eval-author/pyproject.toml +++ b/plugins/nemo-eval-author/pyproject.toml @@ -5,12 +5,14 @@ description = "Eval Author agent for NeMo Platform (hard-depends on Experimental requires-python = ">=3.12,<3.14" dependencies = [ "pydantic>=2", - "harbor>=0.16", + # Harbor 0.18 provides the discovery APIs used by this plugin. + "harbor>=0.18", "nooa", "nemo-experimentalist-plugin", "nemo-insights-plugin", "nemo-platform", "nemo-platform-plugin", + "pyyaml>=6.0.3", "tomlkit>=0.13.3", ] diff --git a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/cli.py b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/cli.py index 3f35bcb7a3..5f5db9b429 100644 --- a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/cli.py +++ b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/cli.py @@ -1,35 +1,51 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Eval Author plugin CLI — ``nemo agents eval-author ...`` subcommands. +"""Eval Author commands under ``nemo agents eval-author``.""" -Registered under ``nemo.cli.agents`` and mounted by ``AgentsCLI`` as -``nemo agents eval-author ``. - -Scaffolding. Every verb is registered under its final name so the command tree is -discoverable and the child tickets have a landing spot, and each body exits non-zero -until its own ticket lands. Flags belong to those tickets, so nothing here declares -options yet. - -This module imports nothing from ``eval_author`` while the command bodies remain -placeholders. The completed runner resolves the active Platform model pair at run time. -""" - -from typing import ClassVar, NoReturn +import asyncio +from pathlib import Path +from typing import Annotated, ClassVar, NoReturn import typer +from nemo_eval_author_plugin.discovery import run as discovery +from nemo_insights_plugin.contracts.checks import format_report from nemo_platform_plugin.cli import NemoCLI def _not_implemented(ctx: typer.Context, ticket: str) -> NoReturn: - """Fail loudly, so a placeholder verb can never be mistaken for a successful run. - - The message quotes ``ctx.command_path`` rather than a hardcoded path. - """ typer.echo(f"`{ctx.command_path}` is not implemented yet ({ticket}).", err=True) raise typer.Exit(code=1) +def _report_discovery(result: discovery.DiscoverResult) -> None: + status = format_report(result.report.checks) + if status: + typer.echo(status) + if result.report.run_command: + typer.echo("") + typer.echo(f"Run: {result.report.run_command}") + + typer.echo("") + if result.dry_run: + typer.echo("Dry run: no files were uploaded.") + typer.echo("") + typer.echo(result.markdown, nl=False) + elif result.uploaded: + remote_path = f"{result.report.agent}/{discovery.REPORT_FILENAME}" + typer.echo(f"Uploaded {remote_path} to fileset '{discovery.FILESET_NAME}'.") + else: + typer.echo(f"Upload failed: {result.upload_error or 'unknown error'}", err=True) + + failures = sum(check.status == "fail" for check in result.report.checks) + failures += not result.dry_run and not result.uploaded + warnings = sum(check.status == "warn" for check in result.report.checks) + failure_label = "failure" if failures == 1 else "failures" + warning_label = "warning" if warnings == 1 else "warnings" + status = "passed" if result.ok else "failed" + typer.echo(f"Final overview: Discovery {status} with {failures} {failure_label} and {warnings} {warning_label}.") + + class EvalAuthorCLI(NemoCLI): """``nemo agents eval-author ...`` subcommands.""" @@ -41,17 +57,43 @@ def get_cli(self) -> typer.Typer: @app.callback() def _root() -> None: - """Force subcommand dispatch even when only one verb is registered.""" + """Select an Eval Author command.""" @app.command("discover") - def discover(ctx: typer.Context) -> None: - """Discover candidate evaluation cases from agent traces.""" - # TODO(ASE-677): declare flags and wire discovery. - _not_implemented(ctx, "ASE-677") + def discover( + repo: Annotated[ + Path, + typer.Option("--repo", help="Repository that contains the agent.", exists=True, file_okay=False), + ] = Path(), + agent: Annotated[ + str | None, + typer.Option("--agent", help="Agent name. The default comes from optimizer.yaml or the directory."), + ] = None, + dry_run: Annotated[ + bool, + typer.Option("--dry-run", help="Print discovery.md without an upload."), + ] = False, + ) -> None: + """Inspect the repository and record its Harbor preflight. + + WARNING: Use this command only with a trusted repository. + Agent imports execute module top-level code. + """ + result = asyncio.run( + discovery.discover( + discovery.DiscoverOptions( + repo_root=repo, + agent=agent, + dry_run=dry_run, + ) + ) + ) + _report_discovery(result) + raise typer.Exit(code=0 if result.ok else 1) @app.command("audit") def audit(ctx: typer.Context) -> None: - """Audit an existing eval suite for coverage gaps.""" + """Report coverage gaps in an existing eval suite.""" # TODO(ASE-676): declare flags and wire the audit. _not_implemented(ctx, "ASE-676") @@ -63,13 +105,13 @@ def propose(ctx: typer.Context) -> None: @app.command("run") def run(ctx: typer.Context) -> None: - """Run the Eval Author pipeline end to end.""" + """Run the Eval Author pipeline.""" # TODO(ASE-673): declare flags and wire the pipeline to run_eval_author. _not_implemented(ctx, "ASE-673") @app.command("doctor") def doctor(ctx: typer.Context) -> None: - """Diagnose Eval Author setup: credentials, platform, runtime.""" + """Diagnose credentials, platform access, and the runtime.""" # TODO(ASE-678): report the prerequisites the other verbs gate on. _not_implemented(ctx, "ASE-678") diff --git a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/discovery/report.py b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/discovery/report.py new file mode 100644 index 0000000000..a241c89101 --- /dev/null +++ b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/discovery/report.py @@ -0,0 +1,193 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Discovery report contract and Markdown renderer.""" + +import shlex +from dataclasses import dataclass, field +from datetime import UTC, datetime +from importlib.metadata import version +from pathlib import Path +from typing import Any + +import yaml +from nemo_eval_author_plugin.discovery.scan import RepositoryScan +from nemo_eval_author_plugin.discovery.validate import RequiredEnvVar, ValidationOutcome +from nemo_insights_plugin.contracts.checks import CheckResult, format_report, required_failures + + +@dataclass +class ConfigReport: + """Preflight results for one Harbor config.""" + + name: str + path: Path + required_env_vars: list[RequiredEnvVar] + checks: list[CheckResult] + + @property + def runnable(self) -> bool: + """Return whether the config passed all required checks.""" + return not required_failures(self.checks) + + +@dataclass +class DiscoveryReport: + """Repository facts and current Harbor preflight results.""" + + agent: str + workspace: str + repo_root: Path + configs: list[ConfigReport] + dataset_paths: list[Path] + ethos_path: str | None + harbor_version: str + discovered_at: datetime + fingerprint: str + input_file_count: int + repository_checks: list[CheckResult] + trace_check: CheckResult + schema_version: int = field(init=False, default=1) + + @property + def runnable(self) -> bool: + """Return whether all repository-owned configs passed required checks.""" + return bool(self.configs) and all(config.runnable for config in self.configs) + + @property + def checks(self) -> list[CheckResult]: + """Return repository, config, and trace checks in execution order.""" + checks = list(self.repository_checks) + for config in self.configs: + checks.extend(config.checks) + checks.append(self.trace_check) + return checks + + @property + def run_command(self) -> str | None: + """Return the Harbor command only for one runnable config.""" + if len(self.configs) != 1: + return None + return self.run_command_for(self.configs[0]) + + def run_command_for(self, config: ConfigReport) -> str | None: + """Return the Harbor command for one runnable config.""" + if not config.runnable: + return None + config_path = config.path.resolve().relative_to(self.repo_root.resolve()).as_posix() + cd_command = shlex.join(["cd", str(self.repo_root.resolve())]) + harbor_command = shlex.join(["harbor", "job", "start", "-c", config_path]) + return f"{cd_command} && {harbor_command}" + + +def harbor_version() -> str: + """Return the installed Harbor version.""" + return version("harbor") + + +def build_report( + *, + agent: str, + workspace: str, + repo_root: Path, + scan_result: RepositoryScan, + validations: list[ValidationOutcome], + trace_check: CheckResult, + discovered_at: datetime | None = None, +) -> DiscoveryReport: + """Build one report from the repository scan and config preflights.""" + configs = [ + ConfigReport( + name=candidate.name, + path=candidate.path, + required_env_vars=validation.required_env_vars, + checks=validation.checks, + ) + for candidate, validation in zip(scan_result.configs, validations, strict=True) + ] + return DiscoveryReport( + agent=agent, + workspace=workspace, + repo_root=repo_root.resolve(), + configs=configs, + dataset_paths=scan_result.dataset_paths, + ethos_path=scan_result.ethos_path, + harbor_version=harbor_version(), + discovered_at=discovered_at or datetime.now(UTC), + fingerprint=f"sha256:{scan_result.fingerprint}", + input_file_count=scan_result.input_file_count, + repository_checks=scan_result.checks, + trace_check=trace_check, + ) + + +def render_markdown(report: DiscoveryReport) -> str: + """Render YAML front matter and a concise status report.""" + front = yaml.safe_dump(_front_matter(report), sort_keys=False, default_flow_style=False).rstrip() + lines = [f"# Discovery report for `{report.agent}`"] + status = format_report([*report.repository_checks, report.trace_check]) + if status: + lines.extend(["", "```text", status, "```"]) + if report.configs: + lines.extend(["", "## Harbor entrypoints"]) + for config in report.configs: + path = _display_path(config.path, report.repo_root) + lines.extend( + [ + "", + f"### `{config.name}` (`{path}`)", + "", + f"Runnable: {'true' if config.runnable else 'false'}", + "", + "```text", + format_report(config.checks), + "```", + ] + ) + if command := report.run_command_for(config): + lines.extend(["", "```bash", command, "```"]) + body = "\n".join(lines) + return f"---\n{front}\n---\n\n{body}\n" + + +def _front_matter(report: DiscoveryReport) -> dict[str, Any]: + config = report.configs[0] if len(report.configs) == 1 else None + return { + "schema_version": report.schema_version, + "agent": report.agent, + "workspace": report.workspace, + "repo_root": str(report.repo_root), + "runnable": report.runnable, + "configs": [ + {"name": config.name, "path": _display_path(config.path, report.repo_root)} for config in report.configs + ], + "config_path": _display_path(config.path if config is not None else None, report.repo_root), + "dataset_paths": [_display_path(path, report.repo_root) for path in report.dataset_paths], + "run_command": report.run_command, + "ethos_path": report.ethos_path, + "harbor_version": report.harbor_version, + "required_env_vars": [ + { + "name": item.name, + "default": item.default, + "declared_in": _display_path(item.declared_in, report.repo_root), + } + for config in report.configs + for item in config.required_env_vars + ], + "discovered_at": report.discovered_at.isoformat(), + "fingerprint": report.fingerprint, + "input_file_count": report.input_file_count, + "checks": [check.model_dump(mode="json") for check in report.checks], + } + + +def _display_path(path: Path | None, repo_root: Path) -> str | None: + if path is None: + return None + if not path.is_absolute(): + return path.as_posix() + try: + return path.resolve().relative_to(repo_root.resolve()).as_posix() + except ValueError: + return path.as_posix() diff --git a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/discovery/run.py b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/discovery/run.py new file mode 100644 index 0000000000..016c1e0401 --- /dev/null +++ b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/discovery/run.py @@ -0,0 +1,121 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Orchestration for ``nemo agents eval-author discover``.""" + +import re +from dataclasses import dataclass +from pathlib import Path + +import yaml +from nemo_eval_author_plugin.discovery import report, scan, validate +from nemo_experimentalist_plugin.client import make_client +from nemo_platform import AsyncNeMoPlatform +from nemo_platform.config.config import Config + +FILESET_NAME = "nemo-eval-author" +REPORT_FILENAME = "discovery.md" +_SLUG_PATTERN = re.compile(r"[^a-z0-9]+") + + +@dataclass +class DiscoverOptions: + """Resolved command options.""" + + repo_root: Path + agent: str | None = None + dry_run: bool = False + + +@dataclass +class DiscoverResult: + """The report and upload result for one invocation.""" + + report: report.DiscoveryReport + markdown: str + uploaded: bool = False + dry_run: bool = False + upload_error: str | None = None + + @property + def ok(self) -> bool: + """Return the command exit condition.""" + return self.report.runnable and (self.dry_run or self.uploaded) + + +async def discover(options: DiscoverOptions) -> DiscoverResult: + """Scan, validate, report, and optionally upload one repository.""" + repo_root = options.repo_root.resolve() + agent = _slug(options.agent) if options.agent is not None else _infer_agent_name(repo_root) + workspace = _active_workspace() + client = make_client(None) + try: + return await _discover(client, options, repo_root=repo_root, agent=agent, workspace=workspace) + finally: + await client.close() + + +async def _discover( + client: AsyncNeMoPlatform, + options: DiscoverOptions, + *, + repo_root: Path, + agent: str, + workspace: str, +) -> DiscoverResult: + ref = f"{workspace}/{agent}-spec#AGENT-SPEC.md" + try: + platform_ethos = (ref, await client.files.download_content(remote_path=ref)) + except Exception: + platform_ethos = None + scan_result = scan.scan_repository(repo_root, platform_ethos=platform_ethos) + validations = [await validate.run_ladder(config, repo_root) for config in scan_result.configs] + trace_check = await scan.probe_traces(client, agent=agent, workspace=workspace) + record = report.build_report( + agent=agent, + workspace=workspace, + repo_root=repo_root, + scan_result=scan_result, + validations=validations, + trace_check=trace_check, + ) + markdown = report.render_markdown(record) + result = DiscoverResult(report=record, markdown=markdown, dry_run=options.dry_run) + if options.dry_run: + return result + + try: + await client.files.upload_content( + content=markdown.encode("utf-8"), + remote_path=f"{agent}/{REPORT_FILENAME}", + fileset=FILESET_NAME, + workspace=workspace, + fileset_auto_create=True, + ) + except Exception as exc: + result.upload_error = f"{type(exc).__name__}: {exc}" + else: + result.uploaded = True + return result + + +def _active_workspace() -> str: + return Config.load().resolve().workspace + + +def _infer_agent_name(repo_root: Path) -> str: + """Read a root profile name, or use the repository directory.""" + try: + data = yaml.safe_load((repo_root / "optimizer.yaml").read_text(encoding="utf-8")) + except (OSError, UnicodeError, yaml.YAMLError): + data = None + declared = data.get("agent") if isinstance(data, dict) else None + return _slug(declared) if isinstance(declared, str) and declared.strip() else _slug(repo_root.name) + + +def _slug(value: str) -> str: + slug = _SLUG_PATTERN.sub("-", value.strip().lower()).strip("-") + return slug or "agent" + + +__all__ = ["DiscoverOptions", "DiscoverResult", "discover"] diff --git a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/discovery/scan.py b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/discovery/scan.py new file mode 100644 index 0000000000..be90dcc310 --- /dev/null +++ b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/discovery/scan.py @@ -0,0 +1,220 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Scan repository-owned Harbor inputs.""" + +import hashlib +import json +import os +from collections.abc import Iterator +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml +from nemo_insights_plugin.contracts.checks import CheckResult, CheckSeverity, CheckStatus + +_CONFIG_SUFFIXES = (".yaml", ".yml", ".json") +_MAX_CONFIG_DEPTH = 4 +_PRUNE_DIR_NAMES = frozenset( + { + ".git", + ".venv", + "venv", + "node_modules", + "__pycache__", + ".ruff_cache", + ".pytest_cache", + ".mypy_cache", + ".tox", + ".eggs", + ".cache", + "site-packages", + "vendor", + "cache", + "dist", + "build", + "eval-and-optimize", + ".nemo-optimizer", + "jobs", + } +) + + +@dataclass(frozen=True) +class ConfigCandidate: + """A repository-owned Harbor config file.""" + + path: Path + data: dict[str, Any] + + @property + def name(self) -> str: + """Return the declared job name or the file name.""" + job_name = self.data.get("job_name") + return job_name.strip() if isinstance(job_name, str) and job_name.strip() else self.path.name + + +@dataclass +class RepositoryScan: + """The repository facts that the validation ladder needs.""" + + configs: list[ConfigCandidate] + dataset_paths: list[Path] + ethos_path: str | None + fingerprint: str + input_file_count: int + checks: list[CheckResult] + + +def _check( + name: str, + status: CheckStatus, + message: str, + *, + severity: CheckSeverity = "required", + hint: str | None = None, +) -> CheckResult: + return CheckResult(name=name, group="repository", status=status, severity=severity, message=message, hint=hint) + + +def walk_dirs(root: Path, *, max_depth: int | None = None) -> Iterator[Path]: + """Yield repository directories and skip generated trees.""" + for current, dir_names, _ in os.walk(root): + directory = Path(current) + depth = len(directory.relative_to(root).parts) + dir_names[:] = sorted( + name for name in dir_names if name not in _PRUNE_DIR_NAMES and (max_depth is None or depth < max_depth) + ) + yield directory + + +def scan_repository(repo_root: Path, *, platform_ethos: tuple[str, bytes] | None = None) -> RepositoryScan: + """Find repo-owned configs and local Harbor datasets.""" + repo_root = repo_root.resolve() + configs = _config_candidates(repo_root) + checks: list[CheckResult] = [] + if not configs: + checks.append( + _check( + "config", + "fail", + "No repository-owned Harbor config file exists.", + hint="Add a YAML, YML, or JSON config with a nonempty datasets or tasks list.", + ) + ) + else: + count = len(configs) + checks.append( + _check("config", "pass", f"Found {count} repository-owned Harbor config file{'s' if count != 1 else ''}.") + ) + + ethos = platform_ethos + if ethos is None and (repo_root / "ETHOS.md").is_file(): + ethos = ("ETHOS.md", (repo_root / "ETHOS.md").read_bytes()) + if ethos is not None: + checks.append(_check("ethos", "pass", f"{ethos[0]} defines the agent doctrine.", severity="advisory")) + else: + checks.append( + _check( + "ethos", + "warn", + "ETHOS.md does not exist at the repository root.", + severity="advisory", + hint="Add ETHOS.md to define the agent doctrine.", + ) + ) + + datasets = _dataset_paths(repo_root) + fingerprint, count = _fingerprint(repo_root, [config.path for config in configs], ethos, datasets) + return RepositoryScan(configs, datasets, ethos[0] if ethos else None, fingerprint, count, checks) + + +def _config_candidates(repo_root: Path) -> list[ConfigCandidate]: + candidates: list[ConfigCandidate] = [] + for directory in walk_dirs(repo_root, max_depth=_MAX_CONFIG_DEPTH): + for path in sorted(directory.iterdir()): + if path.is_symlink() or not path.is_file() or path.suffix.lower() not in _CONFIG_SUFFIXES: + continue + data = _load_mapping(path) + if data is not None and _has_work(data): + candidates.append(ConfigCandidate(path=path, data=data)) + return sorted( + candidates, + key=lambda candidate: ( + len(candidate.path.relative_to(repo_root).parts) - 1, + candidate.path.relative_to(repo_root).as_posix(), + ), + ) + + +def _load_mapping(path: Path) -> dict[str, Any] | None: + try: + text = path.read_text(encoding="utf-8") + data = json.loads(text) if path.suffix.lower() == ".json" else yaml.safe_load(text) + except (OSError, UnicodeError, json.JSONDecodeError, yaml.YAMLError): + return None + return data if isinstance(data, dict) else None + + +def _has_work(data: dict[str, Any]) -> bool: + return any(isinstance(data.get(name), list) and data[name] for name in ("datasets", "tasks")) + + +def _dataset_paths(repo_root: Path) -> list[Path]: + datasets: set[Path] = set() + for directory in walk_dirs(repo_root): + if directory != repo_root and directory.name != "task_template" and (directory / "task.toml").is_file(): + datasets.add(directory.parent) + return sorted(datasets) + + +def _fingerprint( + repo_root: Path, + config_paths: list[Path], + ethos: tuple[str, bytes] | None, + datasets: list[Path], +) -> tuple[str, int]: + files = {path for path in [*config_paths, repo_root / "optimizer.yaml"] if path.is_file()} + for dataset in datasets: + if not dataset.is_relative_to(repo_root): + continue + for directory in walk_dirs(dataset): + files.update( + path for path in directory.iterdir() if path.is_file() and path.resolve().is_relative_to(repo_root) + ) + files.discard(repo_root / "ETHOS.md") + + digest = hashlib.sha256() + for path in sorted(files): + digest.update(str(path.relative_to(repo_root)).encode()) + digest.update(b"\0") + digest.update(path.read_bytes()) + digest.update(b"\0") + if ethos is not None: + digest.update(ethos[0].encode() + b"\0" + ethos[1] + b"\0") + return digest.hexdigest(), len(files) + (ethos is not None) + + +async def probe_traces(client: Any, *, agent: str, workspace: str) -> CheckResult: + """Check whether Intake has traces for later authoring steps.""" + try: + page = await client.intake.spans.groups.list( + workspace=workspace, + by="session_id", + page=1, + page_size=1, + filter={"agent_name": agent}, + sort="-span_count", + ) + except Exception as exc: + return _check( + "traces", + "warn", + f"Cannot read traces for {agent}: {type(exc).__name__}: {exc}", + severity="advisory", + ) + total = page.pagination.total_results if page.pagination is not None else len(page.data) + if not total: + return _check("traces", "warn", f"No traces exist for {agent}.", severity="advisory") + return _check("traces", "pass", f"{total} trace sessions exist for {agent}.", severity="advisory") diff --git a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/discovery/validate.py b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/discovery/validate.py new file mode 100644 index 0000000000..3761c8e907 --- /dev/null +++ b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/discovery/validate.py @@ -0,0 +1,309 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run Harbor preflight checks against a repository-owned config.""" + +import contextlib +import shutil +import subprocess +import sys +import tempfile +import tomllib +from collections.abc import Iterator +from dataclasses import dataclass, field +from fnmatch import fnmatchcase +from pathlib import Path + +from harbor.agents.factory import AgentFactory +from harbor.environments.factory import EnvironmentFactory +from harbor.job import Job +from harbor.models.agent.name import AgentName +from harbor.models.job.config import JobConfig +from harbor.models.task.config import TaskConfig +from harbor.models.task.paths import TaskPaths +from harbor.models.task.task import Task +from harbor.utils.env import get_required_host_vars +from harbor.utils.import_path import import_class +from nemo_eval_author_plugin.discovery.scan import ConfigCandidate +from nemo_insights_plugin.contracts.checks import CheckResult, CheckSeverity, CheckStatus +from pydantic import ValidationError + + +@dataclass +class RequiredEnvVar: + """A host variable required by a Harbor config.""" + + name: str + default: str | None + declared_in: Path + + +@dataclass +class ValidationOutcome: + """Results from one Harbor preflight.""" + + checks: list[CheckResult] = field(default_factory=list) + required_env_vars: list[RequiredEnvVar] = field(default_factory=list) + + +def _check( + name: str, + status: CheckStatus, + message: str, + *, + severity: CheckSeverity = "required", + hint: str | None = None, +) -> CheckResult: + return CheckResult(name=name, group="validation", status=status, severity=severity, message=message, hint=hint) + + +async def run_ladder(candidate: ConfigCandidate, repo_root: Path) -> ValidationOutcome: + """Run the complete preflight without caching or skipping any check.""" + outcome = ValidationOutcome() + with contextlib.chdir(repo_root): + try: + config = JobConfig.model_validate(candidate.data) + config.validate_agent_concurrency_limits() + except ValidationError as exc: + errors = exc.errors(include_url=False, include_input=False) + outcome.checks.append(_check("schema", "fail", f"Harbor rejected the job config: {errors}")) + return outcome + except ValueError as exc: + outcome.checks.append(_check("schema", "fail", f"Harbor rejected the job config: {exc}")) + return outcome + outcome.checks.append(_check("schema", "pass", "Harbor accepts the job config schema.")) + + job = await _resolve(config, outcome) + _check_agent(config, outcome) + _check_backend(config, outcome) + outcome.checks.append(check_config_file(candidate.path, repo_root)) + if job is None: + return outcome + + resolved = _resolved_task_paths(job) + if resolved is None: + outcome.checks.append( + _check( + "compatibility", + "fail", + "This Harbor version does not expose Job._task_configs.", + hint="Install a Harbor version that exposes the resolved task list.", + ) + ) + return outcome + task_dirs = _check_tasks(resolved, outcome) + _check_coverage(config, resolved, outcome) + _check_required_env_vars(config, task_dirs, outcome) + return outcome + + +async def _resolve(config: JobConfig, outcome: ValidationOutcome) -> Job | None: + try: + with tempfile.TemporaryDirectory(prefix="eval-author-jobs-") as scratch: + job = await Job.create(config.model_copy(update={"jobs_dir": Path(scratch)})) + job._close_logger_handlers() + except Exception as exc: + outcome.checks.append( + _check( + "resolution", + "fail", + f"Harbor could not resolve the job: {type(exc).__name__}: {exc}", + hint="This error occurs before Harbor starts a container.", + ) + ) + return None + outcome.checks.append(_check("resolution", "pass", "Harbor resolved the job.")) + return job + + +def _resolved_task_paths(job: Job) -> list[Path] | None: + task_configs = getattr(job, "_task_configs", None) + if task_configs is None: + return None + paths: list[Path] = [] + for task_config in task_configs: + try: + paths.append(task_config.get_local_path().resolve()) + except ValueError: + continue + return paths + + +def _check_tasks(resolved: list[Path], outcome: ValidationOutcome) -> list[Path]: + valid = [path for path in resolved if Task.is_valid_dir(path)] + if not resolved: + outcome.checks.append(_check("tasks", "fail", "The config resolves to zero tasks.")) + return [] + outcome.checks.append( + _check( + "tasks", + "fail" if len(valid) != len(resolved) else "pass", + f"{len(valid)} of {len(resolved)} task dirs are valid Harbor tasks.", + ) + ) + return valid + + +def _check_coverage(config: JobConfig, resolved: list[Path], outcome: ValidationOutcome) -> None: + resolved_set = {path.resolve() for path in resolved} + dropped_any = False + for dataset in config.datasets: + if dataset.path is None or not dataset.path.is_dir(): + continue + on_disk = [ + child + for child in sorted(dataset.path.iterdir()) + if child.is_dir() and child.name != "task_template" and (child / "task.toml").is_file() + ] + dropped = [child for child in on_disk if child.resolve() not in resolved_set] + if not dropped: + continue + dropped_any = True + selected_dropped = [ + path + for path in dropped + if (not dataset.task_names or any(fnmatchcase(path.name, pattern) for pattern in dataset.task_names)) + and not any(fnmatchcase(path.name, pattern) for pattern in dataset.exclude_task_names or []) + ] + required = bool(selected_dropped) and dataset.n_tasks is None + filtered = bool(dataset.task_names or dataset.exclude_task_names) + reported = selected_dropped if required else dropped + names = ", ".join(path.name for path in reported) + outcome.checks.append( + _check( + "coverage", + "fail" if required else "warn", + f"Harbor did not resolve {len(reported)} task dirs: {names}.", + severity="required" if required else "advisory", + hint=( + "Harbor skipped a task selected by the dataset filters." + if required and filtered + else "Harbor skips these task dirs silently." + if required + else "The dataset filters or n_tasks select a task subset." + ), + ) + ) + if not dropped_any: + outcome.checks.append(_check("coverage", "pass", "Harbor dropped no local task dirs.")) + + +def _check_required_env_vars(config: JobConfig, task_dirs: list[Path], outcome: ValidationOutcome) -> None: + required: dict[str, RequiredEnvVar] = {} + + def collect(env: dict[str, str], declared_in: Path) -> None: + for name, default in get_required_host_vars(env): + required.setdefault(name, RequiredEnvVar(name, default, declared_in)) + + for task_dir in task_dirs: + task_config = _task_config(task_dir) + if task_config is None: + continue + path = TaskPaths(task_dir).config_path + collect(task_config.environment.env, path) + collect(task_config.verifier.env, path) + collect(task_config.solution.env, path) + collect(config.environment.env, Path("")) + collect(config.verifier.env, Path("")) + for agent in config.agents: + collect(agent.env, Path("")) + outcome.required_env_vars = sorted(required.values(), key=lambda item: item.name) + names = ", ".join(item.name for item in outcome.required_env_vars) + outcome.checks.append( + _check("credentials", "pass", f"{len(required)} host variables required" + (f": {names}." if names else ".")) + ) + + +def _task_config(task_dir: Path) -> TaskConfig | None: + try: + return TaskConfig.model_validate_toml(TaskPaths(task_dir).config_path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, tomllib.TOMLDecodeError, ValidationError): + return None + + +def _check_agent(config: JobConfig, outcome: ValidationOutcome) -> None: + for agent in config.agents: + if agent.import_path is not None: + with _evict_module_tree(agent.import_path): + try: + imported = import_class(agent.import_path, label="agent") + except (Exception, SystemExit) as exc: + outcome.checks.append( + _check("agent", "fail", f"Cannot import agent {agent.import_path}: {type(exc).__name__}: {exc}") + ) + else: + outcome.checks.append(_check("agent", "pass", f"Agent {imported.__name__} imports as a class.")) + elif agent.name is not None: + try: + AgentFactory.get_agent_class(AgentName(agent.name)) + except Exception as exc: + outcome.checks.append( + _check("agent", "fail", f"Cannot load Harbor agent {agent.name}: {type(exc).__name__}: {exc}") + ) + else: + outcome.checks.append(_check("agent", "pass", f"Built-in agent {agent.name} is available.")) + + +def _check_backend(config: JobConfig, outcome: ValidationOutcome) -> None: + label = config.environment.import_path or (config.environment.type.value if config.environment.type else "docker") + try: + EnvironmentFactory.run_preflight(config.environment.type, config.environment.import_path) + except (Exception, SystemExit) as exc: + outcome.checks.append( + _check("backend", "fail", f"Environment backend {label} is not ready: {type(exc).__name__}: {exc}") + ) + else: + outcome.checks.append(_check("backend", "pass", f"Environment backend {label} passed preflight.")) + + +def check_config_file(config_path: Path, repo_root: Path) -> CheckResult: + """Check the bytes that Harbor receives from its CLI.""" + harbor = _harbor_executable() + if harbor is None: + return _check( + "round-trip", + "warn", + "The Harbor CLI round trip did not run.", + severity="advisory", + hint="No harbor executable exists on PATH.", + ) + try: + completed = subprocess.run( + [harbor, "job", "start", "--print-config", "-c", str(config_path)], + cwd=repo_root, + capture_output=True, + text=True, + timeout=120, + check=False, + ) + except (OSError, subprocess.SubprocessError) as exc: + return _check("round-trip", "warn", f"The Harbor CLI round trip failed: {exc}", severity="advisory") + if completed.returncode: + detail = (completed.stderr or completed.stdout).strip().splitlines() + return _check( + "round-trip", "fail", f"The Harbor CLI rejected the config: {detail[-1] if detail else 'no output'}." + ) + return _check("round-trip", "pass", "The config file loads through the Harbor CLI.") + + +def _harbor_executable() -> str | None: + local = Path(sys.executable).parent / "harbor" + return str(local) if local.is_file() else shutil.which("harbor") + + +@contextlib.contextmanager +def _evict_module_tree(import_path: str) -> Iterator[None]: + """Import without cached modules from another repository.""" + module = import_path.split(":", 1)[0].split(".", 1)[0] + previous = { + name: cached for name, cached in list(sys.modules.items()) if name == module or name.startswith(f"{module}.") + } + for name in previous: + sys.modules.pop(name) + try: + yield + finally: + for name in list(sys.modules): + if name == module or name.startswith(f"{module}."): + sys.modules.pop(name) + sys.modules.update(previous) diff --git a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/README.md b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/README.md index 35c869f87d..56acfce823 100644 --- a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/README.md +++ b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/README.md @@ -5,6 +5,11 @@ SPDX-License-Identifier: Apache-2.0 # Eval Author +> **Eval Author is in active development and is not intended for external use.** +> +> **WARNING:** Use Eval Author only with a trusted repository. +> An agent import executes module top-level code. + The top-level `nemo_eval_author_plugin.eval_author` package is the canonical Eval Author implementation. It turns an Experimentalist Insight and its production trace refs into evaluator dataset changes, creating or augmenting regression signals that @@ -74,10 +79,9 @@ them. The staged template is refreshed on every invocation rather than reused. The returned Python contract is documented in the [Eval Author Python Reference](REFERENCE.md#evalauthorresult). -## Intended Invocation +## Intended Python Invocation -Until a CLI or platform job is wired, Python callers can invoke the runner -directly: +`run_eval_author(...)` remains available to Python callers: ```python import asyncio @@ -105,4 +109,35 @@ async def main() -> None: asyncio.run(main()) ``` -No standalone Eval Author CLI is implemented yet. +## Discovery CLI + +`nemo agents eval-author discover` validates one repository-owned Harbor config and writes no repository files. + +The command accepts these flags: + +- `--repo` selects the repository. The current directory is the default. +- `--agent` sets the name. Without it, a string `agent` in root `optimizer.yaml` takes precedence over the repository directory slug. +- `--dry-run` prints the report and uploads no files. + +Discovery does not infer or generate a config. The preflight includes: + +- Harbor validates the schema, agent concurrency limits, task directories, and dataset coverage. Harbor also resolves the job. +- Discovery identifies required environment variables. Harbor imports the agent and validates the environment backend. +- `harbor job start --print-config` loads the config file. + +The `ETHOS.md` and trace checks are advisory. +Each invocation repeats the preflight and records one `sha256:` fingerprint plus the input file count. The fingerprint never skips validation. + +Without `--dry-run`, the command uploads only `/discovery.md` to the `nemo-eval-author` fileset. +Only a runnable report includes `cd && harbor job start -c `. + +A standard run returns exit code 0 only for a runnable report and a successful upload. +A dry run returns exit code 0 only if the report is runnable. +All other outcomes return exit code 1. The command still uploads a report for an absent or rejected config. + +## Planned Commands + +- `nemo agents eval-author audit` will report coverage gaps against `ETHOS.md` and will not change the current Harbor suite. +- `nemo agents eval-author propose` will draft Harbor tasks and verifier patches for review. +- `nemo agents eval-author run` will run `discover`, `audit`, and `propose` as one pipeline. +- `nemo agents eval-author doctor` will check credentials, platform access, and the runtime for the other commands. diff --git a/plugins/nemo-eval-author/tests/discover/test_command.py b/plugins/nemo-eval-author/tests/discover/test_command.py new file mode 100644 index 0000000000..74f8abea1e --- /dev/null +++ b/plugins/nemo-eval-author/tests/discover/test_command.py @@ -0,0 +1,285 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""End-to-end tests for the discovery command.""" + +import asyncio +from pathlib import Path +from unittest.mock import AsyncMock + +import pytest +import typer +from harbor_fixtures import StubClient, StubFiles, read_front_matter, write_dataset, write_job_config +from nemo_eval_author_plugin import cli +from nemo_eval_author_plugin.discovery import run as discovery +from nemo_eval_author_plugin.discovery import validate +from typer.testing import CliRunner + +runner = CliRunner() +AGENT = "ticket-triage" + + +@pytest.fixture +def app() -> typer.Typer: + return cli.EvalAuthorCLI().get_cli() + + +@pytest.fixture +def client(monkeypatch) -> StubClient: + stub = StubClient() + monkeypatch.setattr(discovery, "make_client", lambda base_url: stub) + return stub + + +@pytest.fixture(autouse=True) +def workspace(monkeypatch, tmp_path): + config = tmp_path / "nmp-config.yaml" + config.touch() + monkeypatch.setenv("NMP_WORKSPACE", "default") + monkeypatch.setenv("NMP_CONFIG_FILE", str(config)) + + +@pytest.fixture(autouse=True) +def successful_external_preflight(monkeypatch): + monkeypatch.setattr(validate.EnvironmentFactory, "run_preflight", lambda *args: None) + monkeypatch.setattr( + validate, + "check_config_file", + lambda path, root: validate._check("round-trip", "pass", "The config file loads through the Harbor CLI."), + ) + + +def _invoke(app: typer.Typer, repo: Path, *extra: str): + return runner.invoke(app, ["discover", "--repo", str(repo), *extra]) + + +def _healthy_repo(root: Path) -> Path: + write_dataset(root / "evals" / "validation") + write_job_config(root / "configs" / "eval.yaml", dataset="evals/validation") + return root + + +def _snapshot(root: Path) -> list[tuple[str, bytes | None]]: + return [ + (path.relative_to(root).as_posix(), path.read_bytes() if path.is_file() else None) + for path in sorted(root.rglob("*")) + ] + + +def test_command_reads_the_canonical_platform_agent_spec(app, client, monkeypatch, tmp_path): + repo = _healthy_repo(tmp_path / "agent-repo") + (repo / "ETHOS.md").write_text("# Local\n", encoding="utf-8") + ref = f"default/{AGENT}-spec#AGENT-SPEC.md" + download = AsyncMock(side_effect=[b"# Platform one\n", b"# Platform two\n"]) + monkeypatch.setattr(client.files, "download_content", download) + + first = _invoke(app, repo, "--agent", AGENT) + first_front = read_front_matter(client.files.stored[f"{AGENT}/discovery.md"]) + second = _invoke(app, repo, "--agent", AGENT) + second_front = read_front_matter(client.files.stored[f"{AGENT}/discovery.md"]) + + assert (first.exit_code, second.exit_code) == (0, 0) + assert first_front["ethos_path"] == ref + assert first_front["fingerprint"] != second_front["fingerprint"] + assert [awaited.kwargs for awaited in download.await_args_list] == [{"remote_path": ref}] * 2 + + +def test_healthy_config_uploads_only_discovery_report_and_does_not_write_to_repo(app, client, tmp_path): + repo = _healthy_repo(tmp_path / "agent-repo") + before = _snapshot(repo) + + result = _invoke(app, repo, "--agent", AGENT) + + assert result.exit_code == 0, result.output + assert "Repository\n ✓ Found 1 repository-owned Harbor config file." in result.output + assert "harbor job start -c configs/eval.yaml" in result.output + assert "Uploaded ticket-triage/discovery.md to fileset 'nemo-eval-author'." in result.output + assert client.files.stored.keys() == {f"{AGENT}/discovery.md"} + assert len(client.files.uploads) == 1 + assert client.files.uploads[0]["fileset"] == "nemo-eval-author" + assert client.files.uploads[0]["workspace"] == "default" + assert client.files.uploads[0]["fileset_auto_create"] is True + assert read_front_matter(client.files.stored[f"{AGENT}/discovery.md"])["runnable"] is True + assert _snapshot(repo) == before + assert client.closed is True + assert result.output.rstrip().endswith("Final overview: Discovery passed with 0 failures and 2 warnings.") + + +def test_missing_config_exits_one_but_uploads_the_report(app, client, tmp_path): + repo = tmp_path / "empty-repo" + repo.mkdir() + (repo / "README.md").write_text("# Empty\n", encoding="utf-8") + + result = _invoke(app, repo, "--agent", AGENT) + + assert result.exit_code == 1, result.output + assert client.files.stored.keys() == {f"{AGENT}/discovery.md"} + front = read_front_matter(client.files.stored[f"{AGENT}/discovery.md"]) + assert front["runnable"] is False + assert front["config_path"] is None + assert front["run_command"] is None + assert "harbor job start" not in result.output + assert result.output.rstrip().endswith("Final overview: Discovery failed with 1 failure and 2 warnings.") + + +def test_rejected_config_exits_one_and_uploads_its_report(app, client, tmp_path): + repo = tmp_path / "rejected-repo" + config = repo / "configs" / "eval.yaml" + config.parent.mkdir(parents=True) + config.write_text("datasets:\n - invalid\n", encoding="utf-8") + + result = _invoke(app, repo, "--agent", AGENT) + + assert result.exit_code == 1, result.output + front = read_front_matter(client.files.stored[f"{AGENT}/discovery.md"]) + assert front["runnable"] is False + assert any(check["name"] == "schema" and check["status"] == "fail" for check in front["checks"]) + assert front["run_command"] is None + assert result.output.rstrip().endswith("Final overview: Discovery failed with 1 failure and 2 warnings.") + + +def test_preflights_every_config_sequentially_and_uploads_all_results_after_a_failure( + app, client, monkeypatch, tmp_path +): + repo = tmp_path / "multi-config-repo" + first = repo / "first.yaml" + second = repo / "nested" / "second.yml" + for path, text in ( + (first, "job_name: first-entry\ndatasets:\n- path: first\n"), + (second, "datasets:\n- path: second\n"), + ): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + calls: list[str] = [] + active = False + + async def preflight(candidate, root): + nonlocal active + assert root == repo + assert not active + active = True + await asyncio.sleep(0) + relative = candidate.path.relative_to(repo).as_posix() + calls.append(relative) + active = False + if candidate.path == first: + return validate.ValidationOutcome( + checks=[validate._check("schema", "fail", "Harbor rejected the first config.")] + ) + return validate.ValidationOutcome( + checks=[validate._check("schema", "pass", "Harbor accepted the second config.")] + ) + + monkeypatch.setattr(validate, "run_ladder", preflight) + + result = _invoke(app, repo, "--agent", AGENT) + + assert result.exit_code == 1, result.output + assert calls == ["first.yaml", "nested/second.yml"] + uploaded = client.files.stored[f"{AGENT}/discovery.md"].decode() + assert "Harbor rejected the first config." in uploaded + assert "Harbor accepted the second config." in uploaded + + +def test_schema_failure_does_not_upload_the_rejected_input_value(app, client, tmp_path): + secret = "nvapi-secret-value-123456789" + repo = tmp_path / "secret-rejected-repo" + config = repo / "configs" / "eval.yaml" + config.parent.mkdir(parents=True) + config.write_text(f"datasets:\n - {secret}\n", encoding="utf-8") + + result = _invoke(app, repo, "--agent", AGENT) + + assert result.exit_code == 1, result.output + uploaded = client.files.stored[f"{AGENT}/discovery.md"] + schema_check = next(check for check in read_front_matter(uploaded)["checks"] if check["name"] == "schema") + assert secret not in schema_check["message"] + assert secret.encode() not in uploaded + assert "errors.pydantic.dev" not in schema_check["message"] + + +def test_dry_run_uploads_nothing_and_prints_the_report(app, client, tmp_path): + repo = _healthy_repo(tmp_path / "dry-repo") + + result = _invoke(app, repo, "--agent", AGENT, "--dry-run") + + assert result.exit_code == 0, result.output + assert "Dry run: no files were uploaded." in result.output + assert "---\nschema_version: 1" in result.output + assert "runnable: true" in result.output + assert client.files.uploads == [] + assert result.output.rstrip().endswith("Final overview: Discovery passed with 0 failures and 2 warnings.") + + +def test_upload_error_exits_one(app, monkeypatch, tmp_path): + repo = _healthy_repo(tmp_path / "upload-error-repo") + client = StubClient(files=StubFiles(fail=True)) + monkeypatch.setattr(discovery, "make_client", lambda base_url: client) + + result = _invoke(app, repo, "--agent", AGENT) + + assert result.exit_code == 1, result.output + assert "Upload failed: RuntimeError: fileset unavailable" in result.output + assert "harbor job start -c configs/eval.yaml" in result.output + assert client.files.stored == {} + assert result.output.rstrip().endswith("Final overview: Discovery failed with 1 failure and 2 warnings.") + + +def test_agent_name_precedence_uses_explicit_profile_then_directory(app, client, tmp_path): + explicit = _healthy_repo(tmp_path / "explicit-checkout") + (explicit / "optimizer.yaml").write_text("agent: ignored\n", encoding="utf-8") + profiled = _healthy_repo(tmp_path / "profile-checkout") + (profiled / "optimizer.yaml").write_text("agent: Profile Agent\n", encoding="utf-8") + defaulted = _healthy_repo(tmp_path / "Directory Agent") + + results = [ + _invoke(app, explicit, "--agent", "explicit-agent"), + _invoke(app, profiled), + _invoke(app, defaulted), + ] + + assert all(result.exit_code == 0 for result in results), [result.output for result in results] + assert client.files.stored.keys() == { + "explicit-agent/discovery.md", + "profile-agent/discovery.md", + "directory-agent/discovery.md", + } + + +def test_explicit_agent_is_slugged_for_traces_and_remote_paths(app, client, tmp_path, monkeypatch): + repo = _healthy_repo(tmp_path / "explicit-agent-repo") + trace_probe = AsyncMock( + return_value=discovery.scan._check("traces", "warn", "No traces exist.", severity="advisory") + ) + monkeypatch.setattr(discovery.scan, "probe_traces", trace_probe) + + result = _invoke(app, repo, "--agent", "../Ticket Agent#production") + + assert result.exit_code == 0, result.output + trace_probe.assert_awaited_once_with(client, agent="ticket-agent-production", workspace="default") + assert client.files.stored.keys() == {"ticket-agent-production/discovery.md"} + assert ( + read_front_matter(client.files.stored["ticket-agent-production/discovery.md"])["agent"] + == "ticket-agent-production" + ) + + +def test_every_invocation_revalidates_the_repository_config(app, client, tmp_path): + repo = _healthy_repo(tmp_path / "changing-repo") + config = repo / "configs" / "eval.yaml" + + first = _invoke(app, repo, "--agent", AGENT) + first_front = read_front_matter(client.files.stored[f"{AGENT}/discovery.md"]) + config.write_text("datasets:\n - invalid\n", encoding="utf-8") + second = _invoke(app, repo, "--agent", AGENT) + + assert first.exit_code == 0, first.output + assert second.exit_code == 1, second.output + second_front = read_front_matter(client.files.stored[f"{AGENT}/discovery.md"]) + assert second_front["runnable"] is False + assert any(check["name"] == "schema" and check["status"] == "fail" for check in second_front["checks"]) + assert { + "discovered_at": second_front["discovered_at"] != first_front["discovered_at"], + "fingerprint": second_front["fingerprint"] != first_front["fingerprint"], + } == {"discovered_at": True, "fingerprint": True} diff --git a/plugins/nemo-eval-author/tests/discover/test_report.py b/plugins/nemo-eval-author/tests/discover/test_report.py new file mode 100644 index 0000000000..caf999e1ea --- /dev/null +++ b/plugins/nemo-eval-author/tests/discover/test_report.py @@ -0,0 +1,211 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Discovery report contract tests.""" + +import shlex +from datetime import UTC, datetime + +from harbor_fixtures import read_front_matter +from nemo_eval_author_plugin.discovery import report, scan, validate +from nemo_eval_author_plugin.discovery.validate import RequiredEnvVar +from nemo_insights_plugin.contracts.checks import CheckResult, CheckStatus, format_report + + +def _check(name: str = "config", status: CheckStatus = "pass", message: str = "Found the config.") -> CheckResult: + return CheckResult( + name=name, + group="repository" if name == "config" else "validation", + status=status, + severity="required", + message=message, + ) + + +_TRACE_CHECK = scan._check("traces", "pass", "Trace sessions exist.", severity="advisory") + + +def _config( + root, + *, + path: str = "configs/eval.yaml", + name: str = "evaluation", + checks: list[CheckResult] | None = None, +): + root = root.resolve() + return report.ConfigReport( + name=name, + path=root / path, + required_env_vars=[ + RequiredEnvVar(name="HF_TOKEN", default=None, declared_in=root / "evals" / "validation" / "task.toml") + ], + checks=checks if checks is not None else [_check()], + ) + + +def _record(tmp_path, *, configs=None, repository_checks: list[CheckResult] | None = None): + root = tmp_path.resolve() + return report.DiscoveryReport( + agent="ticket-triage", + workspace="default", + repo_root=root, + configs=[_config(root)] if configs is None else configs, + dataset_paths=[root / "evals" / "validation"], + ethos_path="ETHOS.md", + harbor_version="0.18.0", + discovered_at=datetime(2026, 8, 10, 15, tzinfo=UTC), + fingerprint="sha256:abc123", + input_file_count=4, + repository_checks=repository_checks or [], + trace_check=_TRACE_CHECK, + ) + + +def test_front_matter_records_the_complete_repository_contract(tmp_path): + check = _check() + record = _record(tmp_path, configs=[_config(tmp_path, checks=[check])]) + + markdown = report.render_markdown(record) + front = read_front_matter(markdown) + + assert front == { + "schema_version": 1, + "agent": "ticket-triage", + "workspace": "default", + "repo_root": str(tmp_path.resolve()), + "runnable": True, + "configs": [{"name": "evaluation", "path": "configs/eval.yaml"}], + "config_path": "configs/eval.yaml", + "dataset_paths": ["evals/validation"], + "run_command": f"cd {shlex.quote(str(tmp_path.resolve()))} && harbor job start -c configs/eval.yaml", + "ethos_path": "ETHOS.md", + "harbor_version": "0.18.0", + "required_env_vars": [ + { + "name": "HF_TOKEN", + "default": None, + "declared_in": "evals/validation/task.toml", + } + ], + "discovered_at": "2026-08-10T15:00:00+00:00", + "fingerprint": "sha256:abc123", + "input_file_count": 4, + "checks": [check.model_dump(mode="json"), _TRACE_CHECK.model_dump(mode="json")], + } + assert format_report([check]) in markdown + + +def test_a_rejected_config_is_blocked_and_has_no_command(tmp_path): + failure = _check("resolution", "fail", "Harbor could not resolve the job.") + record = _record(tmp_path, configs=[_config(tmp_path, checks=[_check(), failure])]) + + markdown = report.render_markdown(record) + front = read_front_matter(markdown) + + assert front["runnable"] is False + assert front["run_command"] is None + assert "harbor job start" not in markdown + assert failure.message in markdown + + +def test_a_report_without_a_repository_config_is_not_runnable(tmp_path): + record = _record( + tmp_path, + configs=[], + repository_checks=[_check("config", "fail", "No repository-owned Harbor config file exists.")], + ) + + front = read_front_matter(report.render_markdown(record)) + + assert front["runnable"] is False + assert front["config_path"] is None + assert front["run_command"] is None + + +def test_the_run_command_changes_to_the_repo_and_quotes_shell_paths(tmp_path): + repo = tmp_path / "repo $(touch unsafe); name" + record = _record( + repo, + configs=[_config(repo, path="configs/eval $(touch unsafe); suite.yaml")], + ) + + cd_command, harbor_command = record.run_command.split(" && ") + assert shlex.split(cd_command) == ["cd", str(repo.resolve())] + assert shlex.split(harbor_command) == [ + "harbor", + "job", + "start", + "-c", + "configs/eval $(touch unsafe); suite.yaml", + ] + + +def test_multi_config_report_uses_names_paths_stable_sections_and_one_command_each(tmp_path, monkeypatch): + root = tmp_path.resolve() + candidates = [ + scan.ConfigCandidate(root / "a.yaml", {"job_name": "shared", "tasks": ["a"]}), + scan.ConfigCandidate(root / "nested" / "b.yml", {"job_name": "shared", "tasks": ["b"]}), + scan.ConfigCandidate(root / "nested" / "fallback.json", {"job_name": " ", "tasks": ["c"]}), + ] + scan_result = scan.RepositoryScan( + configs=candidates, + dataset_paths=[], + ethos_path=None, + fingerprint="abc123", + input_file_count=3, + checks=[_check("config", "pass", "Found 3 repository-owned Harbor config files.")], + ) + validations = [ + validate.ValidationOutcome(checks=[_check("schema", "pass", f"Schema {index} passed.")]) for index in range(3) + ] + trace_check = scan._check("traces", "warn", "No traces exist.", severity="advisory") + monkeypatch.setattr(report, "harbor_version", lambda: "0.18.0") + + record = report.build_report( + agent="ticket-triage", + workspace="default", + repo_root=root, + scan_result=scan_result, + validations=validations, + trace_check=trace_check, + ) + markdown = report.render_markdown(record) + front = read_front_matter(markdown) + + assert front["configs"] == [ + {"name": "shared", "path": "a.yaml"}, + {"name": "shared", "path": "nested/b.yml"}, + {"name": "fallback.json", "path": "nested/fallback.json"}, + ] + assert (front["runnable"], front["config_path"], front["run_command"]) == (True, None, None) + headings = [ + "### `shared` (`a.yaml`)", + "### `shared` (`nested/b.yml`)", + "### `fallback.json` (`nested/fallback.json`)", + ] + positions = [markdown.index(heading) for heading in headings] + assert positions == sorted(positions) + for index, (position, candidate) in enumerate(zip(positions, candidates, strict=True)): + end = positions[index + 1] if index + 1 < len(positions) else len(markdown) + section = markdown[position:end] + assert f"Schema {index} passed." in section + assert f"harbor job start -c {candidate.path.relative_to(root).as_posix()}" in section + assert markdown.count("harbor job start -c") == 3 + + +def test_multi_config_report_is_not_runnable_when_one_config_fails(tmp_path): + failure = _check("schema", "fail", "Harbor rejected one config.") + record = _record( + tmp_path, + configs=[ + _config(tmp_path, path="first.yaml", name="first", checks=[failure]), + _config(tmp_path, path="second.yaml", name="second"), + ], + ) + + markdown = report.render_markdown(record) + front = read_front_matter(markdown) + + assert (front["runnable"], front["config_path"], front["run_command"]) == (False, None, None) + assert "harbor job start -c first.yaml" not in markdown + assert "harbor job start -c second.yaml" in markdown diff --git a/plugins/nemo-eval-author/tests/discover/test_scan.py b/plugins/nemo-eval-author/tests/discover/test_scan.py new file mode 100644 index 0000000000..7337367b73 --- /dev/null +++ b/plugins/nemo-eval-author/tests/discover/test_scan.py @@ -0,0 +1,171 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Repository scan contract tests.""" + +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock + +from nemo_eval_author_plugin.discovery import scan + + +def _config(path: Path, text: str = "datasets:\n- path: evals\n") -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + +def _task(path: Path) -> None: + path.mkdir(parents=True, exist_ok=True) + (path / "task.toml").write_text('version = "1.0"\n', encoding="utf-8") + + +def test_finds_configs_through_depth_four_in_stable_order_and_prunes_jobs(tmp_path): + expected = [ + _config(tmp_path / "root.yaml"), + _config(tmp_path / "z" / "depth-one.yml"), + _config(tmp_path / "a" / "a" / "depth-two.json", '{"tasks": ["task"]}'), + _config(tmp_path / "b" / "b" / "b" / "depth-three.yaml"), + _config(tmp_path / "c" / "c" / "c" / "c" / "depth-four.yaml"), + ] + _config(tmp_path / "d" / "d" / "d" / "d" / "d" / "depth-five.yaml") + + result = scan.scan_repository(tmp_path) + + assert [candidate.path for candidate in result.configs] == expected + assert {check.status for check in result.checks if check.name == "config"} == {"pass"} + assert any(check.name == "ethos" and check.status == "warn" for check in result.checks) + + +def test_reads_yaml_yml_and_json_configs(tmp_path): + for suffix, text in ( + (".yaml", "datasets:\n- path: evals\n"), + (".yml", "datasets:\n- path: evals\n"), + (".json", '{"datasets": [{"path": "evals"}]}'), + ): + repo = tmp_path / suffix[1:] + expected = _config(repo / "configs" / f"job{suffix}", text) + + result = scan.scan_repository(repo) + + assert [candidate.path for candidate in result.configs] == [expected] + + +def test_rejects_a_config_symlink_that_resolves_outside_the_repository(tmp_path): + repo = tmp_path / "repo" + outside = _config(tmp_path / "outside.yaml") + link = repo / "configs" / "eval.yaml" + link.parent.mkdir(parents=True) + link.symlink_to(outside) + + result = scan.scan_repository(repo) + + assert result.configs == [] + assert next(check for check in result.checks if check.name == "config").status == "fail" + + +def test_rejects_a_config_file_symlink_inside_the_repository(tmp_path): + config = _config(tmp_path / "configs" / "eval.yaml") + link = config.with_name("alias.yaml") + link.symlink_to(config.name) + + result = scan.scan_repository(tmp_path) + + assert [candidate.path for candidate in result.configs] == [config] + + +def test_profile_prior_job_and_task_layout_never_become_config_candidates(tmp_path): + (tmp_path / "optimizer.yaml").write_text( + "agent: ticket-triage\ndatasets:\n validation: evals/validation\n", + encoding="utf-8", + ) + prior_job = tmp_path / "jobs" / "run-1" + _config(prior_job / "config.json", '{"datasets": [{"path": "evals/validation"}]}') + (prior_job / "lock.json").write_text('{"harbor_version": "0.18.0"}\n', encoding="utf-8") + dataset = tmp_path / "evals" / "validation" + _task(dataset / "task-0") + + result = scan.scan_repository(tmp_path) + + assert (result.configs, result.dataset_paths) == ([], [dataset]) + + +def test_uses_only_ethos_for_the_doctrine_contract(tmp_path): + _config(tmp_path / "harbor-job.yaml") + (tmp_path / "README.md").write_text("# Readme\n", encoding="utf-8") + (tmp_path / "AGENT-SPEC.md").write_text("# Old\n", encoding="utf-8") + + without_ethos = scan.scan_repository(tmp_path) + assert without_ethos.ethos_path is None + assert any(check.name == "ethos" and check.status == "warn" for check in without_ethos.checks) + + ethos = tmp_path / "ETHOS.md" + ethos.write_text("# Agent doctrine\n", encoding="utf-8") + with_ethos = scan.scan_repository(tmp_path) + + assert with_ethos.ethos_path == "ETHOS.md" + assert any(check.name == "ethos" and check.status == "pass" for check in with_ethos.checks) + + +def test_discovers_local_datasets_and_prunes_generated_trees(tmp_path): + _config(tmp_path / "harbor-job.yaml") + _task(tmp_path) + _task(tmp_path / "evals" / "suite" / "task-one") + _task(tmp_path / "evals" / "suite" / "task_template") + _task(tmp_path / ".nemo-optimizer" / "output" / "task-two") + _task(tmp_path / "node_modules" / "package" / "task-three") + _task(tmp_path / "vendor" / "package" / "task-four") + _task(tmp_path / "cache" / "package" / "task-five") + _task(tmp_path / "jobs" / "prior-run" / "task-six") + + result = scan.scan_repository(tmp_path) + + assert result.dataset_paths == [tmp_path / "evals" / "suite"] + assert result.input_file_count == 3 + + +def test_fingerprint_covers_config_ethos_optimizer_and_dataset_files(tmp_path): + config = _config(tmp_path / "harbor-job.yaml") + other_config = _config(tmp_path / "nested" / "other.json", '{"tasks": ["task"]}') + ethos = tmp_path / "ETHOS.md" + optimizer = tmp_path / "optimizer.yaml" + ethos.write_text("# One\n", encoding="utf-8") + optimizer.write_text("model: one\n", encoding="utf-8") + task = tmp_path / "evals" / "suite" / "task-one" + _task(task) + dataset_file = task / "notes.txt" + dataset_file.write_text("one\n", encoding="utf-8") + + first = scan.scan_repository(tmp_path) + + assert first.input_file_count == 6 + assert [candidate.path for candidate in first.configs] == [config, other_config] + for path, replacement in ( + (config, "datasets:\n- path: another-evals\n"), + (other_config, '{"tasks": ["another-task"]}'), + (ethos, "# Two\n"), + (optimizer, "model: two\n"), + (task / "task.toml", 'version = "2.0"\n'), + (dataset_file, "two\n"), + ): + original = path.read_text(encoding="utf-8") + path.write_text(replacement, encoding="utf-8") + assert scan.scan_repository(tmp_path).fingerprint != first.fingerprint + path.write_text(original, encoding="utf-8") + + +async def test_trace_probe_handles_exception_empty_and_positive_totals(): + for total, status in ((None, "warn"), (0, "warn"), (2, "pass")): + list_groups = ( + AsyncMock(side_effect=RuntimeError("intake unavailable")) + if total is None + else AsyncMock(return_value=SimpleNamespace(pagination=SimpleNamespace(total_results=total))) + ) + client = SimpleNamespace( + intake=SimpleNamespace(spans=SimpleNamespace(groups=SimpleNamespace(list=list_groups))) + ) + + finding = await scan.probe_traces(client, agent="ticket-triage", workspace="default") + + assert (finding.status, finding.severity) == (status, "advisory") diff --git a/plugins/nemo-eval-author/tests/discover/test_validate.py b/plugins/nemo-eval-author/tests/discover/test_validate.py new file mode 100644 index 0000000000..2c2acffe30 --- /dev/null +++ b/plugins/nemo-eval-author/tests/discover/test_validate.py @@ -0,0 +1,389 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Focused Harbor preflight contract tests.""" + +import subprocess +import sys +from pathlib import Path + +import pytest +from harbor.utils.logger import logger as harbor_logger +from harbor_fixtures import write_dataset, write_task, write_wrapper +from nemo_eval_author_plugin.discovery import scan, validate + + +def _candidate(path: Path, data: dict) -> scan.ConfigCandidate: + return scan.ConfigCandidate(path=path, data=data) + + +def _check(outcome: validate.ValidationOutcome, name: str): + matches = [item for item in outcome.checks if item.name == name] + assert matches, f"no {name!r} check in {[item.name for item in outcome.checks]}" + return matches[0] + + +def _patch_external_preflight(monkeypatch) -> None: + monkeypatch.setattr(validate.EnvironmentFactory, "run_preflight", lambda *args: None) + monkeypatch.setattr( + validate, + "check_config_file", + lambda path, root: validate._check("round-trip", "pass", "The config file loads through the Harbor CLI"), + ) + + +async def test_a_well_formed_repo_passes_the_ladder(tmp_path, monkeypatch): + dataset = write_dataset(tmp_path / "evals" / "validation") + _patch_external_preflight(monkeypatch) + + outcome = await validate.run_ladder( + _candidate( + tmp_path / "harbor-job.yaml", {"agents": [{"name": "oracle"}], "datasets": [{"path": str(dataset)}]} + ), + tmp_path, + ) + + assert not [check for check in outcome.checks if check.status == "fail"] + assert {check.name for check in outcome.checks} == { + "schema", + "resolution", + "agent", + "backend", + "round-trip", + "tasks", + "coverage", + "credentials", + } + assert _check(outcome, "tasks").message.startswith("2 of 2") + assert _check(outcome, "coverage").status == "pass" + + +async def test_ladder_closes_added_harbor_logger_handlers(tmp_path, monkeypatch): + dataset = write_dataset(tmp_path / "evals" / "validation") + _patch_external_preflight(monkeypatch) + handlers_before = tuple(harbor_logger.handlers) + + await validate.run_ladder( + _candidate( + tmp_path / "harbor-job.yaml", + {"agents": [{"name": "oracle"}], "datasets": [{"path": str(dataset)}]}, + ), + tmp_path, + ) + + assert tuple(harbor_logger.handlers) == handlers_before + + +async def test_schema_failure_stops_the_ladder(tmp_path): + outcome = await validate.run_ladder(_candidate(tmp_path / "harbor-job.yaml", {"datasets": "not-a-list"}), tmp_path) + + assert _check(outcome, "schema").status == "fail" + assert [item.name for item in outcome.checks] == ["schema"] + + +async def test_resolution_failure_reports_the_real_error(tmp_path, monkeypatch): + _patch_external_preflight(monkeypatch) + outcome = await validate.run_ladder( + _candidate(tmp_path / "harbor-job.yaml", {"datasets": [{"path": str(tmp_path / "nope")}]}), tmp_path + ) + + finding = _check(outcome, "resolution") + assert finding.status == "fail" + assert "nope" in finding.message + + +async def test_missing_resolved_task_attribute_is_a_compatibility_failure(tmp_path, monkeypatch): + dataset = write_dataset(tmp_path / "evals" / "validation", count=1) + _patch_external_preflight(monkeypatch) + + class _JobWithoutTaskConfigs: + @classmethod + async def create(cls, _config): + return cls() + + def _close_logger_handlers(self): + pass + + monkeypatch.setattr(validate, "Job", _JobWithoutTaskConfigs) + outcome = await validate.run_ladder( + _candidate(tmp_path / "harbor-job.yaml", {"datasets": [{"path": str(dataset)}]}), tmp_path + ) + + assert _check(outcome, "compatibility").status == "fail" + assert "Job._task_configs" in _check(outcome, "compatibility").message + + +async def test_invalid_resolved_task_fails_tasks_check(tmp_path, monkeypatch): + dataset = write_dataset(tmp_path / "evals" / "validation", count=1) + _patch_external_preflight(monkeypatch) + create_job = validate.Job.create + + async def resolve_then_invalidate(config): + job = await create_job(config) + monkeypatch.setattr(validate.Task, "is_valid_dir", lambda _path: False) + return job + + monkeypatch.setattr(validate.Job, "create", staticmethod(resolve_then_invalidate)) + + outcome = await validate.run_ladder( + _candidate(tmp_path / "harbor-job.yaml", {"datasets": [{"path": str(dataset)}]}), tmp_path + ) + + assert _check(outcome, "tasks").status == "fail" + + +async def test_silent_task_drop_fails_concrete_coverage(tmp_path, monkeypatch): + dataset = tmp_path / "evals" / "validation" + write_task(dataset / "task-0") + write_task(dataset / "task-1", instruction=None) + _patch_external_preflight(monkeypatch) + + outcome = await validate.run_ladder( + _candidate(tmp_path / "harbor-job.yaml", {"datasets": [{"path": str(dataset)}]}), tmp_path + ) + + assert _check(outcome, "coverage").status == "fail" + assert "task-1" in _check(outcome, "coverage").message + + +async def test_empty_task_names_does_not_make_a_dropped_task_advisory(tmp_path, monkeypatch): + dataset = tmp_path / "evals" / "validation" + write_task(dataset / "task-0") + write_task(dataset / "task-1", instruction=None) + _patch_external_preflight(monkeypatch) + + outcome = await validate.run_ladder( + _candidate( + tmp_path / "harbor-job.yaml", + {"datasets": [{"path": str(dataset), "task_names": []}]}, + ), + tmp_path, + ) + + coverage = _check(outcome, "coverage") + assert (coverage.status, coverage.severity) == ("fail", "required") + assert "task-1" in coverage.message + assert coverage.hint == "Harbor skips these task dirs silently." + + +async def test_malformed_explicit_task_name_fails_coverage(tmp_path, monkeypatch): + dataset = tmp_path / "evals" / "validation" + write_task(dataset / "task-0") + write_task(dataset / "task-1", instruction=None) + _patch_external_preflight(monkeypatch) + + outcome = await validate.run_ladder( + _candidate( + tmp_path / "harbor-job.yaml", + {"datasets": [{"path": str(dataset), "task_names": ["task-0", "task-1"]}]}, + ), + tmp_path, + ) + + assert (_check(outcome, "coverage").status, _check(outcome, "coverage").severity) == ("fail", "required") + assert "task-1" in _check(outcome, "coverage").message + + +async def test_excluded_task_drop_is_advisory(tmp_path, monkeypatch): + dataset = tmp_path / "evals" / "validation" + write_task(dataset / "task-0") + write_task(dataset / "task-1", instruction=None) + _patch_external_preflight(monkeypatch) + + outcome = await validate.run_ladder( + _candidate( + tmp_path / "harbor-job.yaml", + {"datasets": [{"path": str(dataset), "exclude_task_names": ["task-1"]}]}, + ), + tmp_path, + ) + + assert (_check(outcome, "coverage").status, _check(outcome, "coverage").severity) == ("warn", "advisory") + + +async def test_non_excluded_invalid_task_fails_coverage_with_exclude_filter(tmp_path, monkeypatch): + dataset = tmp_path / "evals" / "validation" + write_task(dataset / "task-0") + write_task(dataset / "task-excluded") + write_task(dataset / "task-invalid", instruction=None) + _patch_external_preflight(monkeypatch) + + outcome = await validate.run_ladder( + _candidate( + tmp_path / "harbor-job.yaml", + {"datasets": [{"path": str(dataset), "exclude_task_names": ["task-excluded"]}]}, + ), + tmp_path, + ) + + coverage = _check(outcome, "coverage") + assert (coverage.status, coverage.severity) == ("fail", "required") + assert "task-invalid" in coverage.message + assert "task-excluded" not in coverage.message + assert coverage.hint == "Harbor skipped a task selected by the dataset filters." + + +async def test_n_tasks_subset_drop_is_advisory(tmp_path, monkeypatch): + dataset = write_dataset(tmp_path / "evals" / "validation") + _patch_external_preflight(monkeypatch) + + outcome = await validate.run_ladder( + _candidate( + tmp_path / "harbor-job.yaml", + {"datasets": [{"path": str(dataset), "n_tasks": 1}]}, + ), + tmp_path, + ) + + assert (_check(outcome, "coverage").status, _check(outcome, "coverage").severity) == ("warn", "advisory") + + +async def test_required_host_variables_are_recorded(tmp_path, monkeypatch): + dataset = tmp_path / "evals" / "validation" + write_task( + dataset / "task-0", + task_toml='\n[environment.env]\nHF_TOKEN = "${HF_TOKEN}"\nREGION = "${AWS_REGION:-us-west-2}"\n', + ) + _patch_external_preflight(monkeypatch) + + outcome = await validate.run_ladder( + _candidate(tmp_path / "harbor-job.yaml", {"datasets": [{"path": str(dataset)}]}), tmp_path + ) + + assert {item.name: item.default for item in outcome.required_env_vars} == { + "HF_TOKEN": None, + "AWS_REGION": "us-west-2", + } + assert _check(outcome, "credentials").status == "pass" + + +async def test_missing_custom_agent_import_is_recorded(tmp_path, monkeypatch): + dataset = write_dataset(tmp_path / "evals" / "validation", count=1) + write_wrapper(tmp_path) + monkeypatch.setattr(sys, "path", [path for path in sys.path if path not in {"", str(tmp_path)}]) + monkeypatch.delitem(sys.modules, "harbor_wrapper", raising=False) + _patch_external_preflight(monkeypatch) + + outcome = await validate.run_ladder( + _candidate( + tmp_path / "harbor-job.yaml", + {"agents": [{"import_path": "harbor_wrapper:WrappedAgent"}], "datasets": [{"path": str(dataset)}]}, + ), + tmp_path, + ) + + assert _check(outcome, "agent").status == "fail" + + +async def test_non_class_custom_agent_import_is_recorded(tmp_path, monkeypatch): + dataset = write_dataset(tmp_path / "evals" / "validation", count=1) + (tmp_path / "invalid_wrapper.py").write_text("def not_a_class():\n return None\n", encoding="utf-8") + monkeypatch.syspath_prepend(str(tmp_path)) + _patch_external_preflight(monkeypatch) + + outcome = await validate.run_ladder( + _candidate( + tmp_path / "harbor-job.yaml", + {"agents": [{"import_path": "invalid_wrapper:not_a_class"}], "datasets": [{"path": str(dataset)}]}, + ), + tmp_path, + ) + + assert _check(outcome, "agent").status == "fail" + + +def test_custom_agent_import_system_exit_is_recorded(monkeypatch): + config = validate.JobConfig.model_validate({"agents": [{"import_path": "agent_module:Agent"}]}) + outcome = validate.ValidationOutcome() + + def exit_import(*_args, **_kwargs): + raise SystemExit("agent stopped") + + monkeypatch.setattr(validate, "import_class", exit_import) + + validate._check_agent(config, outcome) + + assert (_check(outcome, "agent").status, _check(outcome, "agent").message) == ( + "fail", + "Cannot import agent agent_module:Agent: SystemExit: agent stopped", + ) + + +def test_custom_agent_import_keyboard_interrupt_propagates(monkeypatch): + config = validate.JobConfig.model_validate({"agents": [{"import_path": "agent_module:Agent"}]}) + outcome = validate.ValidationOutcome() + + def interrupt_import(*_args, **_kwargs): + raise KeyboardInterrupt + + monkeypatch.setattr(validate, "import_class", interrupt_import) + + with pytest.raises(KeyboardInterrupt): + validate._check_agent(config, outcome) + + +async def test_custom_agent_import_does_not_reuse_another_repositorys_module(tmp_path, monkeypatch): + first, second = tmp_path / "first", tmp_path / "second" + _patch_external_preflight(monkeypatch) + for repo, class_name in ((first, "FirstAgent"), (second, "SecondAgent")): + dataset = write_dataset(repo / "evals" / "validation", count=1) + write_wrapper(repo, class_name=class_name) + monkeypatch.syspath_prepend(str(repo)) + outcome = await validate.run_ladder( + _candidate( + repo / "harbor-job.yaml", + { + "agents": [{"import_path": f"harbor_wrapper:{class_name}"}], + "datasets": [{"path": str(dataset)}], + }, + ), + repo, + ) + assert _check(outcome, "agent").status == "pass" + + +async def test_backend_failure_is_recorded(tmp_path, monkeypatch): + dataset = write_dataset(tmp_path / "evals" / "validation", count=1) + _patch_external_preflight(monkeypatch) + monkeypatch.setattr( + validate.EnvironmentFactory, + "run_preflight", + lambda *args: (_ for _ in ()).throw(RuntimeError("no Docker")), + ) + + outcome = await validate.run_ladder( + _candidate(tmp_path / "harbor-job.yaml", {"datasets": [{"path": str(dataset)}]}), tmp_path + ) + + assert _check(outcome, "backend").status == "fail" + + +def test_config_file_round_trip_runs_harbor_cli(tmp_path, monkeypatch): + config_path = tmp_path / "harbor-job.yaml" + config_path.write_text("datasets: []\n", encoding="utf-8") + monkeypatch.setattr(validate, "_harbor_executable", lambda: "harbor") + calls = [] + + def run(command, **kwargs): + calls.append((command, kwargs["cwd"])) + return subprocess.CompletedProcess(command, 0, "", "") + + monkeypatch.setattr(subprocess, "run", run) + + assert validate.check_config_file(config_path, tmp_path).status == "pass" + assert calls == [(["harbor", "job", "start", "--print-config", "-c", str(config_path)], tmp_path)] + + +def test_config_file_round_trip_reports_harbor_rejection(tmp_path, monkeypatch): + config_path = tmp_path / "harbor-job.yaml" + config_path.write_text("datasets: []\n", encoding="utf-8") + monkeypatch.setattr(validate, "_harbor_executable", lambda: "harbor") + + def reject(command, **_kwargs): + return subprocess.CompletedProcess(command, 1, "", "invalid config\n") + + monkeypatch.setattr(subprocess, "run", reject) + + finding = validate.check_config_file(config_path, tmp_path) + + assert (finding.status, finding.message) == ("fail", "The Harbor CLI rejected the config: invalid config.") diff --git a/plugins/nemo-eval-author/tests/harbor_fixtures.py b/plugins/nemo-eval-author/tests/harbor_fixtures.py new file mode 100644 index 0000000000..c3f21b92e2 --- /dev/null +++ b/plugins/nemo-eval-author/tests/harbor_fixtures.py @@ -0,0 +1,80 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Harbor task builders and platform stubs.""" + +from pathlib import Path +from typing import Any + +import yaml + + +def read_front_matter(text: str | bytes) -> dict[str, Any]: + if isinstance(text, bytes): + text = text.decode("utf-8") + assert text.startswith("---\n"), f"no front matter in {text[:40]!r}" + payload = yaml.safe_load(text[4:].partition("\n---\n")[0]) + assert isinstance(payload, dict) + return payload + + +def write_task( + task_dir: Path, + *, + task_toml: str = "", + instruction: str | None = "Do the thing.\n", +) -> None: + task_dir.mkdir(parents=True, exist_ok=True) + (task_dir / "task.toml").write_text(f'version = "1.0"\n{task_toml}', encoding="utf-8") + + if instruction is not None: + (task_dir / "instruction.md").write_text(instruction, encoding="utf-8") + (task_dir / "environment").mkdir(exist_ok=True) + (task_dir / "environment" / "Dockerfile").write_text("FROM ubuntu:24.04\n\nWORKDIR /app\n", encoding="utf-8") + (task_dir / "tests").mkdir(exist_ok=True) + (task_dir / "tests" / "test.sh").write_text("#!/bin/bash\necho 1 > /logs/verifier/reward.txt\n", encoding="utf-8") + + +def write_dataset(root: Path, *, count: int = 2) -> Path: + for index in range(count): + write_task(root / f"task-{index}") + return root + + +def write_wrapper(wrapper_dir: Path, *, class_name: str = "WrappedAgent") -> None: + wrapper_dir.mkdir(parents=True, exist_ok=True) + (wrapper_dir / "harbor_wrapper.py").write_text( + f"from harbor.agents.base import BaseAgent\n\n\nclass {class_name}(BaseAgent):\n pass\n", + encoding="utf-8", + ) + + +def write_job_config(path: Path, *, dataset: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + payload = {"agents": [{"name": "oracle"}], "datasets": [{"path": dataset}]} + path.write_text(yaml.safe_dump(payload), encoding="utf-8") + + +class StubFiles: + def __init__(self, fail: bool = False) -> None: + self.stored: dict[str, bytes] = {} + self.uploads: list[dict[str, Any]] = [] + self.fail = fail + + async def download_content(self, *, remote_path: str) -> bytes: + raise FileNotFoundError(remote_path) + + async def upload_content(self, *, content: bytes, remote_path: str, **kwargs: Any) -> None: + if self.fail: + raise RuntimeError("fileset unavailable") + self.uploads.append({"remote_path": remote_path, "content": content, **kwargs}) + self.stored[remote_path] = content + + +class StubClient: + def __init__(self, files: StubFiles | None = None) -> None: + self.files = files or StubFiles() + self.closed = False + + async def close(self) -> None: + self.closed = True diff --git a/plugins/nemo-eval-author/tests/test_cli.py b/plugins/nemo-eval-author/tests/test_cli.py index ae6e71218b..d42d8b0f79 100644 --- a/plugins/nemo-eval-author/tests/test_cli.py +++ b/plugins/nemo-eval-author/tests/test_cli.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Scaffolding tests: the command tree exists, and every verb still refuses to run.""" +"""Command-tree and placeholder tests.""" import pytest import typer @@ -10,50 +10,27 @@ runner = CliRunner() -# Each verb is a placeholder owned by the child ticket named beside it. -_PLACEHOLDER_VERBS = [ - ("discover", "ASE-677"), - ("audit", "ASE-676"), - ("propose", "ASE-675"), - ("run", "ASE-673"), - ("doctor", "ASE-678"), -] - @pytest.fixture def app() -> typer.Typer: return cli.EvalAuthorCLI().get_cli() -def test_help_lists_every_verb(app: typer.Typer) -> None: +def test_help_lists_discover(app: typer.Typer) -> None: result = runner.invoke(app, ["--help"]) assert result.exit_code == 0, result.output - for command, _ in _PLACEHOLDER_VERBS: - assert command in result.output - - -@pytest.mark.parametrize(("command", "ticket"), _PLACEHOLDER_VERBS) -def test_verb_refuses_to_run_and_names_its_ticket(app: typer.Typer, command: str, ticket: str) -> None: - result = runner.invoke(app, [command]) - - assert result.exit_code == 1, result.output - assert ticket in result.output - - -def test_not_implemented_quotes_the_invoked_command_path() -> None: - """Placeholder messages use ``ctx.command_path``, not a hardcoded CLI string.""" - app = typer.Typer() - - @app.callback() - def _root() -> None: - """Force subcommand dispatch.""" + assert "discover" in result.output - @app.command("probe") - def probe(ctx: typer.Context) -> None: - cli._not_implemented(ctx, "ASE-000") - result = runner.invoke(app, ["probe"], prog_name="nemo") +def test_placeholder_verbs_refuse_to_run_and_name_their_tickets(app: typer.Typer) -> None: + for command, ticket in ( + ("audit", "ASE-676"), + ("propose", "ASE-675"), + ("run", "ASE-673"), + ("doctor", "ASE-678"), + ): + result = runner.invoke(app, [command]) - assert result.exit_code == 1, result.output - assert "`nemo probe` is not implemented yet (ASE-000)." in result.output + assert result.exit_code == 1, result.output + assert ticket in result.output diff --git a/uv.lock b/uv.lock index c5600f2419..3b3e5fa965 100644 --- a/uv.lock +++ b/uv.lock @@ -4463,18 +4463,20 @@ dependencies = [ { name = "nemo-platform-plugin", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nooa", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "pydantic", extra = ["email"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pyyaml", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "tomlkit", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] [package.metadata] requires-dist = [ - { name = "harbor", specifier = ">=0.16" }, + { name = "harbor", specifier = ">=0.18" }, { name = "nemo-experimentalist-plugin", editable = "plugins/nemo-experimentalist" }, { name = "nemo-insights-plugin", editable = "plugins/nemo-insights" }, { name = "nemo-platform", editable = "packages/nemo_platform" }, { name = "nemo-platform-plugin", editable = "packages/nemo_platform_plugin" }, { name = "nooa", git = "https://github.com/NVIDIA-NeMo/labs-OO-Agents.git?rev=6e0274dd03f883254a084cfb9f871ea580e03434" }, { name = "pydantic", specifier = ">=2" }, + { name = "pyyaml", specifier = ">=6.0.3" }, { name = "tomlkit", specifier = ">=0.13.3" }, ]