feat(eval-author): implement the discover command

plugins/nemo-eval-author/README.md

@@ -54,7 +54,6 @@ For non-interactive and isolated environments, `NEMO_DEFAULT_MODEL` and
 `workspace/model-name` and refer to Model Entities on the target Platform.
 
 A `nemo agents eval-author` CLI is registered under `nemo.cli.agents` and
-mounted by the agents plugin. Verb scaffolding is in place
-(`discover`, `audit`, `propose`, `run`, `doctor`); bodies are still
-placeholders until ASE-673–678 land. The library runner already uses the
+mounted by the agents plugin. `discover` is implemented; `audit`, `propose`,
+`run`, and `doctor` remain placeholders. The library runner already uses the
 configured Platform model pair.

plugins/nemo-eval-author/pyproject.toml

@@ -5,12 +5,14 @@ description = "Eval Author agent for NeMo Platform (hard-depends on Experimental
 requires-python = ">=3.12,<3.14"
 dependencies = [
     "pydantic>=2",
-    "harbor>=0.16",
+    # Harbor 0.18 provides the discovery APIs used by this plugin.
+    "harbor>=0.18",
     "nooa",
     "nemo-experimentalist-plugin",
     "nemo-insights-plugin",
     "nemo-platform",
     "nemo-platform-plugin",
+    "pyyaml>=6.0.3",
     "tomlkit>=0.13.3",
 ]
 

plugins/nemo-eval-author/src/nemo_eval_author_plugin/cli.py

@@ -1,35 +1,51 @@
 # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-"""Eval Author plugin CLI — ``nemo agents eval-author ...`` subcommands.
+"""Eval Author commands under ``nemo agents eval-author``."""
 
-Registered under ``nemo.cli.agents`` and mounted by ``AgentsCLI`` as
-``nemo agents eval-author <verb>``.
-
-Scaffolding. Every verb is registered under its final name so the command tree is
-discoverable and the child tickets have a landing spot, and each body exits non-zero
-until its own ticket lands. Flags belong to those tickets, so nothing here declares
-options yet.
-
-This module imports nothing from ``eval_author`` while the command bodies remain
-placeholders. The completed runner resolves the active Platform model pair at run time.
-"""
-
-from typing import ClassVar, NoReturn
+import asyncio
+from pathlib import Path
+from typing import Annotated, ClassVar, NoReturn
 
 import typer
+from nemo_eval_author_plugin.discovery import run as discovery
+from nemo_insights_plugin.contracts.checks import format_report
 from nemo_platform_plugin.cli import NemoCLI
 
 
 def _not_implemented(ctx: typer.Context, ticket: str) -> NoReturn:
-    """Fail loudly, so a placeholder verb can never be mistaken for a successful run.
-
-    The message quotes ``ctx.command_path`` rather than a hardcoded path.
-    """
     typer.echo(f"`{ctx.command_path}` is not implemented yet ({ticket}).", err=True)
     raise typer.Exit(code=1)
 
 
+def _report_discovery(result: discovery.DiscoverResult) -> None:
+    status = format_report(result.report.checks)
+    if status:
+        typer.echo(status)
+    if result.report.run_command:
+        typer.echo("")
+        typer.echo(f"Run: {result.report.run_command}")
+
+    typer.echo("")
+    if result.dry_run:
+        typer.echo("Dry run: no files were uploaded.")
+        typer.echo("")
+        typer.echo(result.markdown, nl=False)
+    elif result.uploaded:
+        remote_path = f"{result.report.agent}/{discovery.REPORT_FILENAME}"
+        typer.echo(f"Uploaded {remote_path} to fileset '{discovery.FILESET_NAME}'.")
+    else:
+        typer.echo(f"Upload failed: {result.upload_error or 'unknown error'}", err=True)
+
+    failures = sum(check.status == "fail" for check in result.report.checks)
+    failures += not result.dry_run and not result.uploaded
+    warnings = sum(check.status == "warn" for check in result.report.checks)
+    failure_label = "failure" if failures == 1 else "failures"
+    warning_label = "warning" if warnings == 1 else "warnings"
+    status = "passed" if result.ok else "failed"
+    typer.echo(f"Final overview: Discovery {status} with {failures} {failure_label} and {warnings} {warning_label}.")
+
+
 class EvalAuthorCLI(NemoCLI):
     """``nemo agents eval-author ...`` subcommands."""
 
@@ -41,17 +57,43 @@ def get_cli(self) -> typer.Typer:
 
         @app.callback()
         def _root() -> None:
-            """Force subcommand dispatch even when only one verb is registered."""
+            """Select an Eval Author command."""
 
         @app.command("discover")
-        def discover(ctx: typer.Context) -> None:
-            """Discover candidate evaluation cases from agent traces."""
-            # TODO(ASE-677): declare flags and wire discovery.
-            _not_implemented(ctx, "ASE-677")
+        def discover(
+            repo: Annotated[
+                Path,
+                typer.Option("--repo", help="Repository that contains the agent.", exists=True, file_okay=False),
+            ] = Path(),
+            agent: Annotated[
+                str | None,
+                typer.Option("--agent", help="Agent name. The default comes from optimizer.yaml or the directory."),
+            ] = None,
+            dry_run: Annotated[
+                bool,
+                typer.Option("--dry-run", help="Print discovery.md without an upload."),
+            ] = False,
+        ) -> None:
+            """Inspect the repository and record its Harbor preflight.
+
+            WARNING: Use this command only with a trusted repository.
+            Agent imports execute module top-level code.
+            """
+            result = asyncio.run(
+                discovery.discover(
+                    discovery.DiscoverOptions(
+                        repo_root=repo,
+                        agent=agent,
+                        dry_run=dry_run,
+                    )
+                )
+            )
+            _report_discovery(result)
+            raise typer.Exit(code=0 if result.ok else 1)
 
         @app.command("audit")
         def audit(ctx: typer.Context) -> None:
-            """Audit an existing eval suite for coverage gaps."""
+            """Report coverage gaps in an existing eval suite."""
             # TODO(ASE-676): declare flags and wire the audit.
             _not_implemented(ctx, "ASE-676")
 
@@ -63,13 +105,13 @@ def propose(ctx: typer.Context) -> None:
 
         @app.command("run")
         def run(ctx: typer.Context) -> None:
-            """Run the Eval Author pipeline end to end."""
+            """Run the Eval Author pipeline."""
             # TODO(ASE-673): declare flags and wire the pipeline to run_eval_author.
             _not_implemented(ctx, "ASE-673")
 
         @app.command("doctor")
         def doctor(ctx: typer.Context) -> None:
-            """Diagnose Eval Author setup: credentials, platform, runtime."""
+            """Diagnose credentials, platform access, and the runtime."""
             # TODO(ASE-678): report the prerequisites the other verbs gate on.
             _not_implemented(ctx, "ASE-678")
 

plugins/nemo-eval-author/src/nemo_eval_author_plugin/discovery/report.py

@@ -0,0 +1,193 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Discovery report contract and Markdown renderer."""
+
+import shlex
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from importlib.metadata import version
+from pathlib import Path
+from typing import Any
+
+import yaml
+from nemo_eval_author_plugin.discovery.scan import RepositoryScan
+from nemo_eval_author_plugin.discovery.validate import RequiredEnvVar, ValidationOutcome
+from nemo_insights_plugin.contracts.checks import CheckResult, format_report, required_failures
+
+
+@dataclass
+class ConfigReport:
+    """Preflight results for one Harbor config."""
+
+    name: str
+    path: Path
+    required_env_vars: list[RequiredEnvVar]
+    checks: list[CheckResult]
+
+    @property
+    def runnable(self) -> bool:
+        """Return whether the config passed all required checks."""
+        return not required_failures(self.checks)
+
+
+@dataclass
+class DiscoveryReport:
+    """Repository facts and current Harbor preflight results."""
+
+    agent: str
+    workspace: str
+    repo_root: Path
+    configs: list[ConfigReport]
+    dataset_paths: list[Path]
+    ethos_path: str | None
+    harbor_version: str
+    discovered_at: datetime
+    fingerprint: str
+    input_file_count: int
+    repository_checks: list[CheckResult]
+    trace_check: CheckResult
+    schema_version: int = field(init=False, default=1)
+
+    @property
+    def runnable(self) -> bool:
+        """Return whether all repository-owned configs passed required checks."""
+        return bool(self.configs) and all(config.runnable for config in self.configs)
+
+    @property
+    def checks(self) -> list[CheckResult]:
+        """Return repository, config, and trace checks in execution order."""
+        checks = list(self.repository_checks)
+        for config in self.configs:
+            checks.extend(config.checks)
+        checks.append(self.trace_check)
+        return checks
+
+    @property
+    def run_command(self) -> str | None:
+        """Return the Harbor command only for one runnable config."""
+        if len(self.configs) != 1:
+            return None
+        return self.run_command_for(self.configs[0])
+
+    def run_command_for(self, config: ConfigReport) -> str | None:
+        """Return the Harbor command for one runnable config."""
+        if not config.runnable:
+            return None
+        config_path = config.path.resolve().relative_to(self.repo_root.resolve()).as_posix()
+        cd_command = shlex.join(["cd", str(self.repo_root.resolve())])
+        harbor_command = shlex.join(["harbor", "job", "start", "-c", config_path])
+        return f"{cd_command} && {harbor_command}"
+
+
+def harbor_version() -> str:
+    """Return the installed Harbor version."""
+    return version("harbor")
+
+
+def build_report(
+    *,
+    agent: str,
+    workspace: str,
+    repo_root: Path,
+    scan_result: RepositoryScan,
+    validations: list[ValidationOutcome],
+    trace_check: CheckResult,
+    discovered_at: datetime | None = None,
+) -> DiscoveryReport:
+    """Build one report from the repository scan and config preflights."""
+    configs = [
+        ConfigReport(
+            name=candidate.name,
+            path=candidate.path,
+            required_env_vars=validation.required_env_vars,
+            checks=validation.checks,
+        )
+        for candidate, validation in zip(scan_result.configs, validations, strict=True)
+    ]
+    return DiscoveryReport(
+        agent=agent,
+        workspace=workspace,
+        repo_root=repo_root.resolve(),
+        configs=configs,
+        dataset_paths=scan_result.dataset_paths,
+        ethos_path=scan_result.ethos_path,
+        harbor_version=harbor_version(),
+        discovered_at=discovered_at or datetime.now(UTC),
+        fingerprint=f"sha256:{scan_result.fingerprint}",
+        input_file_count=scan_result.input_file_count,
+        repository_checks=scan_result.checks,
+        trace_check=trace_check,
+    )
+
+
+def render_markdown(report: DiscoveryReport) -> str:
+    """Render YAML front matter and a concise status report."""
+    front = yaml.safe_dump(_front_matter(report), sort_keys=False, default_flow_style=False).rstrip()
+    lines = [f"# Discovery report for `{report.agent}`"]
+    status = format_report([*report.repository_checks, report.trace_check])
+    if status:
+        lines.extend(["", "```text", status, "```"])
+    if report.configs:
+        lines.extend(["", "## Harbor entrypoints"])
+    for config in report.configs:
+        path = _display_path(config.path, report.repo_root)
+        lines.extend(
+            [
+                "",
+                f"### `{config.name}` (`{path}`)",
+                "",
+                f"Runnable: {'true' if config.runnable else 'false'}",
+                "",
+                "```text",
+                format_report(config.checks),
+                "```",
+            ]
+        )
+        if command := report.run_command_for(config):
+            lines.extend(["", "```bash", command, "```"])
+    body = "\n".join(lines)
+    return f"---\n{front}\n---\n\n{body}\n"
+
+
+def _front_matter(report: DiscoveryReport) -> dict[str, Any]:
+    config = report.configs[0] if len(report.configs) == 1 else None
+    return {
+        "schema_version": report.schema_version,
+        "agent": report.agent,
+        "workspace": report.workspace,
+        "repo_root": str(report.repo_root),
+        "runnable": report.runnable,
+        "configs": [
+            {"name": config.name, "path": _display_path(config.path, report.repo_root)} for config in report.configs
+        ],
+        "config_path": _display_path(config.path if config is not None else None, report.repo_root),
+        "dataset_paths": [_display_path(path, report.repo_root) for path in report.dataset_paths],
+        "run_command": report.run_command,
+        "ethos_path": report.ethos_path,
+        "harbor_version": report.harbor_version,
+        "required_env_vars": [
+            {
+                "name": item.name,
+                "default": item.default,
+                "declared_in": _display_path(item.declared_in, report.repo_root),
+            }
+            for config in report.configs
+            for item in config.required_env_vars
+        ],
+        "discovered_at": report.discovered_at.isoformat(),
+        "fingerprint": report.fingerprint,
+        "input_file_count": report.input_file_count,
+        "checks": [check.model_dump(mode="json") for check in report.checks],
+    }
+
+
+def _display_path(path: Path | None, repo_root: Path) -> str | None:
+    if path is None:
+        return None
+    if not path.is_absolute():
+        return path.as_posix()
+    try:
+        return path.resolve().relative_to(repo_root.resolve()).as_posix()
+    except ValueError:
+        return path.as_posix()