diff --git a/docs/get-started/example-agent.mdx b/docs/get-started/example-agent.mdx index edffb4eb27..4b37b33074 100644 --- a/docs/get-started/example-agent.mdx +++ b/docs/get-started/example-agent.mdx @@ -22,14 +22,16 @@ make bootstrap ## 2. Start NeMo Platform -Once you have the platform installed and configured, you can start it. Run this block from the repository root. This will start the platform in a background process: +Agent traces are stored in ClickHouse, which runs as its own container. Start it first, from the repository root: ```bash -set -a -source packages/nmp_platform/config/local.env -set +a services/intake/scripts/spans/run_clickhouse.sh -uv run nemo services start --config packages/nmp_platform/config/local.yaml +``` + +Then run setup. This starts the platform services in the background and walks you through the configuration required to run the platform: + +```bash +uv run nemo setup ``` You should now be able to navigate to `http://localhost:8080` and see the NeMo Platform web UI. @@ -72,7 +74,7 @@ If the analyst discovers issues in your application, it will create what we call Let's run it: ```bash -uv run --frozen nemo insights analyze \ +uv run --frozen nemo agents analyst run \ --agent nemo-experimentalist-tau3-nooa \ --workspace tau3-airline \ --base-url "$NMP_BASE_URL" @@ -100,7 +102,7 @@ uv run --frozen nemo workspaces create canonical-tau3-airline \ Now we're ready to run the experimentalist! ```bash -uv run --frozen nemo experimentalist run \ +uv run --frozen nemo agents experimentalist run \ --no-insight \ --agent plugins/nemo-experimentalist/examples/tau3-nooa-agent \ --agent-spec plugins/nemo-experimentalist/examples/tau3-nooa-agent/AGENT-SPEC.md \ diff --git a/plugins/nemo-eval-author/pyproject.toml b/plugins/nemo-eval-author/pyproject.toml index 870d2dcefc..65dedcd6c3 100644 --- a/plugins/nemo-eval-author/pyproject.toml +++ b/plugins/nemo-eval-author/pyproject.toml @@ -17,6 +17,9 @@ dependencies = [ [project.entry-points."nemo.cli"] eval-author = "nemo_eval_author_plugin.cli:EvalAuthorCLI" +[project.entry-points."nemo.cli.agents"] +eval-author = "nemo_eval_author_plugin.cli:EvalAuthorCLI" + [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/cli.py b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/cli.py index d7c5667414..7a4d4401b0 100644 --- a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/cli.py +++ b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/cli.py @@ -1,7 +1,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Eval Author plugin CLI — ``nemo eval-author ...`` subcommands. +"""Eval Author plugin CLI — ``nemo agents eval-author ...`` subcommands. + +The same class is registered under both ``nemo.cli.agents`` and ``nemo.cli``, so every +verb is reachable as ``nemo agents eval-author `` (canonical) and as ``nemo +eval-author `` (retained for backward compatibility). Scaffolding. Every verb is registered under its final name so the command tree is discoverable and the child tickets have a landing spot, and each body exits non-zero @@ -21,14 +25,18 @@ from nemo_platform_plugin.cli import NemoCLI -def _not_implemented(command: str, ticket: str) -> NoReturn: - """Fail loudly, so a placeholder verb can never be mistaken for a successful run.""" - typer.echo(f"`nemo eval-author {command}` is not implemented yet ({ticket}).", err=True) +def _not_implemented(ctx: typer.Context, ticket: str) -> NoReturn: + """Fail loudly, so a placeholder verb can never be mistaken for a successful run. + + The message quotes ``ctx.command_path`` rather than a hardcoded path, so it names + whichever of the two mount points the caller actually used. + """ + typer.echo(f"`{ctx.command_path}` is not implemented yet ({ticket}).", err=True) raise typer.Exit(code=1) class EvalAuthorCLI(NemoCLI): - """``nemo eval-author ...`` subcommands.""" + """``nemo agents eval-author ...`` subcommands.""" name: ClassVar[str] = "eval-author" description: ClassVar[str] = "NeMo Eval Author commands." @@ -41,33 +49,33 @@ def _root() -> None: """Force subcommand dispatch even when only one verb is registered.""" @app.command("discover") - def discover() -> None: + def discover(ctx: typer.Context) -> None: """Discover candidate evaluation cases from agent traces.""" # TODO(ASE-677): declare flags and wire discovery. - _not_implemented("discover", "ASE-677") + _not_implemented(ctx, "ASE-677") @app.command("audit") - def audit() -> None: + def audit(ctx: typer.Context) -> None: """Audit an existing eval suite for coverage gaps.""" # TODO(ASE-676): declare flags and wire the audit. - _not_implemented("audit", "ASE-676") + _not_implemented(ctx, "ASE-676") @app.command("propose") - def propose() -> None: + def propose(ctx: typer.Context) -> None: """Propose eval suite additions for review.""" # TODO(ASE-675): declare flags and wire the proposal. - _not_implemented("propose", "ASE-675") + _not_implemented(ctx, "ASE-675") @app.command("run") - def run() -> None: + def run(ctx: typer.Context) -> None: """Run the Eval Author pipeline end to end.""" # TODO(ASE-673): declare flags and wire the pipeline to run_eval_author. - _not_implemented("run", "ASE-673") + _not_implemented(ctx, "ASE-673") @app.command("doctor") - def doctor() -> None: + def doctor(ctx: typer.Context) -> None: """Diagnose Eval Author setup: credentials, platform, runtime.""" # TODO(ASE-678): report the prerequisites the other verbs gate on. - _not_implemented("doctor", "ASE-678") + _not_implemented(ctx, "ASE-678") return app diff --git a/plugins/nemo-eval-author/tests/test_cli.py b/plugins/nemo-eval-author/tests/test_cli.py index 663d0f15f1..7f15ddff5a 100644 --- a/plugins/nemo-eval-author/tests/test_cli.py +++ b/plugins/nemo-eval-author/tests/test_cli.py @@ -4,8 +4,10 @@ """Scaffolding tests: the command tree exists, and every verb still refuses to run. The entry-point cases cover the ``pyproject.toml`` wiring that nothing else exercises. A -typo in the key or the import path does not fail an import; it just makes ``nemo +typo in the key or the import path does not fail an import; it just makes ``nemo agents eval-author`` quietly missing from the CLI, which no unit test of this module would catch. +The class is registered twice — canonically under ``nemo.cli.agents`` and, for backward +compatibility, under ``nemo.cli`` — so both groups are asserted. """ from importlib.metadata import EntryPoint, entry_points @@ -32,9 +34,9 @@ def app() -> typer.Typer: return cli.EvalAuthorCLI().get_cli() -def _eval_author_entry_point() -> EntryPoint: - matches = [entry for entry in entry_points(group="nemo.cli") if entry.name == "eval-author"] - assert matches, "no nemo.cli entry point named 'eval-author'; reinstall the plugin with uv sync" +def _eval_author_entry_point(group: str = "nemo.cli") -> EntryPoint: + matches = [entry for entry in entry_points(group=group) if entry.name == "eval-author"] + assert matches, f"no {group} entry point named 'eval-author'; reinstall the plugin with uv sync" return matches[0] @@ -54,11 +56,42 @@ def test_verb_refuses_to_run_and_names_its_ticket(app: typer.Typer, command: str assert ticket in result.output -def test_entry_point_key_matches_the_cli_name() -> None: +@pytest.mark.parametrize("group", ["nemo.cli.agents", "nemo.cli"]) +def test_entry_point_key_matches_the_cli_name(group: str) -> None: """Discovery rejects a plugin whose entry-point key differs from its ``name``.""" - assert _eval_author_entry_point().value == "nemo_eval_author_plugin.cli:EvalAuthorCLI" + assert _eval_author_entry_point(group).value == "nemo_eval_author_plugin.cli:EvalAuthorCLI" assert cli.EvalAuthorCLI.name == "eval-author" -def test_entry_point_loads_the_cli_class() -> None: - assert _eval_author_entry_point().load() is cli.EvalAuthorCLI +@pytest.mark.parametrize("group", ["nemo.cli.agents", "nemo.cli"]) +def test_entry_point_loads_the_cli_class(group: str) -> None: + assert _eval_author_entry_point(group).load() is cli.EvalAuthorCLI + + +def _mounted_under_agents() -> typer.Typer: + """The mount `AgentsCLI` performs, without importing the agents plugin.""" + agents = typer.Typer() + agents.add_typer(cli.EvalAuthorCLI().get_cli(), name="eval-author") + root = typer.Typer() + root.add_typer(agents, name="agents") + return root + + +@pytest.mark.parametrize(("command", "ticket"), _PLACEHOLDER_VERBS) +def test_verb_is_reachable_under_agents_and_names_that_path(command: str, ticket: str) -> None: + """The placeholder message must quote the path the caller typed, not a hardcoded one.""" + result = runner.invoke(_mounted_under_agents(), ["agents", "eval-author", command], prog_name="nemo") + + assert result.exit_code == 1, result.output + assert f"`nemo agents eval-author {command}` is not implemented yet ({ticket})." in result.output + + +@pytest.mark.parametrize(("command", "ticket"), _PLACEHOLDER_VERBS) +def test_legacy_top_level_path_still_works(command: str, ticket: str) -> None: + root = typer.Typer() + root.add_typer(cli.EvalAuthorCLI().get_cli(), name="eval-author") + + result = runner.invoke(root, ["eval-author", command], prog_name="nemo") + + assert result.exit_code == 1, result.output + assert f"`nemo eval-author {command}` is not implemented yet ({ticket})." in result.output diff --git a/plugins/nemo-experimentalist/AGENTS.md b/plugins/nemo-experimentalist/AGENTS.md index c8e9df1c43..3b70c51f4b 100644 --- a/plugins/nemo-experimentalist/AGENTS.md +++ b/plugins/nemo-experimentalist/AGENTS.md @@ -13,6 +13,21 @@ Inherited from the NeMo Platform monorepo that now hosts this plugin: ## Active migrations +### 2026-07-31: Command group nested under `nemo agents` + +The canonical path is `nemo agents experimentalist `. `ExperimentalistCLI` is +registered under the `nemo.cli.agents` entry-point group, which the `nemo-agents` +plugin's `AgentsCLI` discovers and mounts. + +The `nemo.cli` registration stays, so `nemo experimentalist ` keeps working. Both +groups point at the same class, so a new verb is written once and appears under both — +do not add a second implementation for the legacy path. Docs, help text, and error +messages should name the `nemo agents` form; prefer `ctx.command_path` over a hardcoded +path when a message quotes the command back to the user. + +The analyst and Eval Author moved in the same change: `nemo agents analyst run` (was +`nemo insights analyze`) and `nemo agents eval-author `. + ### 2026-07-28: Eval Author extracted to its own plugin, heading for standalone `plugins/nemo-eval-author/` (`nemo-eval-author-plugin`) owns the Eval Author agent package @@ -69,16 +84,16 @@ breaking rename with no compatibility aliases: Two names deliberately did **not** change. `optimizer.yaml` and the `.nemo-optimizer/` state directory are a shared contract with `nemo-insights-plugin`: `PROFILE_FILENAME` and `discover_profile()` live in -`nemo_insights_plugin.contracts.profile`, and `nemo insights analyze` writes +`nemo_insights_plugin.contracts.profile`, and `nemo agents analyst run` writes `/.nemo-optimizer/insights.yaml`, which this plugin reads as the default insight. Rename them only in lockstep with a Platform change to that contract. `EvolutionaryOptimizer` and `EvolutionaryOptimizerConfig` also keep their names — they describe the optimization algorithm, not the product. -The command group is top-level (`nemo experimentalist`) rather than the -eventual `nemo agents experimentalist`. The platform's `nemo.cli` entry-point -group is flat — only `nemo.jobs` and `nemo.functions` are dot-scoped — so -nesting under `nemo agents` needs a Platform-side change first. +At the time of this rename the command group was top-level (`nemo +experimentalist`), because the platform's `nemo.cli` entry-point group was flat +and nesting under `nemo agents` needed a Platform-side change first. That change +has since landed — see the `nemo agents` entry above for the current path. ### 2026-07-21: Curator renamed to Eval Author diff --git a/plugins/nemo-experimentalist/pyproject.toml b/plugins/nemo-experimentalist/pyproject.toml index e0add47d4f..4b1eb0033e 100644 --- a/plugins/nemo-experimentalist/pyproject.toml +++ b/plugins/nemo-experimentalist/pyproject.toml @@ -21,6 +21,9 @@ dependencies = [ [project.entry-points."nemo.cli"] experimentalist = "nemo_experimentalist_plugin.cli:ExperimentalistCLI" +[project.entry-points."nemo.cli.agents"] +experimentalist = "nemo_experimentalist_plugin.cli:ExperimentalistCLI" + [project.entry-points."nemo.skills"] experimentalist = "nemo_experimentalist_plugin.skills:skills_dir" diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/cli.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/cli.py index f8049e0b4a..3e324d6a77 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/cli.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/cli.py @@ -1,7 +1,12 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Experimentalist plugin CLI — ``nemo experimentalist ...`` subcommands.""" +"""Experimentalist plugin CLI — ``nemo agents experimentalist ...`` subcommands. + +The same class is registered under both ``nemo.cli.agents`` and ``nemo.cli``, so every +verb is reachable as ``nemo agents experimentalist `` (canonical) and as ``nemo +experimentalist `` (retained for backward compatibility). +""" import asyncio import os @@ -50,7 +55,7 @@ # Lazily imported in the experiment command: importing experimentalist.run reaches model # construction that requires EXPERIMENTALIST_API_* env at import time, and this module -# must import env-less so `nemo experimentalist doctor` can diagnose the missing creds. +# must import env-less so `nemo agents experimentalist doctor` can diagnose the missing creds. # Tests monkeypatch this global with a recorder, which bypasses the lazy import. run_experimentalist = None @@ -74,7 +79,7 @@ def _default_experiment_dir(profile: AgentProfile | None) -> Path: class ExperimentalistCLI(NemoCLI): - """``nemo experimentalist ...`` subcommands.""" + """``nemo agents experimentalist ...`` subcommands.""" name: ClassVar[str] = "experimentalist" description: ClassVar[str] = "NeMo Experimentalist commands." @@ -112,7 +117,7 @@ def run( "(surfaced in Studio). A path that exists on disk is read locally; " "otherwise it is fetched from the platform. Default: " "/.nemo-optimizer/insights.yaml when it exists (where " - "`nemo insights analyze` writes by default)." + "`nemo agents analyst run` writes by default)." ), ), insight_id: str | None = typer.Option( @@ -227,7 +232,7 @@ async def _flow() -> str: if no_insight: typer.echo("Insight disabled: --no-insight (Mode 2)", err=True) elif effective_insight.is_profile_default: - # `nemo insights analyze` writes here by default: the verbs connect flag-free. + # `nemo agents analyst run` writes here by default: the verbs connect flag-free. typer.echo( f"Insight file: {effective_insight.ref} (default; pass --insight to override)", err=True, diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/profile.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/profile.py index 8a61bbce48..89c6ce4d18 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/profile.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/profile.py @@ -4,9 +4,9 @@ """Experimentalist-owned models for the shared ``optimizer.yaml`` profile. The profile is the shared per-agent contract: the Platform-owned -``nemo insights analyze`` producer writes +``nemo agents analyst run`` producer writes ``/.nemo-optimizer/insights.yaml``, and -``nemo experimentalist run`` reads it by default. NeMo Experimentalist validates +``nemo agents experimentalist run`` reads it by default. NeMo Experimentalist validates the full experiment schema; NeMo Insights consumes only its analysis subset. """ diff --git a/plugins/nemo-experimentalist/tests/test_cli_profile.py b/plugins/nemo-experimentalist/tests/test_cli_profile.py index 5f25fc7b78..c48ce1172e 100644 --- a/plugins/nemo-experimentalist/tests/test_cli_profile.py +++ b/plugins/nemo-experimentalist/tests/test_cli_profile.py @@ -1203,5 +1203,5 @@ def test_experiment_help_names_insights_writer(app) -> None: assert result.exit_code == 0, result.output help_text = " ".join(result.output.replace("│", " ").split()) - assert "nemo insights analyze" in help_text + assert "nemo agents analyst run" in help_text assert "writes by default" in help_text diff --git a/plugins/nemo-insights/pyproject.toml b/plugins/nemo-insights/pyproject.toml index c1538d7e8f..3985d8422a 100644 --- a/plugins/nemo-insights/pyproject.toml +++ b/plugins/nemo-insights/pyproject.toml @@ -22,6 +22,9 @@ dependencies = [ [project.entry-points."nemo.cli"] insights = "nemo_insights_plugin.cli:InsightsCLI" +[project.entry-points."nemo.cli.agents"] +analyst = "nemo_insights_plugin.analyst.cli:AnalystCLI" + [project.entry-points."nemo.services"] insights = "nemo_insights_plugin.service:InsightsService" diff --git a/plugins/nemo-insights/src/nemo_insights_plugin/analyst/cli.py b/plugins/nemo-insights/src/nemo_insights_plugin/analyst/cli.py new file mode 100644 index 0000000000..8196eeb2b1 --- /dev/null +++ b/plugins/nemo-insights/src/nemo_insights_plugin/analyst/cli.py @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Analyst CLI — ``nemo agents analyst ...`` subcommands. + +The verbs are the module-level callbacks that back ``nemo insights analyze`` +and ``nemo insights doctor``, so the two command trees cannot drift. The +periodic-analysis and job surfaces stay on ``nemo insights``: they manage the +plugin's scheduled runs rather than driving the analyst itself. +""" + +from typing import ClassVar + +import typer +from nemo_insights_plugin.cli import analyze, doctor +from nemo_platform_plugin.cli import NemoCLI + + +class AnalystCLI(NemoCLI): + """``nemo agents analyst ...`` subcommands.""" + + name: ClassVar[str] = "analyst" + description: ClassVar[str] = "Analyze agent telemetry and record what the agent gets wrong." + + def get_cli(self) -> typer.Typer: + app = typer.Typer(help=self.description, no_args_is_help=True) + + @app.callback() + def _root() -> None: + """Force subcommand dispatch even when only one verb is registered.""" + + app.command("run")(analyze) + app.command("doctor")(doctor) + return app diff --git a/plugins/nemo-insights/src/nemo_insights_plugin/cli.py b/plugins/nemo-insights/src/nemo_insights_plugin/cli.py index 918002c4f1..cb110bd628 100644 --- a/plugins/nemo-insights/src/nemo_insights_plugin/cli.py +++ b/plugins/nemo-insights/src/nemo_insights_plugin/cli.py @@ -1,7 +1,13 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Insights CLI and contributed subcommands.""" +"""Insights CLI and contributed subcommands. + +The module-level :func:`analyze` and :func:`doctor` callbacks are shared with +:class:`nemo_insights_plugin.analyst.cli.AnalystCLI`, which mounts them as the canonical +``nemo agents analyst run`` and ``nemo agents analyst doctor``. They stay registered here +as ``nemo insights analyze`` / ``nemo insights doctor`` for backward compatibility. +""" import asyncio import json @@ -184,6 +190,135 @@ async def _run_analysis(analysis: _ResolvedAnalysis, *, verbose: bool) -> str: raise typer.Exit(1) from None +def analyze( + agent: str | None = typer.Option( + None, + "--agent", + help="Name of the agent (agent under test) the analyst should focus on.", + ), + agent_spec: Path | None = typer.Option( + None, + "--agent-spec", + help="Path to a markdown file describing the agent under test (its spec).", + exists=True, + readable=True, + ), + workspace: str | None = typer.Option( + None, + "--workspace", + help="Workspace the analyst should operate in.", + ), + base_url: str | None = typer.Option( + None, + "--base-url", + help="Base URL of the running NMP instance the analyst's tools should call.", + ), + profile_path: Path | None = typer.Option( + None, + "--profile", + help="Path to optimizer.yaml. Default: discovered by walking up from cwd.", + exists=True, + dir_okay=False, + readable=True, + ), + insights_output: Path | None = typer.Option( + None, + "--insights-file-output", + help=( + "Read and write insights from this local YAML file instead " + "of the Insights plugin API. Lets the analyst run against a " + "deployment that hosts observability data but not this " + "plugin; each run merges into the file. Trace/feedback reads " + "still hit --base-url." + ), + ), + verbose: bool = typer.Option( + False, + "--verbose", + "-v", + help=( + "Stream the analyst's tool calls and reasoning to stderr " + "while it runs. Off by default so that stdout stays clean " + "for piping the final answer." + ), + ), +) -> None: + """Run the analyst agent against a running NMP instance. + + Builds the analyst agent with ``--agent`` (and optional + ``--agent-spec``) formatted into its instructions and tools scoped + to ``--agent`` / ``--workspace`` / ``--base-url``, runs it, and + prints whatever the agent returns. + """ + try: + analysis = _resolve_analysis( + agent=agent, + agent_spec=agent_spec, + workspace=workspace, + base_url=base_url, + profile_path=profile_path, + insights_output=insights_output, + ) + output = asyncio.run(_run_analysis(analysis, verbose=verbose)) + except (ProfileError, EnvFileError, InsightsFileError, OSError, UnicodeError) as exc: + typer.echo(f"Error: {_one_line_error(exc)}", err=True) + raise typer.Exit(1) from None + typer.echo(output) + + +def doctor( + profile_path: Path | None = typer.Option( + None, + "--profile", + help="Path to optimizer.yaml. Default: discovered by walking up from cwd.", + exists=True, + dir_okay=False, + readable=True, + ), + base_url: str | None = typer.Option( + None, + "--base-url", + help="Base URL of the running NMP instance to check.", + ), +) -> None: + """Check whether the current profile is ready for analysis.""" + try: + try: + profile, profile_error = _load_profile_or_error(profile_path) + except ProfileError as exc: + profile, profile_error = None, str(exc) + spec_path: Path | None = None + spec_error: str | None = None + if profile is not None: + try: + spec_path = pick_agent_spec(profile) + except ProfileError as exc: + spec_error = str(exc) + _, spec_results = read_agent_spec(spec_path, spec_error) + + async def _flow() -> list[CheckResult]: + results = check_profile(profile, profile_error) + results.extend(spec_results) + results.extend( + await check_environment( + agent=profile.agent if profile is not None else None, + workspace=profile.workspace if profile is not None else None, + base_url=resolve_base_url(base_url), + profile_dir=profile.profile_dir if profile is not None else None, + probes=_PREFLIGHT_PROBES, + ) + ) + return results + + results = asyncio.run(_flow()) + except (EnvFileError, OSError, UnicodeError) as exc: + typer.echo(f"Error: {_one_line_error(exc)}", err=True) + raise typer.Exit(1) from None + typer.echo(format_report(results)) + if required_failures(results): + raise typer.Exit(code=1) + + class InsightsCLI(NemoCLI): """``nemo insights ...`` subcommands.""" @@ -203,134 +338,8 @@ def _root() -> None: ) app.add_typer(analysis_app, name="analysis") - @app.command("analyze") - def analyze( - agent: str | None = typer.Option( - None, - "--agent", - help="Name of the agent (agent under test) the analyst should focus on.", - ), - agent_spec: Path | None = typer.Option( - None, - "--agent-spec", - help="Path to a markdown file describing the agent under test (its spec).", - exists=True, - readable=True, - ), - workspace: str | None = typer.Option( - None, - "--workspace", - help="Workspace the analyst should operate in.", - ), - base_url: str | None = typer.Option( - None, - "--base-url", - help="Base URL of the running NMP instance the analyst's tools should call.", - ), - profile_path: Path | None = typer.Option( - None, - "--profile", - help="Path to optimizer.yaml. Default: discovered by walking up from cwd.", - exists=True, - dir_okay=False, - readable=True, - ), - insights_output: Path | None = typer.Option( - None, - "--insights-file-output", - help=( - "Read and write insights from this local YAML file instead " - "of the Insights plugin API. Lets the analyst run against a " - "deployment that hosts observability data but not this " - "plugin; each run merges into the file. Trace/feedback reads " - "still hit --base-url." - ), - ), - verbose: bool = typer.Option( - False, - "--verbose", - "-v", - help=( - "Stream the analyst's tool calls and reasoning to stderr " - "while it runs. Off by default so that stdout stays clean " - "for piping the final answer." - ), - ), - ) -> None: - """Run the analyst agent against a running NMP instance. - - Builds the analyst agent with ``--agent`` (and optional - ``--agent-spec``) formatted into its instructions and tools scoped - to ``--agent`` / ``--workspace`` / ``--base-url``, runs it, and - prints whatever the agent returns. - """ - try: - analysis = _resolve_analysis( - agent=agent, - agent_spec=agent_spec, - workspace=workspace, - base_url=base_url, - profile_path=profile_path, - insights_output=insights_output, - ) - output = asyncio.run(_run_analysis(analysis, verbose=verbose)) - except (ProfileError, EnvFileError, InsightsFileError, OSError, UnicodeError) as exc: - typer.echo(f"Error: {_one_line_error(exc)}", err=True) - raise typer.Exit(1) from None - typer.echo(output) - - @app.command("doctor") - def doctor( - profile_path: Path | None = typer.Option( - None, - "--profile", - help="Path to optimizer.yaml. Default: discovered by walking up from cwd.", - exists=True, - dir_okay=False, - readable=True, - ), - base_url: str | None = typer.Option( - None, - "--base-url", - help="Base URL of the running NMP instance to check.", - ), - ) -> None: - """Check whether the current profile is ready for analysis.""" - try: - try: - profile, profile_error = _load_profile_or_error(profile_path) - except ProfileError as exc: - profile, profile_error = None, str(exc) - spec_path: Path | None = None - spec_error: str | None = None - if profile is not None: - try: - spec_path = pick_agent_spec(profile) - except ProfileError as exc: - spec_error = str(exc) - _, spec_results = read_agent_spec(spec_path, spec_error) - - async def _flow() -> list[CheckResult]: - results = check_profile(profile, profile_error) - results.extend(spec_results) - results.extend( - await check_environment( - agent=profile.agent if profile is not None else None, - workspace=profile.workspace if profile is not None else None, - base_url=resolve_base_url(base_url), - profile_dir=profile.profile_dir if profile is not None else None, - probes=_PREFLIGHT_PROBES, - ) - ) - return results - - results = asyncio.run(_flow()) - except (EnvFileError, OSError, UnicodeError) as exc: - typer.echo(f"Error: {_one_line_error(exc)}", err=True) - raise typer.Exit(1) from None - typer.echo(format_report(results)) - if required_failures(results): - raise typer.Exit(code=1) + app.command("analyze")(analyze) + app.command("doctor")(doctor) @analysis_app.command("enable") def enable_analysis(