From e9ca859fd9d08dcc93517d2c3e874e7f9c952b8b Mon Sep 17 00:00:00 2001 From: Jash Gulabrai Date: Wed, 27 May 2026 17:25:04 -0400 Subject: [PATCH 01/10] feat(guardrails): Add Guardrails plugin benchmark harness to CI Signed-off-by: Jash Gulabrai --- .github/workflows/ci.yaml | 40 ++ Makefile | 6 + plugins/nemo-guardrails/benchmarks/README.md | 171 ++++++++ .../benchmarks/artifacts/.gitignore | 2 + .../nmp_igw_guardrails_sweep_concurrency.yaml | 30 ++ plugins/nemo-guardrails/pyproject.toml | 14 + .../benchmarks/aiperf_runner.py | 160 +++++++ .../benchmarks/bootstrap.py | 117 ++++++ .../benchmarks/constants.py | 36 ++ .../benchmarks/paths.py | 95 +++++ .../benchmarks/processes.py | 148 +++++++ .../benchmarks/report.py | 107 +++++ .../nemo_guardrails_plugin/benchmarks/run.py | 390 ++++++++++++++++++ .../benchmarks/seeding.py | 268 ++++++++++++ .../nemo_guardrails_plugin/benchmarks/shim.py | 140 +++++++ .../unit/benchmarks/test_aiperf_runner.py | 131 ++++++ .../tests/unit/benchmarks/test_paths.py | 108 +++++ .../tests/unit/benchmarks/test_report.py | 90 ++++ .../tests/unit/benchmarks/test_seeding.py | 217 ++++++++++ .../tests/unit/benchmarks/test_shim.py | 105 +++++ uv.lock | 9 + 21 files changed, 2384 insertions(+) create mode 100644 plugins/nemo-guardrails/benchmarks/README.md create mode 100644 plugins/nemo-guardrails/benchmarks/artifacts/.gitignore create mode 100644 plugins/nemo-guardrails/benchmarks/configs/nmp_igw_guardrails_sweep_concurrency.yaml create mode 100644 plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/aiperf_runner.py create mode 100644 plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/bootstrap.py create mode 100644 plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/constants.py create mode 100644 plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/paths.py create mode 100644 plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/processes.py create mode 100644 plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/report.py create mode 100644 plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/run.py create mode 100644 plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/seeding.py create mode 100644 plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/shim.py create mode 100644 plugins/nemo-guardrails/tests/unit/benchmarks/test_aiperf_runner.py create mode 100644 plugins/nemo-guardrails/tests/unit/benchmarks/test_paths.py create mode 100644 plugins/nemo-guardrails/tests/unit/benchmarks/test_report.py create mode 100644 plugins/nemo-guardrails/tests/unit/benchmarks/test_seeding.py create mode 100644 plugins/nemo-guardrails/tests/unit/benchmarks/test_shim.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0973fc23e9..5a0b98cb29 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -381,6 +381,46 @@ jobs: ;; esac + benchmark-guardrails: + name: NeMo Guardrails benchmark + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout nemo-platform + uses: actions/checkout@v6 + with: + path: nemo-platform + - name: Checkout NeMo-Guardrails + uses: actions/checkout@v6 + with: + repository: NVIDIA/NeMo-Guardrails + path: NeMo-Guardrails + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + python-version: "3.11" + enable-cache: true + - name: Bootstrap Python environment + working-directory: nemo-platform + run: make bootstrap-python + env: + PYTORCH_DEPS: cpu + - name: Run benchmark sweep + working-directory: nemo-platform + run: make benchmark-guardrails + env: + NEMO_GUARDRAILS_REPO_ROOT: ${{ github.workspace }}/NeMo-Guardrails + _TYPER_FORCE_DISABLE_TERMINAL: "1" + - name: Upload benchmark artifacts + if: always() + uses: actions/upload-artifact@v6 + with: + name: benchmark-guardrails-results + retention-days: 30 + path: | + nemo-platform/report.xml + nemo-platform/plugins/nemo-guardrails/benchmarks/artifacts/runs/ + coverage-comment: name: Post coverage comment needs: [python-unit-test, python-integration-test] diff --git a/Makefile b/Makefile index 7709b62d2d..7e4a4930d6 100644 --- a/Makefile +++ b/Makefile @@ -440,3 +440,9 @@ test-e2e-kubernetes-gpu: ## Run GPU e2e tests against Kubernetes (requires GPU n test-e2e-kubernetes-gpu-customizer: ## Run GPU customizer e2e tests against Kubernetes (requires GPU nodes; set NMP_E2E_CLUSTER_URL) @echo "Running GPU customizer e2e tests with Kubernetes..." uv run --frozen pytest e2e/test_customizer.py --kubernetes --feature gpu --feature customizer --log-cli-level=INFO -v --junitxml=report-kubernetes-gpu-customizer.xml + +.PHONY: benchmark-guardrails +benchmark-guardrails: ## Run nemo-guardrails IGW benchmark sweep (set BENCHMARK_ARGS for extra flags) + @echo "Running nemo-guardrails IGW benchmark..." + uv run --frozen --package nemo-guardrails-plugin --extra bench \ + python -m nemo_guardrails_plugin.benchmarks.run $(BENCHMARK_ARGS) diff --git a/plugins/nemo-guardrails/benchmarks/README.md b/plugins/nemo-guardrails/benchmarks/README.md new file mode 100644 index 0000000000..3b3881f96e --- /dev/null +++ b/plugins/nemo-guardrails/benchmarks/README.md @@ -0,0 +1,171 @@ +# NeMo Guardrails Plugin Benchmarks + +Local harness for benchmarking the `nemo-guardrails` Inference Gateway +middleware against the upstream NeMo Guardrails benchmark suite (mock LLMs + +AIPerf sweep). + +The implementation lives in `nemo_guardrails_plugin.benchmarks` (under +`plugins/nemo-guardrails/src/`). The harness does **not** copy benchmark code +from the NeMo Guardrails repository; it expects a local checkout and runs its +benchmark modules with `PYTHONPATH` pointed at that checkout. + +## Layout + +```text +plugins/nemo-guardrails/benchmarks/ + configs/ + nmp_igw_guardrails_sweep_concurrency.yaml # AIPerf sweep template + artifacts/ # per-run outputs (gitignored) + generated/ # placeholder; per-run artifacts live under artifacts/runs/ +plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/ + run.py # entry point: `python -m nemo_guardrails_plugin.benchmarks.run` + paths.py # filesystem layout + constants.py # workspace / VM / provider names + processes.py # subprocess supervision (process groups + ExitStack) + seeding.py # NMP SDK calls (workspace, providers, GuardrailConfig, VM) + aiperf_runner.py # rewrite AIPerf config + invoke upstream sweep + collect results + bootstrap.py # manage the isolated venv that hosts the `aiperf` CLI + shim.py # tiny HTTP shim that satisfies AIPerf's `/v1/models` pre-check + report.py # emit JUnit XML +``` + +## Prerequisites + +- This repo bootstrapped via `make bootstrap-python` (the harness runs in + `.venv` and imports the NMP SDK from the workspace). +- A local NeMo Guardrails checkout. By default the harness looks at + `../NeMo-Guardrails` relative to the NMP repo root. +- `uv` available on `PATH`. +- Ports `8000`, `8001`, `8080`, and `8090` available — unless you opt into + reusing an already-running local NMP via `--reuse-services`. Port `8090` is + used by an internal shim that satisfies AIPerf's hard-coded `/v1/models` + health probe. + +The harness has its own `pyyaml` + `httpx` requirements that are declared as +the `bench` extra on `nemo-guardrails-plugin`. The `make benchmark-guardrails` +target installs them automatically via `uv run --extra bench`; they are not +part of the plugin's runtime install. + +The upstream `aiperf` CLI itself pins `aiofiles<24.2`, which conflicts with +NMP's evaluator-service. To avoid downgrading the shared workspace venv, the +harness creates an isolated venv at +`plugins/nemo-guardrails/benchmarks/artifacts/venvs/aiperf/` on first run and +reuses it on subsequent runs. CI gets a fresh one each invocation; locally +this caches across runs for fast iteration. + +## Run locally + +From the NMP repo root: + +```bash +make benchmark-guardrails +``` + +If your NeMo Guardrails checkout is somewhere else: + +```bash +NEMO_GUARDRAILS_REPO_ROOT=/path/to/NeMo-Guardrails make benchmark-guardrails +``` + +To pass through arbitrary harness flags: + +```bash +make benchmark-guardrails BENCHMARK_ARGS="--verbose --reuse-services" +``` + +The default sweep runs concurrency levels: + +```text +1, 2, 4, 8, 16, 32, 64 +``` + +With the default 60-second benchmark duration, expect ~10 minutes after +service bootstrap. + +## What the harness starts + +- The upstream benchmark **mock app LLM** on `http://localhost:8000`, +- The upstream benchmark **mock content-safety LLM** on `http://localhost:8001`, +- Local **NMP services** on `http://localhost:8080`, unless `--reuse-services`. + +It then seeds NMP via the SDK with: + +- workspace `benchmark`, +- app model provider `benchmark-app-llm`, +- content-safety model provider `benchmark-content-safety-llm`, +- guardrail config `content-safety-local`, +- VirtualModel `benchmark/guardrails-vm` with `nemo-guardrails` attached to + both request and response middleware. + +The benchmark target is: + +```text +http://localhost:8080/apis/inference-gateway/v2/workspaces/benchmark/openai/-/v1/chat/completions +``` + +## Useful flags / environment + +The harness accepts both CLI flags and environment variables: + +| CLI flag | Environment variable | Default | +|-----------------------------------|----------------------------------|------------------------| +| `--nemo-guardrails-repo-root` | `NEMO_GUARDRAILS_REPO_ROOT` | `../NeMo-Guardrails` | +| `--reuse-services` | `NMP_BENCHMARK_REUSE_SERVICES=1` | start `nemo services run` | +| `--keep-running` | `NMP_BENCHMARK_KEEP_RUNNING=1` | tear down on exit | +| `--mock-workers` | `NMP_BENCHMARK_MOCK_WORKERS` | `4` | +| `--junit-path` | _n/a_ | `/report.xml` | +| `--run-id` | _n/a_ | current timestamp | + +`--keep-running` leaves child processes alive for post-mortem inspection; the +list of PIDs is recorded in the per-run directory's `pids.txt`. + +## Outputs + +Each run writes artifacts under: + +```text +plugins/nemo-guardrails/benchmarks/artifacts/runs// + logs/ + mock-app-llm.log + mock-content-safety-llm.log + nmp-services.log + aiperf.log + generated/ + app_provider.json + content_safety_provider.json + virtual_model.json + content_safety_local_nmp_request.json + nmp_igw_guardrails_sweep_concurrency.yaml # runtime AIPerf config + aiperf_results//// + run_metadata.json + process_result.json + profile_export*.json # written by aiperf + pids.txt +``` + +`report.xml` is written to the repo root by default so CI's +`actions/upload-artifact` step picks it up at the same path as other test +suites. Override with `--junit-path`. + +## CI + +A `benchmark-guardrails` job in `.github/workflows/ci.yaml` checks out both +this repo and `NVIDIA/NeMo-Guardrails`, runs `make bootstrap-python` and +`make benchmark-guardrails`, and uploads the per-run artifacts directory plus +`report.xml` (so GitHub renders sweep pass/fail in the PR view). + +Pass/fail is currently driven purely by `aiperf` exit code; no latency +thresholds are enforced. + +## Cleanup + +By default the harness only stops processes it started. It does not kill +unrelated processes on ports `8000`, `8001`, or `8080`. + +Local NMP state is isolated by default under: + +```text +plugins/nemo-guardrails/benchmarks/artifacts/nmp-data +``` + +Delete that directory for a completely fresh local benchmark state. diff --git a/plugins/nemo-guardrails/benchmarks/artifacts/.gitignore b/plugins/nemo-guardrails/benchmarks/artifacts/.gitignore new file mode 100644 index 0000000000..d6b7ef32c8 --- /dev/null +++ b/plugins/nemo-guardrails/benchmarks/artifacts/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/plugins/nemo-guardrails/benchmarks/configs/nmp_igw_guardrails_sweep_concurrency.yaml b/plugins/nemo-guardrails/benchmarks/configs/nmp_igw_guardrails_sweep_concurrency.yaml new file mode 100644 index 0000000000..1c141f3ec6 --- /dev/null +++ b/plugins/nemo-guardrails/benchmarks/configs/nmp_igw_guardrails_sweep_concurrency.yaml @@ -0,0 +1,30 @@ +# Benchmark NMP IGW VirtualModel with nemo-guardrails request/response middleware. + +batch_name: nmp_igw_guardrails_sweep_concurrency +output_base_dir: plugins/nemo-guardrails/benchmarks/artifacts/aiperf_results + +base_config: + model: benchmark/guardrails-vm + tokenizer: meta-llama/Llama-3.3-70B-Instruct + # AIPerf's pre-check does GET urljoin(url, "/v1/models"); we run a thin + # shim on :8090 that satisfies that probe and reverse-proxies + # /v1/chat/completions through to NMP's IGW. See + # nemo_guardrails_plugin.benchmarks.shim for the implementation. + url: "http://localhost:8090" + endpoint: "/v1/chat/completions" + endpoint_type: chat + streaming: false + + warmup_request_count: 10 + benchmark_duration: 60 + concurrency: 0 + request_rate_mode: constant + + random_seed: 12345 + prompt_input_tokens_mean: 100 + prompt_input_tokens_stddev: 10 + prompt_output_tokens_mean: 50 + prompt_output_tokens_stddev: 5 + +sweeps: + concurrency: [1, 2, 4, 8, 16, 32, 64] diff --git a/plugins/nemo-guardrails/pyproject.toml b/plugins/nemo-guardrails/pyproject.toml index 72b07b447d..4a52e7fc6f 100644 --- a/plugins/nemo-guardrails/pyproject.toml +++ b/plugins/nemo-guardrails/pyproject.toml @@ -12,6 +12,20 @@ dependencies = [ "dataclasses-json>=0.6.7", ] +[project.optional-dependencies] +# Dependencies for the local IGW benchmark harness in +# `nemo_guardrails_plugin.benchmarks`. Activate with `uv sync --extra bench` or +# `uv run --extra bench ...` (see `make benchmark-guardrails`). +bench = [ + "httpx>=0.27", + "pyyaml>=6.0", + # NOTE: aiperf itself is *not* listed here. Its 0.x line pins + # aiofiles<24.2 which conflicts with evaluator-service's + # aiofiles>=25.1. To avoid downgrading the shared workspace venv, the + # harness installs aiperf into a dedicated venv at run time; see + # `nemo_guardrails_plugin.benchmarks.bootstrap`. +] + [project.entry-points."nemo.inference_middleware"] nemo-guardrails = "nemo_guardrails_plugin.middleware:GuardrailsMiddleware" diff --git a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/aiperf_runner.py b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/aiperf_runner.py new file mode 100644 index 0000000000..8c61fd66d0 --- /dev/null +++ b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/aiperf_runner.py @@ -0,0 +1,160 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Invoke the upstream NeMo Guardrails AIPerf sweep runner as a subprocess. + +The upstream code lives in ``${NEMO_GUARDRAILS_REPO_ROOT}/benchmark/`` and is not +installed as a package; we set ``PYTHONPATH`` so ``python -m benchmark.aiperf`` +resolves. +""" + +from __future__ import annotations + +import json +import logging +import os +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml + +log = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class SweepRunResult: + """Outcome of a single ``aiperf profile`` invocation (one concurrency level).""" + + sweep_label: str + output_dir: Path + return_code: int + duration_seconds: float + metadata_path: Path | None + process_result_path: Path | None + + @property + def passed(self) -> bool: + return self.return_code == 0 + + +def rewrite_aiperf_config(*, template: Path, output: Path, output_base_dir: Path) -> dict[str, Any]: + """Copy the checked-in template to ``output`` with ``output_base_dir`` overridden. + + The runtime copy is written under the per-run directory so concurrent or + historical runs cannot stomp on each other. Returns the parsed config dict. + """ + if not template.is_file(): + raise FileNotFoundError(f"AIPerf template not found: {template}") + config = yaml.safe_load(template.read_text(encoding="utf-8")) + if not isinstance(config, dict): + raise ValueError(f"AIPerf template {template} did not parse as a mapping") + config["output_base_dir"] = str(output_base_dir) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(yaml.safe_dump(config, sort_keys=False), encoding="utf-8") + return config + + +def run_aiperf_sweep( + *, + ng_repo_root: Path, + runtime_config: Path, + log_path: Path, + python_executable: str | None = None, + extra_env: dict[str, str] | None = None, +) -> int: + """Run ``python -m benchmark.aiperf --config-file ...`` and tee output. + + Returns the subprocess exit code. The caller decides whether to treat a + non-zero code as a sweep-level failure or as a fail-fast. + + ``python_executable`` should point at the python in the dedicated aiperf + venv (see :mod:`nemo_guardrails_plugin.benchmarks.bootstrap`). ``extra_env`` + is used to prepend that venv's ``bin/`` to ``PATH`` so the ``aiperf`` CLI + is resolvable when the wrapper shells out to it. + + Note: AIPerf's built-in pre-flight check does a GET on + ``urljoin(config.base_config.url, "/v1/models")`` with no override + available upstream. The harness runs a tiny shim that satisfies this + probe; see :mod:`nemo_guardrails_plugin.benchmarks.shim`. + """ + python_bin = python_executable or sys.executable + # Upstream `benchmark.aiperf` is a single-command Typer app; invoking + # `python -m benchmark.aiperf --config-file ...` runs the only command. + # Passing a literal `run` subcommand confuses Typer. + cmd = [ + python_bin, + "-m", + "benchmark.aiperf", + "--config-file", + str(runtime_config), + ] + + env = {**os.environ, "PYTHONPATH": str(ng_repo_root)} + if extra_env: + env.update(extra_env) + log_path.parent.mkdir(parents=True, exist_ok=True) + log.info("Running aiperf sweep: %s (cwd=%s)", " ".join(cmd), ng_repo_root) + with log_path.open("wb") as log_fh: + proc = subprocess.run( # noqa: S603 - command is constructed internally + cmd, + cwd=str(ng_repo_root), + env=env, + stdout=log_fh, + stderr=subprocess.STDOUT, + check=False, + ) + return proc.returncode + + +def collect_sweep_results(aiperf_output_dir: Path) -> list[SweepRunResult]: + """Walk the AIPerf output tree and surface per-sweep exit status. + + Layout: + ``////{run_metadata.json,process_result.json}`` + + Missing ``process_result.json`` is treated as a failure for that sweep so a + crashed AIPerf subprocess does not silently pass. + """ + if not aiperf_output_dir.is_dir(): + return [] + + results: list[SweepRunResult] = [] + for batch_dir in sorted(p for p in aiperf_output_dir.iterdir() if p.is_dir()): + for timestamp_dir in sorted(p for p in batch_dir.iterdir() if p.is_dir()): + for sweep_dir in sorted(p for p in timestamp_dir.iterdir() if p.is_dir()): + results.append(_load_sweep_result(sweep_dir)) + return results + + +def _load_sweep_result(sweep_dir: Path) -> SweepRunResult: + metadata_path = sweep_dir / "run_metadata.json" + process_result_path = sweep_dir / "process_result.json" + + return_code = 1 + duration = 0.0 + + if process_result_path.is_file(): + try: + data = json.loads(process_result_path.read_text(encoding="utf-8")) + return_code = int(data.get("returncode", 1)) + except (json.JSONDecodeError, OSError, ValueError, TypeError) as exc: + log.warning("Could not parse %s: %s", process_result_path, exc) + + if metadata_path.is_file(): + try: + md = json.loads(metadata_path.read_text(encoding="utf-8")) + duration = float(md.get("duration_seconds", 0.0) or 0.0) + except (json.JSONDecodeError, OSError, ValueError, TypeError) as exc: + log.warning("Could not parse %s: %s", metadata_path, exc) + + return SweepRunResult( + sweep_label=sweep_dir.name, + output_dir=sweep_dir, + return_code=return_code, + duration_seconds=duration, + metadata_path=metadata_path if metadata_path.is_file() else None, + process_result_path=process_result_path if process_result_path.is_file() else None, + ) diff --git a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/bootstrap.py b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/bootstrap.py new file mode 100644 index 0000000000..9253277592 --- /dev/null +++ b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/bootstrap.py @@ -0,0 +1,117 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Bootstrap an isolated venv for the upstream AIPerf load generator. + +`aiperf` (the PyPI package) pins ``aiofiles<24.2`` while NMP's +evaluator-service requires ``aiofiles>=25.1``. We can't add `aiperf` to the +workspace lockfile without downgrading the shared venv and breaking other +services, so the harness manages a dedicated, lock-free venv just for the +AIPerf binary. + +The venv is created on first use (idempotent) and reused on subsequent runs. +The path is deterministic so local dev iterations are fast; in CI the cache is +discarded each run, which is acceptable since `aiperf` is a small install. +""" + +from __future__ import annotations + +import logging +import os +import subprocess +import sys +from pathlib import Path + +log = logging.getLogger("nemo_guardrails_plugin.benchmarks.bootstrap") + +# Versions pinned to what the upstream NeMo-Guardrails README installs alongside +# `python -m benchmark.aiperf`. We don't tighten further; aiperf's own metadata +# pins its dependencies. +_AIPERF_PACKAGES = ("aiperf", "huggingface_hub", "typer>=0.9", "httpx>=0.27") + + +class BootstrapError(RuntimeError): + """Raised when we can't materialise the aiperf venv.""" + + +def _venv_python(venv_dir: Path) -> Path: + """Return the venv's python binary regardless of platform.""" + if os.name == "nt": + return venv_dir / "Scripts" / "python.exe" + return venv_dir / "bin" / "python" + + +def _venv_bin(venv_dir: Path) -> Path: + if os.name == "nt": + return venv_dir / "Scripts" + return venv_dir / "bin" + + +def ensure_aiperf_venv(venv_dir: Path, *, force: bool = False) -> Path: + """Ensure ``venv_dir`` contains a usable ``aiperf`` install. + + Returns the path to the venv's python interpreter so the caller can invoke + `` -m benchmark.aiperf`` with the right environment. + """ + aiperf_bin = _venv_bin(venv_dir) / ("aiperf.exe" if os.name == "nt" else "aiperf") + python_bin = _venv_python(venv_dir) + + if not force and aiperf_bin.exists() and python_bin.exists(): + log.info("Reusing existing aiperf venv at %s", venv_dir) + return python_bin + + log.info("Creating aiperf venv at %s", venv_dir) + venv_dir.parent.mkdir(parents=True, exist_ok=True) + + # Prefer `uv venv` for speed and to match the rest of the project; fall + # back to stdlib `venv` if `uv` isn't on PATH. + try: + subprocess.run( # noqa: S603 - command is constructed internally + ["uv", "venv", "--python", "3.11", str(venv_dir)], + check=True, + capture_output=True, + ) + except FileNotFoundError: + log.warning("uv not found on PATH; falling back to `python -m venv`") + subprocess.run( # noqa: S603 - command is constructed internally + [sys.executable, "-m", "venv", str(venv_dir)], + check=True, + capture_output=True, + ) + except subprocess.CalledProcessError as e: + raise BootstrapError(f"Failed to create aiperf venv at {venv_dir}: {e.stderr.decode(errors='replace')}") from e + + log.info("Installing %s into %s", ", ".join(_AIPERF_PACKAGES), venv_dir) + try: + # `uv pip install --python ` is hermetic: it installs into + # the target venv without touching the workspace lockfile. + subprocess.run( # noqa: S603 - command is constructed internally + ["uv", "pip", "install", "--python", str(python_bin), *_AIPERF_PACKAGES], + check=True, + ) + except FileNotFoundError: + log.warning("uv not found; falling back to `pip install`") + subprocess.run( # noqa: S603 - command is constructed internally + [str(python_bin), "-m", "pip", "install", *_AIPERF_PACKAGES], + check=True, + ) + except subprocess.CalledProcessError as e: + raise BootstrapError(f"Failed to install aiperf into {venv_dir}: {e}") from e + + if not aiperf_bin.exists(): + raise BootstrapError(f"aiperf install completed but {aiperf_bin} is missing") + + return python_bin + + +def env_with_venv_on_path(venv_dir: Path, base_env: dict[str, str] | None = None) -> dict[str, str]: + """Return a copy of ``base_env`` with the venv's bin dir prepended to PATH. + + The upstream `python -m benchmark.aiperf` wrapper shells out to a literal + ``aiperf`` command, so the venv's bin dir has to come before whatever was + on PATH inherited from the parent. + """ + env = dict(base_env if base_env is not None else os.environ) + venv_bin = str(_venv_bin(venv_dir)) + env["PATH"] = venv_bin + os.pathsep + env.get("PATH", "") + return env diff --git a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/constants.py b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/constants.py new file mode 100644 index 0000000000..fd6aff89b0 --- /dev/null +++ b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/constants.py @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Shared constants for the benchmark harness.""" + +from __future__ import annotations + +WORKSPACE = "benchmark" +GUARDRAIL_CONFIG = "content-safety-local" +VM_NAME = "guardrails-vm" + +# ModelProvider that proxies requests to the mock main model +APP_PROVIDER = "benchmark-app-llm" +APP_PROVIDER_URL = "http://localhost:8000" +APP_MODEL_NAME = "meta/llama-3.3-70b-instruct" + +# ModelProvider that proxies requests to the mock content-safety model +CS_PROVIDER = "benchmark-content-safety-llm" +CS_PROVIDER_URL = "http://localhost:8001" +CS_MODEL_NAME = "nvidia/llama-3.1-nemoguard-8b-content-safety" + +NMP_BASE_URL = "http://localhost:8080" +NMP_HEALTH_PATH = "/health/ready" +IGW_CHAT_PATH = f"/apis/inference-gateway/v2/workspaces/{WORKSPACE}/openai/-/v1/chat/completions" + +# Local shim that satisfies AIPerf's pre-check and reverse-proxies chat +# completion requests through to NMP's IGW. See +# `nemo_guardrails_plugin.benchmarks.shim` for the implementation. +AIPERF_SHIM_HOST = "127.0.0.1" +AIPERF_SHIM_PORT = 8090 +AIPERF_SHIM_BASE_URL = f"http://{AIPERF_SHIM_HOST}:{AIPERF_SHIM_PORT}" + +GUARDRAILS_MIDDLEWARE_NAME = "nemo-guardrails" +GUARDRAILS_MIDDLEWARE_CONFIG_TYPE = "guardrail_config" + +JUNIT_SUITE_NAME = "nemo_guardrails_plugin.benchmarks" diff --git a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/paths.py b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/paths.py new file mode 100644 index 0000000000..676b6c8198 --- /dev/null +++ b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/paths.py @@ -0,0 +1,95 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Path resolution for the nemo-guardrails IGW benchmark harness.""" + +from __future__ import annotations + +import datetime as dt +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class RunPaths: + """Filesystem layout for a single benchmark invocation.""" + + nmp_repo_root: Path + ng_repo_root: Path + benchmark_dir: Path + run_dir: Path + log_dir: Path + generated_dir: Path + aiperf_output_dir: Path + pids_file: Path + nmp_data_dir: Path + config_template: Path + runtime_config: Path + junit_path: Path + aiperf_venv_dir: Path + + @property + def run_id(self) -> str: + return self.run_dir.name + + def ensure_directories(self) -> None: + for path in ( + self.log_dir, + self.generated_dir, + self.aiperf_output_dir, + self.nmp_data_dir, + ): + path.mkdir(parents=True, exist_ok=True) + + +def _now_run_id() -> str: + return dt.datetime.now().strftime("%Y%m%d_%H%M%S") + + +def discover_nmp_repo_root(start: Path | None = None) -> Path: + """Walk up from ``start`` until a directory containing pyproject.toml + plugins/ is found.""" + candidate = (start or Path(__file__)).resolve() + for parent in (candidate, *candidate.parents): + if (parent / "pyproject.toml").is_file() and (parent / "plugins").is_dir(): + return parent + raise RuntimeError(f"Could not locate NMP repo root from {start or Path(__file__)}") + + +def default_ng_repo_root(nmp_repo_root: Path) -> Path: + return (nmp_repo_root.parent / "NeMo-Guardrails").resolve() + + +def build_run_paths( + *, + nmp_repo_root: Path, + ng_repo_root: Path, + junit_dir: Path | None = None, + run_id: str | None = None, +) -> RunPaths: + """Compose the standard benchmark filesystem layout under the plugin's artifacts dir. + + ``junit_dir`` controls where ``report.xml`` is written. CI passes the repo root so + that the artifact upload step finds it at the expected location. + """ + benchmark_dir = nmp_repo_root / "plugins" / "nemo-guardrails" / "benchmarks" + artifacts_dir = benchmark_dir / "artifacts" + run_dir = artifacts_dir / "runs" / (run_id or _now_run_id()) + junit_target = (junit_dir or nmp_repo_root) / "report.xml" + + return RunPaths( + nmp_repo_root=nmp_repo_root, + ng_repo_root=ng_repo_root, + benchmark_dir=benchmark_dir, + run_dir=run_dir, + log_dir=run_dir / "logs", + generated_dir=run_dir / "generated", + aiperf_output_dir=run_dir / "aiperf_results", + pids_file=run_dir / "pids.txt", + nmp_data_dir=artifacts_dir / "nmp-data", + config_template=benchmark_dir / "configs" / "nmp_igw_guardrails_sweep_concurrency.yaml", + runtime_config=run_dir / "generated" / "nmp_igw_guardrails_sweep_concurrency.yaml", + junit_path=junit_target, + # Cached aiperf venv lives outside the per-run dir so it's reused + # across local runs but lives under the gitignored artifacts dir. + aiperf_venv_dir=artifacts_dir / "venvs" / "aiperf", + ) diff --git a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/processes.py b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/processes.py new file mode 100644 index 0000000000..47c774dbee --- /dev/null +++ b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/processes.py @@ -0,0 +1,148 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Process supervision for the benchmark harness. + +Each child runs in its own session/process group so termination cleans up forked +worker processes (notably ``uvicorn --workers N`` and ``nemo services run``). +""" + +from __future__ import annotations + +import logging +import os +import signal +import subprocess +import time +from contextlib import ExitStack, contextmanager +from dataclasses import dataclass, field +from pathlib import Path +from typing import IO, Iterator + +import httpx + +log = logging.getLogger(__name__) + +_TERMINATE_TIMEOUT_SECONDS = 20 + + +@dataclass +class SupervisedProcess: + """A long-lived child managed as a context manager. + + Stdout and stderr are merged into a single log file. The child is placed in a + new session via ``start_new_session=True`` so ``os.killpg`` reaps any workers + it forks. + """ + + name: str + cmd: list[str] + log_path: Path + cwd: Path + env: dict[str, str] | None = None + _proc: subprocess.Popen[bytes] | None = field(default=None, init=False, repr=False) + _log_fh: IO[bytes] | None = field(default=None, init=False, repr=False) + + @property + def pid(self) -> int | None: + return self._proc.pid if self._proc else None + + def start(self) -> None: + if self._proc is not None: + raise RuntimeError(f"Process {self.name!r} already started") + + self.log_path.parent.mkdir(parents=True, exist_ok=True) + self._log_fh = self.log_path.open("wb") + + merged_env = {**os.environ, **(self.env or {})} + log.info("Starting %s; log=%s", self.name, self.log_path) + self._proc = subprocess.Popen( # noqa: S603 - command is constructed internally + self.cmd, + cwd=str(self.cwd), + env=merged_env, + stdout=self._log_fh, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + + def stop(self) -> None: + proc = self._proc + if proc is None: + return + if proc.poll() is not None: + self._close_log() + return + + try: + pgid = os.getpgid(proc.pid) + except ProcessLookupError: + self._close_log() + return + + log.info("Stopping %s pid=%d (pgid=%d)", self.name, proc.pid, pgid) + try: + os.killpg(pgid, signal.SIGTERM) + except ProcessLookupError: + self._close_log() + return + + try: + proc.wait(timeout=_TERMINATE_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired: + log.warning( + "%s did not exit after %ds; sending SIGKILL", + self.name, + _TERMINATE_TIMEOUT_SECONDS, + ) + try: + os.killpg(pgid, signal.SIGKILL) + except ProcessLookupError: + pass + proc.wait() + + self._close_log() + + def _close_log(self) -> None: + if self._log_fh is not None: + self._log_fh.close() + self._log_fh = None + + def __enter__(self) -> "SupervisedProcess": + self.start() + return self + + def __exit__(self, *exc: object) -> None: + self.stop() + + +@contextmanager +def supervised_processes(specs: list[SupervisedProcess]) -> Iterator[list[SupervisedProcess]]: + """Enter every process spec in order; stop them in reverse on exit.""" + with ExitStack() as stack: + for spec in specs: + stack.enter_context(spec) + yield specs + + +def write_pids_file(pids_file: Path, processes: list[SupervisedProcess]) -> None: + pids_file.parent.mkdir(parents=True, exist_ok=True) + with pids_file.open("w", encoding="utf-8") as f: + for p in processes: + if p.pid is not None: + f.write(f"{p.name}:{p.pid}\n") + + +def wait_http(url: str, *, timeout_seconds: float, label: str, poll_interval: float = 1.0) -> None: + """Poll ``url`` until it returns 2xx or ``timeout_seconds`` elapses.""" + deadline = time.monotonic() + timeout_seconds + last_error: Exception | None = None + while time.monotonic() < deadline: + try: + r = httpx.get(url, timeout=5.0) + if r.status_code < 400: + return + last_error = RuntimeError(f"HTTP {r.status_code}: {r.text[:200]}") + except httpx.HTTPError as exc: + last_error = exc + time.sleep(poll_interval) + raise TimeoutError(f"Timed out waiting for {label} at {url}: {last_error}") diff --git a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/report.py b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/report.py new file mode 100644 index 0000000000..27d160b5b8 --- /dev/null +++ b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/report.py @@ -0,0 +1,107 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Emit a minimal JUnit XML report for the benchmark harness. + +We deliberately use ``xml.etree`` rather than a third-party JUnit library to +avoid adding a dependency just for this one consumer. Schema is the standard +``...`` shape that GitHub Actions and most CI dashboards +render natively. +""" + +from __future__ import annotations + +import xml.etree.ElementTree as ET +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from xml.dom import minidom + +from nemo_guardrails_plugin.benchmarks.aiperf_runner import SweepRunResult +from nemo_guardrails_plugin.benchmarks.constants import JUNIT_SUITE_NAME + + +@dataclass(frozen=True) +class JUnitCase: + name: str + classname: str + time_seconds: float + passed: bool + failure_message: str | None = None + system_out: str | None = None + + +def cases_from_sweep_results(results: list[SweepRunResult]) -> list[JUnitCase]: + """Translate AIPerf per-sweep outcomes into JUnit test cases. + + Pass criterion is just ``return_code == 0``; downstream tooling can layer + on threshold-based failures later. + """ + cases: list[JUnitCase] = [] + for r in results: + message = None if r.passed else f"aiperf exited with code {r.return_code}" + cases.append( + JUnitCase( + name=r.sweep_label, + classname=JUNIT_SUITE_NAME, + time_seconds=r.duration_seconds, + passed=r.passed, + failure_message=message, + system_out=f"output_dir={r.output_dir}", + ) + ) + return cases + + +def write_junit_report(path: Path, *, suite_name: str, cases: list[JUnitCase]) -> None: + """Render a single-suite JUnit XML file at ``path``.""" + failure_count = sum(1 for c in cases if not c.passed) + total_time = sum(c.time_seconds for c in cases) + + testsuites = ET.Element( + "testsuites", + attrib={ + "name": suite_name, + "tests": str(len(cases)), + "failures": str(failure_count), + "errors": "0", + "time": f"{total_time:.3f}", + }, + ) + testsuite = ET.SubElement( + testsuites, + "testsuite", + attrib={ + "name": suite_name, + "tests": str(len(cases)), + "failures": str(failure_count), + "errors": "0", + "time": f"{total_time:.3f}", + "timestamp": datetime.now(timezone.utc).isoformat(timespec="seconds"), + }, + ) + + for case in cases: + tc = ET.SubElement( + testsuite, + "testcase", + attrib={ + "name": case.name, + "classname": case.classname, + "time": f"{case.time_seconds:.3f}", + }, + ) + if not case.passed: + failure = ET.SubElement( + tc, + "failure", + attrib={"message": case.failure_message or "failed", "type": "BenchmarkFailure"}, + ) + failure.text = case.failure_message or "" + if case.system_out: + ET.SubElement(tc, "system-out").text = case.system_out + + path.parent.mkdir(parents=True, exist_ok=True) + rough = ET.tostring(testsuites, encoding="unicode") + pretty = minidom.parseString(rough).toprettyxml(indent=" ") + path.write_text(pretty, encoding="utf-8") diff --git a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/run.py b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/run.py new file mode 100644 index 0000000000..78ea03e5b9 --- /dev/null +++ b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/run.py @@ -0,0 +1,390 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Top-level entry point for the nemo-guardrails IGW benchmark harness. + +Replaces the previous ``run_igw_guardrails_benchmark.sh`` shell flow with a +single Python orchestrator. Phases: + +1. Resolve paths and validate the upstream NeMo Guardrails checkout. +2. Write a per-run AIPerf config under ``runs//generated/``. +3. Start the two mock LLM servers from ``${NEMO_GUARDRAILS_REPO_ROOT}/benchmark``. +4. Start (or reuse) ``nemo services run``. +5. Wait for health, seed NMP resources via the SDK, smoke-test the VirtualModel. +6. Invoke ``python -m benchmark.aiperf run --config-file ...`` for the sweep. +7. Collect per-sweep results, emit ``report.xml``, exit non-zero on any failure. + +Process supervision uses session-scoped subprocesses and an ``ExitStack`` so a +``SIGTERM`` from CI cleans up forked workers (e.g. ``uvicorn --workers 4``). +""" + +from __future__ import annotations + +import argparse +import logging +import os +import sys +import time +from contextlib import ExitStack +from pathlib import Path + +import httpx +import yaml +from nemo_guardrails_plugin.benchmarks.aiperf_runner import ( + collect_sweep_results, + rewrite_aiperf_config, + run_aiperf_sweep, +) +from nemo_guardrails_plugin.benchmarks.bootstrap import ( + ensure_aiperf_venv, + env_with_venv_on_path, +) +from nemo_guardrails_plugin.benchmarks.constants import ( + AIPERF_SHIM_BASE_URL, + IGW_CHAT_PATH, + JUNIT_SUITE_NAME, + NMP_BASE_URL, + NMP_HEALTH_PATH, +) +from nemo_guardrails_plugin.benchmarks.paths import ( + RunPaths, + build_run_paths, + default_ng_repo_root, + discover_nmp_repo_root, +) +from nemo_guardrails_plugin.benchmarks.processes import ( + SupervisedProcess, + supervised_processes, + wait_http, + write_pids_file, +) +from nemo_guardrails_plugin.benchmarks.report import ( + cases_from_sweep_results, + write_junit_report, +) +from nemo_guardrails_plugin.benchmarks.seeding import SeededResources, seed_benchmark +from nemo_platform import NeMoPlatform + +log = logging.getLogger("nemo_guardrails_plugin.benchmarks") + +_MOCK_START_TIMEOUT_SECONDS = 60 +_NMP_START_TIMEOUT_SECONDS = 180 + + +_REQUIRED_NG_FILES = ( + Path("benchmark/aiperf/__main__.py"), + Path("benchmark/aiperf/run_aiperf.py"), + Path("benchmark/mock_llm_server/run_server.py"), + Path("benchmark/mock_llm_server/configs/meta-llama-3.3-70b-instruct.env"), + Path("benchmark/mock_llm_server/configs/nvidia-llama-3.1-nemoguard-8b-content-safety.env"), + Path("examples/configs/content_safety_local/config.yml"), + Path("examples/configs/content_safety_local/prompts.yml"), +) + + +def _configure_logging(verbose: bool) -> None: + logging.basicConfig( + level=logging.DEBUG if verbose else logging.INFO, + format="%(asctime)s [%(name)s] %(levelname)s %(message)s", + datefmt="%Y-%m-%dT%H:%M:%S", + ) + + +def _validate_ng_repo(ng_repo_root: Path) -> None: + missing = [p for p in _REQUIRED_NG_FILES if not (ng_repo_root / p).is_file()] + if missing: + bullet = "\n - ".join(str(ng_repo_root / p) for p in missing) + raise FileNotFoundError(f"NeMo Guardrails checkout at {ng_repo_root} is missing required files:\n - {bullet}") + + +def _build_mock_processes(paths: RunPaths, workers: int) -> list[SupervisedProcess]: + """Spawn ``python -m benchmark.mock_llm_server.run_server`` for both mocks. + + Each child is given its own log file and a ``PYTHONPATH`` pointing at the + upstream checkout so its imports resolve. + """ + env = {"PYTHONPATH": str(paths.ng_repo_root)} + workdir = paths.ng_repo_root / "benchmark" + + def spec(name: str, port: int, env_file: Path) -> SupervisedProcess: + return SupervisedProcess( + name=name, + cmd=[ + sys.executable, + "-m", + "benchmark.mock_llm_server.run_server", + "--workers", + str(workers), + "--port", + str(port), + "--config-file", + str(env_file), + ], + log_path=paths.log_dir / f"{name}.log", + cwd=workdir, + env=env, + ) + + return [ + spec( + "mock-app-llm", + 8000, + paths.ng_repo_root / "benchmark/mock_llm_server/configs/meta-llama-3.3-70b-instruct.env", + ), + spec( + "mock-content-safety-llm", + 8001, + paths.ng_repo_root / "benchmark/mock_llm_server/configs/nvidia-llama-3.1-nemoguard-8b-content-safety.env", + ), + ] + + +def _build_nmp_process(paths: RunPaths) -> SupervisedProcess: + return SupervisedProcess( + name="nmp-services", + cmd=["nemo", "services", "run"], + log_path=paths.log_dir / "nmp-services.log", + cwd=paths.nmp_repo_root, + env={"NMP_BASE_URL": NMP_BASE_URL, "NMP_DATA_DIR": str(paths.nmp_data_dir)}, + ) + + +def _build_aiperf_shim_process(paths: RunPaths) -> SupervisedProcess: + """Run the shim that satisfies AIPerf's `/v1/models` pre-check. + + Without this, AIPerf's hard-coded `urljoin(base_url, "/v1/models")` probe + would 404 against NMP and the sweep would never start. See + `nemo_guardrails_plugin.benchmarks.shim` for details. + """ + return SupervisedProcess( + name="aiperf-shim", + cmd=[sys.executable, "-m", "nemo_guardrails_plugin.benchmarks.shim"], + log_path=paths.log_dir / "aiperf-shim.log", + cwd=paths.nmp_repo_root, + ) + + +_SMOKE_TEST_TIMEOUT_SECONDS = 60 +_SMOKE_TEST_POLL_INTERVAL_SECONDS = 1.0 + + +def _smoke_test(seeded: SeededResources) -> None: + """POST one chat-completion through the IGW VirtualModel before sweeping. + + Catches misconfigured guardrails / middleware wiring early so a sweep + failure isn't ambiguous between "harness broken" and "benchmark regressed". + We hit the IGW URL directly with ``httpx`` rather than going through the + NMP SDK so the smoke test exercises the same path AIPerf will. + + IGW's VirtualModel cache refreshes asynchronously after creation, so the + first few requests can 404 even though seeding succeeded. We retry on + 404/503 for up to ~60s before failing. + """ + url = f"{NMP_BASE_URL}{IGW_CHAT_PATH}" + payload = { + "model": seeded.vm_ref, + "messages": [{"role": "user", "content": "Hello, what can you do?"}], + "max_tokens": 64, + "stream": False, + } + + deadline = time.monotonic() + _SMOKE_TEST_TIMEOUT_SECONDS + last_response: httpx.Response | None = None + while time.monotonic() < deadline: + last_response = httpx.post(url, json=payload, timeout=60.0) + if last_response.status_code < 400: + body = last_response.json() + if not body.get("choices"): + raise RuntimeError(f"Smoke test response missing choices: {body}") + return + if last_response.status_code in (404, 503): + log.info( + "Smoke test got HTTP %d; waiting for IGW cache refresh", + last_response.status_code, + ) + time.sleep(_SMOKE_TEST_POLL_INTERVAL_SECONDS) + continue + break + + code = last_response.status_code if last_response is not None else "no response" + text = last_response.text[:500] if last_response is not None else "" + raise RuntimeError(f"Smoke test failed with HTTP {code}: {text}") + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + prog="nemo-guardrails-benchmark", + description="Run the nemo-guardrails IGW benchmark sweep.", + ) + parser.add_argument( + "--nemo-guardrails-repo-root", + type=Path, + default=Path( + os.environ.get( + "NEMO_GUARDRAILS_REPO_ROOT", + str(default_ng_repo_root(discover_nmp_repo_root())), + ) + ), + help="Path to a local NeMo Guardrails checkout (default: $NEMO_GUARDRAILS_REPO_ROOT or ../NeMo-Guardrails).", + ) + parser.add_argument( + "--reuse-services", + action="store_true", + default=os.environ.get("NMP_BENCHMARK_REUSE_SERVICES", "0") == "1", + help="Skip starting `nemo services run` and reuse an existing local NMP at :8080.", + ) + parser.add_argument( + "--keep-running", + action="store_true", + default=os.environ.get("NMP_BENCHMARK_KEEP_RUNNING", "0") == "1", + help="Leave started processes alive after the sweep (debugging).", + ) + parser.add_argument( + "--mock-workers", + type=int, + default=int(os.environ.get("NMP_BENCHMARK_MOCK_WORKERS", "4")), + help="uvicorn worker count for each mock LLM server.", + ) + parser.add_argument( + "--junit-path", + type=Path, + default=None, + help="Path to write report.xml (default: /report.xml for CI compatibility).", + ) + parser.add_argument( + "--run-id", + default=None, + help="Override the per-run directory name (default: current timestamp).", + ) + parser.add_argument("--verbose", "-v", action="store_true") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + _configure_logging(args.verbose) + + nmp_repo_root = discover_nmp_repo_root() + ng_repo_root = args.nemo_guardrails_repo_root.resolve() + _validate_ng_repo(ng_repo_root) + + paths = build_run_paths( + nmp_repo_root=nmp_repo_root, + ng_repo_root=ng_repo_root, + junit_dir=args.junit_path.parent if args.junit_path else None, + run_id=args.run_id, + ) + if args.junit_path: + paths = RunPaths(**{**paths.__dict__, "junit_path": args.junit_path.resolve()}) + paths.ensure_directories() + + log.info("Run directory: %s", paths.run_dir) + log.info("NeMo Guardrails repo: %s", paths.ng_repo_root) + + rewrite_aiperf_config( + template=paths.config_template, + output=paths.runtime_config, + output_base_dir=paths.aiperf_output_dir, + ) + sweep_config = yaml.safe_load(paths.runtime_config.read_text(encoding="utf-8")) + log.info( + "AIPerf sweep: concurrency=%s, duration=%ss", + sweep_config.get("sweeps", {}).get("concurrency"), + sweep_config.get("base_config", {}).get("benchmark_duration"), + ) + + # Ensure the dedicated aiperf venv exists *before* we start any supervised + # processes. A first-time install can take ~30s and we'd rather pay that + # cost up front than during the NMP-services startup race. + aiperf_python = ensure_aiperf_venv(paths.aiperf_venv_dir) + log.info("Using aiperf python at %s", aiperf_python) + + processes_to_start: list[SupervisedProcess] = _build_mock_processes(paths, args.mock_workers) + if not args.reuse_services: + processes_to_start.append(_build_nmp_process(paths)) + # The AIPerf shim is harness-local; we always start it (it talks to NMP + # over HTTP, so it doesn't care whether nemo services run is supervised + # by us or already running externally). + processes_to_start.append(_build_aiperf_shim_process(paths)) + + with ExitStack() as stack: + started = stack.enter_context(supervised_processes(processes_to_start)) + write_pids_file(paths.pids_file, started) + if args.keep_running: + # Pop the cleanup so processes outlive this script. + stack.pop_all() + + wait_http( + "http://localhost:8000/health", + timeout_seconds=_MOCK_START_TIMEOUT_SECONDS, + label="mock app LLM", + ) + wait_http( + "http://localhost:8001/health", + timeout_seconds=_MOCK_START_TIMEOUT_SECONDS, + label="mock content-safety LLM", + ) + wait_http( + f"{NMP_BASE_URL}{NMP_HEALTH_PATH}", + timeout_seconds=_NMP_START_TIMEOUT_SECONDS, + label="NMP services", + ) + wait_http( + f"{AIPERF_SHIM_BASE_URL}/__shim/health", + timeout_seconds=_MOCK_START_TIMEOUT_SECONDS, + label="AIPerf shim", + ) + + client = NeMoPlatform(base_url=NMP_BASE_URL) + seeded = seed_benchmark( + client, + ng_repo_root=paths.ng_repo_root, + generated_dir=paths.generated_dir, + ) + + log.info("Smoke testing %s", seeded.vm_ref) + _smoke_test(seeded) + + log.info( + "Starting AIPerf sweep against %s -> shim -> %s%s", + AIPERF_SHIM_BASE_URL, + NMP_BASE_URL, + IGW_CHAT_PATH, + ) + aiperf_exit = run_aiperf_sweep( + ng_repo_root=paths.ng_repo_root, + runtime_config=paths.runtime_config, + log_path=paths.log_dir / "aiperf.log", + python_executable=str(aiperf_python), + extra_env=env_with_venv_on_path(paths.aiperf_venv_dir, {}), + ) + + sweep_results = collect_sweep_results(paths.aiperf_output_dir) + cases = cases_from_sweep_results(sweep_results) + if not cases: + # AIPerf failed before producing per-sweep dirs; emit a synthetic failure + # so CI surfaces something actionable instead of an empty report. + from nemo_guardrails_plugin.benchmarks.report import JUnitCase + + cases = [ + JUnitCase( + name="aiperf", + classname=JUNIT_SUITE_NAME, + time_seconds=0.0, + passed=aiperf_exit == 0, + failure_message=(f"aiperf exited with code {aiperf_exit} and produced no per-sweep results"), + system_out=f"aiperf_output_dir={paths.aiperf_output_dir}", + ) + ] + write_junit_report(paths.junit_path, suite_name=JUNIT_SUITE_NAME, cases=cases) + log.info("Wrote JUnit report to %s", paths.junit_path) + + failures = sum(1 for c in cases if not c.passed) + log.info("Sweep summary: %d run(s), %d failure(s)", len(cases), failures) + if failures or aiperf_exit != 0: + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/seeding.py b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/seeding.py new file mode 100644 index 0000000000..555a7aaf51 --- /dev/null +++ b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/seeding.py @@ -0,0 +1,268 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Seed NMP with the workspace, providers, guardrail config, and VirtualModel +required by the IGW guardrails benchmark. + +Replaces the previous ``setup_nmp_guardrails_benchmark.sh`` flow with direct +NMP SDK calls (``client.workspaces``, ``client.inference``, ``client.guardrail``). +""" + +from __future__ import annotations + +import json +import logging +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml +from nemo_guardrails_plugin.benchmarks.constants import ( + APP_MODEL_NAME, + APP_PROVIDER, + APP_PROVIDER_URL, + CS_MODEL_NAME, + CS_PROVIDER, + CS_PROVIDER_URL, + GUARDRAIL_CONFIG, + GUARDRAILS_MIDDLEWARE_CONFIG_TYPE, + GUARDRAILS_MIDDLEWARE_NAME, + VM_NAME, + WORKSPACE, +) +from nemo_platform import NeMoPlatform, NotFoundError +from nemo_platform.types.inference.middleware_call_param import MiddlewareCallParam +from nemo_platform.types.inference.virtual_model_inference_config_param import ( + VirtualModelInferenceConfigParam, +) + +log = logging.getLogger(__name__) + +_PROVIDER_WAIT_TIMEOUT_SECONDS = 60 +_PROVIDER_POLL_INTERVAL_SECONDS = 1.0 + + +@dataclass(frozen=True) +class SeededResources: + workspace: str + app_provider_name: str + cs_provider_name: str + app_model_entity: str + cs_model_entity: str + guardrail_config_name: str + vm_name: str + + @property + def vm_ref(self) -> str: + return f"{self.workspace}/{self.vm_name}" + + @property + def guardrail_config_ref(self) -> str: + return f"{self.workspace}/{self.guardrail_config_name}" + + +def seed_benchmark( + client: NeMoPlatform, + *, + ng_repo_root: Path, + generated_dir: Path, + provider_wait_timeout: float = _PROVIDER_WAIT_TIMEOUT_SECONDS, +) -> SeededResources: + """Create workspace, providers, GuardrailConfig, and VirtualModel. + + All ``create`` calls are idempotent (``exist_ok=True``) so this is safe to + rerun against a reused NMP instance. + """ + generated_dir.mkdir(parents=True, exist_ok=True) + + log.info("Creating workspace %s", WORKSPACE) + client.workspaces.create( + name=WORKSPACE, + description="Local IGW guardrails benchmark workspace", + exist_ok=True, + ) + + log.info("Registering app mock provider %s", APP_PROVIDER) + client.inference.providers.create( + workspace=WORKSPACE, + name=APP_PROVIDER, + host_url=APP_PROVIDER_URL, + enabled_models=[APP_MODEL_NAME], + description=f"Benchmark mock app LLM on {APP_PROVIDER_URL}", + exist_ok=True, + ) + + log.info("Registering content-safety mock provider %s", CS_PROVIDER) + client.inference.providers.create( + workspace=WORKSPACE, + name=CS_PROVIDER, + host_url=CS_PROVIDER_URL, + enabled_models=[CS_MODEL_NAME], + description=f"Benchmark mock content-safety LLM on {CS_PROVIDER_URL}", + exist_ok=True, + ) + + log.info("Waiting for provider discovery") + app_provider = _wait_for_served_model( + client, + provider_name=APP_PROVIDER, + served_model_name=APP_MODEL_NAME, + timeout_seconds=provider_wait_timeout, + ) + cs_provider = _wait_for_served_model( + client, + provider_name=CS_PROVIDER, + served_model_name=CS_MODEL_NAME, + timeout_seconds=provider_wait_timeout, + ) + + app_entity = _extract_model_entity(app_provider, APP_MODEL_NAME, provider_name=APP_PROVIDER) + cs_entity = _extract_model_entity(cs_provider, CS_MODEL_NAME, provider_name=CS_PROVIDER) + + _dump_model(generated_dir / "app_provider.json", app_provider) + _dump_model(generated_dir / "content_safety_provider.json", cs_provider) + + log.info("Building GuardrailConfig payload from %s", ng_repo_root) + config_data = build_guardrail_config_data( + source_config_dir=ng_repo_root / "examples" / "configs" / "content_safety_local", + content_safety_model_entity=cs_entity, + ) + # Persist the same payload shape the old shell harness produced for debuggability. + (generated_dir / "content_safety_local_nmp_request.json").write_text( + json.dumps( + { + "name": GUARDRAIL_CONFIG, + "description": "Benchmark content_safety_local config routed through IGW", + "data": config_data, + "exist_ok": True, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + + log.info("Creating GuardrailConfig %s", GUARDRAIL_CONFIG) + client.guardrail.configs.create( + workspace=WORKSPACE, + name=GUARDRAIL_CONFIG, + description="Benchmark content_safety_local config routed through IGW", + data=config_data, + exist_ok=True, + ) + + middleware_call: MiddlewareCallParam = { + "name": GUARDRAILS_MIDDLEWARE_NAME, + "config_type": GUARDRAILS_MIDDLEWARE_CONFIG_TYPE, + "config_id": f"{WORKSPACE}/{GUARDRAIL_CONFIG}", + } + vm_models: list[VirtualModelInferenceConfigParam] = [{"model": app_entity, "backend_format": "OPENAI_CHAT"}] + + log.info("Creating VirtualModel %s/%s", WORKSPACE, VM_NAME) + vm = client.inference.virtual_models.create( + workspace=WORKSPACE, + name=VM_NAME, + default_model_entity=app_entity, + models=vm_models, + request_middleware=[middleware_call], + response_middleware=[middleware_call], + exist_ok=True, + ) + _dump_model(generated_dir / "virtual_model.json", vm) + + return SeededResources( + workspace=WORKSPACE, + app_provider_name=APP_PROVIDER, + cs_provider_name=CS_PROVIDER, + app_model_entity=app_entity, + cs_model_entity=cs_entity, + guardrail_config_name=GUARDRAIL_CONFIG, + vm_name=VM_NAME, + ) + + +def _wait_for_served_model( + client: NeMoPlatform, + *, + provider_name: str, + served_model_name: str, + timeout_seconds: float, +) -> Any: + """Poll a provider until ``served_models`` lists the expected entry. + + Gateway readiness alone is not enough: VirtualModel creation needs the + discovered ``model_entity_id``, which only appears once the provider has + enumerated its models. + """ + deadline = time.monotonic() + timeout_seconds + last_error: Exception | None = None + while time.monotonic() < deadline: + try: + provider = client.inference.providers.retrieve(provider_name, workspace=WORKSPACE) + except NotFoundError as exc: + last_error = exc + time.sleep(_PROVIDER_POLL_INTERVAL_SECONDS) + continue + served_models = getattr(provider, "served_models", None) or [] + for m in served_models: + if getattr(m, "served_model_name", None) == served_model_name and getattr(m, "model_entity_id", None): + return provider + time.sleep(_PROVIDER_POLL_INTERVAL_SECONDS) + + raise TimeoutError( + f"Provider {provider_name!r} did not surface served model " + f"{served_model_name!r} within {timeout_seconds}s: {last_error}" + ) + + +def _extract_model_entity(provider: Any, served_model_name: str, *, provider_name: str) -> str: + for m in getattr(provider, "served_models", None) or []: + if getattr(m, "served_model_name", None) == served_model_name: + entity = getattr(m, "model_entity_id", None) + if entity: + return entity + raise RuntimeError(f"Provider {provider_name!r} does not expose served model {served_model_name!r}") + + +def build_guardrail_config_data( + *, + source_config_dir: Path, + content_safety_model_entity: str, +) -> dict[str, Any]: + """Read the upstream content_safety_local config and rewrite it for NMP. + + The upstream config references an HTTP base_url; in NMP we instead route by + ``model_entity_id`` resolved via the inference gateway. Prompts are inlined + from the sibling ``prompts.yml`` file. + """ + config_yaml = source_config_dir / "config.yml" + prompts_yaml = source_config_dir / "prompts.yml" + if not config_yaml.is_file(): + raise FileNotFoundError(f"Expected guardrails config at {config_yaml}") + if not prompts_yaml.is_file(): + raise FileNotFoundError(f"Expected guardrails prompts at {prompts_yaml}") + + config: dict[str, Any] = yaml.safe_load(config_yaml.read_text(encoding="utf-8")) + prompts: dict[str, Any] = yaml.safe_load(prompts_yaml.read_text(encoding="utf-8")) + + config["models"] = [ + { + "type": "content_safety", + "engine": "nim", + "model": content_safety_model_entity, + } + ] + config["prompts"] = prompts.get("prompts", []) + return config + + +def _dump_model(path: Path, model: Any) -> None: + """Best-effort serialize an SDK response model to JSON.""" + if hasattr(model, "model_dump"): + payload = model.model_dump(mode="json") + elif hasattr(model, "to_dict"): + payload = model.to_dict() + else: + payload = json.loads(json.dumps(model, default=str)) + path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") diff --git a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/shim.py b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/shim.py new file mode 100644 index 0000000000..d304fcd8bc --- /dev/null +++ b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/shim.py @@ -0,0 +1,140 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""HTTP shim that satisfies AIPerf's pre-check and proxies chat completions. + +AIPerf's ``_check_service`` issues ``GET urljoin(base_url, "/v1/models")`` and +expects a 200, but NMP doesn't serve ``/v1/models`` at the IGW workspace root +the way OpenAI does, and upstream AIPerf doesn't expose a knob to override the +probe path. To unblock the benchmark without patching AIPerf or NMP, we run +this tiny shim on a separate port: + +- ``GET /v1/models`` -> ``200 {"object":"list","data":[]}`` +- ``POST /v1/chat/completions`` -> reverse-proxy to NMP IGW +- ``GET /__shim/health`` -> ``200 {"status":"ok"}`` +- Any other path -> 404 + +This module is meant to be invoked as ``python -m +nemo_guardrails_plugin.benchmarks.shim`` so it can be supervised by the same +process-group machinery the harness uses for the mock LLMs and ``nemo +services run``. +""" + +from __future__ import annotations + +import argparse +import json +import logging +import sys +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import httpx +from nemo_guardrails_plugin.benchmarks.constants import ( + AIPERF_SHIM_HOST, + AIPERF_SHIM_PORT, + IGW_CHAT_PATH, + NMP_BASE_URL, +) + +log = logging.getLogger("nemo_guardrails_plugin.benchmarks.shim") + + +_MODELS_RESPONSE = json.dumps({"object": "list", "data": []}).encode("utf-8") +_HEALTH_RESPONSE = json.dumps({"status": "ok"}).encode("utf-8") + + +class _ShimHandler(BaseHTTPRequestHandler): + """Minimal handler that routes the two paths AIPerf actually touches.""" + + # Allow override via class attr so tests can swap in a mock httpx client. + upstream_url: str = f"{NMP_BASE_URL}{IGW_CHAT_PATH}" + + def log_message(self, format: str, *args: object) -> None: # noqa: A002 + log.debug("shim: " + format, *args) + + def _send_json(self, status: int, body: bytes) -> None: + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self) -> None: # noqa: N802 - http.server API + if self.path == "/v1/models": + self._send_json(200, _MODELS_RESPONSE) + return + if self.path == "/__shim/health": + self._send_json(200, _HEALTH_RESPONSE) + return + self._send_json(404, json.dumps({"detail": "not found"}).encode("utf-8")) + + def do_POST(self) -> None: # noqa: N802 - http.server API + if self.path != "/v1/chat/completions": + self._send_json(404, json.dumps({"detail": "not found"}).encode("utf-8")) + return + + length = int(self.headers.get("Content-Length", "0") or "0") + body = self.rfile.read(length) if length else b"" + + forwarded_headers = { + k: v for k, v in self.headers.items() if k.lower() not in {"host", "content-length", "connection"} + } + + try: + response = httpx.post( + self.upstream_url, + content=body, + headers=forwarded_headers, + timeout=httpx.Timeout(connect=10.0, read=300.0, write=10.0, pool=10.0), + ) + except httpx.HTTPError as e: + log.warning("shim: upstream request failed: %s", e) + self._send_json( + 502, + json.dumps({"detail": f"upstream error: {e}"}).encode("utf-8"), + ) + return + + self.send_response(response.status_code) + for header, value in response.headers.items(): + if header.lower() in {"transfer-encoding", "connection", "content-length"}: + continue + self.send_header(header, value) + self.send_header("Content-Length", str(len(response.content))) + self.end_headers() + self.wfile.write(response.content) + + +def serve(host: str = AIPERF_SHIM_HOST, port: int = AIPERF_SHIM_PORT) -> None: + """Run the shim until the process is signalled.""" + httpd = ThreadingHTTPServer((host, port), _ShimHandler) + log.info("AIPerf shim listening on http://%s:%d", host, port) + try: + httpd.serve_forever() + finally: + httpd.server_close() + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="nemo-guardrails-benchmark-shim", + description=__doc__, + ) + parser.add_argument("--host", default=AIPERF_SHIM_HOST) + parser.add_argument("--port", type=int, default=AIPERF_SHIM_PORT) + parser.add_argument( + "--log-level", + default="INFO", + choices=["DEBUG", "INFO", "WARNING", "ERROR"], + ) + args = parser.parse_args(argv) + logging.basicConfig( + level=args.log_level, + format="%(asctime)s [%(name)s] %(levelname)s %(message)s", + ) + serve(host=args.host, port=args.port) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/plugins/nemo-guardrails/tests/unit/benchmarks/test_aiperf_runner.py b/plugins/nemo-guardrails/tests/unit/benchmarks/test_aiperf_runner.py new file mode 100644 index 0000000000..7b7df9d9f1 --- /dev/null +++ b/plugins/nemo-guardrails/tests/unit/benchmarks/test_aiperf_runner.py @@ -0,0 +1,131 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import json +from pathlib import Path + +import pytest +import yaml +from nemo_guardrails_plugin.benchmarks.aiperf_runner import ( + SweepRunResult, + collect_sweep_results, + rewrite_aiperf_config, +) + + +def _write_template(path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + yaml.safe_dump( + { + "batch_name": "nmp_igw_guardrails_sweep_concurrency", + "output_base_dir": "plugins/nemo-guardrails/benchmarks/artifacts/aiperf_results", + "base_config": {"model": "benchmark/guardrails-vm"}, + "sweeps": {"concurrency": [1, 2, 4]}, + } + ), + encoding="utf-8", + ) + + +class TestRewriteAiperfConfig: + def test_overrides_output_base_dir(self, tmp_path: Path) -> None: + template = tmp_path / "template.yaml" + _write_template(template) + output = tmp_path / "out" / "runtime.yaml" + target_dir = tmp_path / "results" + + config = rewrite_aiperf_config(template=template, output=output, output_base_dir=target_dir) + + assert config["output_base_dir"] == str(target_dir) + written = yaml.safe_load(output.read_text(encoding="utf-8")) + assert written["output_base_dir"] == str(target_dir) + assert written["base_config"]["model"] == "benchmark/guardrails-vm" + assert written["sweeps"]["concurrency"] == [1, 2, 4] + + def test_missing_template_raises(self, tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError): + rewrite_aiperf_config( + template=tmp_path / "absent.yaml", + output=tmp_path / "out.yaml", + output_base_dir=tmp_path / "results", + ) + + def test_non_mapping_template_raises(self, tmp_path: Path) -> None: + template = tmp_path / "bad.yaml" + template.write_text("- just\n- a\n- list\n", encoding="utf-8") + with pytest.raises(ValueError, match="mapping"): + rewrite_aiperf_config( + template=template, + output=tmp_path / "out.yaml", + output_base_dir=tmp_path / "results", + ) + + +def _make_sweep_dir(parent: Path, sweep_label: str, *, returncode: int | None, duration: float | None) -> Path: + sweep_dir = parent / sweep_label + sweep_dir.mkdir(parents=True) + if returncode is not None: + (sweep_dir / "process_result.json").write_text(json.dumps({"returncode": returncode}), encoding="utf-8") + if duration is not None: + (sweep_dir / "run_metadata.json").write_text(json.dumps({"duration_seconds": duration}), encoding="utf-8") + return sweep_dir + + +class TestCollectSweepResults: + def test_empty_dir_returns_empty(self, tmp_path: Path) -> None: + assert collect_sweep_results(tmp_path / "missing") == [] + + def test_collects_all_sweeps_with_status(self, tmp_path: Path) -> None: + batch = tmp_path / "nmp_igw_guardrails_sweep_concurrency" / "20260527_120000" + _make_sweep_dir(batch, "concurrency1", returncode=0, duration=70.5) + _make_sweep_dir(batch, "concurrency2", returncode=1, duration=70.5) + + results = collect_sweep_results(tmp_path) + + assert len(results) == 2 + results_by_label = {r.sweep_label: r for r in results} + assert results_by_label["concurrency1"].passed + assert results_by_label["concurrency1"].duration_seconds == 70.5 + assert not results_by_label["concurrency2"].passed + assert results_by_label["concurrency2"].return_code == 1 + + def test_missing_process_result_is_failure(self, tmp_path: Path) -> None: + batch = tmp_path / "batch" / "ts" + _make_sweep_dir(batch, "concurrency1", returncode=None, duration=10.0) + + results = collect_sweep_results(tmp_path) + + assert len(results) == 1 + assert not results[0].passed + + def test_malformed_json_treated_as_failure_without_crashing(self, tmp_path: Path) -> None: + batch = tmp_path / "batch" / "ts" + sweep = _make_sweep_dir(batch, "concurrency1", returncode=None, duration=None) + (sweep / "process_result.json").write_text("not-json", encoding="utf-8") + (sweep / "run_metadata.json").write_text("not-json", encoding="utf-8") + + results = collect_sweep_results(tmp_path) + assert results[0].return_code == 1 + assert results[0].duration_seconds == 0.0 + + +class TestSweepRunResult: + def test_passed_property(self) -> None: + passing = SweepRunResult( + sweep_label="x", + output_dir=Path("."), + return_code=0, + duration_seconds=1.0, + metadata_path=None, + process_result_path=None, + ) + assert passing.passed + assert not SweepRunResult( + sweep_label="x", + output_dir=Path("."), + return_code=2, + duration_seconds=1.0, + metadata_path=None, + process_result_path=None, + ).passed diff --git a/plugins/nemo-guardrails/tests/unit/benchmarks/test_paths.py b/plugins/nemo-guardrails/tests/unit/benchmarks/test_paths.py new file mode 100644 index 0000000000..1603616b99 --- /dev/null +++ b/plugins/nemo-guardrails/tests/unit/benchmarks/test_paths.py @@ -0,0 +1,108 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path + +import pytest +from nemo_guardrails_plugin.benchmarks.paths import ( + build_run_paths, + default_ng_repo_root, + discover_nmp_repo_root, +) + + +def _make_fake_repo(root: Path) -> Path: + root.mkdir(parents=True, exist_ok=True) + (root / "pyproject.toml").write_text("", encoding="utf-8") + (root / "plugins").mkdir() + return root + + +class TestDiscoverNmpRepoRoot: + def test_finds_root_from_nested_path(self, tmp_path: Path) -> None: + repo = _make_fake_repo(tmp_path / "repo") + nested = repo / "plugins" / "foo" / "src" / "deep" + nested.mkdir(parents=True) + + assert discover_nmp_repo_root(nested) == repo + + def test_returns_repo_when_pointed_directly_at_it(self, tmp_path: Path) -> None: + repo = _make_fake_repo(tmp_path / "repo") + assert discover_nmp_repo_root(repo) == repo + + def test_raises_when_no_repo_in_ancestry(self, tmp_path: Path) -> None: + with pytest.raises(RuntimeError, match="repo root"): + discover_nmp_repo_root(tmp_path) + + +class TestDefaultNgRepoRoot: + def test_sibling_of_nmp_root(self, tmp_path: Path) -> None: + nmp = _make_fake_repo(tmp_path / "nemo-platform") + ng = tmp_path / "NeMo-Guardrails" + + assert default_ng_repo_root(nmp) == ng.resolve() + + +class TestBuildRunPaths: + def test_layout_matches_documented_structure(self, tmp_path: Path) -> None: + nmp = _make_fake_repo(tmp_path / "nemo-platform") + ng = tmp_path / "NeMo-Guardrails" + ng.mkdir() + + paths = build_run_paths(nmp_repo_root=nmp, ng_repo_root=ng, run_id="20260527_120000") + + assert paths.run_dir == nmp / "plugins/nemo-guardrails/benchmarks/artifacts/runs/20260527_120000" + assert paths.log_dir == paths.run_dir / "logs" + assert paths.generated_dir == paths.run_dir / "generated" + assert paths.aiperf_output_dir == paths.run_dir / "aiperf_results" + assert paths.pids_file == paths.run_dir / "pids.txt" + assert paths.nmp_data_dir == nmp / "plugins/nemo-guardrails/benchmarks/artifacts/nmp-data" + assert ( + paths.config_template + == nmp / "plugins/nemo-guardrails/benchmarks/configs/nmp_igw_guardrails_sweep_concurrency.yaml" + ) + assert paths.runtime_config == paths.generated_dir / "nmp_igw_guardrails_sweep_concurrency.yaml" + assert paths.junit_path == nmp / "report.xml" + + def test_junit_dir_override(self, tmp_path: Path) -> None: + nmp = _make_fake_repo(tmp_path / "nemo-platform") + ng = tmp_path / "NeMo-Guardrails" + ng.mkdir() + junit_dir = tmp_path / "ci-artifacts" + + paths = build_run_paths(nmp_repo_root=nmp, ng_repo_root=ng, junit_dir=junit_dir, run_id="x") + + assert paths.junit_path == junit_dir / "report.xml" + + def test_ensure_directories_creates_required_dirs(self, tmp_path: Path) -> None: + nmp = _make_fake_repo(tmp_path / "nemo-platform") + ng = tmp_path / "NeMo-Guardrails" + ng.mkdir() + + paths = build_run_paths(nmp_repo_root=nmp, ng_repo_root=ng, run_id="x") + paths.ensure_directories() + + assert paths.log_dir.is_dir() + assert paths.generated_dir.is_dir() + assert paths.aiperf_output_dir.is_dir() + assert paths.nmp_data_dir.is_dir() + + def test_run_id_uses_timestamp_when_not_given(self, tmp_path: Path) -> None: + nmp = _make_fake_repo(tmp_path / "nemo-platform") + ng = tmp_path / "NeMo-Guardrails" + ng.mkdir() + + paths = build_run_paths(nmp_repo_root=nmp, ng_repo_root=ng) + # Timestamp format: YYYYmmdd_HHMMSS = 15 chars including underscore. + assert len(paths.run_id) == 15 + assert paths.run_id[8] == "_" + + def test_aiperf_venv_dir_is_outside_run_dir(self, tmp_path: Path) -> None: + """The cached aiperf venv must be shared across runs, not under run_dir.""" + nmp = _make_fake_repo(tmp_path / "nemo-platform") + ng = tmp_path / "NeMo-Guardrails" + ng.mkdir() + + paths = build_run_paths(nmp_repo_root=nmp, ng_repo_root=ng, run_id="x") + assert paths.run_dir not in paths.aiperf_venv_dir.parents + assert paths.aiperf_venv_dir.parent.name == "venvs" diff --git a/plugins/nemo-guardrails/tests/unit/benchmarks/test_report.py b/plugins/nemo-guardrails/tests/unit/benchmarks/test_report.py new file mode 100644 index 0000000000..b9c36306bd --- /dev/null +++ b/plugins/nemo-guardrails/tests/unit/benchmarks/test_report.py @@ -0,0 +1,90 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import xml.etree.ElementTree as ET +from pathlib import Path + +from nemo_guardrails_plugin.benchmarks.aiperf_runner import SweepRunResult +from nemo_guardrails_plugin.benchmarks.report import ( + JUnitCase, + cases_from_sweep_results, + write_junit_report, +) + + +def _result(label: str, *, returncode: int, duration: float = 60.0) -> SweepRunResult: + return SweepRunResult( + sweep_label=label, + output_dir=Path("/tmp") / label, + return_code=returncode, + duration_seconds=duration, + metadata_path=None, + process_result_path=None, + ) + + +class TestCasesFromSweepResults: + def test_passing_case_has_no_failure_message(self) -> None: + cases = cases_from_sweep_results([_result("concurrency1", returncode=0)]) + assert len(cases) == 1 + assert cases[0].passed + assert cases[0].failure_message is None + assert cases[0].time_seconds == 60.0 + + def test_failing_case_includes_exit_code(self) -> None: + cases = cases_from_sweep_results([_result("concurrency1", returncode=3)]) + assert not cases[0].passed + assert "code 3" in (cases[0].failure_message or "") + + +class TestWriteJunitReport: + def test_basic_report_structure(self, tmp_path: Path) -> None: + cases = [ + JUnitCase(name="concurrency1", classname="suite", time_seconds=70.0, passed=True), + JUnitCase( + name="concurrency2", + classname="suite", + time_seconds=72.5, + passed=False, + failure_message="boom", + system_out="output_dir=/tmp/concurrency2", + ), + ] + + path = tmp_path / "report.xml" + write_junit_report(path, suite_name="suite", cases=cases) + + tree = ET.parse(path) + root = tree.getroot() + assert root.tag == "testsuites" + assert root.attrib["tests"] == "2" + assert root.attrib["failures"] == "1" + + testsuite = root.find("testsuite") + assert testsuite is not None + assert testsuite.attrib["name"] == "suite" + assert testsuite.attrib["failures"] == "1" + + testcases = testsuite.findall("testcase") + assert [tc.attrib["name"] for tc in testcases] == ["concurrency1", "concurrency2"] + + passing, failing = testcases + assert passing.find("failure") is None + failure = failing.find("failure") + assert failure is not None + assert failure.attrib["message"] == "boom" + system_out = failing.find("system-out") + assert system_out is not None + assert system_out.text == "output_dir=/tmp/concurrency2" + + def test_writes_pretty_xml(self, tmp_path: Path) -> None: + path = tmp_path / "report.xml" + write_junit_report( + path, + suite_name="suite", + cases=[JUnitCase(name="x", classname="suite", time_seconds=0.0, passed=True)], + ) + + text = path.read_text(encoding="utf-8") + assert text.startswith(" SimpleNamespace: + return SimpleNamespace( + name=provider_name, + served_models=[ + SimpleNamespace( + served_model_name=served_model_name, + model_entity_id=f"{WORKSPACE}/{served_model_name.replace('/', '-')}-{entity_suffix}", + ) + ], + ) + + +def _write_upstream_configs(ng_root: Path) -> Path: + cs_dir = ng_root / "examples" / "configs" / "content_safety_local" + cs_dir.mkdir(parents=True) + (cs_dir / "config.yml").write_text( + yaml.safe_dump( + { + "models": [ + { + "type": "main", + "engine": "nim", + "model": "meta/llama-3.3-70b-instruct", + "parameters": {"base_url": "http://localhost:8000"}, + }, + ], + "rails": {"input": {"flows": ["content safety check input $model=content_safety"]}}, + } + ), + encoding="utf-8", + ) + (cs_dir / "prompts.yml").write_text( + yaml.safe_dump({"prompts": [{"task": "content_safety_check_input", "content": "..."}]}), + encoding="utf-8", + ) + return cs_dir + + +@pytest.fixture +def fake_client() -> MagicMock: + client = MagicMock() + client.inference.providers.create = MagicMock() + client.inference.providers.retrieve = MagicMock( + side_effect=lambda name, workspace=None: _make_provider( + provider_name=name, + served_model_name=APP_MODEL_NAME if name == APP_PROVIDER else CS_MODEL_NAME, + ) + ) + client.inference.virtual_models.create = MagicMock( + return_value=SimpleNamespace(name=VM_NAME, default_model_entity=f"{WORKSPACE}/app") + ) + client.guardrail.configs.create = MagicMock(return_value=SimpleNamespace(name=GUARDRAIL_CONFIG)) + client.workspaces.create = MagicMock(return_value=SimpleNamespace(name=WORKSPACE)) + return client + + +# --------------------------------------------------------------------------- +# build_guardrail_config_data +# --------------------------------------------------------------------------- + + +class TestBuildGuardrailConfigData: + def test_rewrites_models_and_inlines_prompts(self, tmp_path: Path) -> None: + cs_dir = _write_upstream_configs(tmp_path) + + data = build_guardrail_config_data( + source_config_dir=cs_dir, + content_safety_model_entity="benchmark/cs-entity", + ) + + assert data["models"] == [{"type": "content_safety", "engine": "nim", "model": "benchmark/cs-entity"}] + assert data["prompts"] == [{"task": "content_safety_check_input", "content": "..."}] + # Non-models fields preserved. + assert data["rails"]["input"]["flows"] == ["content safety check input $model=content_safety"] + + def test_missing_config_raises(self, tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError, match="config.yml"): + build_guardrail_config_data( + source_config_dir=tmp_path / "nope", + content_safety_model_entity="x/y", + ) + + +# --------------------------------------------------------------------------- +# seed_benchmark +# --------------------------------------------------------------------------- + + +class TestSeedBenchmark: + def test_calls_sdk_with_expected_payloads(self, fake_client: MagicMock, tmp_path: Path) -> None: + ng_root = tmp_path / "NeMo-Guardrails" + _write_upstream_configs(ng_root) + generated_dir = tmp_path / "generated" + + seeded = seed_benchmark( + fake_client, + ng_repo_root=ng_root, + generated_dir=generated_dir, + provider_wait_timeout=1.0, + ) + + fake_client.workspaces.create.assert_called_once_with( + name=WORKSPACE, + description="Local IGW guardrails benchmark workspace", + exist_ok=True, + ) + # Both providers registered. + provider_create_names = [c.kwargs["name"] for c in fake_client.inference.providers.create.call_args_list] + assert sorted(provider_create_names) == sorted([APP_PROVIDER, CS_PROVIDER]) + + # Guardrail config payload uses the discovered content-safety entity. + gc_call = fake_client.guardrail.configs.create.call_args + assert gc_call.kwargs["name"] == GUARDRAIL_CONFIG + assert gc_call.kwargs["workspace"] == WORKSPACE + assert gc_call.kwargs["exist_ok"] is True + cs_entity = seeded.cs_model_entity + assert gc_call.kwargs["data"]["models"][0]["model"] == cs_entity + + # VirtualModel uses the discovered app entity and points middleware at the + # guardrail config we just created. + vm_call = fake_client.inference.virtual_models.create.call_args + assert vm_call.kwargs["name"] == VM_NAME + assert vm_call.kwargs["default_model_entity"] == seeded.app_model_entity + assert vm_call.kwargs["models"] == [{"model": seeded.app_model_entity, "backend_format": "OPENAI_CHAT"}] + expected_middleware = [ + { + "name": "nemo-guardrails", + "config_type": "guardrail_config", + "config_id": f"{WORKSPACE}/{GUARDRAIL_CONFIG}", + } + ] + assert vm_call.kwargs["request_middleware"] == expected_middleware + assert vm_call.kwargs["response_middleware"] == expected_middleware + + def test_generated_dir_contains_artifacts(self, fake_client: MagicMock, tmp_path: Path) -> None: + ng_root = tmp_path / "NeMo-Guardrails" + _write_upstream_configs(ng_root) + generated_dir = tmp_path / "generated" + + seed_benchmark( + fake_client, + ng_repo_root=ng_root, + generated_dir=generated_dir, + provider_wait_timeout=1.0, + ) + + assert (generated_dir / "app_provider.json").is_file() + assert (generated_dir / "content_safety_provider.json").is_file() + assert (generated_dir / "virtual_model.json").is_file() + + request_payload = json.loads( + (generated_dir / "content_safety_local_nmp_request.json").read_text(encoding="utf-8") + ) + assert request_payload["name"] == GUARDRAIL_CONFIG + assert request_payload["exist_ok"] is True + assert request_payload["data"]["models"][0]["type"] == "content_safety" + + def test_returns_seeded_resources(self, fake_client: MagicMock, tmp_path: Path) -> None: + ng_root = tmp_path / "NeMo-Guardrails" + _write_upstream_configs(ng_root) + + seeded = seed_benchmark( + fake_client, + ng_repo_root=ng_root, + generated_dir=tmp_path / "generated", + provider_wait_timeout=1.0, + ) + + assert seeded.workspace == WORKSPACE + assert seeded.vm_ref == f"{WORKSPACE}/{VM_NAME}" + assert seeded.guardrail_config_ref == f"{WORKSPACE}/{GUARDRAIL_CONFIG}" + + def test_raises_if_served_models_never_populated(self, tmp_path: Path) -> None: + ng_root = tmp_path / "NeMo-Guardrails" + _write_upstream_configs(ng_root) + + client = MagicMock() + client.workspaces.create = MagicMock() + client.inference.providers.create = MagicMock() + client.inference.providers.retrieve = MagicMock(return_value=SimpleNamespace(served_models=[])) + + with pytest.raises(TimeoutError, match="served model"): + seed_benchmark( + client, + ng_repo_root=ng_root, + generated_dir=tmp_path / "generated", + provider_wait_timeout=0.1, + ) diff --git a/plugins/nemo-guardrails/tests/unit/benchmarks/test_shim.py b/plugins/nemo-guardrails/tests/unit/benchmarks/test_shim.py new file mode 100644 index 0000000000..f100d93b82 --- /dev/null +++ b/plugins/nemo-guardrails/tests/unit/benchmarks/test_shim.py @@ -0,0 +1,105 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Smoke tests for the AIPerf shim. + +We stand up the shim on a real local port and a stub upstream HTTP server, then +exercise the three routes AIPerf and the harness rely on. +""" + +from __future__ import annotations + +import json +import socket +import threading +from contextlib import closing +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Generator + +import httpx +import pytest +from nemo_guardrails_plugin.benchmarks import shim as shim_module + + +def _free_port() -> int: + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +class _UpstreamHandler(BaseHTTPRequestHandler): + received_body: bytes = b"" + + def log_message(self, *args: object, **kwargs: object) -> None: # noqa: A002 + return + + def do_POST(self) -> None: # noqa: N802 + length = int(self.headers.get("Content-Length", "0") or "0") + type(self).received_body = self.rfile.read(length) if length else b"" + body = json.dumps({"choices": [{"message": {"content": "hi"}}]}).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +@pytest.fixture +def upstream_server() -> Generator[str, None, None]: + port = _free_port() + server = ThreadingHTTPServer(("127.0.0.1", port), _UpstreamHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{port}/chat" + finally: + server.shutdown() + server.server_close() + + +@pytest.fixture +def shim_server(upstream_server: str) -> Generator[str, None, None]: + port = _free_port() + # Override the class attr so the shim proxies to our stub upstream. + previous = shim_module._ShimHandler.upstream_url + shim_module._ShimHandler.upstream_url = upstream_server + server = ThreadingHTTPServer(("127.0.0.1", port), shim_module._ShimHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{port}" + finally: + shim_module._ShimHandler.upstream_url = previous + server.shutdown() + server.server_close() + + +def test_models_returns_200_with_openai_shape(shim_server: str) -> None: + response = httpx.get(f"{shim_server}/v1/models", timeout=2.0) + assert response.status_code == 200 + body = response.json() + assert body == {"object": "list", "data": []} + + +def test_health_endpoint_returns_200(shim_server: str) -> None: + response = httpx.get(f"{shim_server}/__shim/health", timeout=2.0) + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + + +def test_chat_completions_proxies_to_upstream(shim_server: str) -> None: + payload = {"model": "x", "messages": [{"role": "user", "content": "hi"}]} + response = httpx.post( + f"{shim_server}/v1/chat/completions", + json=payload, + timeout=5.0, + ) + assert response.status_code == 200 + assert response.json()["choices"][0]["message"]["content"] == "hi" + # Body should be forwarded byte-for-byte (ignoring possible whitespace). + assert json.loads(_UpstreamHandler.received_body) == payload + + +def test_unknown_path_returns_404(shim_server: str) -> None: + response = httpx.get(f"{shim_server}/something/else", timeout=2.0) + assert response.status_code == 404 diff --git a/uv.lock b/uv.lock index 9a43dcd91e..52c13d4806 100644 --- a/uv.lock +++ b/uv.lock @@ -3912,6 +3912,12 @@ dependencies = [ { name = "nemoguardrails", extra = ["tracing"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] +[package.optional-dependencies] +bench = [ + { name = "httpx", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" }, + { name = "pyyaml", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" }, +] + [package.dev-dependencies] dev = [ { name = "pytest", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, @@ -3923,10 +3929,13 @@ dev = [ [package.metadata] requires-dist = [ { name = "dataclasses-json", specifier = ">=0.6.7" }, + { name = "httpx", marker = "extra == 'bench'", specifier = ">=0.27" }, { name = "nemo-platform", editable = "packages/nemo_platform" }, { name = "nemo-platform-plugin", editable = "packages/nemo_platform_plugin" }, { name = "nemoguardrails", extras = ["tracing"], specifier = "==0.21.0" }, + { name = "pyyaml", marker = "extra == 'bench'", specifier = ">=6.0" }, ] +provides-extras = ["bench"] [package.metadata.requires-dev] dev = [ From 776bdcf1742a041846b383e7b67314a2a5b41a5e Mon Sep 17 00:00:00 2001 From: Jash Gulabrai Date: Wed, 27 May 2026 18:09:39 -0400 Subject: [PATCH 02/10] Fix working_directory Signed-off-by: Jash Gulabrai --- .github/workflows/ci.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5a0b98cb29..fa148f50fa 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -398,6 +398,10 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v7 with: + # Point setup-uv at the nemo-platform checkout so it picks up the + # repo's `required-version` constraint from pyproject.toml. Other + # jobs check out into the workspace root and get this implicitly. + working-directory: nemo-platform python-version: "3.11" enable-cache: true - name: Bootstrap Python environment From 3853c63c103247e5496c5443d017d1098e163e44 Mon Sep 17 00:00:00 2001 From: Jash Gulabrai Date: Wed, 27 May 2026 22:06:41 -0400 Subject: [PATCH 03/10] Use non-gated tokenizer Signed-off-by: Jash Gulabrai --- .../configs/nmp_igw_guardrails_sweep_concurrency.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/plugins/nemo-guardrails/benchmarks/configs/nmp_igw_guardrails_sweep_concurrency.yaml b/plugins/nemo-guardrails/benchmarks/configs/nmp_igw_guardrails_sweep_concurrency.yaml index 1c141f3ec6..8a5c38e4b3 100644 --- a/plugins/nemo-guardrails/benchmarks/configs/nmp_igw_guardrails_sweep_concurrency.yaml +++ b/plugins/nemo-guardrails/benchmarks/configs/nmp_igw_guardrails_sweep_concurrency.yaml @@ -5,7 +5,13 @@ output_base_dir: plugins/nemo-guardrails/benchmarks/artifacts/aiperf_results base_config: model: benchmark/guardrails-vm - tokenizer: meta-llama/Llama-3.3-70B-Instruct + # AIPerf only needs the tokenizer to count input/output tokens for its + # metrics; the actual model under test is the IGW VirtualModel above. We + # use Qwen3-8B (Apache-2.0, ungated) instead of Llama 3.3 so CI can fetch + # the tokenizer without an HF_TOKEN. Token counts will differ slightly + # from Llama, but middleware-overhead and latency trends remain valid as + # long as the tokenizer stays consistent across runs. + tokenizer: Qwen/Qwen3-8B # AIPerf's pre-check does GET urljoin(url, "/v1/models"); we run a thin # shim on :8090 that satisfies that probe and reverse-proxies # /v1/chat/completions through to NMP's IGW. See From 80cb5df5f4fd980ec03851d4ba97fff79d7fb805 Mon Sep 17 00:00:00 2001 From: Jash Gulabrai Date: Thu, 28 May 2026 11:47:02 -0400 Subject: [PATCH 04/10] Minor refactor Signed-off-by: Jash Gulabrai --- .github/workflows/ci.yaml | 1 - plugins/nemo-guardrails/benchmarks/README.md | 55 +++-- .../nmp_igw_guardrails_sweep_concurrency.yaml | 15 +- plugins/nemo-guardrails/pyproject.toml | 3 +- .../benchmarks/aiperf_runner.py | 74 ++++-- .../benchmarks/bootstrap.py | 111 +++------ .../benchmarks/constants.py | 2 - .../benchmarks/paths.py | 60 +++-- .../benchmarks/processes.py | 162 ++++++++----- .../benchmarks/report.py | 107 --------- .../nemo_guardrails_plugin/benchmarks/run.py | 218 ++++++++---------- .../benchmarks/seeding.py | 6 +- .../unit/benchmarks/test_aiperf_runner.py | 46 ++-- .../tests/unit/benchmarks/test_paths.py | 24 +- .../tests/unit/benchmarks/test_report.py | 90 -------- .../tests/unit/benchmarks/test_seeding.py | 8 +- 16 files changed, 416 insertions(+), 566 deletions(-) delete mode 100644 plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/report.py delete mode 100644 plugins/nemo-guardrails/tests/unit/benchmarks/test_report.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fa148f50fa..d4dd969571 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -422,7 +422,6 @@ jobs: name: benchmark-guardrails-results retention-days: 30 path: | - nemo-platform/report.xml nemo-platform/plugins/nemo-guardrails/benchmarks/artifacts/runs/ coverage-comment: diff --git a/plugins/nemo-guardrails/benchmarks/README.md b/plugins/nemo-guardrails/benchmarks/README.md index 3b3881f96e..bc47ad43bc 100644 --- a/plugins/nemo-guardrails/benchmarks/README.md +++ b/plugins/nemo-guardrails/benchmarks/README.md @@ -16,17 +16,15 @@ plugins/nemo-guardrails/benchmarks/ configs/ nmp_igw_guardrails_sweep_concurrency.yaml # AIPerf sweep template artifacts/ # per-run outputs (gitignored) - generated/ # placeholder; per-run artifacts live under artifacts/runs/ plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/ - run.py # entry point: `python -m nemo_guardrails_plugin.benchmarks.run` + run.py # entrypoint: `python -m nemo_guardrails_plugin.benchmarks.run` paths.py # filesystem layout constants.py # workspace / VM / provider names processes.py # subprocess supervision (process groups + ExitStack) - seeding.py # NMP SDK calls (workspace, providers, GuardrailConfig, VM) + seeding.py # NMP SDK calls to create the required entities aiperf_runner.py # rewrite AIPerf config + invoke upstream sweep + collect results bootstrap.py # manage the isolated venv that hosts the `aiperf` CLI shim.py # tiny HTTP shim that satisfies AIPerf's `/v1/models` pre-check - report.py # emit JUnit XML ``` ## Prerequisites @@ -41,7 +39,7 @@ plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/ used by an internal shim that satisfies AIPerf's hard-coded `/v1/models` health probe. -The harness has its own `pyyaml` + `httpx` requirements that are declared as +The harness has its own dependencies that are declared as the `bench` extra on `nemo-guardrails-plugin`. The `make benchmark-guardrails` target installs them automatically via `uv run --extra bench`; they are not part of the plugin's runtime install. @@ -79,8 +77,7 @@ The default sweep runs concurrency levels: 1, 2, 4, 8, 16, 32, 64 ``` -With the default 60-second benchmark duration, expect ~10 minutes after -service bootstrap. +With the default 60-second benchmark duration, expect the benchmark to run for ~10 minutes after service bootstrap. ## What the harness starts @@ -97,7 +94,7 @@ It then seeds NMP via the SDK with: - VirtualModel `benchmark/guardrails-vm` with `nemo-guardrails` attached to both request and response middleware. -The benchmark target is: +The benchmark target for inference requests is: ```text http://localhost:8080/apis/inference-gateway/v2/workspaces/benchmark/openai/-/v1/chat/completions @@ -113,11 +110,10 @@ The harness accepts both CLI flags and environment variables: | `--reuse-services` | `NMP_BENCHMARK_REUSE_SERVICES=1` | start `nemo services run` | | `--keep-running` | `NMP_BENCHMARK_KEEP_RUNNING=1` | tear down on exit | | `--mock-workers` | `NMP_BENCHMARK_MOCK_WORKERS` | `4` | -| `--junit-path` | _n/a_ | `/report.xml` | | `--run-id` | _n/a_ | current timestamp | `--keep-running` leaves child processes alive for post-mortem inspection; the -list of PIDs is recorded in the per-run directory's `pids.txt`. +harness logs each child's PID as it starts. ## Outputs @@ -140,32 +136,45 @@ plugins/nemo-guardrails/benchmarks/artifacts/runs// run_metadata.json process_result.json profile_export*.json # written by aiperf - pids.txt ``` -`report.xml` is written to the repo root by default so CI's -`actions/upload-artifact` step picks it up at the same path as other test -suites. Override with `--junit-path`. - ## CI A `benchmark-guardrails` job in `.github/workflows/ci.yaml` checks out both this repo and `NVIDIA/NeMo-Guardrails`, runs `make bootstrap-python` and -`make benchmark-guardrails`, and uploads the per-run artifacts directory plus -`report.xml` (so GitHub renders sweep pass/fail in the PR view). +`make benchmark-guardrails`, and uploads the per-run artifacts directory +(`logs/`, `generated/`, `aiperf_results/`) on success or failure. -Pass/fail is currently driven purely by `aiperf` exit code; no latency -thresholds are enforced. +Pass/fail is driven by the harness's exit code, which is non-zero if `aiperf` +itself exits non-zero or any sweep returns a non-zero exit code. No latency +thresholds are enforced — those can be layered on later by a separate +analyzer that reads the per-sweep CSVs. ## Cleanup -By default the harness only stops processes it started. It does not kill -unrelated processes on ports `8000`, `8001`, or `8080`. +The harness only stops the processes it started. It will not kill unrelated +processes on ports `8000`, `8001`, `8080`, or `8090`. -Local NMP state is isolated by default under: +Across runs, NMP's data dir is reused so subsequent benchmarks start +faster. The harness redirects NMP's writes via the `NMP_DATA_DIR` env var to +a per-checkout directory: ```text plugins/nemo-guardrails/benchmarks/artifacts/nmp-data ``` -Delete that directory for a completely fresh local benchmark state. +This holds NMP's SQLite database, Secrets vault, and other persistent +service state. It is gitignored and isolated from `~/.nmp/`, so the +benchmark will not pollute your normal local NMP setup. Reuse is what +lets repeat runs skip the workspace / provider / VirtualModel creation +work (the harness treats `409 Conflict` as "already exists, carry on"). + +If a benchmark misbehaves and you suspect stale state (ex. after pulling +a schema change), delete that directory for a fully fresh run: + +```bash +rm -rf plugins/nemo-guardrails/benchmarks/artifacts/nmp-data +``` + +In CI this is automatic — every job gets a fresh runner, so `nmp-data` +does not exist. diff --git a/plugins/nemo-guardrails/benchmarks/configs/nmp_igw_guardrails_sweep_concurrency.yaml b/plugins/nemo-guardrails/benchmarks/configs/nmp_igw_guardrails_sweep_concurrency.yaml index 8a5c38e4b3..6fe53c5c93 100644 --- a/plugins/nemo-guardrails/benchmarks/configs/nmp_igw_guardrails_sweep_concurrency.yaml +++ b/plugins/nemo-guardrails/benchmarks/configs/nmp_igw_guardrails_sweep_concurrency.yaml @@ -6,16 +6,13 @@ output_base_dir: plugins/nemo-guardrails/benchmarks/artifacts/aiperf_results base_config: model: benchmark/guardrails-vm # AIPerf only needs the tokenizer to count input/output tokens for its - # metrics; the actual model under test is the IGW VirtualModel above. We - # use Qwen3-8B (Apache-2.0, ungated) instead of Llama 3.3 so CI can fetch - # the tokenizer without an HF_TOKEN. Token counts will differ slightly - # from Llama, but middleware-overhead and latency trends remain valid as - # long as the tokenizer stays consistent across runs. + # metrics; the actual model under test is the mock IGW VirtualModel above. + # We use Qwen3-8B (instead of Llama 3.3, which is gated) so CI can fetch + # the tokenizer without an HF Token. tokenizer: Qwen/Qwen3-8B - # AIPerf's pre-check does GET urljoin(url, "/v1/models"); we run a thin - # shim on :8090 that satisfies that probe and reverse-proxies - # /v1/chat/completions through to NMP's IGW. See - # nemo_guardrails_plugin.benchmarks.shim for the implementation. + # The benchmark runs a required health check at `/v1/models`. Until this + # endpoint is configurable, we run a thin shim on :8090 that satisfies that probe. + # See nemo_guardrails_plugin.benchmarks.shim for the implementation. url: "http://localhost:8090" endpoint: "/v1/chat/completions" endpoint_type: chat diff --git a/plugins/nemo-guardrails/pyproject.toml b/plugins/nemo-guardrails/pyproject.toml index 4a52e7fc6f..83a6db4797 100644 --- a/plugins/nemo-guardrails/pyproject.toml +++ b/plugins/nemo-guardrails/pyproject.toml @@ -14,8 +14,7 @@ dependencies = [ [project.optional-dependencies] # Dependencies for the local IGW benchmark harness in -# `nemo_guardrails_plugin.benchmarks`. Activate with `uv sync --extra bench` or -# `uv run --extra bench ...` (see `make benchmark-guardrails`). +# `nemo_guardrails_plugin.benchmarks` bench = [ "httpx>=0.27", "pyyaml>=6.0", diff --git a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/aiperf_runner.py b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/aiperf_runner.py index 8c61fd66d0..c14484e694 100644 --- a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/aiperf_runner.py +++ b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/aiperf_runner.py @@ -28,11 +28,26 @@ class SweepRunResult: """Outcome of a single ``aiperf profile`` invocation (one concurrency level).""" + # Name of the per-sweep subdirectory AIPerf created, e.g. ``"concurrency16"``. + # Surfaced in the harness's summary log so failures point at a directory name. sweep_label: str + # Absolute path to the sweep's output directory under + # ``aiperf_results////``. Contains the AIPerf + # CSV, per-request JSONL, run metadata, and the wrapper's process_result.json. output_dir: Path + # Exit code of the AIPerf subprocess for this sweep (0 = success). Sourced from + # ``process_result.json``; defaults to 1 if that file is missing or unparseable + # so a crashed subprocess does not silently pass. return_code: int + # Wall-clock duration of the sweep in seconds, read from ``run_metadata.json``. + # 0.0 when AIPerf never wrote metadata (typically because it crashed early). duration_seconds: float + # Path to ``run_metadata.json`` if AIPerf wrote it, else ``None``. Kept on the + # result for downstream analyzers that want to inspect AIPerf's view of the run. metadata_path: Path | None + # Path to ``process_result.json`` if AIPerf wrote it, else ``None``. This is + # the upstream wrapper's record of the subprocess exit; useful when + # ``return_code`` itself looks suspicious. process_result_path: Path | None @property @@ -40,26 +55,42 @@ def passed(self) -> bool: return self.return_code == 0 -def rewrite_aiperf_config(*, template: Path, output: Path, output_base_dir: Path) -> dict[str, Any]: - """Copy the checked-in template to ``output`` with ``output_base_dir`` overridden. - - The runtime copy is written under the per-run directory so concurrent or - historical runs cannot stomp on each other. Returns the parsed config dict. +def prepare_runtime_aiperf_config( + *, + template_path: Path, + runtime_config_path: Path, + aiperf_output_dir: Path, +) -> dict[str, Any]: + """Materialize the AIPerf config this run will use. + + Reads the checked-in ``template_path`` config, overrides its + ``output_base_dir`` to point inside the current run's directory, and writes + the result to ``runtime_config_path``. AIPerf is later invoked with + ``--config-file `` so every artifact lands under a + separate per-run directory. + + Returns the parsed config dict so callers can log fields (sweep params, + benchmark_duration) without re-reading the file. """ - if not template.is_file(): - raise FileNotFoundError(f"AIPerf template not found: {template}") - config = yaml.safe_load(template.read_text(encoding="utf-8")) + if not template_path.is_file(): + raise FileNotFoundError(f"AIPerf template not found: {template_path}") + + config = yaml.safe_load(template_path.read_text(encoding="utf-8")) if not isinstance(config, dict): - raise ValueError(f"AIPerf template {template} did not parse as a mapping") - config["output_base_dir"] = str(output_base_dir) - output.parent.mkdir(parents=True, exist_ok=True) - output.write_text(yaml.safe_dump(config, sort_keys=False), encoding="utf-8") + raise ValueError(f"Failed to parse AIPerf template {template_path}. Ensure it is valid YAML.") + + # Point AIPerf's output_base_dir at this run's directory so its results + # nest under our per-run artifacts tree. + config["output_base_dir"] = str(aiperf_output_dir) + runtime_config_path.parent.mkdir(parents=True, exist_ok=True) + runtime_config_path.write_text(yaml.safe_dump(config, sort_keys=False), encoding="utf-8") + return config def run_aiperf_sweep( *, - ng_repo_root: Path, + nemoguardrails_repo_root: Path, runtime_config: Path, log_path: Path, python_executable: str | None = None, @@ -77,7 +108,7 @@ def run_aiperf_sweep( Note: AIPerf's built-in pre-flight check does a GET on ``urljoin(config.base_config.url, "/v1/models")`` with no override - available upstream. The harness runs a tiny shim that satisfies this + available upstream yet. The harness runs a tiny shim that satisfies this probe; see :mod:`nemo_guardrails_plugin.benchmarks.shim`. """ python_bin = python_executable or sys.executable @@ -92,20 +123,24 @@ def run_aiperf_sweep( str(runtime_config), ] - env = {**os.environ, "PYTHONPATH": str(ng_repo_root)} + env = {**os.environ, "PYTHONPATH": str(nemoguardrails_repo_root)} if extra_env: env.update(extra_env) + log_path.parent.mkdir(parents=True, exist_ok=True) - log.info("Running aiperf sweep: %s (cwd=%s)", " ".join(cmd), ng_repo_root) + + log.info("Running aiperf sweep: %s (cwd=%s)", " ".join(cmd), nemoguardrails_repo_root) + with log_path.open("wb") as log_fh: proc = subprocess.run( # noqa: S603 - command is constructed internally cmd, - cwd=str(ng_repo_root), + cwd=str(nemoguardrails_repo_root), env=env, stdout=log_fh, stderr=subprocess.STDOUT, check=False, ) + return proc.returncode @@ -130,6 +165,11 @@ def collect_sweep_results(aiperf_output_dir: Path) -> list[SweepRunResult]: def _load_sweep_result(sweep_dir: Path) -> SweepRunResult: + """Load the sweep result from the given directory. + + Layout: + ``////{run_metadata.json,process_result.json}`` + """ metadata_path = sweep_dir / "run_metadata.json" process_result_path = sweep_dir / "process_result.json" diff --git a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/bootstrap.py b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/bootstrap.py index 9253277592..04fc9bb0a4 100644 --- a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/bootstrap.py +++ b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/bootstrap.py @@ -3,15 +3,10 @@ """Bootstrap an isolated venv for the upstream AIPerf load generator. -`aiperf` (the PyPI package) pins ``aiofiles<24.2`` while NMP's -evaluator-service requires ``aiofiles>=25.1``. We can't add `aiperf` to the -workspace lockfile without downgrading the shared venv and breaking other -services, so the harness manages a dedicated, lock-free venv just for the -AIPerf binary. - -The venv is created on first use (idempotent) and reused on subsequent runs. -The path is deterministic so local dev iterations are fast; in CI the cache is -discarded each run, which is acceptable since `aiperf` is a small install. +``aiperf`` pins ``aiofiles<24.2`` which conflicts with NMP's evaluator-service +requirement of ``aiofiles>=25.1``, so we install it into a dedicated venv +instead of the shared workspace one. The venv is reused across local runs; +CI gets a fresh one each invocation. """ from __future__ import annotations @@ -19,99 +14,55 @@ import logging import os import subprocess -import sys from pathlib import Path log = logging.getLogger("nemo_guardrails_plugin.benchmarks.bootstrap") -# Versions pinned to what the upstream NeMo-Guardrails README installs alongside -# `python -m benchmark.aiperf`. We don't tighten further; aiperf's own metadata -# pins its dependencies. +# Mirrors the upstream NeMo-Guardrails AIPerf README. We don't pin further; +# aiperf itself pins its transitives. _AIPERF_PACKAGES = ("aiperf", "huggingface_hub", "typer>=0.9", "httpx>=0.27") -class BootstrapError(RuntimeError): - """Raised when we can't materialise the aiperf venv.""" +def ensure_aiperf_venv(venv_dir: Path) -> Path: + """Idempotently create the aiperf venv. Returns the venv's python path. - -def _venv_python(venv_dir: Path) -> Path: - """Return the venv's python binary regardless of platform.""" - if os.name == "nt": - return venv_dir / "Scripts" / "python.exe" - return venv_dir / "bin" / "python" - - -def _venv_bin(venv_dir: Path) -> Path: - if os.name == "nt": - return venv_dir / "Scripts" - return venv_dir / "bin" - - -def ensure_aiperf_venv(venv_dir: Path, *, force: bool = False) -> Path: - """Ensure ``venv_dir`` contains a usable ``aiperf`` install. - - Returns the path to the venv's python interpreter so the caller can invoke - `` -m benchmark.aiperf`` with the right environment. + Uses ``uv venv`` + ``uv pip install`` since the harness is only ever invoked + via ``make benchmark-guardrails``, which already requires ``uv`` to be on + PATH. Skips both steps if the venv and the ``aiperf`` binary already exist. """ - aiperf_bin = _venv_bin(venv_dir) / ("aiperf.exe" if os.name == "nt" else "aiperf") - python_bin = _venv_python(venv_dir) + python_bin = venv_dir / "bin" / "python" + aiperf_bin = venv_dir / "bin" / "aiperf" - if not force and aiperf_bin.exists() and python_bin.exists(): + if aiperf_bin.exists() and python_bin.exists(): log.info("Reusing existing aiperf venv at %s", venv_dir) return python_bin log.info("Creating aiperf venv at %s", venv_dir) venv_dir.parent.mkdir(parents=True, exist_ok=True) - - # Prefer `uv venv` for speed and to match the rest of the project; fall - # back to stdlib `venv` if `uv` isn't on PATH. - try: - subprocess.run( # noqa: S603 - command is constructed internally - ["uv", "venv", "--python", "3.11", str(venv_dir)], - check=True, - capture_output=True, - ) - except FileNotFoundError: - log.warning("uv not found on PATH; falling back to `python -m venv`") - subprocess.run( # noqa: S603 - command is constructed internally - [sys.executable, "-m", "venv", str(venv_dir)], - check=True, - capture_output=True, - ) - except subprocess.CalledProcessError as e: - raise BootstrapError(f"Failed to create aiperf venv at {venv_dir}: {e.stderr.decode(errors='replace')}") from e + subprocess.run( # noqa: S603 - command is constructed internally + ["uv", "venv", "--python", "3.11", str(venv_dir)], + check=True, + capture_output=True, + ) log.info("Installing %s into %s", ", ".join(_AIPERF_PACKAGES), venv_dir) - try: - # `uv pip install --python ` is hermetic: it installs into - # the target venv without touching the workspace lockfile. - subprocess.run( # noqa: S603 - command is constructed internally - ["uv", "pip", "install", "--python", str(python_bin), *_AIPERF_PACKAGES], - check=True, - ) - except FileNotFoundError: - log.warning("uv not found; falling back to `pip install`") - subprocess.run( # noqa: S603 - command is constructed internally - [str(python_bin), "-m", "pip", "install", *_AIPERF_PACKAGES], - check=True, - ) - except subprocess.CalledProcessError as e: - raise BootstrapError(f"Failed to install aiperf into {venv_dir}: {e}") from e + subprocess.run( # noqa: S603 - command is constructed internally + ["uv", "pip", "install", "--python", str(python_bin), *_AIPERF_PACKAGES], + check=True, + ) if not aiperf_bin.exists(): - raise BootstrapError(f"aiperf install completed but {aiperf_bin} is missing") - + raise RuntimeError(f"aiperf install completed but {aiperf_bin} is missing") return python_bin -def env_with_venv_on_path(venv_dir: Path, base_env: dict[str, str] | None = None) -> dict[str, str]: - """Return a copy of ``base_env`` with the venv's bin dir prepended to PATH. +def env_with_venv_on_path(venv_dir: Path) -> dict[str, str]: + """Return ``os.environ`` with the venv's ``bin/`` prepended to ``PATH``. - The upstream `python -m benchmark.aiperf` wrapper shells out to a literal - ``aiperf`` command, so the venv's bin dir has to come before whatever was - on PATH inherited from the parent. + The upstream ``python -m benchmark.aiperf`` wrapper shells out to a literal + ``aiperf`` binary via ``subprocess.run``, so the venv's bin dir must be + discoverable on ``PATH`` before whatever was inherited from the parent. """ - env = dict(base_env if base_env is not None else os.environ) - venv_bin = str(_venv_bin(venv_dir)) - env["PATH"] = venv_bin + os.pathsep + env.get("PATH", "") + env = dict(os.environ) + env["PATH"] = f"{venv_dir / 'bin'}{os.pathsep}{env.get('PATH', '')}" return env diff --git a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/constants.py b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/constants.py index fd6aff89b0..cbe4b19960 100644 --- a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/constants.py +++ b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/constants.py @@ -32,5 +32,3 @@ GUARDRAILS_MIDDLEWARE_NAME = "nemo-guardrails" GUARDRAILS_MIDDLEWARE_CONFIG_TYPE = "guardrail_config" - -JUNIT_SUITE_NAME = "nemo_guardrails_plugin.benchmarks" diff --git a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/paths.py b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/paths.py index 676b6c8198..b1113fa2c2 100644 --- a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/paths.py +++ b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/paths.py @@ -1,7 +1,17 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Path resolution for the nemo-guardrails IGW benchmark harness.""" +"""Filesystem layout for the nemo-guardrails IGW benchmark harness. + +A benchmark invocation produces a `runs//` directory that holds +everything generated by that run: AIPerf results, supervised-process logs, +and the materialized AIPerf config. Some artifacts are intentionally kept +*outside* the per-run dir so they can be reused across runs (the NMP data +dir, the aiperf venv). + +`RunPaths` is the single source of truth for those locations; `build_run_paths` +composes it from the two repo roots the harness needs (NMP and NeMo-Guardrails). +""" from __future__ import annotations @@ -14,18 +24,42 @@ class RunPaths: """Filesystem layout for a single benchmark invocation.""" + # Root of the nemo-platform repo we're running from. Used as the cwd for + # `nemo services run`. nmp_repo_root: Path - ng_repo_root: Path + # Root of a local NeMo-Guardrails checkout. The harness reads mock-LLM + # configs and the content_safety_local example from here. + nemoguardrails_repo_root: Path + # Convenience anchor for everything under + # `plugins/nemo-guardrails/benchmarks/` in the NMP repo. benchmark_dir: Path + # Per-invocation directory under `benchmarks/artifacts/runs//`. + # Everything produced by this run lives here so CI artifact upload and + # local cleanup can treat it as a single unit. run_dir: Path + # Stdout/stderr for every supervised child (mocks, nmp-services, shim, + # aiperf orchestrator). Lives under `run_dir`. log_dir: Path + # Scratch dir for files the harness materializes at runtime: the + # rewritten AIPerf config, NMP API request bodies we logged for debugging, + # etc. generated_dir: Path + # Where AIPerf writes its sweep outputs (CSVs, JSONL records, metadata). + # The harness rewrites the AIPerf config's `output_base_dir` to point here. aiperf_output_dir: Path - pids_file: Path + # NMP's persistent data directory (SQLite, Secrets vault, etc.). Lives + # *outside* `run_dir` and is reused across runs so subsequent benchmarks + # skip workspace / provider / VM creation work; see the README's Cleanup + # section. Passed to `nemo services run` via `NMP_DATA_DIR`. nmp_data_dir: Path + # Checked-in YAML template for the AIPerf sweep config. config_template: Path + # Per-run materialized copy of `config_template` with `output_base_dir` + # overridden to `aiperf_output_dir`. AIPerf is invoked against this file. runtime_config: Path - junit_path: Path + # Isolated venv for the upstream `aiperf` CLI. Lives outside `run_dir` + # (sibling to `nmp_data_dir`) so subsequent local runs reuse the install; + # see `bootstrap.py` for why we don't share the workspace venv. aiperf_venv_dir: Path @property @@ -55,41 +89,35 @@ def discover_nmp_repo_root(start: Path | None = None) -> Path: raise RuntimeError(f"Could not locate NMP repo root from {start or Path(__file__)}") -def default_ng_repo_root(nmp_repo_root: Path) -> Path: +def default_nemoguardrails_repo_root(nmp_repo_root: Path) -> Path: + """Default to a sibling ``NeMo-Guardrails`` checkout next to ``nmp_repo_root``.""" return (nmp_repo_root.parent / "NeMo-Guardrails").resolve() def build_run_paths( *, nmp_repo_root: Path, - ng_repo_root: Path, - junit_dir: Path | None = None, + nemoguardrails_repo_root: Path, run_id: str | None = None, ) -> RunPaths: - """Compose the standard benchmark filesystem layout under the plugin's artifacts dir. + """Compose the benchmark filesystem layout under the plugin's artifacts dir. - ``junit_dir`` controls where ``report.xml`` is written. CI passes the repo root so - that the artifact upload step finds it at the expected location. + ``run_id`` overrides the per-run directory name (default: current timestamp). """ benchmark_dir = nmp_repo_root / "plugins" / "nemo-guardrails" / "benchmarks" artifacts_dir = benchmark_dir / "artifacts" run_dir = artifacts_dir / "runs" / (run_id or _now_run_id()) - junit_target = (junit_dir or nmp_repo_root) / "report.xml" return RunPaths( nmp_repo_root=nmp_repo_root, - ng_repo_root=ng_repo_root, + nemoguardrails_repo_root=nemoguardrails_repo_root, benchmark_dir=benchmark_dir, run_dir=run_dir, log_dir=run_dir / "logs", generated_dir=run_dir / "generated", aiperf_output_dir=run_dir / "aiperf_results", - pids_file=run_dir / "pids.txt", nmp_data_dir=artifacts_dir / "nmp-data", config_template=benchmark_dir / "configs" / "nmp_igw_guardrails_sweep_concurrency.yaml", runtime_config=run_dir / "generated" / "nmp_igw_guardrails_sweep_concurrency.yaml", - junit_path=junit_target, - # Cached aiperf venv lives outside the per-run dir so it's reused - # across local runs but lives under the gitignored artifacts dir. aiperf_venv_dir=artifacts_dir / "venvs" / "aiperf", ) diff --git a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/processes.py b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/processes.py index 47c774dbee..5cf69e3a15 100644 --- a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/processes.py +++ b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/processes.py @@ -3,8 +3,22 @@ """Process supervision for the benchmark harness. -Each child runs in its own session/process group so termination cleans up forked -worker processes (notably ``uvicorn --workers N`` and ``nemo services run``). +The harness fans out several long-lived child processes per run (two mock LLM +``uvicorn`` servers, ``nemo services run``, and the AIPerf shim) and must clean +them all up — including any workers they fork — even when the parent dies +mid-run. This module wraps ``subprocess.Popen`` in a context-managed +``SupervisedProcess`` that: + +* runs each child in a new session (``start_new_session=True``) so we can send + signals to the whole process group via ``os.killpg`` and reap workers that + ``uvicorn --workers N`` and ``nemo services`` fork off, +* merges stdout/stderr into a per-process log file under ``run_dir/logs/``, +* escalates SIGTERM → SIGKILL on shutdown if the child doesn't exit in time. + +``supervised_processes`` starts the children in order and guarantees every +already-started child is stopped if a later one fails to come up. ``wait_http`` +is the readiness probe the caller uses between starts, so we only move on to +the next child once the previous one is actually serving requests. """ from __future__ import annotations @@ -23,6 +37,8 @@ log = logging.getLogger(__name__) +# How long to wait for a child (and its process group) to exit after SIGTERM +# before we escalate to SIGKILL. _TERMINATE_TIMEOUT_SECONDS = 20 @@ -35,107 +51,144 @@ class SupervisedProcess: it forks. """ + # Human-readable identifier used in log messages (e.g. ``"mock-llama"``, + # ``"nmp-services"``). Also drives the log filename via ``log_path``. name: str + # The argv list passed to ``subprocess.Popen``. Constructed by the caller; + # never assembled from user input here (hence the ``noqa: S603`` in ``start``). cmd: list[str] + # File to merge stdout + stderr into. Parent dirs are created on ``start``. + # The harness uploads these as CI artifacts on failure. log_path: Path + # Working directory for the child process. Set per-process so e.g. the mock + # LLM servers run from inside the NeMo-Guardrails checkout where their + # configs live. cwd: Path + # Extra env vars to overlay on top of the parent's ``os.environ`` (e.g. + # ``PYTHONPATH``, ``NMP_DATA_DIR``). ``None`` means "inherit unchanged". env: dict[str, str] | None = None + # The live ``Popen`` handle, populated by ``start()`` and consulted by + # ``stop()``. Excluded from ``__init__`` and ``repr`` since it's pure state. _proc: subprocess.Popen[bytes] | None = field(default=None, init=False, repr=False) + # Open file handle for ``log_path``, held so ``stop()`` can close it in a + # ``finally`` block regardless of how termination unwinds. _log_fh: IO[bytes] | None = field(default=None, init=False, repr=False) - @property - def pid(self) -> int | None: - return self._proc.pid if self._proc else None - def start(self) -> None: + """Spawn the child, redirecting stdout+stderr to ``log_path``. + + Raises ``RuntimeError`` if called twice on the same instance. A + ``SupervisedProcess`` wraps exactly one child process over its lifetime; + to restart, build a new ``SupervisedProcess`` rather than calling + ``start()`` again on a stopped one. + """ if self._proc is not None: raise RuntimeError(f"Process {self.name!r} already started") + # Ensure the log file's parent directory exists. self.log_path.parent.mkdir(parents=True, exist_ok=True) + # Open the log file for writing. self._log_fh = self.log_path.open("wb") - merged_env = {**os.environ, **(self.env or {})} log.info("Starting %s; log=%s", self.name, self.log_path) - self._proc = subprocess.Popen( # noqa: S603 - command is constructed internally + # Spawn the child process. + self._proc = subprocess.Popen( self.cmd, - cwd=str(self.cwd), - env=merged_env, - stdout=self._log_fh, - stderr=subprocess.STDOUT, - start_new_session=True, + cwd=str(self.cwd), # Working directory + env={**os.environ, **(self.env or {})}, # Extra environment variables + stdout=self._log_fh, # Redirect stdout to the log file + stderr=subprocess.STDOUT, # Redirect stderr to stdout + start_new_session=True, # Create a new session for the child ) def stop(self) -> None: + """Terminate the child's process group and close the log file. + + Sends SIGTERM to the whole group, waits up to ``_TERMINATE_TIMEOUT_SECONDS``, + then escalates to SIGKILL. Safe to call when the child was never started, + has already exited on its own, or races with us between signals — every + such case becomes a no-op that still closes the log handle. + """ proc = self._proc if proc is None: return - if proc.poll() is not None: - self._close_log() - return try: - pgid = os.getpgid(proc.pid) - except ProcessLookupError: - self._close_log() - return - - log.info("Stopping %s pid=%d (pgid=%d)", self.name, proc.pid, pgid) - try: - os.killpg(pgid, signal.SIGTERM) - except ProcessLookupError: - self._close_log() - return + # Child already exited on its own (crashed, finished early, etc.) — + # nothing to signal, just fall through to the log-close in `finally`. + if proc.poll() is not None: + return - try: - proc.wait(timeout=_TERMINATE_TIMEOUT_SECONDS) - except subprocess.TimeoutExpired: - log.warning( - "%s did not exit after %ds; sending SIGKILL", - self.name, - _TERMINATE_TIMEOUT_SECONDS, - ) + # Look up the process group id so we can signal the child *and* + # every worker it forked (uvicorn --workers, nemo services). try: - os.killpg(pgid, signal.SIGKILL) + pgid = os.getpgid(proc.pid) + log.info("Stopping %s pid=%d (pgid=%d)", self.name, proc.pid, pgid) + os.killpg(pgid, signal.SIGTERM) except ProcessLookupError: - pass - proc.wait() - - self._close_log() + return - def _close_log(self) -> None: - if self._log_fh is not None: - self._log_fh.close() - self._log_fh = None + # Give the group a chance to shut down gracefully. If SIGTERM + # doesn't take effect in time, escalate to SIGKILL on the whole + # group and then wait unconditionally. + try: + proc.wait(timeout=_TERMINATE_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired: + log.warning( + "%s did not exit after %ds; sending SIGKILL", + self.name, + _TERMINATE_TIMEOUT_SECONDS, + ) + try: + os.killpg(pgid, signal.SIGKILL) + except ProcessLookupError: + pass + proc.wait() + finally: + # Always close the log handle, even if we returned early above or + # the wait/signal calls raised something unexpected. Idempotent: + # subsequent stop() calls (ex. via __exit__ after manual stop) are + # no-ops once the handle is None. + if self._log_fh is not None: + self._log_fh.close() + self._log_fh = None def __enter__(self) -> "SupervisedProcess": + """Context-manager entry: start the child and return ``self``.""" self.start() return self def __exit__(self, *exc: object) -> None: + """Context-manager exit: stop the child regardless of exception state.""" self.stop() @contextmanager def supervised_processes(specs: list[SupervisedProcess]) -> Iterator[list[SupervisedProcess]]: - """Enter every process spec in order; stop them in reverse on exit.""" + """Start every spec in order; stop them in reverse on exit. + + Backed by ``ExitStack`` so that if any spec's ``start()`` raises, every + already-started child gets ``stop()``-ed before the exception propagates. + On clean exit, children are torn down in LIFO order (i.e. NMP services + stop before the mock LLMs they depend on). + """ with ExitStack() as stack: for spec in specs: stack.enter_context(spec) yield specs -def write_pids_file(pids_file: Path, processes: list[SupervisedProcess]) -> None: - pids_file.parent.mkdir(parents=True, exist_ok=True) - with pids_file.open("w", encoding="utf-8") as f: - for p in processes: - if p.pid is not None: - f.write(f"{p.name}:{p.pid}\n") - - def wait_http(url: str, *, timeout_seconds: float, label: str, poll_interval: float = 1.0) -> None: - """Poll ``url`` until it returns 2xx or ``timeout_seconds`` elapses.""" + """Poll ``url`` until it returns < 400 or ``timeout_seconds`` elapses. + + Used as a readiness gate between starting one supervised process and the + next. ``label`` is included in the ``TimeoutError`` message so it's clear + which dependency failed to come up. Raises ``TimeoutError`` with the last + HTTP error or transport exception attached. + """ deadline = time.monotonic() + timeout_seconds last_error: Exception | None = None + while time.monotonic() < deadline: try: r = httpx.get(url, timeout=5.0) @@ -145,4 +198,5 @@ def wait_http(url: str, *, timeout_seconds: float, label: str, poll_interval: fl except httpx.HTTPError as exc: last_error = exc time.sleep(poll_interval) + raise TimeoutError(f"Timed out waiting for {label} at {url}: {last_error}") diff --git a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/report.py b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/report.py deleted file mode 100644 index 27d160b5b8..0000000000 --- a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/report.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Emit a minimal JUnit XML report for the benchmark harness. - -We deliberately use ``xml.etree`` rather than a third-party JUnit library to -avoid adding a dependency just for this one consumer. Schema is the standard -``...`` shape that GitHub Actions and most CI dashboards -render natively. -""" - -from __future__ import annotations - -import xml.etree.ElementTree as ET -from dataclasses import dataclass -from datetime import datetime, timezone -from pathlib import Path -from xml.dom import minidom - -from nemo_guardrails_plugin.benchmarks.aiperf_runner import SweepRunResult -from nemo_guardrails_plugin.benchmarks.constants import JUNIT_SUITE_NAME - - -@dataclass(frozen=True) -class JUnitCase: - name: str - classname: str - time_seconds: float - passed: bool - failure_message: str | None = None - system_out: str | None = None - - -def cases_from_sweep_results(results: list[SweepRunResult]) -> list[JUnitCase]: - """Translate AIPerf per-sweep outcomes into JUnit test cases. - - Pass criterion is just ``return_code == 0``; downstream tooling can layer - on threshold-based failures later. - """ - cases: list[JUnitCase] = [] - for r in results: - message = None if r.passed else f"aiperf exited with code {r.return_code}" - cases.append( - JUnitCase( - name=r.sweep_label, - classname=JUNIT_SUITE_NAME, - time_seconds=r.duration_seconds, - passed=r.passed, - failure_message=message, - system_out=f"output_dir={r.output_dir}", - ) - ) - return cases - - -def write_junit_report(path: Path, *, suite_name: str, cases: list[JUnitCase]) -> None: - """Render a single-suite JUnit XML file at ``path``.""" - failure_count = sum(1 for c in cases if not c.passed) - total_time = sum(c.time_seconds for c in cases) - - testsuites = ET.Element( - "testsuites", - attrib={ - "name": suite_name, - "tests": str(len(cases)), - "failures": str(failure_count), - "errors": "0", - "time": f"{total_time:.3f}", - }, - ) - testsuite = ET.SubElement( - testsuites, - "testsuite", - attrib={ - "name": suite_name, - "tests": str(len(cases)), - "failures": str(failure_count), - "errors": "0", - "time": f"{total_time:.3f}", - "timestamp": datetime.now(timezone.utc).isoformat(timespec="seconds"), - }, - ) - - for case in cases: - tc = ET.SubElement( - testsuite, - "testcase", - attrib={ - "name": case.name, - "classname": case.classname, - "time": f"{case.time_seconds:.3f}", - }, - ) - if not case.passed: - failure = ET.SubElement( - tc, - "failure", - attrib={"message": case.failure_message or "failed", "type": "BenchmarkFailure"}, - ) - failure.text = case.failure_message or "" - if case.system_out: - ET.SubElement(tc, "system-out").text = case.system_out - - path.parent.mkdir(parents=True, exist_ok=True) - rough = ET.tostring(testsuites, encoding="unicode") - pretty = minidom.parseString(rough).toprettyxml(indent=" ") - path.write_text(pretty, encoding="utf-8") diff --git a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/run.py b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/run.py index 78ea03e5b9..21ba87b8ec 100644 --- a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/run.py +++ b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/run.py @@ -12,7 +12,7 @@ 4. Start (or reuse) ``nemo services run``. 5. Wait for health, seed NMP resources via the SDK, smoke-test the VirtualModel. 6. Invoke ``python -m benchmark.aiperf run --config-file ...`` for the sweep. -7. Collect per-sweep results, emit ``report.xml``, exit non-zero on any failure. +7. Collect per-sweep results and exit non-zero on any failure. Process supervision uses session-scoped subprocesses and an ``ExitStack`` so a ``SIGTERM`` from CI cleans up forked workers (e.g. ``uvicorn --workers 4``). @@ -28,11 +28,9 @@ from contextlib import ExitStack from pathlib import Path -import httpx -import yaml from nemo_guardrails_plugin.benchmarks.aiperf_runner import ( collect_sweep_results, - rewrite_aiperf_config, + prepare_runtime_aiperf_config, run_aiperf_sweep, ) from nemo_guardrails_plugin.benchmarks.bootstrap import ( @@ -42,28 +40,23 @@ from nemo_guardrails_plugin.benchmarks.constants import ( AIPERF_SHIM_BASE_URL, IGW_CHAT_PATH, - JUNIT_SUITE_NAME, NMP_BASE_URL, NMP_HEALTH_PATH, + WORKSPACE, ) from nemo_guardrails_plugin.benchmarks.paths import ( RunPaths, build_run_paths, - default_ng_repo_root, + default_nemoguardrails_repo_root, discover_nmp_repo_root, ) from nemo_guardrails_plugin.benchmarks.processes import ( SupervisedProcess, supervised_processes, wait_http, - write_pids_file, -) -from nemo_guardrails_plugin.benchmarks.report import ( - cases_from_sweep_results, - write_junit_report, ) from nemo_guardrails_plugin.benchmarks.seeding import SeededResources, seed_benchmark -from nemo_platform import NeMoPlatform +from nemo_platform import APIStatusError, NeMoPlatform log = logging.getLogger("nemo_guardrails_plugin.benchmarks") @@ -71,7 +64,7 @@ _NMP_START_TIMEOUT_SECONDS = 180 -_REQUIRED_NG_FILES = ( +_REQUIRED_NEMOGUARDRAILS_FILES = ( Path("benchmark/aiperf/__main__.py"), Path("benchmark/aiperf/run_aiperf.py"), Path("benchmark/mock_llm_server/run_server.py"), @@ -90,22 +83,27 @@ def _configure_logging(verbose: bool) -> None: ) -def _validate_ng_repo(ng_repo_root: Path) -> None: - missing = [p for p in _REQUIRED_NG_FILES if not (ng_repo_root / p).is_file()] +def _validate_nemoguardrails_repo(nemoguardrails_repo_root: Path) -> None: + """Fail fast if the upstream checkout is missing files the harness depends on. + """ + missing = [p for p in _REQUIRED_NEMOGUARDRAILS_FILES if not (nemoguardrails_repo_root / p).is_file()] if missing: - bullet = "\n - ".join(str(ng_repo_root / p) for p in missing) - raise FileNotFoundError(f"NeMo Guardrails checkout at {ng_repo_root} is missing required files:\n - {bullet}") + bullet = "\n - ".join(str(p) for p in missing) + raise FileNotFoundError( + f"NeMo Guardrails checkout at {nemoguardrails_repo_root} is missing required files:\n - {bullet}" + ) -def _build_mock_processes(paths: RunPaths, workers: int) -> list[SupervisedProcess]: +def _build_mock_nim_processes(paths: RunPaths, workers: int) -> list[SupervisedProcess]: """Spawn ``python -m benchmark.mock_llm_server.run_server`` for both mocks. Each child is given its own log file and a ``PYTHONPATH`` pointing at the upstream checkout so its imports resolve. """ - env = {"PYTHONPATH": str(paths.ng_repo_root)} - workdir = paths.ng_repo_root / "benchmark" + env = {"PYTHONPATH": str(paths.nemoguardrails_repo_root)} + workdir = paths.nemoguardrails_repo_root / "benchmark" + # Helper to build a ``SupervisedProcess`` for one of the mock LLM servers. def spec(name: str, port: int, env_file: Path) -> SupervisedProcess: return SupervisedProcess( name=name, @@ -126,20 +124,27 @@ def spec(name: str, port: int, env_file: Path) -> SupervisedProcess: ) return [ + # Main LLM mock server spec( "mock-app-llm", 8000, - paths.ng_repo_root / "benchmark/mock_llm_server/configs/meta-llama-3.3-70b-instruct.env", + paths.nemoguardrails_repo_root / "benchmark/mock_llm_server/configs/meta-llama-3.3-70b-instruct.env", ), + # Content-safety LLM mock server spec( "mock-content-safety-llm", 8001, - paths.ng_repo_root / "benchmark/mock_llm_server/configs/nvidia-llama-3.1-nemoguard-8b-content-safety.env", + paths.nemoguardrails_repo_root / "benchmark/mock_llm_server/configs/nvidia-llama-3.1-nemoguard-8b-content-safety.env", ), ] def _build_nmp_process(paths: RunPaths) -> SupervisedProcess: + """Start ``nemo services run`` in a supervised process. + + The harness sets ``NMP_BASE_URL`` and ``NMP_DATA_DIR`` env vars so the + child process can talk to NMP over HTTP and write state to the per-run data dir. + """ return SupervisedProcess( name="nmp-services", cmd=["nemo", "services", "run"], @@ -152,8 +157,8 @@ def _build_nmp_process(paths: RunPaths) -> SupervisedProcess: def _build_aiperf_shim_process(paths: RunPaths) -> SupervisedProcess: """Run the shim that satisfies AIPerf's `/v1/models` pre-check. - Without this, AIPerf's hard-coded `urljoin(base_url, "/v1/models")` probe - would 404 against NMP and the sweep would never start. See + Without this, AIPerf's hard-coded health check against the `/v1/models` + endpoint would 404, and the sweep would never start. See `nemo_guardrails_plugin.benchmarks.shim` for details. """ return SupervisedProcess( @@ -164,51 +169,34 @@ def _build_aiperf_shim_process(paths: RunPaths) -> SupervisedProcess: ) -_SMOKE_TEST_TIMEOUT_SECONDS = 60 -_SMOKE_TEST_POLL_INTERVAL_SECONDS = 1.0 - - -def _smoke_test(seeded: SeededResources) -> None: - """POST one chat-completion through the IGW VirtualModel before sweeping. - - Catches misconfigured guardrails / middleware wiring early so a sweep - failure isn't ambiguous between "harness broken" and "benchmark regressed". - We hit the IGW URL directly with ``httpx`` rather than going through the - NMP SDK so the smoke test exercises the same path AIPerf will. - - IGW's VirtualModel cache refreshes asynchronously after creation, so the - first few requests can 404 even though seeding succeeded. We retry on - 404/503 for up to ~60s before failing. +def _smoke_test(client: NeMoPlatform, seeded: SeededResources) -> None: + """Verify the VirtualModel is reachable and returns a chat completion, + before running the AIPerf sweep. """ - url = f"{NMP_BASE_URL}{IGW_CHAT_PATH}" payload = { "model": seeded.vm_ref, - "messages": [{"role": "user", "content": "Hello, what can you do?"}], - "max_tokens": 64, - "stream": False, + "messages": [{"role": "user", "content": "Hello"}], + "max_tokens": 16, } - deadline = time.monotonic() + _SMOKE_TEST_TIMEOUT_SECONDS - last_response: httpx.Response | None = None - while time.monotonic() < deadline: - last_response = httpx.post(url, json=payload, timeout=60.0) - if last_response.status_code < 400: - body = last_response.json() - if not body.get("choices"): - raise RuntimeError(f"Smoke test response missing choices: {body}") - return - if last_response.status_code in (404, 503): - log.info( - "Smoke test got HTTP %d; waiting for IGW cache refresh", - last_response.status_code, + last_error: str = "no attempts made" + + for attempt in range(60): + try: + body = client.inference.gateway.openai.post( + "v1/chat/completions", + workspace=WORKSPACE, + body=payload, ) - time.sleep(_SMOKE_TEST_POLL_INTERVAL_SECONDS) - continue - break + if body.get("choices"): + return + last_error = f"response missing choices: {body}" + except APIStatusError as exc: + last_error = f"HTTP {exc.status_code}: {str(exc)[:500]}" + log.info("Smoke test attempt %d: %s; retrying", attempt + 1, last_error) + time.sleep(1.0) - code = last_response.status_code if last_response is not None else "no response" - text = last_response.text[:500] if last_response is not None else "" - raise RuntimeError(f"Smoke test failed with HTTP {code}: {text}") + raise RuntimeError(f"Smoke test failed after 60 attempts: {last_error}") def parse_args(argv: list[str] | None = None) -> argparse.Namespace: @@ -222,7 +210,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: default=Path( os.environ.get( "NEMO_GUARDRAILS_REPO_ROOT", - str(default_ng_repo_root(discover_nmp_repo_root())), + str(default_nemoguardrails_repo_root(discover_nmp_repo_root())), ) ), help="Path to a local NeMo Guardrails checkout (default: $NEMO_GUARDRAILS_REPO_ROOT or ../NeMo-Guardrails).", @@ -245,12 +233,6 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: default=int(os.environ.get("NMP_BENCHMARK_MOCK_WORKERS", "4")), help="uvicorn worker count for each mock LLM server.", ) - parser.add_argument( - "--junit-path", - type=Path, - default=None, - help="Path to write report.xml (default: /report.xml for CI compatibility).", - ) parser.add_argument( "--run-id", default=None, @@ -264,29 +246,28 @@ def main(argv: list[str] | None = None) -> int: args = parse_args(argv) _configure_logging(args.verbose) - nmp_repo_root = discover_nmp_repo_root() - ng_repo_root = args.nemo_guardrails_repo_root.resolve() - _validate_ng_repo(ng_repo_root) + # Validate the NeMo Guardrails checkout before attempting to run the benchmark. + nemoguardrails_repo_root = args.nemo_guardrails_repo_root.resolve() + _validate_nemoguardrails_repo(nemoguardrails_repo_root) + log.info("Validated NeMo Guardrails local checkout at: %s", nemoguardrails_repo_root) + + # Build the directory structure that will contain the benchmark results. + nmp_repo_root = discover_nmp_repo_root() paths = build_run_paths( nmp_repo_root=nmp_repo_root, - ng_repo_root=ng_repo_root, - junit_dir=args.junit_path.parent if args.junit_path else None, + nemoguardrails_repo_root=nemoguardrails_repo_root, run_id=args.run_id, ) - if args.junit_path: - paths = RunPaths(**{**paths.__dict__, "junit_path": args.junit_path.resolve()}) paths.ensure_directories() - log.info("Run directory: %s", paths.run_dir) - log.info("NeMo Guardrails repo: %s", paths.ng_repo_root) + log.info("Created directory for benchmark results at: %s", paths.run_dir) - rewrite_aiperf_config( - template=paths.config_template, - output=paths.runtime_config, - output_base_dir=paths.aiperf_output_dir, + sweep_config = prepare_runtime_aiperf_config( + template_path=paths.config_template, + runtime_config_path=paths.runtime_config, + aiperf_output_dir=paths.aiperf_output_dir, ) - sweep_config = yaml.safe_load(paths.runtime_config.read_text(encoding="utf-8")) log.info( "AIPerf sweep: concurrency=%s, duration=%ss", sweep_config.get("sweeps", {}).get("concurrency"), @@ -294,25 +275,24 @@ def main(argv: list[str] | None = None) -> int: ) # Ensure the dedicated aiperf venv exists *before* we start any supervised - # processes. A first-time install can take ~30s and we'd rather pay that - # cost up front than during the NMP-services startup race. + # processes. aiperf_python = ensure_aiperf_venv(paths.aiperf_venv_dir) log.info("Using aiperf python at %s", aiperf_python) - processes_to_start: list[SupervisedProcess] = _build_mock_processes(paths, args.mock_workers) + processes = _build_mock_nim_processes(paths, args.mock_workers) if not args.reuse_services: - processes_to_start.append(_build_nmp_process(paths)) - # The AIPerf shim is harness-local; we always start it (it talks to NMP - # over HTTP, so it doesn't care whether nemo services run is supervised - # by us or already running externally). - processes_to_start.append(_build_aiperf_shim_process(paths)) + processes.append(_build_nmp_process(paths)) + + processes.append(_build_aiperf_shim_process(paths)) + # Start the processes and wait for them to be ready before seeding NMP. with ExitStack() as stack: - started = stack.enter_context(supervised_processes(processes_to_start)) - write_pids_file(paths.pids_file, started) + stack.enter_context(supervised_processes(processes)) if args.keep_running: # Pop the cleanup so processes outlive this script. stack.pop_all() + + log.info("Waiting for all services to be ready...") wait_http( "http://localhost:8000/health", @@ -335,15 +315,17 @@ def main(argv: list[str] | None = None) -> int: label="AIPerf shim", ) + log.info(f"All services are ready. Seeding benchmark resources in workspace {WORKSPACE}...") + client = NeMoPlatform(base_url=NMP_BASE_URL) seeded = seed_benchmark( client, - ng_repo_root=paths.ng_repo_root, + nemoguardrails_repo_root=paths.nemoguardrails_repo_root, generated_dir=paths.generated_dir, ) - log.info("Smoke testing %s", seeded.vm_ref) - _smoke_test(seeded) + log.info("Waiting for VirtualModel %s to be ready...", seeded.vm_ref) + _smoke_test(client, seeded) log.info( "Starting AIPerf sweep against %s -> shim -> %s%s", @@ -352,37 +334,35 @@ def main(argv: list[str] | None = None) -> int: IGW_CHAT_PATH, ) aiperf_exit = run_aiperf_sweep( - ng_repo_root=paths.ng_repo_root, + nemoguardrails_repo_root=paths.nemoguardrails_repo_root, runtime_config=paths.runtime_config, log_path=paths.log_dir / "aiperf.log", python_executable=str(aiperf_python), - extra_env=env_with_venv_on_path(paths.aiperf_venv_dir, {}), + extra_env=env_with_venv_on_path(paths.aiperf_venv_dir), ) sweep_results = collect_sweep_results(paths.aiperf_output_dir) - cases = cases_from_sweep_results(sweep_results) - if not cases: - # AIPerf failed before producing per-sweep dirs; emit a synthetic failure - # so CI surfaces something actionable instead of an empty report. - from nemo_guardrails_plugin.benchmarks.report import JUnitCase - - cases = [ - JUnitCase( - name="aiperf", - classname=JUNIT_SUITE_NAME, - time_seconds=0.0, - passed=aiperf_exit == 0, - failure_message=(f"aiperf exited with code {aiperf_exit} and produced no per-sweep results"), - system_out=f"aiperf_output_dir={paths.aiperf_output_dir}", - ) - ] - write_junit_report(paths.junit_path, suite_name=JUNIT_SUITE_NAME, cases=cases) - log.info("Wrote JUnit report to %s", paths.junit_path) + failures = sum(1 for r in sweep_results if not r.passed) + + if not sweep_results: + # AIPerf exited before producing any per-sweep dirs. Surface that + # explicitly so the log isn't ambiguous about why we're failing. + log.error( + "aiperf exited with code %d and produced no per-sweep results in %s", + aiperf_exit, + paths.aiperf_output_dir, + ) + else: + log.info( + "Sweep summary: %d run(s), %d failure(s); per-sweep outputs under %s", + len(sweep_results), + failures, + paths.aiperf_output_dir, + ) - failures = sum(1 for c in cases if not c.passed) - log.info("Sweep summary: %d run(s), %d failure(s)", len(cases), failures) - if failures or aiperf_exit != 0: + if failures or aiperf_exit != 0 or not sweep_results: return 1 + return 0 diff --git a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/seeding.py b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/seeding.py index 555a7aaf51..6c05e98c05 100644 --- a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/seeding.py +++ b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/seeding.py @@ -65,7 +65,7 @@ def guardrail_config_ref(self) -> str: def seed_benchmark( client: NeMoPlatform, *, - ng_repo_root: Path, + nemoguardrails_repo_root: Path, generated_dir: Path, provider_wait_timeout: float = _PROVIDER_WAIT_TIMEOUT_SECONDS, ) -> SeededResources: @@ -123,9 +123,9 @@ def seed_benchmark( _dump_model(generated_dir / "app_provider.json", app_provider) _dump_model(generated_dir / "content_safety_provider.json", cs_provider) - log.info("Building GuardrailConfig payload from %s", ng_repo_root) + log.info("Building GuardrailConfig payload from %s", nemoguardrails_repo_root) config_data = build_guardrail_config_data( - source_config_dir=ng_repo_root / "examples" / "configs" / "content_safety_local", + source_config_dir=nemoguardrails_repo_root / "examples" / "configs" / "content_safety_local", content_safety_model_entity=cs_entity, ) # Persist the same payload shape the old shell harness produced for debuggability. diff --git a/plugins/nemo-guardrails/tests/unit/benchmarks/test_aiperf_runner.py b/plugins/nemo-guardrails/tests/unit/benchmarks/test_aiperf_runner.py index 7b7df9d9f1..1c81252934 100644 --- a/plugins/nemo-guardrails/tests/unit/benchmarks/test_aiperf_runner.py +++ b/plugins/nemo-guardrails/tests/unit/benchmarks/test_aiperf_runner.py @@ -9,7 +9,7 @@ from nemo_guardrails_plugin.benchmarks.aiperf_runner import ( SweepRunResult, collect_sweep_results, - rewrite_aiperf_config, + prepare_runtime_aiperf_config, ) @@ -28,37 +28,41 @@ def _write_template(path: Path) -> None: ) -class TestRewriteAiperfConfig: +class TestPrepareRuntimeAiperfConfig: def test_overrides_output_base_dir(self, tmp_path: Path) -> None: - template = tmp_path / "template.yaml" - _write_template(template) - output = tmp_path / "out" / "runtime.yaml" - target_dir = tmp_path / "results" - - config = rewrite_aiperf_config(template=template, output=output, output_base_dir=target_dir) + template_path = tmp_path / "template.yaml" + _write_template(template_path) + runtime_config_path = tmp_path / "out" / "runtime.yaml" + aiperf_output_dir = tmp_path / "results" + + config = prepare_runtime_aiperf_config( + template_path=template_path, + runtime_config_path=runtime_config_path, + aiperf_output_dir=aiperf_output_dir, + ) - assert config["output_base_dir"] == str(target_dir) - written = yaml.safe_load(output.read_text(encoding="utf-8")) - assert written["output_base_dir"] == str(target_dir) + assert config["output_base_dir"] == str(aiperf_output_dir) + written = yaml.safe_load(runtime_config_path.read_text(encoding="utf-8")) + assert written["output_base_dir"] == str(aiperf_output_dir) assert written["base_config"]["model"] == "benchmark/guardrails-vm" assert written["sweeps"]["concurrency"] == [1, 2, 4] def test_missing_template_raises(self, tmp_path: Path) -> None: with pytest.raises(FileNotFoundError): - rewrite_aiperf_config( - template=tmp_path / "absent.yaml", - output=tmp_path / "out.yaml", - output_base_dir=tmp_path / "results", + prepare_runtime_aiperf_config( + template_path=tmp_path / "absent.yaml", + runtime_config_path=tmp_path / "out.yaml", + aiperf_output_dir=tmp_path / "results", ) def test_non_mapping_template_raises(self, tmp_path: Path) -> None: - template = tmp_path / "bad.yaml" - template.write_text("- just\n- a\n- list\n", encoding="utf-8") + template_path = tmp_path / "bad.yaml" + template_path.write_text("- just\n- a\n- list\n", encoding="utf-8") with pytest.raises(ValueError, match="mapping"): - rewrite_aiperf_config( - template=template, - output=tmp_path / "out.yaml", - output_base_dir=tmp_path / "results", + prepare_runtime_aiperf_config( + template_path=template_path, + runtime_config_path=tmp_path / "out.yaml", + aiperf_output_dir=tmp_path / "results", ) diff --git a/plugins/nemo-guardrails/tests/unit/benchmarks/test_paths.py b/plugins/nemo-guardrails/tests/unit/benchmarks/test_paths.py index 1603616b99..53b2b42e36 100644 --- a/plugins/nemo-guardrails/tests/unit/benchmarks/test_paths.py +++ b/plugins/nemo-guardrails/tests/unit/benchmarks/test_paths.py @@ -6,7 +6,7 @@ import pytest from nemo_guardrails_plugin.benchmarks.paths import ( build_run_paths, - default_ng_repo_root, + default_nemoguardrails_repo_root, discover_nmp_repo_root, ) @@ -40,7 +40,7 @@ def test_sibling_of_nmp_root(self, tmp_path: Path) -> None: nmp = _make_fake_repo(tmp_path / "nemo-platform") ng = tmp_path / "NeMo-Guardrails" - assert default_ng_repo_root(nmp) == ng.resolve() + assert default_nemoguardrails_repo_root(nmp) == ng.resolve() class TestBuildRunPaths: @@ -49,37 +49,25 @@ def test_layout_matches_documented_structure(self, tmp_path: Path) -> None: ng = tmp_path / "NeMo-Guardrails" ng.mkdir() - paths = build_run_paths(nmp_repo_root=nmp, ng_repo_root=ng, run_id="20260527_120000") + paths = build_run_paths(nmp_repo_root=nmp, nemoguardrails_repo_root=ng, run_id="20260527_120000") assert paths.run_dir == nmp / "plugins/nemo-guardrails/benchmarks/artifacts/runs/20260527_120000" assert paths.log_dir == paths.run_dir / "logs" assert paths.generated_dir == paths.run_dir / "generated" assert paths.aiperf_output_dir == paths.run_dir / "aiperf_results" - assert paths.pids_file == paths.run_dir / "pids.txt" assert paths.nmp_data_dir == nmp / "plugins/nemo-guardrails/benchmarks/artifacts/nmp-data" assert ( paths.config_template == nmp / "plugins/nemo-guardrails/benchmarks/configs/nmp_igw_guardrails_sweep_concurrency.yaml" ) assert paths.runtime_config == paths.generated_dir / "nmp_igw_guardrails_sweep_concurrency.yaml" - assert paths.junit_path == nmp / "report.xml" - - def test_junit_dir_override(self, tmp_path: Path) -> None: - nmp = _make_fake_repo(tmp_path / "nemo-platform") - ng = tmp_path / "NeMo-Guardrails" - ng.mkdir() - junit_dir = tmp_path / "ci-artifacts" - - paths = build_run_paths(nmp_repo_root=nmp, ng_repo_root=ng, junit_dir=junit_dir, run_id="x") - - assert paths.junit_path == junit_dir / "report.xml" def test_ensure_directories_creates_required_dirs(self, tmp_path: Path) -> None: nmp = _make_fake_repo(tmp_path / "nemo-platform") ng = tmp_path / "NeMo-Guardrails" ng.mkdir() - paths = build_run_paths(nmp_repo_root=nmp, ng_repo_root=ng, run_id="x") + paths = build_run_paths(nmp_repo_root=nmp, nemoguardrails_repo_root=ng, run_id="x") paths.ensure_directories() assert paths.log_dir.is_dir() @@ -92,7 +80,7 @@ def test_run_id_uses_timestamp_when_not_given(self, tmp_path: Path) -> None: ng = tmp_path / "NeMo-Guardrails" ng.mkdir() - paths = build_run_paths(nmp_repo_root=nmp, ng_repo_root=ng) + paths = build_run_paths(nmp_repo_root=nmp, nemoguardrails_repo_root=ng) # Timestamp format: YYYYmmdd_HHMMSS = 15 chars including underscore. assert len(paths.run_id) == 15 assert paths.run_id[8] == "_" @@ -103,6 +91,6 @@ def test_aiperf_venv_dir_is_outside_run_dir(self, tmp_path: Path) -> None: ng = tmp_path / "NeMo-Guardrails" ng.mkdir() - paths = build_run_paths(nmp_repo_root=nmp, ng_repo_root=ng, run_id="x") + paths = build_run_paths(nmp_repo_root=nmp, nemoguardrails_repo_root=ng, run_id="x") assert paths.run_dir not in paths.aiperf_venv_dir.parents assert paths.aiperf_venv_dir.parent.name == "venvs" diff --git a/plugins/nemo-guardrails/tests/unit/benchmarks/test_report.py b/plugins/nemo-guardrails/tests/unit/benchmarks/test_report.py deleted file mode 100644 index b9c36306bd..0000000000 --- a/plugins/nemo-guardrails/tests/unit/benchmarks/test_report.py +++ /dev/null @@ -1,90 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -import xml.etree.ElementTree as ET -from pathlib import Path - -from nemo_guardrails_plugin.benchmarks.aiperf_runner import SweepRunResult -from nemo_guardrails_plugin.benchmarks.report import ( - JUnitCase, - cases_from_sweep_results, - write_junit_report, -) - - -def _result(label: str, *, returncode: int, duration: float = 60.0) -> SweepRunResult: - return SweepRunResult( - sweep_label=label, - output_dir=Path("/tmp") / label, - return_code=returncode, - duration_seconds=duration, - metadata_path=None, - process_result_path=None, - ) - - -class TestCasesFromSweepResults: - def test_passing_case_has_no_failure_message(self) -> None: - cases = cases_from_sweep_results([_result("concurrency1", returncode=0)]) - assert len(cases) == 1 - assert cases[0].passed - assert cases[0].failure_message is None - assert cases[0].time_seconds == 60.0 - - def test_failing_case_includes_exit_code(self) -> None: - cases = cases_from_sweep_results([_result("concurrency1", returncode=3)]) - assert not cases[0].passed - assert "code 3" in (cases[0].failure_message or "") - - -class TestWriteJunitReport: - def test_basic_report_structure(self, tmp_path: Path) -> None: - cases = [ - JUnitCase(name="concurrency1", classname="suite", time_seconds=70.0, passed=True), - JUnitCase( - name="concurrency2", - classname="suite", - time_seconds=72.5, - passed=False, - failure_message="boom", - system_out="output_dir=/tmp/concurrency2", - ), - ] - - path = tmp_path / "report.xml" - write_junit_report(path, suite_name="suite", cases=cases) - - tree = ET.parse(path) - root = tree.getroot() - assert root.tag == "testsuites" - assert root.attrib["tests"] == "2" - assert root.attrib["failures"] == "1" - - testsuite = root.find("testsuite") - assert testsuite is not None - assert testsuite.attrib["name"] == "suite" - assert testsuite.attrib["failures"] == "1" - - testcases = testsuite.findall("testcase") - assert [tc.attrib["name"] for tc in testcases] == ["concurrency1", "concurrency2"] - - passing, failing = testcases - assert passing.find("failure") is None - failure = failing.find("failure") - assert failure is not None - assert failure.attrib["message"] == "boom" - system_out = failing.find("system-out") - assert system_out is not None - assert system_out.text == "output_dir=/tmp/concurrency2" - - def test_writes_pretty_xml(self, tmp_path: Path) -> None: - path = tmp_path / "report.xml" - write_junit_report( - path, - suite_name="suite", - cases=[JUnitCase(name="x", classname="suite", time_seconds=0.0, passed=True)], - ) - - text = path.read_text(encoding="utf-8") - assert text.startswith(" None: with pytest.raises(TimeoutError, match="served model"): seed_benchmark( client, - ng_repo_root=ng_root, + nemoguardrails_repo_root=ng_root, generated_dir=tmp_path / "generated", provider_wait_timeout=0.1, ) From 4e3f7cf90be28c58030da7275232f114ff1eebb3 Mon Sep 17 00:00:00 2001 From: Jash Gulabrai Date: Thu, 28 May 2026 11:49:54 -0400 Subject: [PATCH 05/10] Make CI step manual Signed-off-by: Jash Gulabrai --- .github/workflows/ci.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d4dd969571..0ba0738b0b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -382,7 +382,8 @@ jobs: esac benchmark-guardrails: - name: NeMo Guardrails benchmark + name: Guardrails plugin benchmark + if: github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest timeout-minutes: 30 steps: From dd043c771c6373c42263780933078e6007dc62f4 Mon Sep 17 00:00:00 2001 From: Jash Gulabrai Date: Thu, 28 May 2026 12:01:04 -0400 Subject: [PATCH 06/10] Fix lint Signed-off-by: Jash Gulabrai --- .../src/nemo_guardrails_plugin/benchmarks/run.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/run.py b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/run.py index 21ba87b8ec..64f676abee 100644 --- a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/run.py +++ b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/run.py @@ -84,8 +84,7 @@ def _configure_logging(verbose: bool) -> None: def _validate_nemoguardrails_repo(nemoguardrails_repo_root: Path) -> None: - """Fail fast if the upstream checkout is missing files the harness depends on. - """ + """Fail fast if the upstream checkout is missing files the harness depends on.""" missing = [p for p in _REQUIRED_NEMOGUARDRAILS_FILES if not (nemoguardrails_repo_root / p).is_file()] if missing: bullet = "\n - ".join(str(p) for p in missing) @@ -134,7 +133,8 @@ def spec(name: str, port: int, env_file: Path) -> SupervisedProcess: spec( "mock-content-safety-llm", 8001, - paths.nemoguardrails_repo_root / "benchmark/mock_llm_server/configs/nvidia-llama-3.1-nemoguard-8b-content-safety.env", + paths.nemoguardrails_repo_root + / "benchmark/mock_llm_server/configs/nvidia-llama-3.1-nemoguard-8b-content-safety.env", ), ] @@ -291,7 +291,7 @@ def main(argv: list[str] | None = None) -> int: if args.keep_running: # Pop the cleanup so processes outlive this script. stack.pop_all() - + log.info("Waiting for all services to be ready...") wait_http( From d5e9376c8152b104cc6870565f04850c601d825e Mon Sep 17 00:00:00 2001 From: Jash Gulabrai Date: Thu, 28 May 2026 16:07:06 -0400 Subject: [PATCH 07/10] Address CodeRabbit Signed-off-by: Jash Gulabrai --- .../benchmarks/aiperf_runner.py | 2 +- .../benchmarks/processes.py | 23 +++++++++++-------- .../benchmarks/seeding.py | 9 ++++++-- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/aiperf_runner.py b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/aiperf_runner.py index c14484e694..11e567e590 100644 --- a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/aiperf_runner.py +++ b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/aiperf_runner.py @@ -77,7 +77,7 @@ def prepare_runtime_aiperf_config( config = yaml.safe_load(template_path.read_text(encoding="utf-8")) if not isinstance(config, dict): - raise ValueError(f"Failed to parse AIPerf template {template_path}. Ensure it is valid YAML.") + raise ValueError(f"Expected a YAML mapping at {template_path}, got {type(config).__name__}") # Point AIPerf's output_base_dir at this run's directory so its results # nest under our per-run artifacts tree. diff --git a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/processes.py b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/processes.py index 5cf69e3a15..e31fd59f36 100644 --- a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/processes.py +++ b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/processes.py @@ -91,15 +91,20 @@ def start(self) -> None: self._log_fh = self.log_path.open("wb") log.info("Starting %s; log=%s", self.name, self.log_path) - # Spawn the child process. - self._proc = subprocess.Popen( - self.cmd, - cwd=str(self.cwd), # Working directory - env={**os.environ, **(self.env or {})}, # Extra environment variables - stdout=self._log_fh, # Redirect stdout to the log file - stderr=subprocess.STDOUT, # Redirect stderr to stdout - start_new_session=True, # Create a new session for the child - ) + try: + self._proc = subprocess.Popen( + self.cmd, + cwd=str(self.cwd), + env={**os.environ, **(self.env or {})}, + stdout=self._log_fh, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + except BaseException: + # Popen failed; close the log handle so we don't leak it. + self._log_fh.close() + self._log_fh = None + raise def stop(self) -> None: """Terminate the child's process group and close the log file. diff --git a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/seeding.py b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/seeding.py index 6c05e98c05..96cd152f0f 100644 --- a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/seeding.py +++ b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/seeding.py @@ -243,8 +243,13 @@ def build_guardrail_config_data( if not prompts_yaml.is_file(): raise FileNotFoundError(f"Expected guardrails prompts at {prompts_yaml}") - config: dict[str, Any] = yaml.safe_load(config_yaml.read_text(encoding="utf-8")) - prompts: dict[str, Any] = yaml.safe_load(prompts_yaml.read_text(encoding="utf-8")) + config = yaml.safe_load(config_yaml.read_text(encoding="utf-8")) + if not isinstance(config, dict): + raise ValueError(f"Expected a YAML mapping at {config_yaml}, got {type(config).__name__}") + + prompts = yaml.safe_load(prompts_yaml.read_text(encoding="utf-8")) or {} + if not isinstance(prompts, dict): + raise ValueError(f"Expected a YAML mapping at {prompts_yaml}, got {type(prompts).__name__}") config["models"] = [ { From 1b2e468e6c3ca768a7734d499c857a48d8ce8502 Mon Sep 17 00:00:00 2001 From: Jash Gulabrai Date: Thu, 28 May 2026 16:10:36 -0400 Subject: [PATCH 08/10] Fix module comment Signed-off-by: Jash Gulabrai --- .../src/nemo_guardrails_plugin/benchmarks/run.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/run.py b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/run.py index 64f676abee..7fe5832e5a 100644 --- a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/run.py +++ b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/run.py @@ -3,8 +3,7 @@ """Top-level entry point for the nemo-guardrails IGW benchmark harness. -Replaces the previous ``run_igw_guardrails_benchmark.sh`` shell flow with a -single Python orchestrator. Phases: +The harness orchestrates the entire benchmark run by running the following steps: 1. Resolve paths and validate the upstream NeMo Guardrails checkout. 2. Write a per-run AIPerf config under ``runs//generated/``. From 40594adf569e1dfda5d4c926ccd4ff5b1ab1936f Mon Sep 17 00:00:00 2001 From: Jash Gulabrai Date: Tue, 2 Jun 2026 14:50:58 -0400 Subject: [PATCH 09/10] Address comments and update ReadME Signed-off-by: Jash Gulabrai --- plugins/nemo-guardrails/benchmarks/README.md | 54 +++++++++++++++++++ .../benchmarks/aiperf_runner.py | 20 ++++--- .../benchmarks/bootstrap.py | 20 ++++--- .../benchmarks/processes.py | 24 +++++++-- .../nemo_guardrails_plugin/benchmarks/run.py | 54 ++++++++----------- 5 files changed, 122 insertions(+), 50 deletions(-) diff --git a/plugins/nemo-guardrails/benchmarks/README.md b/plugins/nemo-guardrails/benchmarks/README.md index bc47ad43bc..4a923f9833 100644 --- a/plugins/nemo-guardrails/benchmarks/README.md +++ b/plugins/nemo-guardrails/benchmarks/README.md @@ -79,6 +79,48 @@ The default sweep runs concurrency levels: With the default 60-second benchmark duration, expect the benchmark to run for ~10 minutes after service bootstrap. +### Monitoring progress + +After bootstrap and seeding, the terminal prints `Running aiperf sweep: ...` while +AIPerf runs. AIPerf stdout/stderr is redirected to `logs/aiperf.log`, not the +harness terminal. When the run finishes successfully, the last harness line looks +like: + +```text +Sweep summary: 7 run(s), 0 failure(s); per-sweep outputs under ... +``` + +**Tail the sweep log** (replace `` with your `--run-id` or timestamp +directory name): + +```bash +tail -f plugins/nemo-guardrails/benchmarks/artifacts/runs//logs/aiperf.log +``` + +Look for lines like `Run 3/7` and `Run 3 completed successfully`. + +**Watch completed sweep directories**: + +```bash +ls plugins/nemo-guardrails/benchmarks/artifacts/runs//aiperf_results/*/*/ +``` + +Each finished level appears as `concurrency1/`, `concurrency2/`, etc., with +`process_result.json` and `profile_export_aiperf.csv` inside. A directory with +an empty `profile_export.jsonl` is usually the sweep currently in progress. + +**Confirm the process is still running**: + +```bash +pgrep -fl "benchmark.aiperf" +``` + +**Normal log noise during bootstrap**: + +- With the `--verbose` flag, `Connection refused` on `:8080` for up to ~1–3 minutes while `nemo services run` starts. +- `409 Conflict` during seeding when resources from a prior run already exist. +- One failed smoke-test attempt (`404` on the VirtualModel) before the IGW route propagates. + ## What the harness starts - The upstream benchmark **mock app LLM** on `http://localhost:8000`, @@ -176,5 +218,17 @@ a schema change), delete that directory for a fully fresh run: rm -rf plugins/nemo-guardrails/benchmarks/artifacts/nmp-data ``` +To remove outputs from a specific run (logs, generated configs, AIPerf results): + +```bash +rm -rf plugins/nemo-guardrails/benchmarks/artifacts/runs/ +``` + +To clear all run outputs: + +```bash +rm -rf plugins/nemo-guardrails/benchmarks/artifacts/runs/* +``` + In CI this is automatic — every job gets a fresh runner, so `nmp-data` does not exist. diff --git a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/aiperf_runner.py b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/aiperf_runner.py index 11e567e590..49360a9498 100644 --- a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/aiperf_runner.py +++ b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/aiperf_runner.py @@ -12,7 +12,6 @@ import json import logging -import os import subprocess import sys from dataclasses import dataclass @@ -21,6 +20,8 @@ import yaml +from nemo_guardrails_plugin.benchmarks.bootstrap import build_env + log = logging.getLogger(__name__) @@ -94,6 +95,7 @@ def run_aiperf_sweep( runtime_config: Path, log_path: Path, python_executable: str | None = None, + venv_bin_path: Path | str | None = None, extra_env: dict[str, str] | None = None, ) -> int: """Run ``python -m benchmark.aiperf --config-file ...`` and tee output. @@ -102,9 +104,9 @@ def run_aiperf_sweep( non-zero code as a sweep-level failure or as a fail-fast. ``python_executable`` should point at the python in the dedicated aiperf - venv (see :mod:`nemo_guardrails_plugin.benchmarks.bootstrap`). ``extra_env`` - is used to prepend that venv's ``bin/`` to ``PATH`` so the ``aiperf`` CLI - is resolvable when the wrapper shells out to it. + venv (see :mod:`nemo_guardrails_plugin.benchmarks.bootstrap`). ``venv_bin_path`` + prepends that venv's ``bin/`` to ``PATH`` so the ``aiperf`` CLI is resolvable + when the wrapper shells out to it. Note: AIPerf's built-in pre-flight check does a GET on ``urljoin(config.base_config.url, "/v1/models")`` with no override @@ -123,9 +125,13 @@ def run_aiperf_sweep( str(runtime_config), ] - env = {**os.environ, "PYTHONPATH": str(nemoguardrails_repo_root)} - if extra_env: - env.update(extra_env) + env = build_env( + venv_bin_path=venv_bin_path, + extra_env={ + "PYTHONPATH": str(nemoguardrails_repo_root), + **(extra_env or {}), + }, + ) log_path.parent.mkdir(parents=True, exist_ok=True) diff --git a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/bootstrap.py b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/bootstrap.py index 04fc9bb0a4..761a5f56c3 100644 --- a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/bootstrap.py +++ b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/bootstrap.py @@ -56,13 +56,19 @@ def ensure_aiperf_venv(venv_dir: Path) -> Path: return python_bin -def env_with_venv_on_path(venv_dir: Path) -> dict[str, str]: - """Return ``os.environ`` with the venv's ``bin/`` prepended to ``PATH``. - - The upstream ``python -m benchmark.aiperf`` wrapper shells out to a literal - ``aiperf`` binary via ``subprocess.run``, so the venv's bin dir must be - discoverable on ``PATH`` before whatever was inherited from the parent. +def build_env( + *, + venv_bin_path: Path | str | None = None, + extra_env: dict[str, str] | None = None, +) -> dict[str, str]: + """Return a child-process environment based on ``os.environ``. + + Optionally prepends ``venv_bin_path`` to ``PATH`` and overlays ``extra_env``. """ env = dict(os.environ) - env["PATH"] = f"{venv_dir / 'bin'}{os.pathsep}{env.get('PATH', '')}" + if venv_bin_path: + bin_path = str(venv_bin_path) + env["PATH"] = f"{bin_path}{os.pathsep}{env.get('PATH', '')}" + if extra_env: + env.update(extra_env) return env diff --git a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/processes.py b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/processes.py index e31fd59f36..2cfa7672db 100644 --- a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/processes.py +++ b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/processes.py @@ -15,10 +15,11 @@ * merges stdout/stderr into a per-process log file under ``run_dir/logs/``, * escalates SIGTERM → SIGKILL on shutdown if the child doesn't exit in time. -``supervised_processes`` starts the children in order and guarantees every -already-started child is stopped if a later one fails to come up. ``wait_http`` -is the readiness probe the caller uses between starts, so we only move on to -the next child once the previous one is actually serving requests. +``supervised_processes`` starts the children in order, polls each spec's +``health_url`` before moving on, and guarantees every already-started child is +stopped if a later one fails to come up. ``wait_http`` implements the readiness +probe used by ``supervised_processes`` and for externally managed dependencies +(ex. ``--reuse-services``). """ from __future__ import annotations @@ -35,6 +36,8 @@ import httpx +from nemo_guardrails_plugin.benchmarks.bootstrap import build_env + log = logging.getLogger(__name__) # How long to wait for a child (and its process group) to exit after SIGTERM @@ -67,6 +70,11 @@ class SupervisedProcess: # Extra env vars to overlay on top of the parent's ``os.environ`` (e.g. # ``PYTHONPATH``, ``NMP_DATA_DIR``). ``None`` means "inherit unchanged". env: dict[str, str] | None = None + # Readiness probe polled after ``start()`` when this process is entered via + # ``supervised_processes``. ``None`` skips the probe (e.g. when reusing an + # externally managed dependency). + health_url: str | None = None + health_timeout_seconds: float = 60.0 # The live ``Popen`` handle, populated by ``start()`` and consulted by # ``stop()``. Excluded from ``__init__`` and ``repr`` since it's pure state. _proc: subprocess.Popen[bytes] | None = field(default=None, init=False, repr=False) @@ -95,7 +103,7 @@ def start(self) -> None: self._proc = subprocess.Popen( self.cmd, cwd=str(self.cwd), - env={**os.environ, **(self.env or {})}, + env=build_env(extra_env=self.env), stdout=self._log_fh, stderr=subprocess.STDOUT, start_new_session=True, @@ -180,6 +188,12 @@ def supervised_processes(specs: list[SupervisedProcess]) -> Iterator[list[Superv with ExitStack() as stack: for spec in specs: stack.enter_context(spec) + if spec.health_url is not None: + wait_http( + spec.health_url, + timeout_seconds=spec.health_timeout_seconds, + label=spec.name, + ) yield specs diff --git a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/run.py b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/run.py index 7fe5832e5a..47114e0195 100644 --- a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/run.py +++ b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/run.py @@ -9,7 +9,7 @@ 2. Write a per-run AIPerf config under ``runs//generated/``. 3. Start the two mock LLM servers from ``${NEMO_GUARDRAILS_REPO_ROOT}/benchmark``. 4. Start (or reuse) ``nemo services run``. -5. Wait for health, seed NMP resources via the SDK, smoke-test the VirtualModel. +5. Wait for per-process health probes, seed NMP resources via the SDK, smoke-test the VirtualModel. 6. Invoke ``python -m benchmark.aiperf run --config-file ...`` for the sweep. 7. Collect per-sweep results and exit non-zero on any failure. @@ -32,12 +32,11 @@ prepare_runtime_aiperf_config, run_aiperf_sweep, ) -from nemo_guardrails_plugin.benchmarks.bootstrap import ( - ensure_aiperf_venv, - env_with_venv_on_path, -) +from nemo_guardrails_plugin.benchmarks.bootstrap import ensure_aiperf_venv from nemo_guardrails_plugin.benchmarks.constants import ( AIPERF_SHIM_BASE_URL, + APP_PROVIDER_URL, + CS_PROVIDER_URL, IGW_CHAT_PATH, NMP_BASE_URL, NMP_HEALTH_PATH, @@ -59,8 +58,8 @@ log = logging.getLogger("nemo_guardrails_plugin.benchmarks") -_MOCK_START_TIMEOUT_SECONDS = 60 -_NMP_START_TIMEOUT_SECONDS = 180 +_MOCK_HEALTH_TIMEOUT_SECONDS = 60.0 +_NMP_HEALTH_TIMEOUT_SECONDS = 180.0 _REQUIRED_NEMOGUARDRAILS_FILES = ( @@ -102,7 +101,7 @@ def _build_mock_nim_processes(paths: RunPaths, workers: int) -> list[SupervisedP workdir = paths.nemoguardrails_repo_root / "benchmark" # Helper to build a ``SupervisedProcess`` for one of the mock LLM servers. - def spec(name: str, port: int, env_file: Path) -> SupervisedProcess: + def spec(name: str, port: int, env_file: Path, *, health_url: str) -> SupervisedProcess: return SupervisedProcess( name=name, cmd=[ @@ -119,6 +118,8 @@ def spec(name: str, port: int, env_file: Path) -> SupervisedProcess: log_path=paths.log_dir / f"{name}.log", cwd=workdir, env=env, + health_url=health_url, + health_timeout_seconds=_MOCK_HEALTH_TIMEOUT_SECONDS, ) return [ @@ -127,6 +128,7 @@ def spec(name: str, port: int, env_file: Path) -> SupervisedProcess: "mock-app-llm", 8000, paths.nemoguardrails_repo_root / "benchmark/mock_llm_server/configs/meta-llama-3.3-70b-instruct.env", + health_url=f"{APP_PROVIDER_URL}/health", ), # Content-safety LLM mock server spec( @@ -134,6 +136,7 @@ def spec(name: str, port: int, env_file: Path) -> SupervisedProcess: 8001, paths.nemoguardrails_repo_root / "benchmark/mock_llm_server/configs/nvidia-llama-3.1-nemoguard-8b-content-safety.env", + health_url=f"{CS_PROVIDER_URL}/health", ), ] @@ -150,6 +153,8 @@ def _build_nmp_process(paths: RunPaths) -> SupervisedProcess: log_path=paths.log_dir / "nmp-services.log", cwd=paths.nmp_repo_root, env={"NMP_BASE_URL": NMP_BASE_URL, "NMP_DATA_DIR": str(paths.nmp_data_dir)}, + health_url=f"{NMP_BASE_URL}{NMP_HEALTH_PATH}", + health_timeout_seconds=_NMP_HEALTH_TIMEOUT_SECONDS, ) @@ -165,6 +170,8 @@ def _build_aiperf_shim_process(paths: RunPaths) -> SupervisedProcess: cmd=[sys.executable, "-m", "nemo_guardrails_plugin.benchmarks.shim"], log_path=paths.log_dir / "aiperf-shim.log", cwd=paths.nmp_repo_root, + health_url=f"{AIPERF_SHIM_BASE_URL}/__shim/health", + health_timeout_seconds=_MOCK_HEALTH_TIMEOUT_SECONDS, ) @@ -291,28 +298,13 @@ def main(argv: list[str] | None = None) -> int: # Pop the cleanup so processes outlive this script. stack.pop_all() - log.info("Waiting for all services to be ready...") - - wait_http( - "http://localhost:8000/health", - timeout_seconds=_MOCK_START_TIMEOUT_SECONDS, - label="mock app LLM", - ) - wait_http( - "http://localhost:8001/health", - timeout_seconds=_MOCK_START_TIMEOUT_SECONDS, - label="mock content-safety LLM", - ) - wait_http( - f"{NMP_BASE_URL}{NMP_HEALTH_PATH}", - timeout_seconds=_NMP_START_TIMEOUT_SECONDS, - label="NMP services", - ) - wait_http( - f"{AIPERF_SHIM_BASE_URL}/__shim/health", - timeout_seconds=_MOCK_START_TIMEOUT_SECONDS, - label="AIPerf shim", - ) + if args.reuse_services: + log.info("Waiting for existing NMP services at %s...", NMP_BASE_URL) + wait_http( + f"{NMP_BASE_URL}{NMP_HEALTH_PATH}", + timeout_seconds=_NMP_HEALTH_TIMEOUT_SECONDS, + label="nmp-services", + ) log.info(f"All services are ready. Seeding benchmark resources in workspace {WORKSPACE}...") @@ -337,7 +329,7 @@ def main(argv: list[str] | None = None) -> int: runtime_config=paths.runtime_config, log_path=paths.log_dir / "aiperf.log", python_executable=str(aiperf_python), - extra_env=env_with_venv_on_path(paths.aiperf_venv_dir), + venv_bin_path=paths.aiperf_venv_dir / "bin", ) sweep_results = collect_sweep_results(paths.aiperf_output_dir) From 49124bc6050301342790e4b82a8e129762fd0c74 Mon Sep 17 00:00:00 2001 From: Jash Gulabrai Date: Tue, 2 Jun 2026 15:30:01 -0400 Subject: [PATCH 10/10] Fix lint Signed-off-by: Jash Gulabrai --- .../src/nemo_guardrails_plugin/benchmarks/aiperf_runner.py | 1 - .../src/nemo_guardrails_plugin/benchmarks/processes.py | 1 - 2 files changed, 2 deletions(-) diff --git a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/aiperf_runner.py b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/aiperf_runner.py index 49360a9498..f7ff6729c1 100644 --- a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/aiperf_runner.py +++ b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/aiperf_runner.py @@ -19,7 +19,6 @@ from typing import Any import yaml - from nemo_guardrails_plugin.benchmarks.bootstrap import build_env log = logging.getLogger(__name__) diff --git a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/processes.py b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/processes.py index 2cfa7672db..7702b4e081 100644 --- a/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/processes.py +++ b/plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/processes.py @@ -35,7 +35,6 @@ from typing import IO, Iterator import httpx - from nemo_guardrails_plugin.benchmarks.bootstrap import build_env log = logging.getLogger(__name__)