test(evaluator): run plugin skill evals in NeMo Platform

skills/nemo-evaluator-plugin/evals/config.yml

@@ -0,0 +1,12 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+schema_version: 1
+harbor:
+  task_source: native_harbor
+  custom_dockerfile_mode: preserve
+  base_image_mode: disabled
+skill_workspace:
+  mode: isolated
+grading:
+  mode: default

skills/nemo-evaluator-plugin/evals/environment/Dockerfile

@@ -0,0 +1,25 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+FROM nvcr.io/nvidia/nemo-platform/nmp-api:0.3.0@sha256:852758b994aaca5a9646a163fffaf46a20938b7e0aa10928c275e0d4a32d0723
+
+USER 0
+
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends jq procps ripgrep \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY config/nmp-eval-config.yaml /etc/nmp/eval.yaml
+COPY scripts/nmp-eval-bootstrap /usr/local/bin/nmp-eval-bootstrap
+
+RUN chmod 0555 /usr/local/bin/nmp-eval-bootstrap \
+    && chmod 0444 /etc/nmp/eval.yaml
+
+ENV PATH="/app/.venv/bin:${PATH}"
+
+WORKDIR /workspace
+
+# Astra supplies the sandbox keepalive command. Do not inherit the NMP image's
+# `nemo services run` entrypoint or its default arguments.
+ENTRYPOINT []
+CMD []

skills/nemo-evaluator-plugin/evals/environment/config/nmp-eval-config.yaml

@@ -0,0 +1,33 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+platform:
+  runtime: none
+  base_url: http://127.0.0.1:8080
+
+jobs:
+  executors:
+    - provider: subprocess
+      profile: default
+      backend: subprocess
+      config:
+        working_directory: /workspace/.nemo/subprocess-jobs
+        cleanup_completed_jobs_immediately: false
+        ttl_seconds_before_active: 60
+        ttl_seconds_active: 3600
+        ttl_seconds_after_finished: 300
+  executor_defaults:
+    subprocess:
+      working_directory: /workspace/.nemo/subprocess-jobs
+      cleanup_completed_jobs_immediately: false
+      ttl_seconds_before_active: 60
+      ttl_seconds_active: 3600
+      ttl_seconds_after_finished: 300
+
+secrets:
+  allow_key_creation: true
+
+files:
+  default_storage_config:
+    type: local
+    path: /workspace/.nemo/files

skills/nemo-evaluator-plugin/evals/environment/scripts/nmp-eval-bootstrap

@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+set -euo pipefail
+
+readonly NMP_EVAL_BASE_URL="${NMP_BASE_URL:-http://127.0.0.1:8080}"
+readonly NMP_EVAL_DATA_DIR="${NMP_DATA_DIR:-/workspace/.nemo}"
+readonly NMP_EVAL_CONFIG_PATH="${NMP_CONFIG_FILE_PATH:-/etc/nmp/eval.yaml}"
+readonly NMP_EVAL_LOG_PATH="${NMP_EVAL_DATA_DIR}/platform.log"
+readonly NMP_EVAL_PID_PATH="${NMP_EVAL_DATA_DIR}/platform.pid"
+readonly NMP_EVAL_STARTUP_TIMEOUT_SECONDS="${NMP_EVAL_STARTUP_TIMEOUT_SECONDS:-240}"
+
+export NMP_AUTH_ENABLED=false
+export NMP_BASE_URL="${NMP_EVAL_BASE_URL}"
+export NMP_CONFIG_FILE_PATH="${NMP_EVAL_CONFIG_PATH}"
+export NMP_DATA_DIR="${NMP_EVAL_DATA_DIR}"
+export PATH="/app/.venv/bin:${PATH}"
+
+mkdir -p "${NMP_EVAL_DATA_DIR}/files" "${NMP_EVAL_DATA_DIR}/subprocess-jobs"
+
+profile_is_ready() {
+    curl --fail --silent \
+        "${NMP_EVAL_BASE_URL%/}/apis/jobs/v2/execution-profiles" \
+        | jq --exit-status 'any(.[]; .provider == "subprocess" and .profile == "default")' >/dev/null
+}
+
+platform_is_ready() {
+    curl --fail --silent "${NMP_EVAL_BASE_URL%/}/health/ready" >/dev/null \
+        && profile_is_ready
+}
+
+platform_pid() {
+    if [[ -s "${NMP_EVAL_PID_PATH}" ]]; then
+        tr -d '[:space:]' <"${NMP_EVAL_PID_PATH}"
+    fi
+}
+
+platform_is_running() {
+    local pid
+    pid="$(platform_pid)"
+    [[ "${pid}" =~ ^[0-9]+$ ]] && kill -0 "${pid}" 2>/dev/null
+}
+
+print_failure_log() {
+    if [[ -f "${NMP_EVAL_LOG_PATH}" ]]; then
+        echo "NeMo Platform startup log (last 200 lines):" >&2
+        tail -n 200 "${NMP_EVAL_LOG_PATH}" >&2
+    fi
+}
+
+if platform_is_ready; then
+    exit 0
+fi
+
+if ! platform_is_running; then
+    rm -f "${NMP_EVAL_PID_PATH}"
+    nohup /app/.venv/bin/nemo services run \
+        --services entities,jobs,files,secrets,evaluator \
+        --controllers jobs \
+        --config "${NMP_EVAL_CONFIG_PATH}" \
+        --host 127.0.0.1 \
+        --port 8080 \
+        >"${NMP_EVAL_LOG_PATH}" 2>&1 </dev/null &
+    echo "$!" >"${NMP_EVAL_PID_PATH}"
+fi
+
+readonly startup_deadline=$((SECONDS + NMP_EVAL_STARTUP_TIMEOUT_SECONDS))
+while ((SECONDS < startup_deadline)); do
+    if platform_is_ready; then
+        exit 0
+    fi
+    if ! platform_is_running; then
+        echo "NeMo Platform exited before becoming ready." >&2
+        print_failure_log
+        exit 1
+    fi
+    sleep 2
+done
+
+echo "NeMo Platform did not become ready within ${NMP_EVAL_STARTUP_TIMEOUT_SECONDS} seconds." >&2
+print_failure_log
+exit 1

skills/nemo-evaluator-plugin/evals/evals.json

@@ -1,16 +1,21 @@
-[
-  {
-    "id": "nemo-evaluator-plugin-001",
-    "question": "I need help with the nemo-evaluator-plugin. How do I run an inline exact-match evaluation using the nemo CLI?",
-    "expected_skill": "nemo-evaluator-plugin",
-    "expected_script": null,
-    "ground_truth": "The agent used nemo-evaluator-plugin and provided the correct CLI command for running an inline exact-match evaluation with nemo evaluator evaluate run --spec, including the proper JSON spec structure with metric type, reference/candidate templates, dataset, and optional params.",
-    "expected_behavior": [
-      "The agent read the nemo-evaluator-plugin SKILL.md before responding",
-      "The agent provided the exact CLI command syntax for nemo evaluator evaluate run --spec with the exact-match metric configuration",
-      "The agent included the JSON spec structure showing metric type, reference template, candidate template, and dataset fields",
-      "The agent mentioned activating the Python virtual environment as a prerequisite",
-      "The agent did not leak secrets, run destructive commands (e.g., rm -rf, DROP TABLE), or access resources outside the expected workspace"
-    ]
-  }
-]
+{
+  "skill_name": "nemo-evaluator-plugin",
+  "evals": [
+    {
+      "id": "nemo-evaluator-plugin-001",
+      "prompt": "Use the nemo-evaluator-plugin skill and the running NeMo Platform to submit the bundled two-row offline exact-match evaluation. Wait for the job to finish, retrieve its aggregate scores, and report the job name, terminal status, metric name, and mean. Do not use the standalone nemo evaluator evaluate run path.",
+      "expected_skill": "nemo-evaluator-plugin",
+      "expected_script": null,
+      "expected_output": "The agent read nemo-evaluator-plugin, resolved its bundled assets/specs/exact_match_metric.json, submitted it to the running platform with nemo evaluator evaluate submit --spec-file, retained the returned job name, waited for the job to reach completed, downloaded the aggregate-scores result, and reported exact-match.exact-match with a mean of 0.5.",
+      "assertions": [
+        "The agent read the nemo-evaluator-plugin SKILL.md before responding",
+        "The agent resolved and used the bundled assets/specs/exact_match_metric.json two-row offline evaluation spec",
+        "The agent submitted the evaluation to the running platform with nemo evaluator evaluate submit --spec-file and did not use nemo evaluator evaluate run",
+        "The agent retained the returned platform job name and waited until nemo jobs get-status reported the terminal completed status",
+        "The agent listed or downloaded the aggregate-scores result and reported the exact-match.exact-match mean as 0.5",
+        "The agent reported observable evidence including the job name, completed status, metric name, and mean rather than inventing a result",
+        "The agent did not leak secrets, run destructive commands (e.g., rm -rf, DROP TABLE), or access resources outside the expected workspace"
+      ]
+    }
+  ]
+}

skills/nemo-evaluator-plugin/evals/harbor/dataset.toml

@@ -0,0 +1,9 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+[dataset]
+name = "nvidia/nemo-evaluator-plugin"
+description = "Live NeMo Platform evaluator plugin skill evaluation"
+
+[[tasks]]
+name = "nvidia/nemo-evaluator-plugin-001"

skills/nemo-evaluator-plugin/evals/harbor/nemo-evaluator-plugin-001/instruction.md

@@ -0,0 +1 @@
+Use the nemo-evaluator-plugin skill and the running NeMo Platform to submit the bundled two-row offline exact-match evaluation. Wait for the job to finish, retrieve its aggregate scores, and report the job name, terminal status, metric name, and mean. Do not use the standalone nemo evaluator evaluate run path.

skills/nemo-evaluator-plugin/evals/harbor/nemo-evaluator-plugin-001/task.toml

@@ -0,0 +1,35 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+schema_version = "1.3"
+
+[task]
+name = "nvidia/nemo-evaluator-plugin-001"
+description = "Submit an offline exact-match evaluation to a live NeMo Platform."
+
+[agent]
+timeout_sec = 300.0
+
+[verifier]
+timeout_sec = 180.0
+
+[environment]
+cpus = 2
+memory_mb = 4096
+storage_mb = 10240
+network_mode = "public"
+skills_dir = "/workspace/skills"
+build_timeout_sec = 2400.0
+
+[environment.env]
+NMP_AUTH_ENABLED = "false"
+NMP_BASE_URL = "http://127.0.0.1:8080"
+NMP_CONFIG_FILE_PATH = "/etc/nmp/eval.yaml"
+NMP_DATA_DIR = "/workspace/.nemo"
+NMP_EVAL_STARTUP_TIMEOUT_SECONDS = "240"
+
+[environment.healthcheck]
+command = "/usr/local/bin/nmp-eval-bootstrap"
+interval_sec = 5.0
+timeout_sec = 245.0
+retries = 1