test(e2e): add onboard inference smoke guard for #3253

.github/workflows/regression-e2e.yaml

@@ -21,7 +21,7 @@ on:
       jobs:
         description: >-
           Comma-separated regression job names to run (empty = all).
-          Valid: dashboard-remote-bind-e2e,gateway-health-honest-e2e,gateway-drift-preflight-e2e,openshell-version-pin-e2e
+          Valid: dashboard-remote-bind-e2e,gateway-health-honest-e2e,gateway-drift-preflight-e2e,openshell-version-pin-e2e,onboard-inference-smoke-e2e
         required: false
         type: string
         default: ""
@@ -48,6 +48,7 @@ jobs:
       gateway: ${{ steps.select.outputs.gateway }}
       gateway_drift_preflight: ${{ steps.select.outputs.gateway_drift_preflight }}
       openshell_version_pin: ${{ steps.select.outputs.openshell_version_pin }}
+      onboard_inference_smoke: ${{ steps.select.outputs.onboard_inference_smoke }}
     steps:
       - id: select
         env:
@@ -87,6 +88,12 @@ jobs:
             echo "openshell_version_pin=false" >> "$GITHUB_OUTPUT"
           fi
 
+          if [ -z "$normalized" ] || includes_job "onboard-inference-smoke-e2e"; then
+            echo "onboard_inference_smoke=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "onboard_inference_smoke=false" >> "$GITHUB_OUTPUT"
+          fi
+
   dashboard-remote-bind-e2e:
     needs: select_regression_jobs
     if: >-
@@ -182,6 +189,42 @@ jobs:
             /tmp/nemoclaw-e2e-openshell-version-pin-downloads.log
           if-no-files-found: ignore
 
+  # ── Onboard inference smoke E2E ─────────────────────────────
+  # Coverage guard for #3253. Onboard must not report installation success
+  # until the configured provider/model route has served a real chat completion.
+  # This simulates a route that is configured but returns HTTP 503 at runtime.
+  onboard-inference-smoke-e2e:
+    needs: select_regression_jobs
+    if: >-
+      github.repository == 'NVIDIA/NemoClaw' &&
+      needs.select_regression_jobs.outputs.onboard_inference_smoke == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Setup Node
+        uses: actions/setup-node@v6
+        with:
+          node-version: "22"
+
+      - name: Run onboard inference smoke E2E test
+        env:
+          NEMOCLAW_NON_INTERACTIVE: "1"
+          NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
+        run: bash test/e2e/test-onboard-inference-smoke.sh
+
+      - name: Upload onboard inference smoke logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: onboard-inference-smoke-logs
+          path: |
+            /tmp/nemoclaw-e2e-onboard-inference-smoke.log
+            /tmp/nemoclaw-e2e-onboard-inference-smoke-node.log
+          if-no-files-found: ignore
+
   # ── Gateway drift preflight E2E ─────────────────────────────
   # Coverage guard for #3399 / #3423. A stale OpenShell gateway image can
   # make sandbox-state RPCs fail with protobuf invalid-wire decode errors.

ACCEPTANCE.md

@@ -0,0 +1,16 @@
+# Definition of Done — E2E Regression Guard
+
+Coverage guard #3595 adds `onboard-inference-smoke-e2e` in `.github/workflows/regression-e2e.yaml`.
+
+The fix PR for #3253 is not complete until this job flips:
+
+- RED on main-equivalent unfixed code: <https://github.com/NVIDIA/NemoClaw/actions/runs/25923084961>
+- GREEN on the fix branch: pending
+
+Dispatch command:
+
+    gh workflow run regression-e2e.yaml --repo NVIDIA/NemoClaw -f jobs=onboard-inference-smoke-e2e --ref <fix-branch>
+
+Expected failure on unfixed code:
+
+    setupInference() accepted a configured route without proving the chat/completions path; onboard would later print Installation complete while the first real request returns HTTP 503 (#3253)

test/e2e/docs/parity-inventory.generated.json

@@ -9844,6 +9844,43 @@
         }
       ]
     },
+    {
+      "script": "test/e2e/test-onboard-inference-smoke.sh",
+      "assertions": [
+        {
+          "script": "test/e2e/test-onboard-inference-smoke.sh",
+          "line": 156,
+          "text": "setupInference() accepted a configured route without proving the chat/completions path; onboard would later print Installation complete while the first real request returns HTTP 503 (#3253)",
+          "polarity": "fail",
+          "normalized_id": "setupinference.accepted.a.configured.route.without.proving.the.chat.completions.path.onboard.would.later.print.installation.complete.while.the.first.real.request.returns.http.503.3253",
+          "mapping_status": "deferred"
+        },
+        {
+          "script": "test/e2e/test-onboard-inference-smoke.sh",
+          "line": 158,
+          "text": "setupInference() did not accept a runtime-broken inference route",
+          "polarity": "pass",
+          "normalized_id": "setupinference.did.not.accept.a.runtime.broken.inference.route",
+          "mapping_status": "deferred"
+        },
+        {
+          "script": "test/e2e/test-onboard-inference-smoke.sh",
+          "line": 161,
+          "text": "onboard did not surface actionable inference smoke diagnostics (expected provider/model/api_base/credential env/upstream 503)",
+          "polarity": "fail",
+          "normalized_id": "onboard.did.not.surface.actionable.inference.smoke.diagnostics.expected.provider.model.api.base.credential.env.upstream.503",
+          "mapping_status": "deferred"
+        },
+        {
+          "script": "test/e2e/test-onboard-inference-smoke.sh",
+          "line": 163,
+          "text": "onboard surfaced actionable inference smoke diagnostics for the broken route",
+          "polarity": "pass",
+          "normalized_id": "onboard.surfaced.actionable.inference.smoke.diagnostics.for.the.broken.route",
+          "mapping_status": "deferred"
+        }
+      ]
+    },
     {
       "script": "test/e2e/test-onboard-repair.sh",
       "assertions": [
@@ -16311,8 +16348,8 @@
     }
   ],
   "totals": {
-    "scripts": 50,
-    "assertions": 2007,
+    "scripts": 51,
+    "assertions": 2011,
     "zero_assertion_scripts": 1
   }
 }

test/e2e/docs/parity-map.yaml

@@ -5,6 +5,31 @@ scripts:
     bucket: final-security-policy-platform-misc
     retirement_evidence: no PASS/FAIL legacy assertions extracted; reviewed 2026-05-13
     assertions: []
+  test-onboard-inference-smoke.sh:
+    scenario: ''
+    status: deferred
+    bucket: inference-onboard-smoke
+    assertions:
+    - legacy: setupInference() accepted a configured route without proving the chat/completions path; onboard would later print Installation complete while the first real request returns HTTP 503 (#3253)
+      status: deferred
+      reason: regression guard validates fix PR #3594 before migration to scenario framework
+      owner: e2e-maintainers
+      runner_requirement: local CLI build with mocked OpenShell runner
+    - legacy: setupInference() did not accept a runtime-broken inference route
+      status: deferred
+      reason: regression guard validates fix PR #3594 before migration to scenario framework
+      owner: e2e-maintainers
+      runner_requirement: local CLI build with mocked OpenShell runner
+    - legacy: onboard did not surface actionable inference smoke diagnostics (expected provider/model/api_base/credential env/upstream 503)
+      status: deferred
+      reason: regression guard validates fix PR #3594 before migration to scenario framework
+      owner: e2e-maintainers
+      runner_requirement: local CLI build with mocked OpenShell runner
+    - legacy: onboard surfaced actionable inference smoke diagnostics for the broken route
+      status: deferred
+      reason: regression guard validates fix PR #3594 before migration to scenario framework
+      owner: e2e-maintainers
+      runner_requirement: local CLI build with mocked OpenShell runner
   test-brave-search-e2e.sh:
     scenario: ubuntu-repo-cloud-openclaw
     status: migrated

test/e2e/test-onboard-inference-smoke.sh

@@ -0,0 +1,163 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Coverage guard for issue #3253 — onboard must not report installation
+# success until the configured inference route has served a real request.
+#
+# Expected RED on main-equivalent code: PASSING inference configuration is
+# treated as enough. setupInference() accepts a provider/model whose route is
+# configured but whose chat/completions endpoint returns HTTP 503, so this test
+# fails because setupInference() resolves successfully and prints only the route
+# success line.
+#
+# Expected GREEN after fix: setupInference() performs a one-shot inference smoke
+# probe, exits non-zero on the upstream 503, and surfaces provider/model/api
+# base/credential-env diagnostics before any "Installation complete" summary.
+
+set -euo pipefail
+
+LOG_FILE="/tmp/nemoclaw-e2e-onboard-inference-smoke.log"
+exec > >(tee "$LOG_FILE") 2>&1
+
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m'
+
+pass() { echo -e "${GREEN}[PASS]${NC} $1"; }
+info() { echo -e "${YELLOW}[INFO]${NC} $1"; }
+diag() { echo -e "${YELLOW}[DIAG]${NC} $1"; }
+fail() {
+  echo -e "${RED}[FAIL]${NC} $1" >&2
+  diag "onboard inference smoke log tail:"
+  tail -120 "$LOG_FILE" 2>/dev/null || true
+  exit 1
+}
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+cd "$REPO_ROOT"
+
+info "Preparing CLI build"
+if [ ! -d node_modules ]; then
+  npm ci --ignore-scripts
+fi
+npm run build:cli
+
+info "Invoking setupInference() with a gateway route that is configured but runtime-broken"
+set +e
+NEMOCLAW_NON_INTERACTIVE=1 \
+  NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \
+  NEMOCLAW_ONBOARD_INFERENCE_SMOKE_E2E=1 \
+  node <<'NODE' 2>&1 | tee /tmp/nemoclaw-e2e-onboard-inference-smoke-node.log
+const Module = require("module");
+const originalLoad = Module._load;
+const calls = [];
+
+Module._load = function patchedLoad(request, parent, isMain) {
+  if (request === "./adapters/openshell/resolve" || request.endsWith("/adapters/openshell/resolve")) {
+    return { resolveOpenshell: () => "/usr/bin/openshell" };
+  }
+  if (request === "./runner" || request.endsWith("/runner")) {
+    const actualRunner = originalLoad.apply(this, arguments);
+    return {
+      ...actualRunner,
+      run: (cmd, opts = {}) => {
+        calls.push(["run", cmd]);
+        if (Array.isArray(cmd) && cmd.includes("provider") && cmd.includes("upsert")) {
+          return { status: 0, stdout: "Created provider compatible-endpoint\n", stderr: "" };
+        }
+        if (Array.isArray(cmd) && cmd.includes("inference") && cmd.includes("set")) {
+          return { status: 0, stdout: "Inference configured\n", stderr: "" };
+        }
+        if (Array.isArray(cmd) && cmd.some((part) => String(part).includes("/chat/completions"))) {
+          return {
+            status: 22,
+            stdout: JSON.stringify({ error: { message: "upstream returned HTTP 503 from compatible-endpoint" } }),
+            stderr: "curl: (22) The requested URL returned error: 503",
+          };
+        }
+        return { status: 0, stdout: "", stderr: "" };
+      },
+      runCapture: (cmd) => {
+        calls.push(["runCapture", cmd]);
+        if (Array.isArray(cmd) && cmd.includes("inference") && cmd.includes("get")) {
+          return JSON.stringify({ provider: "compatible-endpoint", model: "broken-model" });
+        }
+        return "";
+      },
+    };
+  }
+  if (request === "./onboard/providers" || request.endsWith("/onboard/providers")) {
+    return {
+      REMOTE_PROVIDER_CONFIG: {
+        custom: {
+          label: "Other OpenAI-compatible endpoint",
+          providerName: "compatible-endpoint",
+          providerType: "openai",
+          credentialEnv: "COMPATIBLE_API_KEY",
+          endpointUrl: "",
+          helpUrl: null,
+          modelMode: "input",
+          defaultModel: "",
+          skipVerify: true,
+        },
+      },
+      LOCAL_INFERENCE_PROVIDERS: [],
+      providerExistsInGateway: () => true,
+      getProviderLabel: (provider) => provider,
+      upsertProvider: (...args) => {
+        calls.push(["upsertProvider", args]);
+        return { ok: true, status: 0, message: "Created provider compatible-endpoint" };
+      },
+    };
+  }
+  if (request === "./registry" || request.endsWith("/registry")) {
+    return {
+      updateSandbox: (_name, patch) => calls.push(["registry.updateSandbox", patch]),
+      getSandbox: () => null,
+      getDisabledChannels: () => [],
+    };
+  }
+  return originalLoad.apply(this, arguments);
+};
+
+const onboard = require("./dist/lib/onboard");
+const result = onboard.setupInference(
+  "test-sandbox",
+  "broken-model",
+  "compatible-endpoint",
+  "https://broken.example.invalid/v1",
+  "BROKEN_API_KEY",
+);
+
+Promise.resolve(result)
+  .then((value) => {
+    console.log("__SETUP_INFERENCE_RESOLVED__");
+    console.log(JSON.stringify(value));
+    console.log("__CALLS__" + JSON.stringify(calls));
+    process.exit(0);
+  })
+  .catch((error) => {
+    console.error("__SETUP_INFERENCE_REJECTED__");
+    console.error(error && error.stack ? error.stack : error);
+    console.log("__CALLS__" + JSON.stringify(calls));
+    process.exit(3);
+  });
+NODE
+NODE_EXIT=$?
+set -e
+cat /tmp/nemoclaw-e2e-onboard-inference-smoke-node.log
+
+info "node exit code: ${NODE_EXIT}"
+
+if grep -q "__SETUP_INFERENCE_RESOLVED__" /tmp/nemoclaw-e2e-onboard-inference-smoke-node.log || [ "$NODE_EXIT" -eq 0 ]; then
+  fail "setupInference() accepted a configured route without proving the chat/completions path; onboard would later print Installation complete while the first real request returns HTTP 503 (#3253)"
+fi
+pass "setupInference() did not accept a runtime-broken inference route"
+
+if ! grep -qiE "503|upstream|compatible-endpoint|broken-model|BROKEN_API_KEY|broken.example.invalid" /tmp/nemoclaw-e2e-onboard-inference-smoke-node.log; then
+  fail "onboard did not surface actionable inference smoke diagnostics (expected provider/model/api_base/credential env/upstream 503)"
+fi
+pass "onboard surfaced actionable inference smoke diagnostics for the broken route"