diff --git a/.github/workflows/regression-e2e.yaml b/.github/workflows/regression-e2e.yaml index 14ba6151a90..f73305ae1a2 100644 --- a/.github/workflows/regression-e2e.yaml +++ b/.github/workflows/regression-e2e.yaml @@ -21,7 +21,7 @@ on: jobs: description: >- Comma-separated regression job names to run (empty = all). - Valid: dashboard-remote-bind-e2e,gateway-health-honest-e2e,gateway-drift-preflight-e2e,openshell-version-pin-e2e + Valid: dashboard-remote-bind-e2e,gateway-health-honest-e2e,gateway-drift-preflight-e2e,openshell-version-pin-e2e,onboard-inference-smoke-e2e required: false type: string default: "" @@ -48,6 +48,7 @@ jobs: gateway: ${{ steps.select.outputs.gateway }} gateway_drift_preflight: ${{ steps.select.outputs.gateway_drift_preflight }} openshell_version_pin: ${{ steps.select.outputs.openshell_version_pin }} + onboard_inference_smoke: ${{ steps.select.outputs.onboard_inference_smoke }} steps: - id: select env: @@ -87,6 +88,12 @@ jobs: echo "openshell_version_pin=false" >> "$GITHUB_OUTPUT" fi + if [ -z "$normalized" ] || includes_job "onboard-inference-smoke-e2e"; then + echo "onboard_inference_smoke=true" >> "$GITHUB_OUTPUT" + else + echo "onboard_inference_smoke=false" >> "$GITHUB_OUTPUT" + fi + dashboard-remote-bind-e2e: needs: select_regression_jobs if: >- @@ -182,6 +189,42 @@ jobs: /tmp/nemoclaw-e2e-openshell-version-pin-downloads.log if-no-files-found: ignore + # ── Onboard inference smoke E2E ───────────────────────────── + # Coverage guard for #3253. Onboard must not report installation success + # until the configured provider/model route has served a real chat completion. + # This simulates a route that is configured but returns HTTP 503 at runtime. + onboard-inference-smoke-e2e: + needs: select_regression_jobs + if: >- + github.repository == 'NVIDIA/NemoClaw' && + needs.select_regression_jobs.outputs.onboard_inference_smoke == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Setup Node + uses: actions/setup-node@v6 + with: + node-version: "22" + + - name: Run onboard inference smoke E2E test + env: + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + run: bash test/e2e/test-onboard-inference-smoke.sh + + - name: Upload onboard inference smoke logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: onboard-inference-smoke-logs + path: | + /tmp/nemoclaw-e2e-onboard-inference-smoke.log + /tmp/nemoclaw-e2e-onboard-inference-smoke-node.log + if-no-files-found: ignore + # ── Gateway drift preflight E2E ───────────────────────────── # Coverage guard for #3399 / #3423. A stale OpenShell gateway image can # make sandbox-state RPCs fail with protobuf invalid-wire decode errors. diff --git a/ACCEPTANCE.md b/ACCEPTANCE.md new file mode 100644 index 00000000000..d679a403064 --- /dev/null +++ b/ACCEPTANCE.md @@ -0,0 +1,16 @@ +# Definition of Done — E2E Regression Guard + +Coverage guard #3595 adds `onboard-inference-smoke-e2e` in `.github/workflows/regression-e2e.yaml`. + +The fix PR for #3253 is not complete until this job flips: + +- RED on main-equivalent unfixed code: +- GREEN on the fix branch: pending + +Dispatch command: + + gh workflow run regression-e2e.yaml --repo NVIDIA/NemoClaw -f jobs=onboard-inference-smoke-e2e --ref + +Expected failure on unfixed code: + + setupInference() accepted a configured route without proving the chat/completions path; onboard would later print Installation complete while the first real request returns HTTP 503 (#3253) diff --git a/test/e2e/docs/parity-inventory.generated.json b/test/e2e/docs/parity-inventory.generated.json index 5b85bc02436..03ac70c5e78 100644 --- a/test/e2e/docs/parity-inventory.generated.json +++ b/test/e2e/docs/parity-inventory.generated.json @@ -9844,6 +9844,43 @@ } ] }, + { + "script": "test/e2e/test-onboard-inference-smoke.sh", + "assertions": [ + { + "script": "test/e2e/test-onboard-inference-smoke.sh", + "line": 156, + "text": "setupInference() accepted a configured route without proving the chat/completions path; onboard would later print Installation complete while the first real request returns HTTP 503 (#3253)", + "polarity": "fail", + "normalized_id": "setupinference.accepted.a.configured.route.without.proving.the.chat.completions.path.onboard.would.later.print.installation.complete.while.the.first.real.request.returns.http.503.3253", + "mapping_status": "deferred" + }, + { + "script": "test/e2e/test-onboard-inference-smoke.sh", + "line": 158, + "text": "setupInference() did not accept a runtime-broken inference route", + "polarity": "pass", + "normalized_id": "setupinference.did.not.accept.a.runtime.broken.inference.route", + "mapping_status": "deferred" + }, + { + "script": "test/e2e/test-onboard-inference-smoke.sh", + "line": 161, + "text": "onboard did not surface actionable inference smoke diagnostics (expected provider/model/api_base/credential env/upstream 503)", + "polarity": "fail", + "normalized_id": "onboard.did.not.surface.actionable.inference.smoke.diagnostics.expected.provider.model.api.base.credential.env.upstream.503", + "mapping_status": "deferred" + }, + { + "script": "test/e2e/test-onboard-inference-smoke.sh", + "line": 163, + "text": "onboard surfaced actionable inference smoke diagnostics for the broken route", + "polarity": "pass", + "normalized_id": "onboard.surfaced.actionable.inference.smoke.diagnostics.for.the.broken.route", + "mapping_status": "deferred" + } + ] + }, { "script": "test/e2e/test-onboard-repair.sh", "assertions": [ @@ -16311,8 +16348,8 @@ } ], "totals": { - "scripts": 50, - "assertions": 2007, + "scripts": 51, + "assertions": 2011, "zero_assertion_scripts": 1 } } diff --git a/test/e2e/docs/parity-map.yaml b/test/e2e/docs/parity-map.yaml index 2cb6aeb6017..b3cd4605c1f 100644 --- a/test/e2e/docs/parity-map.yaml +++ b/test/e2e/docs/parity-map.yaml @@ -5,6 +5,31 @@ scripts: bucket: final-security-policy-platform-misc retirement_evidence: no PASS/FAIL legacy assertions extracted; reviewed 2026-05-13 assertions: [] + test-onboard-inference-smoke.sh: + scenario: '' + status: deferred + bucket: inference-onboard-smoke + assertions: + - legacy: setupInference() accepted a configured route without proving the chat/completions path; onboard would later print Installation complete while the first real request returns HTTP 503 (#3253) + status: deferred + reason: regression guard validates fix PR #3594 before migration to scenario framework + owner: e2e-maintainers + runner_requirement: local CLI build with mocked OpenShell runner + - legacy: setupInference() did not accept a runtime-broken inference route + status: deferred + reason: regression guard validates fix PR #3594 before migration to scenario framework + owner: e2e-maintainers + runner_requirement: local CLI build with mocked OpenShell runner + - legacy: onboard did not surface actionable inference smoke diagnostics (expected provider/model/api_base/credential env/upstream 503) + status: deferred + reason: regression guard validates fix PR #3594 before migration to scenario framework + owner: e2e-maintainers + runner_requirement: local CLI build with mocked OpenShell runner + - legacy: onboard surfaced actionable inference smoke diagnostics for the broken route + status: deferred + reason: regression guard validates fix PR #3594 before migration to scenario framework + owner: e2e-maintainers + runner_requirement: local CLI build with mocked OpenShell runner test-brave-search-e2e.sh: scenario: ubuntu-repo-cloud-openclaw status: migrated diff --git a/test/e2e/test-onboard-inference-smoke.sh b/test/e2e/test-onboard-inference-smoke.sh new file mode 100755 index 00000000000..b63919a5ed8 --- /dev/null +++ b/test/e2e/test-onboard-inference-smoke.sh @@ -0,0 +1,163 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Coverage guard for issue #3253 — onboard must not report installation +# success until the configured inference route has served a real request. +# +# Expected RED on main-equivalent code: PASSING inference configuration is +# treated as enough. setupInference() accepts a provider/model whose route is +# configured but whose chat/completions endpoint returns HTTP 503, so this test +# fails because setupInference() resolves successfully and prints only the route +# success line. +# +# Expected GREEN after fix: setupInference() performs a one-shot inference smoke +# probe, exits non-zero on the upstream 503, and surfaces provider/model/api +# base/credential-env diagnostics before any "Installation complete" summary. + +set -euo pipefail + +LOG_FILE="/tmp/nemoclaw-e2e-onboard-inference-smoke.log" +exec > >(tee "$LOG_FILE") 2>&1 + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +pass() { echo -e "${GREEN}[PASS]${NC} $1"; } +info() { echo -e "${YELLOW}[INFO]${NC} $1"; } +diag() { echo -e "${YELLOW}[DIAG]${NC} $1"; } +fail() { + echo -e "${RED}[FAIL]${NC} $1" >&2 + diag "onboard inference smoke log tail:" + tail -120 "$LOG_FILE" 2>/dev/null || true + exit 1 +} + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +cd "$REPO_ROOT" + +info "Preparing CLI build" +if [ ! -d node_modules ]; then + npm ci --ignore-scripts +fi +npm run build:cli + +info "Invoking setupInference() with a gateway route that is configured but runtime-broken" +set +e +NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_ONBOARD_INFERENCE_SMOKE_E2E=1 \ + node <<'NODE' 2>&1 | tee /tmp/nemoclaw-e2e-onboard-inference-smoke-node.log +const Module = require("module"); +const originalLoad = Module._load; +const calls = []; + +Module._load = function patchedLoad(request, parent, isMain) { + if (request === "./adapters/openshell/resolve" || request.endsWith("/adapters/openshell/resolve")) { + return { resolveOpenshell: () => "/usr/bin/openshell" }; + } + if (request === "./runner" || request.endsWith("/runner")) { + const actualRunner = originalLoad.apply(this, arguments); + return { + ...actualRunner, + run: (cmd, opts = {}) => { + calls.push(["run", cmd]); + if (Array.isArray(cmd) && cmd.includes("provider") && cmd.includes("upsert")) { + return { status: 0, stdout: "Created provider compatible-endpoint\n", stderr: "" }; + } + if (Array.isArray(cmd) && cmd.includes("inference") && cmd.includes("set")) { + return { status: 0, stdout: "Inference configured\n", stderr: "" }; + } + if (Array.isArray(cmd) && cmd.some((part) => String(part).includes("/chat/completions"))) { + return { + status: 22, + stdout: JSON.stringify({ error: { message: "upstream returned HTTP 503 from compatible-endpoint" } }), + stderr: "curl: (22) The requested URL returned error: 503", + }; + } + return { status: 0, stdout: "", stderr: "" }; + }, + runCapture: (cmd) => { + calls.push(["runCapture", cmd]); + if (Array.isArray(cmd) && cmd.includes("inference") && cmd.includes("get")) { + return JSON.stringify({ provider: "compatible-endpoint", model: "broken-model" }); + } + return ""; + }, + }; + } + if (request === "./onboard/providers" || request.endsWith("/onboard/providers")) { + return { + REMOTE_PROVIDER_CONFIG: { + custom: { + label: "Other OpenAI-compatible endpoint", + providerName: "compatible-endpoint", + providerType: "openai", + credentialEnv: "COMPATIBLE_API_KEY", + endpointUrl: "", + helpUrl: null, + modelMode: "input", + defaultModel: "", + skipVerify: true, + }, + }, + LOCAL_INFERENCE_PROVIDERS: [], + providerExistsInGateway: () => true, + getProviderLabel: (provider) => provider, + upsertProvider: (...args) => { + calls.push(["upsertProvider", args]); + return { ok: true, status: 0, message: "Created provider compatible-endpoint" }; + }, + }; + } + if (request === "./registry" || request.endsWith("/registry")) { + return { + updateSandbox: (_name, patch) => calls.push(["registry.updateSandbox", patch]), + getSandbox: () => null, + getDisabledChannels: () => [], + }; + } + return originalLoad.apply(this, arguments); +}; + +const onboard = require("./dist/lib/onboard"); +const result = onboard.setupInference( + "test-sandbox", + "broken-model", + "compatible-endpoint", + "https://broken.example.invalid/v1", + "BROKEN_API_KEY", +); + +Promise.resolve(result) + .then((value) => { + console.log("__SETUP_INFERENCE_RESOLVED__"); + console.log(JSON.stringify(value)); + console.log("__CALLS__" + JSON.stringify(calls)); + process.exit(0); + }) + .catch((error) => { + console.error("__SETUP_INFERENCE_REJECTED__"); + console.error(error && error.stack ? error.stack : error); + console.log("__CALLS__" + JSON.stringify(calls)); + process.exit(3); + }); +NODE +NODE_EXIT=$? +set -e +cat /tmp/nemoclaw-e2e-onboard-inference-smoke-node.log + +info "node exit code: ${NODE_EXIT}" + +if grep -q "__SETUP_INFERENCE_RESOLVED__" /tmp/nemoclaw-e2e-onboard-inference-smoke-node.log || [ "$NODE_EXIT" -eq 0 ]; then + fail "setupInference() accepted a configured route without proving the chat/completions path; onboard would later print Installation complete while the first real request returns HTTP 503 (#3253)" +fi +pass "setupInference() did not accept a runtime-broken inference route" + +if ! grep -qiE "503|upstream|compatible-endpoint|broken-model|BROKEN_API_KEY|broken.example.invalid" /tmp/nemoclaw-e2e-onboard-inference-smoke-node.log; then + fail "onboard did not surface actionable inference smoke diagnostics (expected provider/model/api_base/credential env/upstream 503)" +fi +pass "onboard surfaced actionable inference smoke diagnostics for the broken route"