test(e2e): migrate test-model-router-provider-routed-inference.sh to vitest [IND-2]

.github/workflows/e2e-vitest-scenarios.yaml

@@ -40,7 +40,7 @@ jobs:
           SCENARIOS: ${{ inputs.scenarios }}
         run: |
           set -euo pipefail
-          allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,inference-routing-vitest,credential-migration-vitest,runtime-overrides-vitest,hermes-e2e-vitest,hermes-root-entrypoint-smoke-vitest,network-policy-vitest,rebuild-openclaw-vitest,token-rotation-vitest,launchable-smoke-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery,double-onboard-vitest,issue-4434-tui-unreachable-inference-vitest"
+          allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,inference-routing-vitest,credential-migration-vitest,runtime-overrides-vitest,hermes-e2e-vitest,hermes-root-entrypoint-smoke-vitest,network-policy-vitest,rebuild-openclaw-vitest,token-rotation-vitest,launchable-smoke-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery,double-onboard-vitest,issue-4434-tui-unreachable-inference-vitest,model-router-provider-routed-inference-vitest"
           if [ -n "${JOBS}" ] && [ -n "${SCENARIOS}" ]; then
             echo "::error::Use either scenarios or jobs, not both." >&2
             exit 1
@@ -93,12 +93,12 @@ jobs:
           SCENARIOS: ${{ inputs.scenarios }}
         run: |
           set -euo pipefail
-          allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,inference-routing-vitest,credential-migration-vitest,runtime-overrides-vitest,hermes-e2e-vitest,hermes-root-entrypoint-smoke-vitest,network-policy-vitest,rebuild-openclaw-vitest,token-rotation-vitest,launchable-smoke-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery,double-onboard-vitest,issue-4434-tui-unreachable-inference-vitest"
+          allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,inference-routing-vitest,credential-migration-vitest,runtime-overrides-vitest,hermes-e2e-vitest,hermes-root-entrypoint-smoke-vitest,network-policy-vitest,rebuild-openclaw-vitest,token-rotation-vitest,launchable-smoke-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery,double-onboard-vitest,issue-4434-tui-unreachable-inference-vitest,model-router-provider-routed-inference-vitest"
           args=(--emit-live-matrix)
           matrix=""
           hermes_selected=false
           registry_scenarios=()
-          free_standing_scenarios=(openshell-version-pin onboard-negative-paths inference-routing runtime-overrides hermes-e2e hermes-root-entrypoint-smoke network-policy rebuild-openclaw token-rotation openclaw-tui-chat-correlation double-onboard issue-4434-tui-unreachable-inference)
+          free_standing_scenarios=(openshell-version-pin onboard-negative-paths inference-routing runtime-overrides hermes-e2e hermes-root-entrypoint-smoke network-policy rebuild-openclaw token-rotation openclaw-tui-chat-correlation double-onboard issue-4434-tui-unreachable-inference model-router-provider-routed-inference)
           is_free_standing_scenario() {
             local id="$1"
             local known
@@ -1117,6 +1117,96 @@ jobs:
           if-no-files-found: ignore
           retention-days: 14
 
+  # Focused coverage slice for the provider-routed Model Router inference
+  # contract. The retained legacy bash lane remains the source for full
+  # closeout until a later PR proves replacement and deletes it.
+  model-router-provider-routed-inference-vitest:
+    needs: [validate-jobs, generate-matrix]
+    if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',model-router-provider-routed-inference-vitest,') || contains(format(',{0},', inputs.scenarios), ',model-router-provider-routed-inference,') }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    env:
+      DOCKER_CONFIG: ${{ runner.temp }}/docker-config-model-router-provider-routed-inference
+      E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/model-router-provider-routed-inference
+      NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
+      NEMOCLAW_RUN_E2E_SCENARIOS: "1"
+      OPENSHELL_GATEWAY: "nemoclaw"
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          persist-credentials: false
+
+      - name: Authenticate to Docker Hub
+        env:
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+          DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          if [[ -z "${DOCKERHUB_USERNAME}" || -z "${DOCKERHUB_TOKEN}" ]]; then
+            echo "::notice::Docker Hub credentials not configured; continuing with anonymous pulls."
+            exit 0
+          fi
+          mkdir -p "${DOCKER_CONFIG}"
+          chmod 700 "${DOCKER_CONFIG}"
+          login_succeeded=0
+          for attempt in 1 2 3; do
+            if echo "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin; then
+              login_succeeded=1
+              break
+            fi
+            if [[ "$attempt" -lt 3 ]]; then
+              echo "::warning::Docker Hub login attempt ${attempt} failed; retrying."
+              sleep 5
+            fi
+          done
+          if [[ "$login_succeeded" -ne 1 ]]; then
+            echo "::warning::Docker Hub login failed after 3 attempts; continuing with anonymous pulls."
+          fi
+
+      - name: Set up Node
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0
+        with:
+          node-version: 22
+          cache: npm
+
+      - name: Install root dependencies
+        run: npm ci --ignore-scripts
+
+      - name: Build CLI
+        run: npm run build:cli
+
+      - name: Run Model Router provider-routed inference live test
+        # Direct Vitest coverage for
+        # test/e2e/test-model-router-provider-routed-inference.sh. It preserves
+        # the real provider-routed onboard, host model-router health, and
+        # sandbox inference.local completion boundaries without adding registry
+        # or migration-ledger wiring.
+        env:
+          NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
+        run: |
+          set -euo pipefail
+          npx vitest run --project e2e-scenarios-live \
+            test/e2e-scenario/live/model-router-provider-routed-inference.test.ts \
+            --silent=false --reporter=default
+
+      - name: Upload Model Router provider-routed inference artifacts
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: e2e-vitest-scenarios-model-router-provider-routed-inference
+          path: e2e-artifacts/vitest/model-router-provider-routed-inference/
+          include-hidden-files: false
+          if-no-files-found: ignore
+          retention-days: 14
+
+      - name: Clean up Docker auth
+        if: always()
+        run: |
+          set -euo pipefail
+          docker logout docker.io || true
+          rm -rf "${DOCKER_CONFIG}"
+
   # Focused coverage slice for the #2603/#3145 OpenClaw websocket
   # protocol/history contract. The retained legacy bash lane remains the
   # source for full closeout until a later PR proves replacement and deletes it.
@@ -1310,6 +1400,7 @@ jobs:
         token-rotation-vitest,
         launchable-smoke-vitest,
         double-onboard-vitest,
+        model-router-provider-routed-inference-vitest,
         openclaw-tui-chat-correlation-vitest,
         gateway-guard-recovery,
         issue-4434-tui-unreachable-inference-vitest,

test/e2e-scenario/live/model-router-provider-routed-inference.test.ts

@@ -0,0 +1,235 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import fs from "node:fs";
+import path from "node:path";
+
+import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts";
+import { expect, test } from "../fixtures/e2e-test.ts";
+import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts";
+
+// Focused Vitest live replacement for
+// test/e2e/test-model-router-provider-routed-inference.sh. Keep this as a
+// direct CLI/sandbox test: the legacy contract is the real provider-routed
+// onboard boundary plus host model-router health and sandbox inference.local
+// completion semantics, not a new scenario registry entry.
+
+const REPO_ROOT = path.resolve(import.meta.dirname, "../../..");
+const CLI_ENTRYPOINT = path.join(REPO_ROOT, "bin", "nemoclaw.js");
+const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-model-router";
+const ONBOARD_TIMEOUT_MS = 25 * 60_000;
+const HEALTH_ATTEMPTS = 20;
+const COMPLETION_ATTEMPTS = 3;
+
+interface ModelRouterHealth {
+  healthy_count?: unknown;
+}
+
+interface ChatCompletionResponse {
+  model?: unknown;
+  choices?: Array<{
+    message?: { content?: unknown };
+    text?: unknown;
+  }>;
+}
+
+function resultText(result: { stdout: string; stderr: string }): string {
+  return [result.stdout, result.stderr].filter(Boolean).join("\n");
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function parseJson<T>(raw: string): T | undefined {
+  try {
+    return JSON.parse(raw) as T;
+  } catch {
+    return undefined;
+  }
+}
+
+function hasHealthyEndpoint(raw: string): boolean {
+  const health = parseJson<ModelRouterHealth>(raw);
+  return typeof health?.healthy_count === "number" && health.healthy_count > 0;
+}
+
+function routedPongReason(raw: string): "ok" | string {
+  const response = parseJson<ChatCompletionResponse>(raw);
+  if (!response) return "response was not JSON";
+  const model = String(response.model ?? "");
+  if (model !== "nvidia-routed" && !model.startsWith("nvidia-routed")) {
+    return "response model was not provider-routed";
+  }
+  const content = (response.choices ?? [])
+    .map((choice) => {
+      if (typeof choice.message?.content === "string") return choice.message.content;
+      if (typeof choice.text === "string") return choice.text;
+      return "";
+    })
+    .join("\n");
+  if (!/\bPONG\b/i.test(content)) return "response missing PONG content";
+  return "ok";
+}
+
+function withProviderRoutedEnv(apiKey: string): NodeJS.ProcessEnv {
+  return {
+    ...buildAvailabilityProbeEnv(),
+    NVIDIA_API_KEY: apiKey,
+    NEMOCLAW_PROVIDER_KEY: apiKey,
+    NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME,
+    NEMOCLAW_NON_INTERACTIVE: "1",
+    NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1",
+    NEMOCLAW_POLICY_TIER: "open",
+    NEMOCLAW_PROVIDER: "routed",
+  };
+}
+
+test.skipIf(!shouldRunLiveE2EScenarios())(
+  "model-router provider-routed onboard returns routed inference.local PONG",
+  async ({ artifacts, cleanup, host, sandbox, secrets, skip }) => {
+    expect(
+      fs.existsSync(CLI_ENTRYPOINT),
+      "run `npm run build:cli` before live repo CLI scenarios",
+    ).toBe(true);
+
+    const docker = await host.command("docker", ["info"], {
+      artifactName: "prereq-docker-info-model-router-provider-routed",
+      env: buildAvailabilityProbeEnv(),
+      timeoutMs: 30_000,
+    });
+    if (docker.exitCode !== 0) {
+      if (process.env.GITHUB_ACTIONS === "true") {
+        throw new Error(
+          `Docker is required for provider-routed Model Router onboarding: ${resultText(docker)}`,
+        );
+      }
+      skip("Docker is required for provider-routed Model Router onboarding");
+    }
+
+    const apiKey = secrets.required("NVIDIA_API_KEY");
+    expect(apiKey.startsWith("nvapi-"), "NVIDIA_API_KEY must start with nvapi-").toBe(true);
+
+    await artifacts.writeJson("scenario.json", {
+      id: "model-router-provider-routed-inference",
+      runner: "vitest",
+      boundary: "direct-cli-onboard-and-sandbox-exec",
+      legacySource: "test/e2e/test-model-router-provider-routed-inference.sh",
+      contract: [
+        "Docker is available before onboarding",
+        "NVIDIA_API_KEY is present and nvapi-prefixed",
+        "nemoclaw onboard --fresh completes with NEMOCLAW_PROVIDER=routed",
+        "host model-router health reports at least one healthy endpoint",
+        "sandbox inference.local returns model nvidia-routed with PONG content",
+      ],
+    });
+
+    const cleanEnv = buildAvailabilityProbeEnv();
+    await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], {
+      artifactName: "pre-cleanup-nemoclaw-destroy-model-router-provider-routed",
+      env: cleanEnv,
+      timeoutMs: 120_000,
+    });
+
+    cleanup.add(`destroy sandbox ${SANDBOX_NAME}`, async () => {
+      await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], {
+        artifactName: "cleanup-nemoclaw-destroy-model-router-provider-routed",
+        env: buildAvailabilityProbeEnv(),
+        timeoutMs: 120_000,
+      });
+    });
+
+    const onboard = await host.command(
+      "node",
+      [
+        CLI_ENTRYPOINT,
+        "onboard",
+        "--fresh",
+        "--non-interactive",
+        "--yes-i-accept-third-party-software",
+      ],
+      {
+        artifactName: "onboard-model-router-provider-routed",
+        env: withProviderRoutedEnv(apiKey),
+        redactionValues: [apiKey],
+        timeoutMs: ONBOARD_TIMEOUT_MS,
+      },
+    );
+    expect(onboard.exitCode, resultText(onboard)).toBe(0);
+
+    let lastHealth = "";
+    for (let attempt = 1; attempt <= HEALTH_ATTEMPTS; attempt += 1) {
+      const health = await host.command(
+        "curl",
+        ["-s", "--max-time", "10", "http://127.0.0.1:4000/health"],
+        {
+          artifactName: `model-router-health-${attempt}`,
+          env: buildAvailabilityProbeEnv(),
+          redactionValues: [apiKey],
+          timeoutMs: 15_000,
+        },
+      );
+      lastHealth = health.stdout || health.stderr;
+      if (health.exitCode === 0 && hasHealthyEndpoint(lastHealth)) break;
+      if (attempt < HEALTH_ATTEMPTS) await sleep(3_000);
+    }
+    expect(
+      hasHealthyEndpoint(lastHealth),
+      `model-router has no healthy endpoints; expected #3255 main-equivalent failure: ${lastHealth.slice(0, 500)}`,
+    ).toBe(true);
+
+    const payload = JSON.stringify({
+      model: "nvidia-routed",
+      messages: [
+        {
+          role: "user",
+          content: "Return only the exact word PONG. Do not include reasoning or any other text.",
+        },
+      ],
+      max_tokens: 128,
+    });
+    let lastCompletion = "";
+    let completionReason = "not attempted";
+    for (let attempt = 1; attempt <= COMPLETION_ATTEMPTS; attempt += 1) {
+      const completion = await sandbox.exec(
+        SANDBOX_NAME,
+        [
+          "curl",
+          "-sk",
+          "--max-time",
+          "90",
+          "https://inference.local/v1/chat/completions",
+          "-H",
+          "Content-Type: application/json",
+          "--data-raw",
+          payload,
+        ],
+        {
+          artifactName: `sandbox-inference-local-routed-completion-${attempt}`,
+          env: buildAvailabilityProbeEnv(),
+          redactionValues: [apiKey],
+          timeoutMs: 120_000,
+        },
+      );
+      lastCompletion = completion.stdout || completion.stderr;
+      completionReason = routedPongReason(lastCompletion);
+      if (completion.exitCode === 0 && completionReason === "ok") break;
+      if (/inference service unavailable|HTTP 503|healthy_count.*0/i.test(lastCompletion)) break;
+      if (attempt < COMPLETION_ATTEMPTS) await sleep(5_000);
+    }
+    expect(
+      completionReason,
+      `Model Router inference.local did not return a routed completion; expected #3255 main-equivalent failure: ${lastCompletion.slice(0, 500)}`,
+    ).toBe("ok");
+
+    await artifacts.writeJson("scenario-result.json", {
+      id: "model-router-provider-routed-inference",
+      assertions: {
+        dockerRunning: docker.exitCode === 0,
+        onboardCompleted: onboard.exitCode === 0,
+        modelRouterHealthy: hasHealthyEndpoint(lastHealth),
+        routedPongCompletion: completionReason === "ok",
+      },
+    });
+  },
+);