fix(e2e): repair main runtime regressions

docs/inference/switch-providers.mdx

@@ -73,6 +73,11 @@ Run the rebuild before relying on the running agent.
 
 Use `--no-verify` only when OpenShell cannot verify the target provider at switch time and you have already confirmed its provider and credential.
 This flag does not bypass shared-gateway compatibility checks.
+When you explicitly supply a direct compatible endpoint at `http://host.openshell.internal:<port>`, NemoClaw skips OpenShell's host-side provider probe because that hostname resolves only inside the sandbox network.
+It then sends one validation request from the target sandbox before persisting the route in NemoClaw state; the request allows up to 16 output tokens.
+If that request fails, NemoClaw attempts to restore the previous OpenShell selection and remove a provider that this switch created.
+If the error reports that rollback could not complete, rerun onboarding before using the route or retrying the switch.
+Endpoint-shape and shared-gateway compatibility checks still apply.
 
 </AgentOnly>
 

docs/reference/commands.mdx

@@ -4100,6 +4100,11 @@ If the in-sandbox config sync fails, NemoClaw keeps the gateway and registry ali
 
 Supported provider names are `nvidia-prod`, `nvidia-nim`, `nvidia-router`, `openai-api`, `anthropic-prod`, `compatible-anthropic-endpoint`, `gemini-api`, `compatible-endpoint`, `hermes-provider`, `ollama-local`, and `vllm-local`.
 Use `--no-verify` only when OpenShell cannot verify the provider at switch time but you have already confirmed the provider and credential.
+When you explicitly supply a direct compatible endpoint at `http://host.openshell.internal:<port>`, NemoClaw skips OpenShell's host-side provider probe because that hostname resolves only inside the sandbox network.
+Before it persists the route in the NemoClaw registry or agent config, the command sends one validation request from the target sandbox with a 16-token output limit.
+If that request fails, the command attempts to restore the previous OpenShell selection and remove a provider that this switch created.
+If the error reports that rollback could not complete, rerun onboarding before using the route or retrying the switch.
+Endpoint-shape and shared-gateway compatibility checks still apply.
 When switching to `compatible-endpoint` or `compatible-anthropic-endpoint` from a different provider family, pass `--endpoint-url` with the trusted custom provider URL and, except for the Hermes case below, `--inference-api` with its API family so NemoClaw can persist a complete route identity for rebuild and shared-gateway checks.
 For a Hermes `compatible-anthropic-endpoint` target, `--inference-api` may be omitted because NemoClaw deterministically selects `openai-completions`; an explicit different API family is rejected.
 NemoClaw rejects loopback, link-local, private, and internal endpoint addresses, including public hostnames that resolve to a private address.

src/lib/actions/inference-set-compatible-provider.test.ts

@@ -616,7 +616,6 @@ describe("runInferenceSet compatible providers", () => {
       {
         provider: "compatible-anthropic-endpoint",
         model: "mock-anthropic-model",
-        noVerify: true,
         endpointUrl: "http://host.openshell.internal:18767/",
         credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY",
         inferenceApi: "anthropic-messages",
@@ -644,6 +643,221 @@ describe("runInferenceSet compatible providers", () => {
       nimContainer: null,
     });
     expect(deps.calls.rewriteConfigUrlsWithDnsPinning).not.toHaveBeenCalled();
+    expect(captureOpenshell).toHaveBeenCalledWith(
+      [
+        "inference",
+        "set",
+        "-g",
+        "nemoclaw",
+        "--provider",
+        "compatible-anthropic-endpoint",
+        "--model",
+        "mock-anthropic-model",
+        "--no-verify",
+      ],
+      expect.objectContaining({ ignoreError: true }),
+    );
+    expect(deps.calls.probeSandboxRoute).toHaveBeenCalledWith({
+      sandboxName: "alpha",
+      provider: "compatible-anthropic-endpoint",
+      model: "mock-anthropic-model",
+      preferredInferenceApi: "anthropic-messages",
+    });
+    expect(deps.calls.probeSandboxRoute.mock.invocationCallOrder[0]).toBeLessThan(
+      deps.calls.updateSandbox.mock.invocationCallOrder[0],
+    );
+  });
+
+  it.each([
+    [
+      "returns a rejection",
+      () => ({
+        ok: false,
+        detail: "sandbox inference invocation probe exited with status 7",
+        httpStatus: null,
+      }),
+      /Sandbox-side verification rejected.*previous OpenShell inference selection was restored/s,
+    ],
+    [
+      "throws",
+      () => {
+        throw new Error("sandbox dial failed");
+      },
+      /sandbox inference invocation probe was unavailable: sandbox dial failed.*previous OpenShell inference selection was restored/s,
+    ],
+  ])("restores the prior route when sandbox-only provider verification %s", async (_failureMode, probeSandboxRoute, expectedError) => {
+    const captureOpenshell = createCompatibleProviderCapture({
+      name: "compatible-anthropic-endpoint",
+      type: "anthropic",
+      credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY",
+      configKey: "ANTHROPIC_BASE_URL",
+      initiallyPresent: false,
+    });
+    const deps = createDeps({
+      config: { agents: { defaults: { model: { primary: "inference/old-model" } } } },
+      entry: {
+        name: "alpha",
+        agent: "openclaw",
+        provider: "nvidia-prod",
+        model: "old-model",
+      },
+      session: baseSession({ provider: "nvidia-prod", model: "old-model" }),
+      captureOpenshell,
+      probeSandboxRoute,
+    });
+
+    await expect(
+      runInferenceSet(
+        {
+          provider: "compatible-anthropic-endpoint",
+          model: "mock-anthropic-model",
+          endpointUrl: "http://host.openshell.internal:18767/",
+          credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY",
+          inferenceApi: "anthropic-messages",
+        },
+        deps,
+      ),
+    ).rejects.toThrow(expectedError);
+
+    expect(
+      captureOpenshell.mock.calls
+        .filter(([args]) => args[0] === "inference" && args[1] === "set")
+        .map(([args]) => args),
+    ).toEqual([
+      [
+        "inference",
+        "set",
+        "-g",
+        "nemoclaw",
+        "--provider",
+        "compatible-anthropic-endpoint",
+        "--model",
+        "mock-anthropic-model",
+        "--no-verify",
+      ],
+      [
+        "inference",
+        "set",
+        "-g",
+        "nemoclaw",
+        "--provider",
+        "nvidia-prod",
+        "--model",
+        "old-model",
+        "--no-verify",
+      ],
+    ]);
+    expect(
+      captureOpenshell.mock.calls.some(
+        ([args]) => args[0] === "provider" && args[1] === "delete",
+      ),
+    ).toBe(true);
+    expect(deps.calls.updateSandbox).not.toHaveBeenCalled();
+    expect(deps.calls.writeSandboxConfig).not.toHaveBeenCalled();
+  });
+
+  it("preserves redacted probe diagnostics when restoring the prior route fails", async () => {
+    const providerCapture = createCompatibleProviderCapture({
+      name: "compatible-anthropic-endpoint",
+      type: "anthropic",
+      credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY",
+      configKey: "ANTHROPIC_BASE_URL",
+      initiallyPresent: false,
+    });
+    const inferenceSetResults = [
+      null,
+      {
+        status: 19,
+        output: "restore rejected",
+        stdout: "",
+        stderr: "restore rejected",
+      },
+    ];
+    let inferenceSetCalls = 0;
+    const captureOpenshell = vi.fn((args: string[]) => {
+      switch (`${args[0]}:${args[1]}`) {
+        case "inference:set":
+          return inferenceSetResults[inferenceSetCalls++] ?? providerCapture(args);
+        default:
+          return providerCapture(args);
+      }
+    });
+    const deps = createDeps({
+      config: { agents: { defaults: { model: { primary: "inference/old-model" } } } },
+      entry: {
+        name: "alpha",
+        agent: "openclaw",
+        provider: "nvidia-prod",
+        model: "old-model",
+      },
+      session: baseSession({ provider: "nvidia-prod", model: "old-model" }),
+      captureOpenshell,
+      probeSandboxRoute: () => {
+        throw new Error("sandbox dial failed; NVIDIA_API_KEY=nvapi-secret-value");
+      },
+    });
+
+    let failure: unknown;
+    try {
+      await runInferenceSet(
+        {
+          provider: "compatible-anthropic-endpoint",
+          model: "mock-anthropic-model",
+          endpointUrl: "http://host.openshell.internal:18767/",
+          credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY",
+          inferenceApi: "anthropic-messages",
+        },
+        deps,
+      );
+    } catch (error) {
+      failure = error;
+    }
+
+    expect(failure).toBeInstanceOf(Error);
+    const failureMessage = (failure as Error).message;
+    expect(failureMessage).toContain(
+      "sandbox inference invocation probe was unavailable: sandbox dial failed",
+    );
+    expect(failureMessage).toContain("NVIDIA_API_KEY=<REDACTED>");
+    expect(failureMessage).not.toContain("nvapi-secret-value");
+    expect(failureMessage).toMatch(
+      /Failed to restore the previous OpenShell inference selection.*status 19.*Re-run onboarding/s,
+    );
+    expect(
+      deps.calls.captureOpenshell.mock.calls
+        .filter(([args]) => args[0] === "inference" && args[1] === "set")
+        .map(([args]) => args),
+    ).toEqual([
+      [
+        "inference",
+        "set",
+        "-g",
+        "nemoclaw",
+        "--provider",
+        "compatible-anthropic-endpoint",
+        "--model",
+        "mock-anthropic-model",
+        "--no-verify",
+      ],
+      [
+        "inference",
+        "set",
+        "-g",
+        "nemoclaw",
+        "--provider",
+        "nvidia-prod",
+        "--model",
+        "old-model",
+        "--no-verify",
+      ],
+    ]);
+    expect(
+      deps.calls.captureOpenshell.mock.calls.some(
+        ([args]) => args[0] === "provider" && args[1] === "delete",
+      ),
+    ).toBe(false);
+    expect(deps.calls.updateSandbox).not.toHaveBeenCalled();
+    expect(deps.calls.writeSandboxConfig).not.toHaveBeenCalled();
   });
 
   for (const provider of ["compatible-endpoint", "compatible-anthropic-endpoint"]) {

src/lib/actions/inference-set-provider.ts

@@ -20,10 +20,31 @@ import {
   openshellReportsProviderNotFound,
 } from "./inference-set-error";
 import type { InferenceSetProviderBinding } from "./inference-set-route-containment";
+import type {
+  SandboxInferenceInvocationInput,
+  SandboxInferenceInvocationResult,
+} from "./sandbox/inference-invocation-probe";
 
 export type { RuntimeProviderBundleRegistry };
 export { RuntimeProviderSelectionError };
 
+export type InferenceSetSandboxRouteProbe = (
+  input: SandboxInferenceInvocationInput,
+) => SandboxInferenceInvocationResult;
+
+export function probeInferenceSetSandboxRoute(
+  input: SandboxInferenceInvocationInput,
+): SandboxInferenceInvocationResult {
+  const probe: typeof import("./sandbox/inference-invocation-probe") = require(
+    "./sandbox/inference-invocation-probe",
+  );
+  return probe.probeSandboxInferenceInvocation(
+    input,
+    {},
+    probe.READINESS_INFERENCE_INVOCATION_TIMEOUT_MS,
+  );
+}
+
 export function requireInferenceSetRuntimeAuthority(
   entry: SandboxEntry,
   providers: RuntimeProviderBundleRegistry = CURRENT_RUNTIME_PROVIDER_BUNDLES,

src/lib/actions/inference-set-route-containment.ts

@@ -73,6 +73,13 @@ export interface HttpsPinProviderBinding extends InferenceSetProviderBinding {
   routeId: string;
 }
 
+/** OpenShell's host verifier cannot resolve routes exposed only on its sandbox bridge. */
+export function isSandboxBridgeProviderBinding(
+  binding: InferenceSetProviderBinding | null,
+): boolean {
+  return binding !== null && isAllowedOpenShellSandboxBridgeUrl(new URL(binding.baseUrl));
+}
+
 type EnsureHttpsPinAdapterRoute = (endpointUrl: string) => Promise<string>;
 
 export interface PreparedInferenceSetRoute {

src/lib/actions/inference-set.test-support.ts

@@ -135,6 +135,7 @@ export function createDeps(options: {
   resolveCredentialValue?: InferenceSetDeps["resolveCredentialValue"];
   ensureHttpsPinRuntimeAdapter?: EnsureHttpsPinRuntimeAdapterFn;
   revokeHttpsPinRuntimeAdapterRoute?: InferenceSetDeps["revokeHttpsPinRuntimeAdapterRoute"];
+  probeSandboxRoute?: InferenceSetDeps["probeSandboxRoute"];
   updateSandbox?: InferenceSetDeps["updateSandbox"];
   restartSandboxGateway?: InferenceSetDeps["restartSandboxGateway"];
   seedHermesDashboardConfigResult?: "converged" | "absent" | "failed";
@@ -158,6 +159,7 @@ export function createDeps(options: {
     resolveCredentialValue: ReturnType<typeof vi.fn>;
     ensureHttpsPinRuntimeAdapter: ReturnType<typeof vi.fn>;
     revokeHttpsPinRuntimeAdapterRoute: ReturnType<typeof vi.fn>;
+    probeSandboxRoute: ReturnType<typeof vi.fn>;
     restartSandboxGateway: ReturnType<typeof vi.fn>;
     withGatewayRouteMutationLock: ReturnType<typeof vi.fn>;
   };
@@ -218,6 +220,7 @@ export function createDeps(options: {
     revokeHttpsPinRuntimeAdapterRoute: vi.fn(
       options.revokeHttpsPinRuntimeAdapterRoute ?? (async () => true),
     ),
+    probeSandboxRoute: vi.fn(options.probeSandboxRoute ?? (() => ({ ok: true }) as const)),
     restartSandboxGateway: vi.fn(
       options.restartSandboxGateway ??
         ((): ReturnType<InferenceSetDeps["restartSandboxGateway"]> => ({
@@ -262,6 +265,7 @@ export function createDeps(options: {
       calls.ensureHttpsPinRuntimeAdapter as unknown as EnsureHttpsPinRuntimeAdapterFn,
     revokeHttpsPinRuntimeAdapterRoute:
       calls.revokeHttpsPinRuntimeAdapterRoute as InferenceSetDeps["revokeHttpsPinRuntimeAdapterRoute"],
+    probeSandboxRoute: calls.probeSandboxRoute as InferenceSetDeps["probeSandboxRoute"],
     withGatewayRouteMutationLock:
       calls.withGatewayRouteMutationLock as InferenceSetDeps["withGatewayRouteMutationLock"],
     restartSandboxGateway: calls.restartSandboxGateway,