fix(e2e): repair main runtime regressions

docs/inference/switch-providers.mdx

@@ -73,6 +73,11 @@ Run the rebuild before relying on the running agent.
 
 Use `--no-verify` only when OpenShell cannot verify the target provider at switch time and you have already confirmed its provider and credential.
 This flag does not bypass shared-gateway compatibility checks.
+For a validated compatible-provider binding at `http://host.openshell.internal:<port>`, NemoClaw skips OpenShell's host-side provider probe because that hostname resolves only inside the sandbox network.
+It then sends one minimal request from the target sandbox before persisting the route in NemoClaw state; the request allows up to 16 output tokens.
+If that request fails, NemoClaw attempts to restore the previous OpenShell selection and remove a provider that this switch created.
+If the error reports that rollback could not complete, re-run onboarding before using or retrying the route.
+Endpoint-shape and shared-gateway compatibility checks still apply.
 
 </AgentOnly>
 

docs/reference/commands.mdx

@@ -4100,6 +4100,11 @@ If the in-sandbox config sync fails, NemoClaw keeps the gateway and registry ali
 
 Supported provider names are `nvidia-prod`, `nvidia-nim`, `nvidia-router`, `openai-api`, `anthropic-prod`, `compatible-anthropic-endpoint`, `gemini-api`, `compatible-endpoint`, `hermes-provider`, `ollama-local`, and `vllm-local`.
 Use `--no-verify` only when OpenShell cannot verify the provider at switch time but you have already confirmed the provider and credential.
+For a validated compatible-provider binding at `http://host.openshell.internal:<port>`, NemoClaw skips OpenShell's host-side provider probe because that hostname resolves only inside the sandbox network.
+Before it persists the route in the NemoClaw registry or agent config, the command sends one request from the target sandbox with a 16-token output limit.
+If that request fails, the command attempts to restore the previous OpenShell selection and remove a provider that this switch created.
+If the error reports that rollback could not complete, re-run onboarding before using or retrying the route.
+Endpoint-shape and shared-gateway compatibility checks still apply.
 When switching to `compatible-endpoint` or `compatible-anthropic-endpoint` from a different provider family, pass `--endpoint-url` with the trusted custom provider URL and, except for the Hermes case below, `--inference-api` with its API family so NemoClaw can persist a complete route identity for rebuild and shared-gateway checks.
 For a Hermes `compatible-anthropic-endpoint` target, `--inference-api` may be omitted because NemoClaw deterministically selects `openai-completions`; an explicit different API family is rejected.
 NemoClaw rejects loopback, link-local, private, and internal endpoint addresses, including public hostnames that resolve to a private address.

src/lib/actions/inference-set-compatible-provider.test.ts

@@ -616,7 +616,6 @@ describe("runInferenceSet compatible providers", () => {
       {
         provider: "compatible-anthropic-endpoint",
         model: "mock-anthropic-model",
-        noVerify: true,
         endpointUrl: "http://host.openshell.internal:18767/",
         credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY",
         inferenceApi: "anthropic-messages",
@@ -644,6 +643,106 @@ describe("runInferenceSet compatible providers", () => {
       nimContainer: null,
     });
     expect(deps.calls.rewriteConfigUrlsWithDnsPinning).not.toHaveBeenCalled();
+    expect(captureOpenshell).toHaveBeenCalledWith(
+      [
+        "inference",
+        "set",
+        "-g",
+        "nemoclaw",
+        "--provider",
+        "compatible-anthropic-endpoint",
+        "--model",
+        "mock-anthropic-model",
+        "--no-verify",
+      ],
+      expect.objectContaining({ ignoreError: true }),
+    );
+    expect(deps.calls.probeSandboxRoute).toHaveBeenCalledWith({
+      sandboxName: "alpha",
+      provider: "compatible-anthropic-endpoint",
+      model: "mock-anthropic-model",
+      preferredInferenceApi: "anthropic-messages",
+    });
+    expect(deps.calls.probeSandboxRoute.mock.invocationCallOrder[0]).toBeLessThan(
+      deps.calls.updateSandbox.mock.invocationCallOrder[0],
+    );
+  });
+
+  it("restores the prior route when sandbox-only provider verification fails", async () => {
+    const captureOpenshell = createCompatibleProviderCapture({
+      name: "compatible-anthropic-endpoint",
+      type: "anthropic",
+      credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY",
+      configKey: "ANTHROPIC_BASE_URL",
+      initiallyPresent: false,
+    });
+    const deps = createDeps({
+      config: { agents: { defaults: { model: { primary: "inference/old-model" } } } },
+      entry: {
+        name: "alpha",
+        agent: "openclaw",
+        provider: "nvidia-prod",
+        model: "old-model",
+      },
+      session: baseSession({ provider: "nvidia-prod", model: "old-model" }),
+      captureOpenshell,
+      probeSandboxRoute: () => ({
+        ok: false,
+        detail: "sandbox inference invocation probe exited with status 7",
+        httpStatus: null,
+      }),
+    });
+
+    await expect(
+      runInferenceSet(
+        {
+          provider: "compatible-anthropic-endpoint",
+          model: "mock-anthropic-model",
+          endpointUrl: "http://host.openshell.internal:18767/",
+          credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY",
+          inferenceApi: "anthropic-messages",
+        },
+        deps,
+      ),
+    ).rejects.toThrow(
+      /Sandbox-side verification rejected.*previous OpenShell inference selection was restored/s,
+    );
+
+    expect(
+      captureOpenshell.mock.calls
+        .filter(([args]) => args[0] === "inference" && args[1] === "set")
+        .map(([args]) => args),
+    ).toEqual([
+      [
+        "inference",
+        "set",
+        "-g",
+        "nemoclaw",
+        "--provider",
+        "compatible-anthropic-endpoint",
+        "--model",
+        "mock-anthropic-model",
+        "--no-verify",
+      ],
+      [
+        "inference",
+        "set",
+        "-g",
+        "nemoclaw",
+        "--provider",
+        "nvidia-prod",
+        "--model",
+        "old-model",
+        "--no-verify",
+      ],
+    ]);
+    expect(
+      captureOpenshell.mock.calls.some(
+        ([args]) => args[0] === "provider" && args[1] === "delete",
+      ),
+    ).toBe(true);
+    expect(deps.calls.updateSandbox).not.toHaveBeenCalled();
+    expect(deps.calls.writeSandboxConfig).not.toHaveBeenCalled();
   });
 
   for (const provider of ["compatible-endpoint", "compatible-anthropic-endpoint"]) {

src/lib/actions/inference-set-provider.ts

@@ -20,10 +20,31 @@ import {
   openshellReportsProviderNotFound,
 } from "./inference-set-error";
 import type { InferenceSetProviderBinding } from "./inference-set-route-containment";
+import type {
+  SandboxInferenceInvocationInput,
+  SandboxInferenceInvocationResult,
+} from "./sandbox/inference-invocation-probe";
 
 export type { RuntimeProviderBundleRegistry };
 export { RuntimeProviderSelectionError };
 
+export type InferenceSetSandboxRouteProbe = (
+  input: SandboxInferenceInvocationInput,
+) => SandboxInferenceInvocationResult;
+
+export function probeInferenceSetSandboxRoute(
+  input: SandboxInferenceInvocationInput,
+): SandboxInferenceInvocationResult {
+  const probe: typeof import("./sandbox/inference-invocation-probe") = require(
+    "./sandbox/inference-invocation-probe",
+  );
+  return probe.probeSandboxInferenceInvocation(
+    input,
+    {},
+    probe.READINESS_INFERENCE_INVOCATION_TIMEOUT_MS,
+  );
+}
+
 export function requireInferenceSetRuntimeAuthority(
   entry: SandboxEntry,
   providers: RuntimeProviderBundleRegistry = CURRENT_RUNTIME_PROVIDER_BUNDLES,

src/lib/actions/inference-set-route-containment.ts

@@ -73,6 +73,13 @@ export interface HttpsPinProviderBinding extends InferenceSetProviderBinding {
   routeId: string;
 }
 
+/** OpenShell's host verifier cannot resolve routes exposed only on its sandbox bridge. */
+export function isSandboxBridgeProviderBinding(
+  binding: InferenceSetProviderBinding | null,
+): boolean {
+  return binding !== null && isAllowedOpenShellSandboxBridgeUrl(new URL(binding.baseUrl));
+}
+
 type EnsureHttpsPinAdapterRoute = (endpointUrl: string) => Promise<string>;
 
 export interface PreparedInferenceSetRoute {

src/lib/actions/inference-set.test-support.ts

@@ -135,6 +135,7 @@ export function createDeps(options: {
   resolveCredentialValue?: InferenceSetDeps["resolveCredentialValue"];
   ensureHttpsPinRuntimeAdapter?: EnsureHttpsPinRuntimeAdapterFn;
   revokeHttpsPinRuntimeAdapterRoute?: InferenceSetDeps["revokeHttpsPinRuntimeAdapterRoute"];
+  probeSandboxRoute?: InferenceSetDeps["probeSandboxRoute"];
   updateSandbox?: InferenceSetDeps["updateSandbox"];
   restartSandboxGateway?: InferenceSetDeps["restartSandboxGateway"];
   seedHermesDashboardConfigResult?: "converged" | "absent" | "failed";
@@ -158,6 +159,7 @@ export function createDeps(options: {
     resolveCredentialValue: ReturnType<typeof vi.fn>;
     ensureHttpsPinRuntimeAdapter: ReturnType<typeof vi.fn>;
     revokeHttpsPinRuntimeAdapterRoute: ReturnType<typeof vi.fn>;
+    probeSandboxRoute: ReturnType<typeof vi.fn>;
     restartSandboxGateway: ReturnType<typeof vi.fn>;
     withGatewayRouteMutationLock: ReturnType<typeof vi.fn>;
   };
@@ -218,6 +220,7 @@ export function createDeps(options: {
     revokeHttpsPinRuntimeAdapterRoute: vi.fn(
       options.revokeHttpsPinRuntimeAdapterRoute ?? (async () => true),
     ),
+    probeSandboxRoute: vi.fn(options.probeSandboxRoute ?? (() => ({ ok: true }) as const)),
     restartSandboxGateway: vi.fn(
       options.restartSandboxGateway ??
         ((): ReturnType<InferenceSetDeps["restartSandboxGateway"]> => ({
@@ -262,6 +265,7 @@ export function createDeps(options: {
       calls.ensureHttpsPinRuntimeAdapter as unknown as EnsureHttpsPinRuntimeAdapterFn,
     revokeHttpsPinRuntimeAdapterRoute:
       calls.revokeHttpsPinRuntimeAdapterRoute as InferenceSetDeps["revokeHttpsPinRuntimeAdapterRoute"],
+    probeSandboxRoute: calls.probeSandboxRoute as InferenceSetDeps["probeSandboxRoute"],
     withGatewayRouteMutationLock:
       calls.withGatewayRouteMutationLock as InferenceSetDeps["withGatewayRouteMutationLock"],
     restartSandboxGateway: calls.restartSandboxGateway,

src/lib/actions/inference-set.ts

@@ -71,7 +71,9 @@ import {
   readPreviousOpenClawInferenceApi,
 } from "./inference-set-gateway-restart";
 import {
+  type InferenceSetSandboxRouteProbe,
   prepareInferenceSetProviderBinding,
+  probeInferenceSetSandboxRoute,
   type RuntimeProviderBundleRegistry,
   RuntimeProviderSelectionError,
   requireInferenceSetRuntimeAuthority,
@@ -85,6 +87,7 @@ import {
   type EnsureHttpsPinRuntimeAdapterFn,
   finalizeInferenceSetRoute,
   type InferenceSetProviderBinding,
+  isSandboxBridgeProviderBinding,
   prepareInferenceSetRoute,
   type RegistryInferenceMetadata,
 } from "./inference-set-route-containment";
@@ -163,6 +166,7 @@ export interface InferenceSetDeps extends InferenceGatewayRestartDeps {
   resolveCredentialValue: (credentialEnv: string) => string;
   ensureHttpsPinRuntimeAdapter: EnsureHttpsPinRuntimeAdapterFn;
   revokeHttpsPinRuntimeAdapterRoute: (routeId: string) => Promise<boolean>;
+  probeSandboxRoute: InferenceSetSandboxRouteProbe;
   withGatewayRouteMutationLock: typeof withGatewayRouteMutationLock;
 }
 
@@ -268,6 +272,7 @@ function defaultDeps(): InferenceSetDeps {
     resolveCredentialValue: (credentialEnv) => process.env[credentialEnv] ?? "",
     ensureHttpsPinRuntimeAdapter,
     revokeHttpsPinRuntimeAdapterRoute,
+    probeSandboxRoute: probeInferenceSetSandboxRoute,
     withGatewayRouteMutationLock,
     restartSandboxGateway: defaultInferenceGatewayRestart,
     isSandboxConfigMutable: (sandboxName) => {
@@ -936,10 +941,15 @@ async function runInferenceSetWithoutHostLock(
   // verify. Only a genuinely-unreachable host stack hard-fails here, before the
   // route is touched.
   let effectiveNoVerify = options.noVerify === true;
-  // The adapter origin resolves only from inside the sandbox network. The
-  // host-side OpenShell verifier cannot resolve host.openshell.internal, so
-  // adapter registration + local health are the verification boundary.
-  if (httpsPinProviderBinding) effectiveNoVerify = true;
+  const probeDirectSandboxBridge = isSandboxBridgeProviderBinding(directProviderBinding);
+  // Adapter routes and explicit custom routes on NemoClaw's sandbox bridge
+  // resolve only from inside the sandbox network. The host-side OpenShell
+  // verifier cannot resolve host.openshell.internal, so its result would be a
+  // guaranteed false negative. HTTPS-pin adapters retain their local-health
+  // verification; direct bridge routes are probed from the sandbox below.
+  if (httpsPinProviderBinding || probeDirectSandboxBridge) {
+    effectiveNoVerify = true;
+  }
   if (deps.isLocalInferenceProvider(provider)) {
     const localValidation = deps.validateLocalProvider(provider);
     if (localValidation.ok) {
@@ -996,11 +1006,43 @@ async function runInferenceSetWithoutHostLock(
   assertReasoningEffortRoute(reasoningEffortRequest, provider, preMutationInferenceApi);
   const previousProvider = typeof entry.provider === "string" ? entry.provider.trim() : "";
   const previousModel = typeof entry.model === "string" ? entry.model.trim() : "";
+  if (probeDirectSandboxBridge && (!previousProvider || !previousModel)) {
+    throw new InferenceSetError(
+      `Cannot verify the sandbox-only provider route because sandbox '${sandboxName}' does not record ` +
+        "the previous provider and model needed to restore its OpenShell inference selection.",
+      2,
+    );
+  }
 
   let appliedProvider = false;
   let appliedInferenceSelection = false;
   let restoredSelectionAfterProviderFailure = false;
   let providerMutation: ReturnType<typeof prepareInferenceSetProviderBinding> | null = null;
+  const restorePreviousInferenceSelection = (): string | null => {
+    let restoreResult: CaptureOpenshellResult;
+    try {
+      restoreResult = deps.captureOpenshell(
+        openshellInferenceSetArgs({
+          gatewayName: preparedRoute.gatewayName,
+          provider: previousProvider,
+          model: previousModel,
+          noVerify: true,
+        }),
+        {
+          ignoreError: true,
+          includeStreams: true,
+          maxBuffer: OPEN_SHELL_FAILURE_CAPTURE_MAX_BUFFER,
+        },
+      );
+    } catch {
+      return "the restore command could not be invoked";
+    }
+    if (restoreResult.status !== 0) {
+      return `the restore command exited with status ${restoreResult.status ?? "unknown"}`;
+    }
+    appliedInferenceSelection = false;
+    return null;
+  };
   try {
     const providerBinding = httpsPinProviderBinding ?? directProviderBinding;
     if (providerBinding) {
@@ -1081,28 +1123,15 @@ async function runInferenceSetWithoutHostLock(
           providerError instanceof Error ? providerError.message : String(providerError);
         const providerExitCode =
           providerError instanceof InferenceSetError ? providerError.exitCode : 1;
-        const restoreResult = deps.captureOpenshell(
-          openshellInferenceSetArgs({
-            gatewayName: preparedRoute.gatewayName,
-            provider: previousProvider,
-            model: previousModel,
-            noVerify: true,
-          }),
-          {
-            ignoreError: true,
-            includeStreams: true,
-            maxBuffer: OPEN_SHELL_FAILURE_CAPTURE_MAX_BUFFER,
-          },
-        );
-        if (restoreResult.status !== 0) {
+        const restoreFailure = restorePreviousInferenceSelection();
+        if (restoreFailure) {
           throw new InferenceSetError(
             `${providerDetail}\n  Failed to restore the previous OpenShell inference selection ` +
-              `'${previousProvider}' / '${previousModel}' (status ${restoreResult.status ?? "unknown"}). ` +
+              `'${previousProvider}' / '${previousModel}': ${restoreFailure}. ` +
               `The live selection and provider binding may be split; re-run onboarding before using this route.`,
             providerExitCode,
           );
         }
-        appliedInferenceSelection = false;
         restoredSelectionAfterProviderFailure = true;
         throw new InferenceSetError(
           `${providerDetail}\n  The previous OpenShell inference selection was restored to ` +
@@ -1113,6 +1142,38 @@ async function runInferenceSetWithoutHostLock(
       }
     }
 
+    if (probeDirectSandboxBridge) {
+      let probe: ReturnType<InferenceSetSandboxRouteProbe>;
+      try {
+        probe = deps.probeSandboxRoute({
+          sandboxName,
+          provider,
+          model,
+          preferredInferenceApi: preMutationInferenceApi,
+        });
+      } catch {
+        probe = {
+          ok: false,
+          detail: "sandbox inference invocation probe was unavailable",
+          httpStatus: null,
+        };
+      }
+      if (!probe.ok) {
+        const restoreFailure = restorePreviousInferenceSelection();
+        if (restoreFailure) {
+          throw new InferenceSetError(
+            `Sandbox-side verification rejected provider '${provider}' / '${model}': ${probe.detail}. ` +
+              `Failed to restore the previous OpenShell inference selection '${previousProvider}' / ` +
+              `'${previousModel}': ${restoreFailure}. Re-run onboarding before using this route.`,
+          );
+        }
+        throw new InferenceSetError(
+          `Sandbox-side verification rejected provider '${provider}' / '${model}': ${probe.detail}. ` +
+            `The previous OpenShell inference selection was restored to '${previousProvider}' / '${previousModel}'.`,
+        );
+      }
+    }
+
     // Write minimal registry state before any sandbox-facing config read so the
     // gateway and registry cannot split if the in-sandbox layer is unavailable.
     const registryFields = (preferredInferenceApi: string | null) =>