fix: let OpenShell routes converge

docs/inference/switch-providers.mdx

@@ -75,7 +75,7 @@ Use `--no-verify` only when OpenShell cannot verify the target provider at switc
 This flag does not bypass shared-gateway compatibility checks.
 When you explicitly supply a direct compatible endpoint at `http://host.openshell.internal:<port>`, NemoClaw skips OpenShell's host-side provider probe because that hostname resolves only inside the sandbox network.
 It then sends a validation request from the target sandbox before persisting the route in NemoClaw state; the request allows up to 16 output tokens.
-When the switch changes the API family and that request returns HTTP `400` or `404`, NemoClaw retries up to two times after delays of one and two seconds.
+When the switch changes the API family and that request returns HTTP `400` or `404`, NemoClaw retries up to two times after delays of 2 and 4 seconds.
 Each retry allows another 16 output tokens.
 Other failures are not retried.
 If that request fails, NemoClaw attempts to restore the previous OpenShell selection and remove a provider that this switch created.

src/lib/actions/inference-set-compatible-provider.test.ts

@@ -11,6 +11,111 @@ import {
   createDeps,
 } from "./inference-set.test-support";
 
+type ProbeSandboxRoute = NonNullable<
+  Parameters<typeof createDeps>[0]["probeSandboxRoute"]
+>;
+
+async function runRejectedCompatibleSwitchScenario(options: {
+  targetFamily: "openai" | "anthropic";
+  probeSandboxRoute: ProbeSandboxRoute;
+  expectedError: RegExp;
+}) {
+  const target =
+    options.targetFamily === "anthropic"
+      ? {
+          provider: "compatible-anthropic-endpoint",
+          model: "mock-anthropic-model",
+          credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY",
+          inferenceApi: "anthropic-messages" as const,
+          captureType: "anthropic" as const,
+          configKey: "ANTHROPIC_BASE_URL" as const,
+        }
+      : {
+          provider: "compatible-endpoint",
+          model: "mock-model",
+          credentialEnv: "COMPATIBLE_API_KEY",
+          inferenceApi: "openai-completions" as const,
+          captureType: "openai" as const,
+          configKey: "OPENAI_BASE_URL" as const,
+        };
+  const captureOpenshell = createCompatibleProviderCapture({
+    name: target.provider,
+    type: target.captureType,
+    credentialEnv: target.credentialEnv,
+    configKey: target.configKey,
+    initiallyPresent: false,
+  });
+  const probeSandboxRoute = vi.fn(options.probeSandboxRoute);
+  const deps = createDeps({
+    config: {
+      agents: { defaults: { model: { primary: "inference/old-model" } } },
+      models: { providers: { inference: { api: "openai-completions", models: [] } } },
+    },
+    entry: {
+      name: "alpha",
+      agent: "openclaw",
+      provider: "nvidia-prod",
+      model: "old-model",
+    },
+    session: baseSession({ provider: "nvidia-prod", model: "old-model" }),
+    captureOpenshell,
+    probeSandboxRoute,
+  });
+
+  await expect(
+    runInferenceSet(
+      {
+        provider: target.provider,
+        model: target.model,
+        endpointUrl: "http://host.openshell.internal:18767/",
+        credentialEnv: target.credentialEnv,
+        inferenceApi: target.inferenceApi,
+      },
+      deps,
+    ),
+  ).rejects.toThrow(options.expectedError);
+
+  expect(
+    captureOpenshell.mock.calls
+      .filter(([args]) => args[0] === "inference" && args[1] === "set")
+      .map(([args]) => args),
+  ).toEqual([
+    [
+      "inference",
+      "set",
+      "-g",
+      "nemoclaw",
+      "--provider",
+      target.provider,
+      "--model",
+      target.model,
+      "--no-verify",
+    ],
+    [
+      "inference",
+      "set",
+      "-g",
+      "nemoclaw",
+      "--provider",
+      "nvidia-prod",
+      "--model",
+      "old-model",
+      "--no-verify",
+    ],
+  ]);
+  expect(
+    captureOpenshell.mock.calls
+      .filter(([args]) => args[0] === "provider" && args[1] === "delete")
+      .map(([args]) => args),
+  ).toEqual([["provider", "delete", "-g", "nemoclaw", target.provider]]);
+  expect(deps.calls.updateSandbox).not.toHaveBeenCalled();
+  expect(deps.calls.writeSandboxConfig).not.toHaveBeenCalled();
+  expect(deps.calls.updateSession).not.toHaveBeenCalled();
+  expect(deps.getSession()).toMatchObject({ provider: "nvidia-prod", model: "old-model" });
+
+  return { deps, probeSandboxRoute };
+}
+
 describe("runInferenceSet compatible providers", () => {
   afterEach(() => vi.unstubAllEnvs());
 
@@ -668,7 +773,7 @@ describe("runInferenceSet compatible providers", () => {
     );
   });
 
-  it("waits for a changed API family to replace the previous sandbox route", async () => {
+  it("waits for a changed API family to replace the previous sandbox route (#9467)", async () => {
     const captureOpenshell = createCompatibleProviderCapture({
       name: "compatible-anthropic-endpoint",
       type: "anthropic",
@@ -716,158 +821,80 @@ describe("runInferenceSet compatible providers", () => {
     );
 
     expect(probeSandboxRoute).toHaveBeenCalledTimes(2);
-    expect(deps.calls.sleep).toHaveBeenCalledWith(1_000);
+    expect(deps.calls.sleep).toHaveBeenCalledWith(2_000);
+    expect(deps.calls.log).toHaveBeenCalledWith(
+      "  Waiting 2s for OpenShell route convergence after HTTP 400 (probe 1/3)...",
+    );
     expect(deps.calls.updateSandbox).toHaveBeenCalled();
   });
 
-  it("restores the prior route after changed-family convergence retries are exhausted", async () => {
-    const captureOpenshell = createCompatibleProviderCapture({
-      name: "compatible-anthropic-endpoint",
-      type: "anthropic",
-      credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY",
-      configKey: "ANTHROPIC_BASE_URL",
-      initiallyPresent: false,
-    });
-    const probeSandboxRoute = vi
-      .fn()
-      .mockReturnValueOnce({
-        ok: false,
-        detail: "sandbox inference invocation probe returned HTTP 400",
-        httpStatus: 400,
-      })
-      .mockReturnValueOnce({
-        ok: false,
-        detail: "sandbox inference invocation probe returned HTTP 404",
-        httpStatus: 404,
-      })
-      .mockReturnValueOnce({
-        ok: false,
-        detail: "sandbox inference invocation probe returned HTTP 400",
-        httpStatus: 400,
-      });
-    const deps = createDeps({
-      config: {
-        agents: { defaults: { model: { primary: "inference/old-model" } } },
-        models: { providers: { inference: { api: "openai-completions", models: [] } } },
-      },
-      entry: {
-        name: "alpha",
-        agent: "openclaw",
-        provider: "nvidia-prod",
-        model: "old-model",
-      },
-      session: baseSession({ provider: "nvidia-prod", model: "old-model" }),
-      captureOpenshell,
-      probeSandboxRoute,
+  it("restores the prior route after changed-family convergence retries are exhausted (#9467)", async () => {
+    const { deps, probeSandboxRoute } = await runRejectedCompatibleSwitchScenario({
+      targetFamily: "anthropic",
+      probeSandboxRoute: vi
+        .fn()
+        .mockReturnValueOnce({
+          ok: false,
+          detail: "sandbox inference invocation probe returned HTTP 400",
+          httpStatus: 400,
+        })
+        .mockReturnValueOnce({
+          ok: false,
+          detail: "sandbox inference invocation probe returned HTTP 404",
+          httpStatus: 404,
+        })
+        .mockReturnValueOnce({
+          ok: false,
+          detail: "sandbox inference invocation probe returned HTTP 400",
+          httpStatus: 400,
+        }),
+      expectedError:
+        /Sandbox-side verification rejected.*previous OpenShell inference selection was restored/s,
     });
 
-    await expect(
-      runInferenceSet(
-        {
-          provider: "compatible-anthropic-endpoint",
-          model: "mock-anthropic-model",
-          endpointUrl: "http://host.openshell.internal:18767/",
-          credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY",
-          inferenceApi: "anthropic-messages",
-        },
-        deps,
-      ),
-    ).rejects.toThrow(
-      /Sandbox-side verification rejected.*previous OpenShell inference selection was restored/s,
+    expect(probeSandboxRoute).toHaveBeenCalledTimes(3);
+    expect(deps.calls.sleep.mock.calls).toEqual([[2_000], [4_000]]);
+    expect(deps.calls.log.mock.calls).toEqual(
+      expect.arrayContaining([
+        ["  Waiting 2s for OpenShell route convergence after HTTP 400 (probe 1/3)..."],
+        ["  Waiting 4s for OpenShell route convergence after HTTP 404 (probe 2/3)..."],
+      ]),
     );
+  });
 
-    expect(probeSandboxRoute).toHaveBeenCalledTimes(3);
-    expect(deps.calls.sleep.mock.calls).toEqual([[1_000], [2_000]]);
-    expect(
-      captureOpenshell.mock.calls
-        .filter(([args]) => args[0] === "inference" && args[1] === "set")
-        .map(([args]) => args),
-    ).toEqual([
-      [
-        "inference",
-        "set",
-        "-g",
-        "nemoclaw",
-        "--provider",
-        "compatible-anthropic-endpoint",
-        "--model",
-        "mock-anthropic-model",
-        "--no-verify",
-      ],
-      [
-        "inference",
-        "set",
-        "-g",
-        "nemoclaw",
-        "--provider",
-        "nvidia-prod",
-        "--model",
-        "old-model",
-        "--no-verify",
-      ],
-    ]);
-    expect(
-      captureOpenshell.mock.calls
-        .filter(([args]) => args[0] === "provider" && args[1] === "delete")
-        .map(([args]) => args),
-    ).toEqual([
-      [
-        "provider",
-        "delete",
-        "-g",
-        "nemoclaw",
-        "compatible-anthropic-endpoint",
-      ],
-    ]);
-    expect(deps.calls.updateSandbox).not.toHaveBeenCalled();
-    expect(deps.calls.writeSandboxConfig).not.toHaveBeenCalled();
+  it.each([
+    ["authentication", 401],
+    ["server", 500],
+  ])("does not retry a changed-family %s failure (#9467)", async (_failureClass, httpStatus) => {
+    const { deps, probeSandboxRoute } = await runRejectedCompatibleSwitchScenario({
+      targetFamily: "anthropic",
+      probeSandboxRoute: () => ({
+        ok: false as const,
+        detail: `sandbox inference invocation probe returned HTTP ${httpStatus}`,
+        httpStatus,
+      }),
+      expectedError:
+        /Sandbox-side verification rejected.*previous OpenShell inference selection was restored/s,
+    });
+
+    expect(probeSandboxRoute).toHaveBeenCalledOnce();
+    expect(deps.calls.sleep).not.toHaveBeenCalled();
+    expect(deps.calls.log).not.toHaveBeenCalledWith(expect.stringContaining("route convergence"));
   });
 
   it("does not retry a target rejection when the API family did not change", async () => {
-    const captureOpenshell = createCompatibleProviderCapture({
-      name: "compatible-endpoint",
-      type: "openai",
-      credentialEnv: "COMPATIBLE_API_KEY",
-      configKey: "OPENAI_BASE_URL",
-      initiallyPresent: false,
-    });
-    const probeSandboxRoute = vi.fn(() => ({
-      ok: false as const,
-      detail: "sandbox inference invocation probe returned HTTP 400",
-      httpStatus: 400,
-    }));
-    const deps = createDeps({
-      config: {
-        agents: { defaults: { model: { primary: "inference/old-model" } } },
-        models: { providers: { inference: { api: "openai-completions", models: [] } } },
-      },
-      entry: {
-        name: "alpha",
-        agent: "openclaw",
-        provider: "nvidia-prod",
-        model: "old-model",
-      },
-      session: baseSession({ provider: "nvidia-prod", model: "old-model" }),
-      captureOpenshell,
-      probeSandboxRoute,
+    const { deps, probeSandboxRoute } = await runRejectedCompatibleSwitchScenario({
+      targetFamily: "openai",
+      probeSandboxRoute: () => ({
+        ok: false as const,
+        detail: "sandbox inference invocation probe returned HTTP 400",
+        httpStatus: 400,
+      }),
+      expectedError: /Sandbox-side verification rejected/,
     });
 
-    await expect(
-      runInferenceSet(
-        {
-          provider: "compatible-endpoint",
-          model: "mock-model",
-          endpointUrl: "http://host.openshell.internal:18767/",
-          credentialEnv: "COMPATIBLE_API_KEY",
-          inferenceApi: "openai-completions",
-        },
-        deps,
-      ),
-    ).rejects.toThrow(/Sandbox-side verification rejected/);
-
     expect(probeSandboxRoute).toHaveBeenCalledOnce();
     expect(deps.calls.sleep).not.toHaveBeenCalled();
-    expect(deps.calls.updateSandbox).not.toHaveBeenCalled();
   });
 
   it.each([

src/lib/actions/inference-set-provider.ts

@@ -33,7 +33,9 @@ export type InferenceSetSandboxRouteProbe = (
   input: SandboxInferenceInvocationInput,
 ) => SandboxInferenceInvocationResult;
 
-const ROUTE_FAMILY_CONVERGENCE_RETRY_DELAYS_MS = [1_000, 2_000] as const;
+// OpenShell 0.0.101 refreshes the sandbox route cache every five seconds.
+// The final probe runs after six seconds so one full refresh can occur.
+const ROUTE_FAMILY_CONVERGENCE_RETRY_DELAYS_MS = [2_000, 4_000] as const;
 
 export function sleepInferenceSetRouteConvergence(milliseconds: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, milliseconds));
@@ -61,6 +63,11 @@ export async function probeInferenceSetSandboxRouteUntilConverged(
   deps: {
     probe: InferenceSetSandboxRouteProbe;
     sleep: (milliseconds: number) => Promise<void>;
+    onRetry?: (
+      result: SandboxInferenceInvocationResult,
+      delayMs: number,
+      attempt: number,
+    ) => void | Promise<void>;
   } = {
     probe: probeInferenceSetSandboxRoute,
     sleep: sleepInferenceSetRouteConvergence,
@@ -73,6 +80,7 @@ export async function probeInferenceSetSandboxRouteUntilConverged(
       !inferenceApiChanged ||
       (result.httpStatus !== 400 && result.httpStatus !== 404),
     retryDelaysMs: ROUTE_FAMILY_CONVERGENCE_RETRY_DELAYS_MS,
+    onRetry: deps.onRetry,
     sleep: deps.sleep,
   });
 }

src/lib/actions/inference-set.ts

@@ -1171,6 +1171,12 @@ async function runInferenceSetWithoutHostLock(
           {
             probe: deps.probeSandboxRoute,
             sleep: deps.sleep,
+            onRetry: (result, delayMs, attempt) => {
+              if (result.ok) return;
+              deps.log(
+                `  Waiting ${delayMs / 1_000}s for OpenShell route convergence after HTTP ${result.httpStatus} (probe ${attempt}/3)...`,
+              );
+            },
           },
         );
       } catch (probeError) {