diff --git a/docs/inference/switch-providers.mdx b/docs/inference/switch-providers.mdx index afadf5ebec2..bd374af78bd 100644 --- a/docs/inference/switch-providers.mdx +++ b/docs/inference/switch-providers.mdx @@ -75,7 +75,7 @@ Use `--no-verify` only when OpenShell cannot verify the target provider at switc This flag does not bypass shared-gateway compatibility checks. When you explicitly supply a direct compatible endpoint at `http://host.openshell.internal:`, NemoClaw skips OpenShell's host-side provider probe because that hostname resolves only inside the sandbox network. It then sends a validation request from the target sandbox before persisting the route in NemoClaw state; the request allows up to 16 output tokens. -When the switch changes the API family and that request returns HTTP `400` or `404`, NemoClaw retries up to two times after delays of one and two seconds. +When the switch changes the API family and that request returns HTTP `400` or `404`, NemoClaw retries up to two times after delays of 2 and 4 seconds. Each retry allows another 16 output tokens. Other failures are not retried. If that request fails, NemoClaw attempts to restore the previous OpenShell selection and remove a provider that this switch created. diff --git a/src/lib/actions/inference-set-compatible-provider.test.ts b/src/lib/actions/inference-set-compatible-provider.test.ts index 81e005eaa6b..9bee0c5187b 100644 --- a/src/lib/actions/inference-set-compatible-provider.test.ts +++ b/src/lib/actions/inference-set-compatible-provider.test.ts @@ -11,6 +11,111 @@ import { createDeps, } from "./inference-set.test-support"; +type ProbeSandboxRoute = NonNullable< + Parameters[0]["probeSandboxRoute"] +>; + +async function runRejectedCompatibleSwitchScenario(options: { + targetFamily: "openai" | "anthropic"; + probeSandboxRoute: ProbeSandboxRoute; + expectedError: RegExp; +}) { + const target = + options.targetFamily === "anthropic" + ? { + provider: "compatible-anthropic-endpoint", + model: "mock-anthropic-model", + credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY", + inferenceApi: "anthropic-messages" as const, + captureType: "anthropic" as const, + configKey: "ANTHROPIC_BASE_URL" as const, + } + : { + provider: "compatible-endpoint", + model: "mock-model", + credentialEnv: "COMPATIBLE_API_KEY", + inferenceApi: "openai-completions" as const, + captureType: "openai" as const, + configKey: "OPENAI_BASE_URL" as const, + }; + const captureOpenshell = createCompatibleProviderCapture({ + name: target.provider, + type: target.captureType, + credentialEnv: target.credentialEnv, + configKey: target.configKey, + initiallyPresent: false, + }); + const probeSandboxRoute = vi.fn(options.probeSandboxRoute); + const deps = createDeps({ + config: { + agents: { defaults: { model: { primary: "inference/old-model" } } }, + models: { providers: { inference: { api: "openai-completions", models: [] } } }, + }, + entry: { + name: "alpha", + agent: "openclaw", + provider: "nvidia-prod", + model: "old-model", + }, + session: baseSession({ provider: "nvidia-prod", model: "old-model" }), + captureOpenshell, + probeSandboxRoute, + }); + + await expect( + runInferenceSet( + { + provider: target.provider, + model: target.model, + endpointUrl: "http://host.openshell.internal:18767/", + credentialEnv: target.credentialEnv, + inferenceApi: target.inferenceApi, + }, + deps, + ), + ).rejects.toThrow(options.expectedError); + + expect( + captureOpenshell.mock.calls + .filter(([args]) => args[0] === "inference" && args[1] === "set") + .map(([args]) => args), + ).toEqual([ + [ + "inference", + "set", + "-g", + "nemoclaw", + "--provider", + target.provider, + "--model", + target.model, + "--no-verify", + ], + [ + "inference", + "set", + "-g", + "nemoclaw", + "--provider", + "nvidia-prod", + "--model", + "old-model", + "--no-verify", + ], + ]); + expect( + captureOpenshell.mock.calls + .filter(([args]) => args[0] === "provider" && args[1] === "delete") + .map(([args]) => args), + ).toEqual([["provider", "delete", "-g", "nemoclaw", target.provider]]); + expect(deps.calls.updateSandbox).not.toHaveBeenCalled(); + expect(deps.calls.writeSandboxConfig).not.toHaveBeenCalled(); + expect(deps.calls.updateSession).not.toHaveBeenCalled(); + expect(deps.getSession()).toMatchObject({ provider: "nvidia-prod", model: "old-model" }); + + return { deps, probeSandboxRoute }; +} + describe("runInferenceSet compatible providers", () => { afterEach(() => vi.unstubAllEnvs()); @@ -668,7 +773,7 @@ describe("runInferenceSet compatible providers", () => { ); }); - it("waits for a changed API family to replace the previous sandbox route", async () => { + it("waits for a changed API family to replace the previous sandbox route (#9467)", async () => { const captureOpenshell = createCompatibleProviderCapture({ name: "compatible-anthropic-endpoint", type: "anthropic", @@ -716,158 +821,80 @@ describe("runInferenceSet compatible providers", () => { ); expect(probeSandboxRoute).toHaveBeenCalledTimes(2); - expect(deps.calls.sleep).toHaveBeenCalledWith(1_000); + expect(deps.calls.sleep).toHaveBeenCalledWith(2_000); + expect(deps.calls.log).toHaveBeenCalledWith( + " Waiting 2s for OpenShell route convergence after HTTP 400 (probe 1/3)...", + ); expect(deps.calls.updateSandbox).toHaveBeenCalled(); }); - it("restores the prior route after changed-family convergence retries are exhausted", async () => { - const captureOpenshell = createCompatibleProviderCapture({ - name: "compatible-anthropic-endpoint", - type: "anthropic", - credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY", - configKey: "ANTHROPIC_BASE_URL", - initiallyPresent: false, - }); - const probeSandboxRoute = vi - .fn() - .mockReturnValueOnce({ - ok: false, - detail: "sandbox inference invocation probe returned HTTP 400", - httpStatus: 400, - }) - .mockReturnValueOnce({ - ok: false, - detail: "sandbox inference invocation probe returned HTTP 404", - httpStatus: 404, - }) - .mockReturnValueOnce({ - ok: false, - detail: "sandbox inference invocation probe returned HTTP 400", - httpStatus: 400, - }); - const deps = createDeps({ - config: { - agents: { defaults: { model: { primary: "inference/old-model" } } }, - models: { providers: { inference: { api: "openai-completions", models: [] } } }, - }, - entry: { - name: "alpha", - agent: "openclaw", - provider: "nvidia-prod", - model: "old-model", - }, - session: baseSession({ provider: "nvidia-prod", model: "old-model" }), - captureOpenshell, - probeSandboxRoute, + it("restores the prior route after changed-family convergence retries are exhausted (#9467)", async () => { + const { deps, probeSandboxRoute } = await runRejectedCompatibleSwitchScenario({ + targetFamily: "anthropic", + probeSandboxRoute: vi + .fn() + .mockReturnValueOnce({ + ok: false, + detail: "sandbox inference invocation probe returned HTTP 400", + httpStatus: 400, + }) + .mockReturnValueOnce({ + ok: false, + detail: "sandbox inference invocation probe returned HTTP 404", + httpStatus: 404, + }) + .mockReturnValueOnce({ + ok: false, + detail: "sandbox inference invocation probe returned HTTP 400", + httpStatus: 400, + }), + expectedError: + /Sandbox-side verification rejected.*previous OpenShell inference selection was restored/s, }); - await expect( - runInferenceSet( - { - provider: "compatible-anthropic-endpoint", - model: "mock-anthropic-model", - endpointUrl: "http://host.openshell.internal:18767/", - credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY", - inferenceApi: "anthropic-messages", - }, - deps, - ), - ).rejects.toThrow( - /Sandbox-side verification rejected.*previous OpenShell inference selection was restored/s, + expect(probeSandboxRoute).toHaveBeenCalledTimes(3); + expect(deps.calls.sleep.mock.calls).toEqual([[2_000], [4_000]]); + expect(deps.calls.log.mock.calls).toEqual( + expect.arrayContaining([ + [" Waiting 2s for OpenShell route convergence after HTTP 400 (probe 1/3)..."], + [" Waiting 4s for OpenShell route convergence after HTTP 404 (probe 2/3)..."], + ]), ); + }); - expect(probeSandboxRoute).toHaveBeenCalledTimes(3); - expect(deps.calls.sleep.mock.calls).toEqual([[1_000], [2_000]]); - expect( - captureOpenshell.mock.calls - .filter(([args]) => args[0] === "inference" && args[1] === "set") - .map(([args]) => args), - ).toEqual([ - [ - "inference", - "set", - "-g", - "nemoclaw", - "--provider", - "compatible-anthropic-endpoint", - "--model", - "mock-anthropic-model", - "--no-verify", - ], - [ - "inference", - "set", - "-g", - "nemoclaw", - "--provider", - "nvidia-prod", - "--model", - "old-model", - "--no-verify", - ], - ]); - expect( - captureOpenshell.mock.calls - .filter(([args]) => args[0] === "provider" && args[1] === "delete") - .map(([args]) => args), - ).toEqual([ - [ - "provider", - "delete", - "-g", - "nemoclaw", - "compatible-anthropic-endpoint", - ], - ]); - expect(deps.calls.updateSandbox).not.toHaveBeenCalled(); - expect(deps.calls.writeSandboxConfig).not.toHaveBeenCalled(); + it.each([ + ["authentication", 401], + ["server", 500], + ])("does not retry a changed-family %s failure (#9467)", async (_failureClass, httpStatus) => { + const { deps, probeSandboxRoute } = await runRejectedCompatibleSwitchScenario({ + targetFamily: "anthropic", + probeSandboxRoute: () => ({ + ok: false as const, + detail: `sandbox inference invocation probe returned HTTP ${httpStatus}`, + httpStatus, + }), + expectedError: + /Sandbox-side verification rejected.*previous OpenShell inference selection was restored/s, + }); + + expect(probeSandboxRoute).toHaveBeenCalledOnce(); + expect(deps.calls.sleep).not.toHaveBeenCalled(); + expect(deps.calls.log).not.toHaveBeenCalledWith(expect.stringContaining("route convergence")); }); it("does not retry a target rejection when the API family did not change", async () => { - const captureOpenshell = createCompatibleProviderCapture({ - name: "compatible-endpoint", - type: "openai", - credentialEnv: "COMPATIBLE_API_KEY", - configKey: "OPENAI_BASE_URL", - initiallyPresent: false, - }); - const probeSandboxRoute = vi.fn(() => ({ - ok: false as const, - detail: "sandbox inference invocation probe returned HTTP 400", - httpStatus: 400, - })); - const deps = createDeps({ - config: { - agents: { defaults: { model: { primary: "inference/old-model" } } }, - models: { providers: { inference: { api: "openai-completions", models: [] } } }, - }, - entry: { - name: "alpha", - agent: "openclaw", - provider: "nvidia-prod", - model: "old-model", - }, - session: baseSession({ provider: "nvidia-prod", model: "old-model" }), - captureOpenshell, - probeSandboxRoute, + const { deps, probeSandboxRoute } = await runRejectedCompatibleSwitchScenario({ + targetFamily: "openai", + probeSandboxRoute: () => ({ + ok: false as const, + detail: "sandbox inference invocation probe returned HTTP 400", + httpStatus: 400, + }), + expectedError: /Sandbox-side verification rejected/, }); - await expect( - runInferenceSet( - { - provider: "compatible-endpoint", - model: "mock-model", - endpointUrl: "http://host.openshell.internal:18767/", - credentialEnv: "COMPATIBLE_API_KEY", - inferenceApi: "openai-completions", - }, - deps, - ), - ).rejects.toThrow(/Sandbox-side verification rejected/); - expect(probeSandboxRoute).toHaveBeenCalledOnce(); expect(deps.calls.sleep).not.toHaveBeenCalled(); - expect(deps.calls.updateSandbox).not.toHaveBeenCalled(); }); it.each([ diff --git a/src/lib/actions/inference-set-provider.ts b/src/lib/actions/inference-set-provider.ts index 1adb71f7f34..e4ac70a4396 100644 --- a/src/lib/actions/inference-set-provider.ts +++ b/src/lib/actions/inference-set-provider.ts @@ -33,7 +33,9 @@ export type InferenceSetSandboxRouteProbe = ( input: SandboxInferenceInvocationInput, ) => SandboxInferenceInvocationResult; -const ROUTE_FAMILY_CONVERGENCE_RETRY_DELAYS_MS = [1_000, 2_000] as const; +// OpenShell 0.0.101 refreshes the sandbox route cache every five seconds. +// The final probe runs after six seconds so one full refresh can occur. +const ROUTE_FAMILY_CONVERGENCE_RETRY_DELAYS_MS = [2_000, 4_000] as const; export function sleepInferenceSetRouteConvergence(milliseconds: number): Promise { return new Promise((resolve) => setTimeout(resolve, milliseconds)); @@ -61,6 +63,11 @@ export async function probeInferenceSetSandboxRouteUntilConverged( deps: { probe: InferenceSetSandboxRouteProbe; sleep: (milliseconds: number) => Promise; + onRetry?: ( + result: SandboxInferenceInvocationResult, + delayMs: number, + attempt: number, + ) => void | Promise; } = { probe: probeInferenceSetSandboxRoute, sleep: sleepInferenceSetRouteConvergence, @@ -73,6 +80,7 @@ export async function probeInferenceSetSandboxRouteUntilConverged( !inferenceApiChanged || (result.httpStatus !== 400 && result.httpStatus !== 404), retryDelaysMs: ROUTE_FAMILY_CONVERGENCE_RETRY_DELAYS_MS, + onRetry: deps.onRetry, sleep: deps.sleep, }); } diff --git a/src/lib/actions/inference-set.ts b/src/lib/actions/inference-set.ts index 7086bf06a0c..d5b1ca84569 100644 --- a/src/lib/actions/inference-set.ts +++ b/src/lib/actions/inference-set.ts @@ -1171,6 +1171,12 @@ async function runInferenceSetWithoutHostLock( { probe: deps.probeSandboxRoute, sleep: deps.sleep, + onRetry: (result, delayMs, attempt) => { + if (result.ok) return; + deps.log( + ` Waiting ${delayMs / 1_000}s for OpenShell route convergence after HTTP ${result.httpStatus} (probe ${attempt}/3)...`, + ); + }, }, ); } catch (probeError) { diff --git a/test/e2e/RETRY_INVENTORY.md b/test/e2e/RETRY_INVENTORY.md index 25f46437ede..7b8516370d5 100644 --- a/test/e2e/RETRY_INVENTORY.md +++ b/test/e2e/RETRY_INVENTORY.md @@ -19,6 +19,7 @@ Exhaustion remains failed. | `github-publication-read` | GitHub API reads; `tools/e2e/base-image-publication.mts` | Fetch error, 408, rate limit, or 5xx | 3 attempts; Retry-After/rate-limit reset or linear delay capped at 10s | Read-only | GitHub API | Returned parsed selection on success; thrown terminal HTTP/fetch error on failure or exhaustion | Caller artifact records the returned publication selection; terminal errors identify exhausted fetch or HTTP status without response content | Eligible bounded read; existing implementation retained | | `trusted-controller-collaborator-permission-read` | Collaborator-permission reads for manual PR dispatch and Launchable E2E dispatch; `.github/workflows/e2e.yaml` | Curl exit 5, 6, 7, 16, 18, 28, 35, 52, 55, 56, 92, 95, or 96; HTTP 408, 429, or 5xx | 3 attempts; linear 1s then 2s | Read-only GitHub API request | GitHub API | Transient API read versus terminal authentication, authorization, actor, or response failure | Operation name, attempt number, and sanitized failure class or HTTP status; no response body, header, or token | Eligible bounded read; HTTP 401, 403, 404, and 422, malformed responses, actor failures, and insufficient roles remain terminal; no cached permission or workflow rerun | | `github-exact-artifact-content-read` | Bound Deep Agents Code contract artifact; `tools/e2e/exact-artifact-download.mts` | Transport failure, HTTP 408, HTTP 429, or HTTP 5xx while reading one pre-bound artifact ID | 3 attempts; Retry-After or linear delay capped at 10s | Read-only request against one immutable artifact ID, name, size, digest, producer run, attempt, and head | GitHub artifact service | `passed-first-attempt`, `passed-after-retry`, `exhausted` for transient exhaustion, or `failed-no-retry` for terminal HTTP; identity, size, digest, archive, and contract failures throw without an aggregate outcome or `failureClass` | Content-read attempts log only the sanitized operation, attempt, HTTP status or transport class, and outcome; thrown validation failures expose only their bounded error message, never headers, body, token, signed URL, or artifact content | Standalone bounded content read; it does not use `retry-policy.ts` or `RetryEvidence`, and all identity, integrity, archive, and contract failures remain terminal | +| `inference-set-route-convergence` | Sandbox inference probe after one OpenShell route selection; `src/lib/actions/inference-set-provider.ts`, `src/lib/actions/inference-set.ts` | HTTP 400 or 404 only when the selected API family changes; authentication, authorization, unsafe or malformed input, every other HTTP status, transport failure, and probe failure are terminal | 3 probes; 2s then 4s | Each retry repeats only the read-only sandbox inference probe after one route mutation | OpenShell route cache | Converged, terminal failure, or exhausted rollback | Retry progress records only HTTP status, attempt number, and delay; the final command error stays redacted, and focused tests assert the exact attempt count and rollback | The 6s final-probe window covers one full 5s OpenShell 0.0.101 cache-refresh interval; exhaustion restores the prior route, removes the uncommitted provider, and remains failed | | `inference-switch-ts` | Verified inference route update; `test/e2e/fixtures/inference-switch-retry.ts` | Timeout, reset, DNS/connectivity/connect error, request transport error, or exact 502/503/504 status; authentication, authorization, policy, malformed-input, and invalid-request signals take precedence | 1-10 attempts; linear 5s | Setting the same desired provider/model is idempotent | Inference provider | Shared `RetryEvidence` classifications | Every attempt classification and aggregate outcome; command artifacts remain separate and redacted | Uses `runBoundedRetry`; deterministic verification mismatches stop; no `--no-verify` exhaustion bypass | | `inference-switch-shell` | Verified shell inference route update; `test/e2e/lib/inference-switch-retry.sh` | Same bounded transient and terminal-precedence signatures as the TypeScript helper | 1-10 attempts; linear 5s | Setting the same desired provider/model is idempotent | Inference provider | Exit status remains failed on exhaustion | Existing command output and retry progress | Bounded compatibility helper; no `--no-verify` exhaustion bypass | | `provider-install-standard` | Provider validation during Brave, cron, device-auth, Hermes-switch, network-policy, and restricted onboarding | `isTransientProviderValidationFailure` allowlist only | 1 local or 3 CI attempts; linear 10s backoff | Repeats the same desired onboarding state; restricted paths destroy the prior sandbox before retry | Inference provider | Transient allowlist versus terminal install failure | Per-attempt command artifacts; restricted paths add a terminal skip artifact | Existing bounded paths; no deterministic install retry |