From fe18311cf0ebc2858a684e463b3f71948706000a Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Thu, 25 Jun 2026 17:45:31 -0700 Subject: [PATCH 1/5] fix(rebuild): abort when gateway provider is missing --- docs/security/credential-storage.mdx | 2 + src/lib/actions/sandbox/rebuild.ts | 49 ++++++++++++++++++++++- test/rebuild-credential-preflight.test.ts | 26 ++++++++++++ 3 files changed, 75 insertions(+), 2 deletions(-) diff --git a/docs/security/credential-storage.mdx b/docs/security/credential-storage.mdx index 036a1507ba8..9970c998f60 100644 --- a/docs/security/credential-storage.mdx +++ b/docs/security/credential-storage.mdx @@ -59,6 +59,8 @@ This means you can: When the host environment is empty, day-two operations such as `$$nemoclaw rebuild` and remote-provider updates can reuse the credential already registered with the OpenShell gateway. Export the credential only when you want to create, replace, or rotate the stored provider value. +For rebuilds that use a non-local upstream provider, the matching OpenShell provider entry must still exist. +If the sandbox registry points at a provider that is missing from OpenShell, `$$nemoclaw rebuild` stops before backup or delete even when the matching credential environment variable is exported; rerun `$$nemoclaw onboard` or re-register the provider first. ## Deploy Reads from Environment Only diff --git a/src/lib/actions/sandbox/rebuild.ts b/src/lib/actions/sandbox/rebuild.ts index 403717b6c95..d2318eca0b6 100644 --- a/src/lib/actions/sandbox/rebuild.ts +++ b/src/lib/actions/sandbox/rebuild.ts @@ -213,6 +213,21 @@ function preflightHermesProviderCredentials( return false; } +function printMissingRebuildGatewayProvider(provider: string, credentialEnv: string | null): void { + console.error(""); + console.error( + ` ${_RD}Rebuild preflight failed:${R} provider '${provider}' is not registered in OpenShell.`, + ); + console.error(" The sandbox registry still points at this upstream provider,"); + console.error(" so rebuild will not recreate it before destroying the sandbox."); + if (credentialEnv) { + console.error(` ${credentialEnv} may be present, but the OpenShell provider is missing.`); + } + console.error(""); + console.error(" Re-register the provider in OpenShell or rerun onboard, then retry rebuild."); + console.error(" Sandbox is untouched — no data was lost."); +} + export async function stageMessagingManifestPlanForRebuild( sandboxName: string, sandboxEntry: registry.SandboxEntry, @@ -434,6 +449,24 @@ function preflightRebuildCredentials( } const rebuildProvider = sb.provider; + const shouldVerifyGatewayProvider = + rebuildProvider && + !isLocalInferenceProvider(rebuildProvider) && + rebuildProvider !== hermesProviderAuth.HERMES_PROVIDER_NAME; + let rebuildProviderRegisteredInGateway: boolean | null = null; + const gatewayProviderExists = (): boolean => { + if (!shouldVerifyGatewayProvider) return false; + if (rebuildProviderRegisteredInGateway === null) { + rebuildProviderRegisteredInGateway = providerExistsInGateway(rebuildProvider, runOpenshell); + log( + `Preflight gateway provider check: provider '${rebuildProvider}' is ${ + rebuildProviderRegisteredInGateway ? "registered" : "missing" + } in OpenShell`, + ); + } + return rebuildProviderRegisteredInGateway; + }; + // Compatibility boundary for GH #2519: pre-fix local-provider sessions could // persist credentialEnv="OPENAI_API_KEY" even though current local-provider // write paths persist null. Only a session for this sandbox plus a local @@ -469,6 +502,11 @@ function preflightRebuildCredentials( } if (!rebuildCredentialEnv) { + if (shouldVerifyGatewayProvider && !gatewayProviderExists()) { + printMissingRebuildGatewayProvider(rebuildProvider, rebuildCredentialEnv); + bail(`Missing gateway provider: ${rebuildProvider}`); + return false; + } log( "Preflight credential check: no credentialEnv in session (local inference or missing session)", ); @@ -479,8 +517,15 @@ function preflightRebuildCredentials( log( `Preflight credential check: ${rebuildCredentialEnv} → ${credentialValue ? "present" : "MISSING"}`, ); - if (credentialValue) return true; - if (rebuildProvider && providerExistsInGateway(rebuildProvider, runOpenshell)) { + if (credentialValue) { + if (shouldVerifyGatewayProvider && !gatewayProviderExists()) { + printMissingRebuildGatewayProvider(rebuildProvider, rebuildCredentialEnv); + bail(`Missing gateway provider: ${rebuildProvider}`); + return false; + } + return true; + } + if (shouldVerifyGatewayProvider && gatewayProviderExists()) { log( `Preflight credential check: provider '${rebuildProvider}' registered in gateway — skipping env check for ${rebuildCredentialEnv}`, ); diff --git a/test/rebuild-credential-preflight.test.ts b/test/rebuild-credential-preflight.test.ts index 66664dfd2dc..bc05cb6d4c2 100644 --- a/test/rebuild-credential-preflight.test.ts +++ b/test/rebuild-credential-preflight.test.ts @@ -508,6 +508,32 @@ describe("Issue #2273: atomic rebuild", () => { expect(output).toContain("Backing up sandbox state"); }); + it("aborts before backup when the gateway provider is missing even with host credential", { + timeout: 60_000, + }, () => { + const f = createFixture({ + credentialEnv: "NVIDIA_INFERENCE_API_KEY", + provider: "nvidia-prod", + providerRegistered: false, + }); + + const result = runRebuild(f, { + NVIDIA_INFERENCE_API_KEY: "nvapi-test-key-for-rebuild", + }); + const output = (result.stderr || "") + (result.stdout || ""); + + expect(result.status).not.toBe(0); + expect(output).toContain("preflight failed"); + expect(output).toContain("provider 'nvidia-prod' is not registered in OpenShell"); + expect(output).toContain("NVIDIA_INFERENCE_API_KEY"); + expect(output).toContain("Sandbox is untouched"); + expect(output).not.toContain("Backing up sandbox state"); + expect(output).not.toContain("Old sandbox deleted"); + expect(output).not.toContain("Creating new sandbox with current image"); + expect(output).not.toContain("missing from gateway; recreating it"); + expect(registryHasSandbox(f)).toBe(true); + }); + it("copies Hermes messaging channels from the registry into the rebuild resume session", { timeout: 60_000, }, () => { From af6186083034627ca57a1eb9e25f7e65b8fe71c8 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Thu, 25 Jun 2026 17:58:16 -0700 Subject: [PATCH 2/5] fix(rebuild): prioritize missing provider preflight --- src/lib/actions/sandbox/rebuild.ts | 20 ++++++++++---------- test/rebuild-credential-preflight.test.ts | 11 ++++++++--- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/lib/actions/sandbox/rebuild.ts b/src/lib/actions/sandbox/rebuild.ts index d2318eca0b6..54547f3c827 100644 --- a/src/lib/actions/sandbox/rebuild.ts +++ b/src/lib/actions/sandbox/rebuild.ts @@ -221,7 +221,7 @@ function printMissingRebuildGatewayProvider(provider: string, credentialEnv: str console.error(" The sandbox registry still points at this upstream provider,"); console.error(" so rebuild will not recreate it before destroying the sandbox."); if (credentialEnv) { - console.error(` ${credentialEnv} may be present, but the OpenShell provider is missing.`); + console.error(` Rebuild cannot rely on ${credentialEnv} while that provider is missing.`); } console.error(""); console.error(" Re-register the provider in OpenShell or rerun onboard, then retry rebuild."); @@ -517,20 +517,20 @@ function preflightRebuildCredentials( log( `Preflight credential check: ${rebuildCredentialEnv} → ${credentialValue ? "present" : "MISSING"}`, ); - if (credentialValue) { - if (shouldVerifyGatewayProvider && !gatewayProviderExists()) { + if (shouldVerifyGatewayProvider) { + if (!gatewayProviderExists()) { printMissingRebuildGatewayProvider(rebuildProvider, rebuildCredentialEnv); bail(`Missing gateway provider: ${rebuildProvider}`); return false; } - return true; - } - if (shouldVerifyGatewayProvider && gatewayProviderExists()) { - log( - `Preflight credential check: provider '${rebuildProvider}' registered in gateway — skipping env check for ${rebuildCredentialEnv}`, - ); - return true; + if (!credentialValue) { + log( + `Preflight credential check: provider '${rebuildProvider}' registered in gateway — skipping env check for ${rebuildCredentialEnv}`, + ); + return true; + } } + if (credentialValue) return true; console.error(""); console.error(` ${_RD}Rebuild preflight failed:${R} provider credential not found.`); diff --git a/test/rebuild-credential-preflight.test.ts b/test/rebuild-credential-preflight.test.ts index bc05cb6d4c2..099cb04c2b4 100644 --- a/test/rebuild-credential-preflight.test.ts +++ b/test/rebuild-credential-preflight.test.ts @@ -657,7 +657,8 @@ describe("Issue #2273: atomic rebuild", () => { expect(result.status).not.toBe(0); expect(output).toContain("preflight failed"); - expect(output).toContain("requires OPENAI_API_KEY"); + expect(output).toContain("provider 'openai-api' is not registered in OpenShell"); + expect(output).toContain("OPENAI_API_KEY"); expect(output).not.toContain("Backing up sandbox state"); expect(output).not.toContain("Old sandbox deleted"); expect(registryHasSandbox(f)).toBe(true); @@ -682,7 +683,8 @@ describe("Issue #2273: atomic rebuild", () => { expect(result.status).not.toBe(0); expect(output).toContain("preflight failed"); - expect(output).toContain("requires OPENAI_API_KEY"); + expect(output).toContain("provider 'openai-api' is not registered in OpenShell"); + expect(output).toContain("OPENAI_API_KEY"); expect(output).not.toContain("Backing up sandbox state"); expect(output).not.toContain("Old sandbox deleted"); expect(registryHasSandbox(f)).toBe(true); @@ -708,7 +710,8 @@ describe("Issue #2273: atomic rebuild", () => { expect(result.status).not.toBe(0); expect(output).toContain("preflight failed"); - expect(output).toContain("requires OPENAI_API_KEY"); + expect(output).toContain("provider 'openai-api' is not registered in OpenShell"); + expect(output).toContain("OPENAI_API_KEY"); expect(output).not.toContain("GH #2519"); expect(output).not.toContain("Backing up sandbox state"); expect(output).not.toContain("Old sandbox deleted"); @@ -812,7 +815,9 @@ describe("Issue #2273: atomic rebuild", () => { expect(result.status).not.toBe(0); expect(output).toContain("preflight failed"); + expect(output).toContain("provider 'nvidia-prod' is not registered in OpenShell"); expect(output).toContain("NVIDIA_INFERENCE_API_KEY"); + expect(output).not.toContain("provider credential not found"); expect(output).toContain("untouched"); expect(registryHasSandbox(f)).toBe(true); }); From 1733fcb762cfda1efc825b8935d431dd9859a609 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Thu, 25 Jun 2026 18:08:13 -0700 Subject: [PATCH 3/5] refactor(rebuild): extract gateway provider preflight --- src/lib/actions/sandbox/rebuild.ts | 72 +++++++++++++++++------------- 1 file changed, 40 insertions(+), 32 deletions(-) diff --git a/src/lib/actions/sandbox/rebuild.ts b/src/lib/actions/sandbox/rebuild.ts index 54547f3c827..dd36c58287e 100644 --- a/src/lib/actions/sandbox/rebuild.ts +++ b/src/lib/actions/sandbox/rebuild.ts @@ -228,6 +228,37 @@ function printMissingRebuildGatewayProvider(provider: string, credentialEnv: str console.error(" Sandbox is untouched — no data was lost."); } +function shouldVerifyRebuildGatewayProvider( + provider: string | null | undefined, +): provider is string { + return Boolean( + provider && + !isLocalInferenceProvider(provider) && + provider !== hermesProviderAuth.HERMES_PROVIDER_NAME, + ); +} + +function checkRebuildGatewayProviderOrBail( + provider: string | null | undefined, + credentialEnv: string | null, + log: (msg: string) => void, + bail: (msg: string, code?: number) => never, +): boolean { + if (!shouldVerifyRebuildGatewayProvider(provider)) return true; + + const providerRegisteredInGateway = providerExistsInGateway(provider, runOpenshell); + log( + `Preflight gateway provider check: provider '${provider}' is ${ + providerRegisteredInGateway ? "registered" : "missing" + } in OpenShell`, + ); + if (providerRegisteredInGateway) return true; + + printMissingRebuildGatewayProvider(provider, credentialEnv); + bail(`Missing gateway provider: ${provider}`); + return false; +} + export async function stageMessagingManifestPlanForRebuild( sandboxName: string, sandboxEntry: registry.SandboxEntry, @@ -449,23 +480,6 @@ function preflightRebuildCredentials( } const rebuildProvider = sb.provider; - const shouldVerifyGatewayProvider = - rebuildProvider && - !isLocalInferenceProvider(rebuildProvider) && - rebuildProvider !== hermesProviderAuth.HERMES_PROVIDER_NAME; - let rebuildProviderRegisteredInGateway: boolean | null = null; - const gatewayProviderExists = (): boolean => { - if (!shouldVerifyGatewayProvider) return false; - if (rebuildProviderRegisteredInGateway === null) { - rebuildProviderRegisteredInGateway = providerExistsInGateway(rebuildProvider, runOpenshell); - log( - `Preflight gateway provider check: provider '${rebuildProvider}' is ${ - rebuildProviderRegisteredInGateway ? "registered" : "missing" - } in OpenShell`, - ); - } - return rebuildProviderRegisteredInGateway; - }; // Compatibility boundary for GH #2519: pre-fix local-provider sessions could // persist credentialEnv="OPENAI_API_KEY" even though current local-provider @@ -502,9 +516,7 @@ function preflightRebuildCredentials( } if (!rebuildCredentialEnv) { - if (shouldVerifyGatewayProvider && !gatewayProviderExists()) { - printMissingRebuildGatewayProvider(rebuildProvider, rebuildCredentialEnv); - bail(`Missing gateway provider: ${rebuildProvider}`); + if (!checkRebuildGatewayProviderOrBail(rebuildProvider, rebuildCredentialEnv, log, bail)) { return false; } log( @@ -517,18 +529,14 @@ function preflightRebuildCredentials( log( `Preflight credential check: ${rebuildCredentialEnv} → ${credentialValue ? "present" : "MISSING"}`, ); - if (shouldVerifyGatewayProvider) { - if (!gatewayProviderExists()) { - printMissingRebuildGatewayProvider(rebuildProvider, rebuildCredentialEnv); - bail(`Missing gateway provider: ${rebuildProvider}`); - return false; - } - if (!credentialValue) { - log( - `Preflight credential check: provider '${rebuildProvider}' registered in gateway — skipping env check for ${rebuildCredentialEnv}`, - ); - return true; - } + if (!checkRebuildGatewayProviderOrBail(rebuildProvider, rebuildCredentialEnv, log, bail)) { + return false; + } + if (!credentialValue && shouldVerifyRebuildGatewayProvider(rebuildProvider)) { + log( + `Preflight credential check: provider '${rebuildProvider}' registered in gateway — skipping env check for ${rebuildCredentialEnv}`, + ); + return true; } if (credentialValue) return true; From 11d499622dfd444340d89ed2605ec3747e08d928 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Thu, 25 Jun 2026 19:08:04 -0700 Subject: [PATCH 4/5] test(rebuild): assert missing provider preflight priority Signed-off-by: Carlos Villela --- test/rebuild-credential-preflight.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/rebuild-credential-preflight.test.ts b/test/rebuild-credential-preflight.test.ts index 099cb04c2b4..18a0c768413 100644 --- a/test/rebuild-credential-preflight.test.ts +++ b/test/rebuild-credential-preflight.test.ts @@ -478,9 +478,12 @@ describe("Issue #2273: atomic rebuild", () => { const result = runRebuild(f); const output = (result.stderr || "") + (result.stdout || ""); - // Should mention preflight failure + // Should prefer the missing-provider abort over the generic missing-env fallback. expect(output).toContain("preflight failed"); + expect(output).toContain("provider 'nvidia-prod' is not registered in OpenShell"); expect(output).toContain("NVIDIA_INFERENCE_API_KEY"); + expect(output).not.toContain("provider credential not found"); + expect(output).not.toContain("export NVIDIA_INFERENCE_API_KEY="); // Should say sandbox is untouched expect(output).toContain("untouched"); // Sandbox should still be in the registry (not destroyed) From 25ed5a210168ef33de582cad7734611380b4629e Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Thu, 25 Jun 2026 19:52:52 -0700 Subject: [PATCH 5/5] refactor(rebuild): extract gateway provider preflight Signed-off-by: Carlos Villela --- .../sandbox/rebuild-provider-preflight.ts | 59 +++++++++++++++++++ src/lib/actions/sandbox/rebuild.ts | 53 ++--------------- 2 files changed, 63 insertions(+), 49 deletions(-) create mode 100644 src/lib/actions/sandbox/rebuild-provider-preflight.ts diff --git a/src/lib/actions/sandbox/rebuild-provider-preflight.ts b/src/lib/actions/sandbox/rebuild-provider-preflight.ts new file mode 100644 index 00000000000..81408aefbe0 --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-provider-preflight.ts @@ -0,0 +1,59 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { runOpenshell } from "../../adapters/openshell/runtime"; +import { RD as _RD, R } from "../../cli/terminal-style"; +import { isLocalInferenceProvider } from "./rebuild-resume-config"; + +const hermesProviderAuth = require("../../hermes-provider-auth") as { + HERMES_PROVIDER_NAME: string; +}; +const { providerExistsInGateway } = require("../../onboard/providers") as { + providerExistsInGateway: (name: string, runOpenshellFn: typeof runOpenshell) => boolean; +}; + +function printMissingRebuildGatewayProvider(provider: string, credentialEnv: string | null): void { + console.error(""); + console.error( + ` ${_RD}Rebuild preflight failed:${R} provider '${provider}' is not registered in OpenShell.`, + ); + console.error(" The sandbox registry still points at this upstream provider,"); + console.error(" so rebuild will not recreate it before destroying the sandbox."); + if (credentialEnv) { + console.error(` Rebuild cannot rely on ${credentialEnv} while that provider is missing.`); + } + console.error(""); + console.error(" Re-register the provider in OpenShell or rerun onboard, then retry rebuild."); + console.error(" Sandbox is untouched — no data was lost."); +} + +export function shouldVerifyRebuildGatewayProvider( + provider: string | null | undefined, +): provider is string { + return Boolean( + provider && + !isLocalInferenceProvider(provider) && + provider !== hermesProviderAuth.HERMES_PROVIDER_NAME, + ); +} + +export function checkRebuildGatewayProviderOrBail( + provider: string | null | undefined, + credentialEnv: string | null, + log: (msg: string) => void, + bail: (msg: string, code?: number) => never, +): boolean { + if (!shouldVerifyRebuildGatewayProvider(provider)) return true; + + const providerRegisteredInGateway = providerExistsInGateway(provider, runOpenshell); + log( + `Preflight gateway provider check: provider '${provider}' is ${ + providerRegisteredInGateway ? "registered" : "missing" + } in OpenShell`, + ); + if (providerRegisteredInGateway) return true; + + printMissingRebuildGatewayProvider(provider, credentialEnv); + bail(`Missing gateway provider: ${provider}`); + return false; +} diff --git a/src/lib/actions/sandbox/rebuild.ts b/src/lib/actions/sandbox/rebuild.ts index dd36c58287e..b4f1f23c4f6 100644 --- a/src/lib/actions/sandbox/rebuild.ts +++ b/src/lib/actions/sandbox/rebuild.ts @@ -22,9 +22,6 @@ const hermesProviderAuth = require("../../hermes-provider-auth") as { baseUrl?: string, ) => void; }; -const { providerExistsInGateway } = require("../../onboard/providers") as { - providerExistsInGateway: (name: string, runOpenshellFn: typeof runOpenshell) => boolean; -}; import { detectOpenShellStateRpcPreflightIssue, @@ -80,6 +77,10 @@ import { resolveRebuildLiveState, } from "./rebuild-flow-helpers"; import { buildRebuildRecreateOnboardOpts } from "./rebuild-gpu-opt-out"; +import { + checkRebuildGatewayProviderOrBail, + shouldVerifyRebuildGatewayProvider, +} from "./rebuild-provider-preflight"; import { getRebuildCredentialEnvFromRegistry, isLocalInferenceProvider, @@ -213,52 +214,6 @@ function preflightHermesProviderCredentials( return false; } -function printMissingRebuildGatewayProvider(provider: string, credentialEnv: string | null): void { - console.error(""); - console.error( - ` ${_RD}Rebuild preflight failed:${R} provider '${provider}' is not registered in OpenShell.`, - ); - console.error(" The sandbox registry still points at this upstream provider,"); - console.error(" so rebuild will not recreate it before destroying the sandbox."); - if (credentialEnv) { - console.error(` Rebuild cannot rely on ${credentialEnv} while that provider is missing.`); - } - console.error(""); - console.error(" Re-register the provider in OpenShell or rerun onboard, then retry rebuild."); - console.error(" Sandbox is untouched — no data was lost."); -} - -function shouldVerifyRebuildGatewayProvider( - provider: string | null | undefined, -): provider is string { - return Boolean( - provider && - !isLocalInferenceProvider(provider) && - provider !== hermesProviderAuth.HERMES_PROVIDER_NAME, - ); -} - -function checkRebuildGatewayProviderOrBail( - provider: string | null | undefined, - credentialEnv: string | null, - log: (msg: string) => void, - bail: (msg: string, code?: number) => never, -): boolean { - if (!shouldVerifyRebuildGatewayProvider(provider)) return true; - - const providerRegisteredInGateway = providerExistsInGateway(provider, runOpenshell); - log( - `Preflight gateway provider check: provider '${provider}' is ${ - providerRegisteredInGateway ? "registered" : "missing" - } in OpenShell`, - ); - if (providerRegisteredInGateway) return true; - - printMissingRebuildGatewayProvider(provider, credentialEnv); - bail(`Missing gateway provider: ${provider}`); - return false; -} - export async function stageMessagingManifestPlanForRebuild( sandboxName: string, sandboxEntry: registry.SandboxEntry,