diff --git a/src/lib/actions/sandbox/rebuild-credential-preflight.ts b/src/lib/actions/sandbox/rebuild-credential-preflight.ts index e4359d25be8..0576b155611 100644 --- a/src/lib/actions/sandbox/rebuild-credential-preflight.ts +++ b/src/lib/actions/sandbox/rebuild-credential-preflight.ts @@ -145,6 +145,7 @@ export function preflightRebuildCredentials( sb: RebuildSandboxEntry, log: RebuildLog, bail: RebuildBail, + options: { allowMissingGatewayProviderWithHostCredential?: boolean } = {}, ): boolean { const rebuildCredentialEnv = getRebuildCredentialEnvFromRegistry(sb.provider, sb.credentialEnv); const rebuildProvider = sb.provider; @@ -171,7 +172,12 @@ export function preflightRebuildCredentials( log( `Preflight credential check: ${rebuildCredentialEnv} → ${credentialValue ? "present" : "MISSING"}`, ); - if (!checkRebuildGatewayProviderOrBail(rebuildProvider, rebuildCredentialEnv, log, bail)) { + if ( + !checkRebuildGatewayProviderOrBail(rebuildProvider, rebuildCredentialEnv, log, bail, { + allowMissingProvider: + options.allowMissingGatewayProviderWithHostCredential === true && Boolean(credentialValue), + }) + ) { return false; } if (!credentialValue && shouldVerifyRebuildGatewayProvider(rebuildProvider)) { diff --git a/src/lib/actions/sandbox/rebuild-onboard-dependencies.ts b/src/lib/actions/sandbox/rebuild-onboard-dependencies.ts index 917243a729d..392cdacc186 100644 --- a/src/lib/actions/sandbox/rebuild-onboard-dependencies.ts +++ b/src/lib/actions/sandbox/rebuild-onboard-dependencies.ts @@ -5,6 +5,7 @@ import type { RebuildDurableConfig } from "./rebuild-durable-config"; import type { RebuildRecreateOnboardOpts } from "./rebuild-gpu-opt-out"; type RebuildAuthoritativePreflightOptions = RebuildRecreateOnboardOpts & { + deferInferenceRouteUntilOnboard?: true; model: string; provider: string; sandboxName: string; diff --git a/src/lib/actions/sandbox/rebuild-preflight-phase.ts b/src/lib/actions/sandbox/rebuild-preflight-phase.ts index e95a3889e63..08a41c3e547 100644 --- a/src/lib/actions/sandbox/rebuild-preflight-phase.ts +++ b/src/lib/actions/sandbox/rebuild-preflight-phase.ts @@ -140,6 +140,7 @@ export async function runRebuildPreflightPhase( // succeeded, matching the previous `skipConfirm || confirmed` contract. autoYes: true, requestedToolDisclosure, + preparedBackupRecovery: recoveryManifest !== null, log, bail, }); diff --git a/src/lib/actions/sandbox/rebuild-preflight-target-phase.ts b/src/lib/actions/sandbox/rebuild-preflight-target-phase.ts index b43b3bf4a43..1bbf3b5a15c 100644 --- a/src/lib/actions/sandbox/rebuild-preflight-target-phase.ts +++ b/src/lib/actions/sandbox/rebuild-preflight-target-phase.ts @@ -49,11 +49,20 @@ export async function prepareRebuildTargetPreflights(args: { rebuildAgent: string | null; autoYes: boolean; requestedToolDisclosure?: ToolDisclosure; + preparedBackupRecovery?: boolean; log: RebuildLog; bail: RebuildBail; }): Promise { - const { sandboxName, sandboxEntry, rebuildAgent, autoYes, requestedToolDisclosure, log, bail } = - args; + const { + sandboxName, + sandboxEntry, + rebuildAgent, + autoYes, + requestedToolDisclosure, + preparedBackupRecovery, + log, + bail, + } = args; hydrateMessagingConfigForRebuild(sandboxName, log); if (!(await ensureRebuildTargetGatewaySelected(sandboxName, sandboxEntry, log, bail))) return null; @@ -115,7 +124,13 @@ export async function prepareRebuildTargetPreflights(args: { bail, }); if ( - !(await preflightAuthoritativeOnboardRuntime(sandboxName, resumeConfig, recreateOptions, bail)) + !(await preflightAuthoritativeOnboardRuntime( + sandboxName, + resumeConfig, + recreateOptions, + bail, + preparedBackupRecovery ? { deferInferenceRouteUntilOnboard: true } : {}, + )) ) { return null; } @@ -142,7 +157,10 @@ export async function prepareRebuildTargetPreflights(args: { recreateOptions, log, bail, - { skipImagePreflight: rebuildsDcodeSandbox }, + { + allowMissingGatewayProviderWithHostCredential: preparedBackupRecovery, + skipImagePreflight: rebuildsDcodeSandbox, + }, ); } finally { restoreBaseImageOverride(); diff --git a/src/lib/actions/sandbox/rebuild-prepared-recovery.test.ts b/src/lib/actions/sandbox/rebuild-prepared-recovery.test.ts index fd5851e2449..65dc481bc9d 100644 --- a/src/lib/actions/sandbox/rebuild-prepared-recovery.test.ts +++ b/src/lib/actions/sandbox/rebuild-prepared-recovery.test.ts @@ -37,6 +37,9 @@ describe("prepared rebuild recovery", () => { ).resolves.toBeUndefined(); expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + expect(harness.preflightAuthoritativeRebuildTargetSpy).toHaveBeenCalledWith( + expect.objectContaining({ deferInferenceRouteUntilOnboard: true }), + ); expect(harness.runOpenshellSpy).toHaveBeenCalledWith( ["sandbox", "delete", "alpha"], expect.objectContaining({ ignoreError: true }), diff --git a/src/lib/actions/sandbox/rebuild-provider-preflight.ts b/src/lib/actions/sandbox/rebuild-provider-preflight.ts index 0bea1e6df54..f8fde74a5cb 100644 --- a/src/lib/actions/sandbox/rebuild-provider-preflight.ts +++ b/src/lib/actions/sandbox/rebuild-provider-preflight.ts @@ -76,6 +76,7 @@ export function checkRebuildGatewayProviderOrBail( credentialEnv: string | null, log: (msg: string) => void, bail: (msg: string, code?: number) => never, + options: { allowMissingProvider?: boolean } = {}, ): boolean { if (!shouldVerifyRebuildGatewayProvider(provider)) return true; @@ -86,6 +87,12 @@ export function checkRebuildGatewayProviderOrBail( } in OpenShell`, ); if (providerRegisteredInGateway) return true; + if (options.allowMissingProvider) { + log( + `Preflight gateway provider check: prepared recovery will recreate missing provider '${provider}' from its explicit host credential`, + ); + return true; + } printMissingRebuildGatewayProvider(provider, credentialEnv); bail(`Missing gateway provider: ${provider}`); diff --git a/src/lib/actions/sandbox/rebuild-target-runtime.ts b/src/lib/actions/sandbox/rebuild-target-runtime.ts index 07182aa3e5f..c713b754d87 100644 --- a/src/lib/actions/sandbox/rebuild-target-runtime.ts +++ b/src/lib/actions/sandbox/rebuild-target-runtime.ts @@ -67,7 +67,10 @@ export async function preflightRebuildTargetRuntime( recreateOptions: RebuildRecreateOnboardOpts, log: RebuildLog, bail: RebuildBail, - options: { skipImagePreflight?: boolean } = {}, + options: { + allowMissingGatewayProviderWithHostCredential?: boolean; + skipImagePreflight?: boolean; + } = {}, ): Promise { const webSearchConfig = target.durableConfig.webSearchConfig; const webSearchProvider = webSearchConfig ? webSearchProviderForConfig(webSearchConfig) : null; @@ -186,6 +189,10 @@ export async function preflightRebuildTargetRuntime( }, log, bail, + { + allowMissingGatewayProviderWithHostCredential: + options.allowMissingGatewayProviderWithHostCredential, + }, ) ) { return { ok: false }; @@ -203,10 +210,12 @@ export async function preflightAuthoritativeOnboardRuntime( resumeConfig: RebuildResumeConfig, recreateOptions: RebuildRecreateOnboardOpts, bail: RebuildBail, + options: { deferInferenceRouteUntilOnboard?: true } = {}, ): Promise { try { await rebuildOnboardDependencies.preflightAuthoritativeRebuildTarget({ ...recreateOptions, + ...options, model: resumeConfig.model, provider: resumeConfig.provider, sandboxName, diff --git a/src/lib/onboard/authoritative-rebuild-target.test.ts b/src/lib/onboard/authoritative-rebuild-target.test.ts index a078d7dfc7c..c715c05b2e3 100644 --- a/src/lib/onboard/authoritative-rebuild-target.test.ts +++ b/src/lib/onboard/authoritative-rebuild-target.test.ts @@ -121,6 +121,21 @@ describe("authoritative rebuild target preflight", () => { ).rejects.toThrow("inference route does not match"); }); + it("defers route validation for prepared recovery until authoritative onboard", async () => { + const targetDeps = deps({ inferenceRouteReady: vi.fn(() => false) }); + + await expect( + preflightAuthoritativeRebuildTarget( + { ...target, deferInferenceRouteUntilOnboard: true }, + targetDeps, + ), + ).resolves.toBeUndefined(); + + expect(targetDeps.inferenceRouteReady).not.toHaveBeenCalled(); + expect(targetDeps.runFatalRuntimePreflight).toHaveBeenCalledOnce(); + expect(targetDeps.ensureOpenshell).toHaveBeenCalledOnce(); + }); + it("rejects a dashboard forward owned by another sandbox", async () => { await expect( preflightAuthoritativeRebuildTarget( diff --git a/src/lib/onboard/authoritative-rebuild-target.ts b/src/lib/onboard/authoritative-rebuild-target.ts index b8b01f37bbf..c6f0fac70bc 100644 --- a/src/lib/onboard/authoritative-rebuild-target.ts +++ b/src/lib/onboard/authoritative-rebuild-target.ts @@ -19,6 +19,7 @@ export type AuthoritativeRebuildPreflightOptions = Pick< "sandboxGpu" | "sandboxGpuDevice" | "noGpu" | "controlUiPort" > & { authoritativeResumeConfig: true; + deferInferenceRouteUntilOnboard?: true; model: string; provider: string; sandboxName: string; @@ -60,6 +61,7 @@ export function resolveAuthoritativeOnboardGatewayBinding( } export type AuthoritativeRebuildTarget = { + deferInferenceRouteUntilOnboard?: true; sandboxName: string; provider: string; model: string; @@ -90,7 +92,14 @@ export async function preflightAuthoritativeRebuildTarget( try { deps.runFatalRuntimePreflight(); deps.ensureOpenshell(); - if (!deps.inferenceRouteReady(target.provider, target.model)) { + // Prepared-backup recovery can run after the installer has replaced a + // legacy gateway. That fresh gateway has no inference route to validate + // yet; authoritative onboarding configures and verifies the pinned route + // before recreating the sandbox. Normal rebuilds must still match here. + if ( + target.deferInferenceRouteUntilOnboard !== true && + !deps.inferenceRouteReady(target.provider, target.model) + ) { fail( `OpenShell inference route does not match provider '${target.provider}' and model '${target.model}'.`, ); diff --git a/test/helpers/rebuild-flow-credential-preflight-cases.ts b/test/helpers/rebuild-flow-credential-preflight-cases.ts index 0e1be8931ae..23d8cdcf186 100644 --- a/test/helpers/rebuild-flow-credential-preflight-cases.ts +++ b/test/helpers/rebuild-flow-credential-preflight-cases.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it } from "vitest"; +import { makePreparedRecoveryManifest } from "../../src/lib/actions/sandbox/rebuild-flow-test-fixtures"; import { createRebuildFlowHarness, installRebuildFlowTestHooks } from "./rebuild-flow-test-harness"; type Harness = ReturnType; @@ -150,6 +151,37 @@ export function registerRebuildFlowCredentialPreflightTests(): void { expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); }); + it("recreates a missing provider from an explicit host credential during prepared recovery", async () => { + const harness = createRebuildFlowHarness({ + sandboxEntry: { + provider: "compatible-endpoint", + model: MODEL, + credentialEnv: "COMPATIBLE_API_KEY", + endpointUrl: "https://inference.example.test/v1", + }, + hydrateCredentialEnv: () => "host-provider-key", + runOpenshell: providerRuntime([]), + sandboxListOutput: "alpha Error", + }); + configureSession(harness, "compatible-endpoint", "COMPATIBLE_API_KEY", { + endpointUrl: "https://inference.example.test/v1", + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { + throwOnError: true, + recoveryManifest: makePreparedRecoveryManifest(), + }), + ).resolves.toBeUndefined(); + + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + expect(harness.onboardSpy).toHaveBeenCalledOnce(); + expect(harness.runOpenshellSpy).toHaveBeenCalledWith( + ["provider", "get", "compatible-endpoint"], + expect.anything(), + ); + }); + it("copies the staged Hermes messaging plan into the rebuild resume session", async () => { const plan = makeMessagingPlan(); const harness = createRebuildFlowHarness({ diff --git a/test/helpers/rebuild-flow-harness.ts b/test/helpers/rebuild-flow-harness.ts index 16324a73e1c..9644bfd6a21 100644 --- a/test/helpers/rebuild-flow-harness.ts +++ b/test/helpers/rebuild-flow-harness.ts @@ -133,6 +133,7 @@ export type RebuildFlowHarness = { markStepFailedSpy: MockInstance; openShieldsSpy: MockInstance; onboardSpy: MockInstance; + preflightAuthoritativeRebuildTargetSpy: MockInstance; preflightMessagingConflictsSpy: MockInstance; preflightDcodeRouteSpy: MockInstance; prepareManagedDcodeRebuildImageSpy: MockInstance; @@ -484,9 +485,9 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): vi.spyOn(rebuildOnboardDependencies, "hydrateCredentialEnv").mockImplementation( (...args: unknown[]) => onboardCredentialEnv.hydrateCredentialEnv(String(args[0] ?? "")), ); - vi.spyOn(rebuildOnboardDependencies, "preflightAuthoritativeRebuildTarget").mockResolvedValue( - undefined, - ); + const preflightAuthoritativeRebuildTargetSpy = vi + .spyOn(rebuildOnboardDependencies, "preflightAuthoritativeRebuildTarget") + .mockResolvedValue(undefined); const applyPresetSpy = vi .spyOn(policies, "applyPreset") .mockImplementation((_sandboxName: unknown, presetName: unknown) => { @@ -556,6 +557,7 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): markStepFailedSpy, openShieldsSpy, onboardSpy, + preflightAuthoritativeRebuildTargetSpy, preflightMessagingConflictsSpy, preflightDcodeRouteSpy, prepareManagedDcodeRebuildImageSpy,