Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion src/lib/actions/sandbox/rebuild-credential-preflight.ts
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,7 @@ export function preflightRebuildCredentials(
sb: RebuildSandboxEntry,
log: RebuildLog,
bail: RebuildBail,
options: { allowMissingGatewayProviderWithHostCredential?: boolean } = {},
): boolean {
const rebuildCredentialEnv = getRebuildCredentialEnvFromRegistry(sb.provider, sb.credentialEnv);
const rebuildProvider = sb.provider;
Expand All @@ -171,7 +172,12 @@ export function preflightRebuildCredentials(
log(
`Preflight credential check: ${rebuildCredentialEnv} → ${credentialValue ? "present" : "MISSING"}`,
);
if (!checkRebuildGatewayProviderOrBail(rebuildProvider, rebuildCredentialEnv, log, bail)) {
if (
!checkRebuildGatewayProviderOrBail(rebuildProvider, rebuildCredentialEnv, log, bail, {
allowMissingProvider:
options.allowMissingGatewayProviderWithHostCredential === true && Boolean(credentialValue),
})
) {
return false;
}
if (!credentialValue && shouldVerifyRebuildGatewayProvider(rebuildProvider)) {
Expand Down
1 change: 1 addition & 0 deletions src/lib/actions/sandbox/rebuild-onboard-dependencies.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import type { RebuildDurableConfig } from "./rebuild-durable-config";
import type { RebuildRecreateOnboardOpts } from "./rebuild-gpu-opt-out";

type RebuildAuthoritativePreflightOptions = RebuildRecreateOnboardOpts & {
deferInferenceRouteUntilOnboard?: true;
model: string;
provider: string;
sandboxName: string;
Expand Down
1 change: 1 addition & 0 deletions src/lib/actions/sandbox/rebuild-preflight-phase.ts
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ export async function runRebuildPreflightPhase(
// succeeded, matching the previous `skipConfirm || confirmed` contract.
autoYes: true,
requestedToolDisclosure,
preparedBackupRecovery: recoveryManifest !== null,
log,
bail,
});
Expand Down
26 changes: 22 additions & 4 deletions src/lib/actions/sandbox/rebuild-preflight-target-phase.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,20 @@ export async function prepareRebuildTargetPreflights(args: {
rebuildAgent: string | null;
autoYes: boolean;
requestedToolDisclosure?: ToolDisclosure;
preparedBackupRecovery?: boolean;
log: RebuildLog;
bail: RebuildBail;
}): Promise<RebuildPreparedTarget | null> {
const { sandboxName, sandboxEntry, rebuildAgent, autoYes, requestedToolDisclosure, log, bail } =
args;
const {
sandboxName,
sandboxEntry,
rebuildAgent,
autoYes,
requestedToolDisclosure,
preparedBackupRecovery,
log,
bail,
} = args;
hydrateMessagingConfigForRebuild(sandboxName, log);
if (!(await ensureRebuildTargetGatewaySelected(sandboxName, sandboxEntry, log, bail)))
return null;
Expand Down Expand Up @@ -115,7 +124,13 @@ export async function prepareRebuildTargetPreflights(args: {
bail,
});
if (
!(await preflightAuthoritativeOnboardRuntime(sandboxName, resumeConfig, recreateOptions, bail))
!(await preflightAuthoritativeOnboardRuntime(
sandboxName,
resumeConfig,
recreateOptions,
bail,
preparedBackupRecovery ? { deferInferenceRouteUntilOnboard: true } : {},
))
) {
return null;
}
Expand All @@ -142,7 +157,10 @@ export async function prepareRebuildTargetPreflights(args: {
recreateOptions,
log,
bail,
{ skipImagePreflight: rebuildsDcodeSandbox },
{
allowMissingGatewayProviderWithHostCredential: preparedBackupRecovery,
skipImagePreflight: rebuildsDcodeSandbox,
},
);
} finally {
restoreBaseImageOverride();
Expand Down
3 changes: 3 additions & 0 deletions src/lib/actions/sandbox/rebuild-prepared-recovery.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@ describe("prepared rebuild recovery", () => {
).resolves.toBeUndefined();

expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled();
expect(harness.preflightAuthoritativeRebuildTargetSpy).toHaveBeenCalledWith(
expect.objectContaining({ deferInferenceRouteUntilOnboard: true }),
);
expect(harness.runOpenshellSpy).toHaveBeenCalledWith(
["sandbox", "delete", "alpha"],
expect.objectContaining({ ignoreError: true }),
Expand Down
7 changes: 7 additions & 0 deletions src/lib/actions/sandbox/rebuild-provider-preflight.ts
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ export function checkRebuildGatewayProviderOrBail(
credentialEnv: string | null,
log: (msg: string) => void,
bail: (msg: string, code?: number) => never,
options: { allowMissingProvider?: boolean } = {},
): boolean {
if (!shouldVerifyRebuildGatewayProvider(provider)) return true;

Expand All @@ -86,6 +87,12 @@ export function checkRebuildGatewayProviderOrBail(
} in OpenShell`,
);
if (providerRegisteredInGateway) return true;
if (options.allowMissingProvider) {
log(
`Preflight gateway provider check: prepared recovery will recreate missing provider '${provider}' from its explicit host credential`,
);
return true;
}

printMissingRebuildGatewayProvider(provider, credentialEnv);
bail(`Missing gateway provider: ${provider}`);
Expand Down
11 changes: 10 additions & 1 deletion src/lib/actions/sandbox/rebuild-target-runtime.ts
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,10 @@ export async function preflightRebuildTargetRuntime(
recreateOptions: RebuildRecreateOnboardOpts,
log: RebuildLog,
bail: RebuildBail,
options: { skipImagePreflight?: boolean } = {},
options: {
allowMissingGatewayProviderWithHostCredential?: boolean;
skipImagePreflight?: boolean;
} = {},
): Promise<RebuildTargetRuntimePreflightResult> {
const webSearchConfig = target.durableConfig.webSearchConfig;
const webSearchProvider = webSearchConfig ? webSearchProviderForConfig(webSearchConfig) : null;
Expand Down Expand Up @@ -186,6 +189,10 @@ export async function preflightRebuildTargetRuntime(
},
log,
bail,
{
allowMissingGatewayProviderWithHostCredential:
options.allowMissingGatewayProviderWithHostCredential,
},
)
) {
return { ok: false };
Expand All @@ -203,10 +210,12 @@ export async function preflightAuthoritativeOnboardRuntime(
resumeConfig: RebuildResumeConfig,
recreateOptions: RebuildRecreateOnboardOpts,
bail: RebuildBail,
options: { deferInferenceRouteUntilOnboard?: true } = {},
): Promise<boolean> {
try {
await rebuildOnboardDependencies.preflightAuthoritativeRebuildTarget({
...recreateOptions,
...options,
model: resumeConfig.model,
provider: resumeConfig.provider,
sandboxName,
Expand Down
15 changes: 15 additions & 0 deletions src/lib/onboard/authoritative-rebuild-target.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,21 @@ describe("authoritative rebuild target preflight", () => {
).rejects.toThrow("inference route does not match");
});

it("defers route validation for prepared recovery until authoritative onboard", async () => {
const targetDeps = deps({ inferenceRouteReady: vi.fn(() => false) });

await expect(
preflightAuthoritativeRebuildTarget(
{ ...target, deferInferenceRouteUntilOnboard: true },
targetDeps,
),
).resolves.toBeUndefined();

expect(targetDeps.inferenceRouteReady).not.toHaveBeenCalled();
expect(targetDeps.runFatalRuntimePreflight).toHaveBeenCalledOnce();
expect(targetDeps.ensureOpenshell).toHaveBeenCalledOnce();
});

it("rejects a dashboard forward owned by another sandbox", async () => {
await expect(
preflightAuthoritativeRebuildTarget(
Expand Down
11 changes: 10 additions & 1 deletion src/lib/onboard/authoritative-rebuild-target.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ export type AuthoritativeRebuildPreflightOptions = Pick<
"sandboxGpu" | "sandboxGpuDevice" | "noGpu" | "controlUiPort"
> & {
authoritativeResumeConfig: true;
deferInferenceRouteUntilOnboard?: true;
model: string;
provider: string;
sandboxName: string;
Expand Down Expand Up @@ -60,6 +61,7 @@ export function resolveAuthoritativeOnboardGatewayBinding(
}

export type AuthoritativeRebuildTarget = {
deferInferenceRouteUntilOnboard?: true;
sandboxName: string;
provider: string;
model: string;
Expand Down Expand Up @@ -90,7 +92,14 @@ export async function preflightAuthoritativeRebuildTarget(
try {
deps.runFatalRuntimePreflight();
deps.ensureOpenshell();
if (!deps.inferenceRouteReady(target.provider, target.model)) {
// Prepared-backup recovery can run after the installer has replaced a
// legacy gateway. That fresh gateway has no inference route to validate
// yet; authoritative onboarding configures and verifies the pinned route
// before recreating the sandbox. Normal rebuilds must still match here.
if (
target.deferInferenceRouteUntilOnboard !== true &&
!deps.inferenceRouteReady(target.provider, target.model)
) {
fail(
`OpenShell inference route does not match provider '${target.provider}' and model '${target.model}'.`,
);
Expand Down
32 changes: 32 additions & 0 deletions test/helpers/rebuild-flow-credential-preflight-cases.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// SPDX-License-Identifier: Apache-2.0

import { describe, expect, it } from "vitest";
import { makePreparedRecoveryManifest } from "../../src/lib/actions/sandbox/rebuild-flow-test-fixtures";
import { createRebuildFlowHarness, installRebuildFlowTestHooks } from "./rebuild-flow-test-harness";

type Harness = ReturnType<typeof createRebuildFlowHarness>;
Expand Down Expand Up @@ -150,6 +151,37 @@ export function registerRebuildFlowCredentialPreflightTests(): void {
expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled();
});

it("recreates a missing provider from an explicit host credential during prepared recovery", async () => {
const harness = createRebuildFlowHarness({
sandboxEntry: {
provider: "compatible-endpoint",
model: MODEL,
credentialEnv: "COMPATIBLE_API_KEY",
endpointUrl: "https://inference.example.test/v1",
},
hydrateCredentialEnv: () => "host-provider-key",
runOpenshell: providerRuntime([]),
sandboxListOutput: "alpha Error",
});
configureSession(harness, "compatible-endpoint", "COMPATIBLE_API_KEY", {
endpointUrl: "https://inference.example.test/v1",
});

await expect(
harness.rebuildSandbox("alpha", ["--yes"], {
throwOnError: true,
recoveryManifest: makePreparedRecoveryManifest(),
}),
).resolves.toBeUndefined();

expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled();
expect(harness.onboardSpy).toHaveBeenCalledOnce();
expect(harness.runOpenshellSpy).toHaveBeenCalledWith(
["provider", "get", "compatible-endpoint"],
expect.anything(),
);
});

it("copies the staged Hermes messaging plan into the rebuild resume session", async () => {
const plan = makeMessagingPlan();
const harness = createRebuildFlowHarness({
Expand Down
8 changes: 5 additions & 3 deletions test/helpers/rebuild-flow-harness.ts
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@ export type RebuildFlowHarness = {
markStepFailedSpy: MockInstance;
openShieldsSpy: MockInstance;
onboardSpy: MockInstance;
preflightAuthoritativeRebuildTargetSpy: MockInstance;
preflightMessagingConflictsSpy: MockInstance;
preflightDcodeRouteSpy: MockInstance;
prepareManagedDcodeRebuildImageSpy: MockInstance;
Expand Down Expand Up @@ -484,9 +485,9 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}):
vi.spyOn(rebuildOnboardDependencies, "hydrateCredentialEnv").mockImplementation(
(...args: unknown[]) => onboardCredentialEnv.hydrateCredentialEnv(String(args[0] ?? "")),
);
vi.spyOn(rebuildOnboardDependencies, "preflightAuthoritativeRebuildTarget").mockResolvedValue(
undefined,
);
const preflightAuthoritativeRebuildTargetSpy = vi
.spyOn(rebuildOnboardDependencies, "preflightAuthoritativeRebuildTarget")
.mockResolvedValue(undefined);
const applyPresetSpy = vi
.spyOn(policies, "applyPreset")
.mockImplementation((_sandboxName: unknown, presetName: unknown) => {
Expand Down Expand Up @@ -556,6 +557,7 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}):
markStepFailedSpy,
openShieldsSpy,
onboardSpy,
preflightAuthoritativeRebuildTargetSpy,
preflightMessagingConflictsSpy,
preflightDcodeRouteSpy,
prepareManagedDcodeRebuildImageSpy,
Expand Down
Loading