Skip to content
Merged
Show file tree
Hide file tree
Changes from 11 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
138 changes: 68 additions & 70 deletions src/lib/onboard/sandbox-create/orchestration.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,17 @@ import { describe, expect, it, vi } from "vitest";

import { PolicyAuthorityRefusalError } from "../../adapters/openshell/policy-authority";
import type { SandboxEntry } from "../../state/registry";
import {
revalidateCreatedSandboxLifecycleRegistration,
type SandboxRecreateObservation,
} from "../sandbox-recreate-transaction";
import { runSandboxProviderPreDeleteCleanup } from "../sandbox-provider-cleanup";
import {
applyManagedSandboxRebuildPolicyCarryForward,
assertApfCreateIntent,
backfillVerifiedExternalSandboxPolicyAuthority,
completeHermesPortableSandboxRegistration,
createOnboardCreatedSandboxRegistrationWithManagedLifecycle,
createProviderEffectBoundary,
finalizeCreatedSandboxBeforeHermesCredentialReconciliation,
hasManagedMcpRebuildHandoff,
Expand All @@ -40,6 +45,69 @@ const UNVERIFIED_RECOVERY_CONTEXT = {
policyCreationReceipt: null,
} as const;

describe("managed bootstrap sandbox registration", () => {
const lifecycleGeneration = "generation-1";
const durableIdentity = "a".repeat(64);
const recordedRegistration = {
lifecycleGeneration,
lifecycleLiveIdentityFingerprint: durableIdentity,
};
const notReady = (liveIdentityFingerprint: string | null): SandboxRecreateObservation => ({
state: "not_ready",
liveIdentityFingerprint,
});

function registrationFixture(managedBootstrap: boolean, observation: SandboxRecreateObservation) {
const publish = vi.fn();
const completeRegistration = createOnboardCreatedSandboxRegistrationWithManagedLifecycle({
sandboxName: "alpha",
managedBootstrap,
sandboxGpuEnabled: false,
createdLifecycle: {
generation: lifecycleGeneration,
recordExactIdentity: vi.fn(),
capture: () => recordedRegistration,
revalidate: (registration, options) =>
revalidateCreatedSandboxLifecycleRegistration(
{ sandboxName: "alpha", gatewayName: "nemoclaw" },
registration,
() => observation,
options,
),
},
getRecordedRegistration: () => recordedRegistration,
createRegistration:
({ createdLifecycle }) =>
async () =>
publish(createdLifecycle.revalidate(createdLifecycle.capture(recordedRegistration))),
registration: {} as never,
Comment thread
sandl99 marked this conversation as resolved.
Resolved
});
return {
complete: () => completeRegistration(null, null),
publish,
};
}

it.each([
["managed matching", true, notReady(durableIdentity), ["published", 1]],
["ordinary matching", false, notReady(durableIdentity), ["rejected", 0]],
["managed missing", true, notReady(null), ["rejected", 0]],
["managed malformed", true, notReady("not-a-fingerprint"), ["rejected", 0]],
["managed changed", true, notReady("b".repeat(64)), ["rejected", 0]],
] as const)(
"publishes only a managed sandbox with its matching valid not Ready identity: %s (#10512)",
async (_case, managedBootstrap, observation, expected) => {
const fixture = registrationFixture(managedBootstrap, observation);
const outcome = await fixture.complete().then(
() => "published",
() => "rejected",
);
Comment thread
sandl99 marked this conversation as resolved.
Resolved

expect([outcome, fixture.publish.mock.calls.length]).toEqual(expected);
},
);
});

describe("created Hermes credential environment reconciliation", () => {
const plan = { agent: "hermes" } as never;

Expand Down Expand Up @@ -867,49 +935,6 @@ describe("sandbox create policy authority checks", () => {
expect(create).not.toHaveBeenCalled();
});

it("checks the named Ready sandbox before registration can continue (#9833)", async () => {
const events: string[] = [];
const result = await runSandboxCreateWithPolicyAuthorityChecks({
sandboxName: "alpha",
revalidate: (sandboxIsLive) => events.push(sandboxIsLive ? "ready-check" : "create-check"),
create: async (verifyCreatedSandbox) => {
events.push("create");
await verifyCreatedSandbox("created");
return "created";
},
captureCreatedSandboxIdentity: () => {
events.push("capture-identity");
return exactIdentity;
},
persistCreatedSandboxIdentity: () => events.push("persist-identity"),
revalidateCreatedSandboxIdentity: () => events.push("identity-check"),
verifyCreatedPolicy: () => {
events.push("policy-check");
return "verified";
},
persistVerifiedPolicy: () => events.push("persist-checkpoint"),
revalidateVerifiedPolicy: () => events.push("revalidate-checkpoint"),
cleanupTemporarySources: vi.fn(),
});
events.push("register");

expect(result).toBe("created");
expect(events).toEqual([
"create-check",
"create",
"capture-identity",
"persist-identity",
"identity-check",
"policy-check",
"identity-check",
"persist-checkpoint",
"revalidate-checkpoint",
"identity-check",
"identity-check",
"register",
]);
});

it("removes temporary sources but preserves the sandbox after final authority failure (#9833)", async () => {
const events: string[] = [];
const revalidate = vi.fn(() => events.push("create-check"));
Expand Down Expand Up @@ -1371,33 +1396,6 @@ describe("sandbox create policy authority checks", () => {
expect(runVerifiedCreateEffects).not.toHaveBeenCalled();
});

it("retains the durable checkpoint when a deferred effect fails (#9833)", async () => {
const persistVerifiedPolicy = vi.fn();

await expect(
runSandboxCreateWithPolicyAuthorityChecks({
sandboxName: "alpha",
revalidate: vi.fn(),
create: async (verifyCreatedSandbox) => {
await verifyCreatedSandbox("created");
return "created";
},
captureCreatedSandboxIdentity: () => exactIdentity,
persistCreatedSandboxIdentity: vi.fn(),
revalidateCreatedSandboxIdentity: vi.fn(),
verifyCreatedPolicy: () => "verified",
persistVerifiedPolicy,
revalidateVerifiedPolicy: vi.fn(),
runVerifiedCreateEffects: async () => {
throw new Error("provider effect failed");
},
cleanupTemporarySources: vi.fn(),
}),
).rejects.toThrow("automatic sandbox cleanup was not safe");

expect(persistVerifiedPolicy).toHaveBeenCalledOnce();
});

it("refuses continuation when identity changes during effective-policy verification (#9833)", async () => {
const continuationEffect = vi.fn();
const persistRetainedSandboxRecovery = vi.fn(() => true);
Expand Down
80 changes: 70 additions & 10 deletions src/lib/onboard/sandbox-create/orchestration.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,10 @@ import type {
} from "../managed-workload/hermes-state-volume";
import type { OwnedSandboxRecreateRuntime } from "../onboard-recreate-journal";
import type { SandboxGpuConfig } from "../sandbox-gpu-mode";
import type {
CreatedSandboxLifecycle,
CreatedSandboxLifecycleRegistration,
} from "../sandbox-recreate-transaction";
import type { PortableOnboardRuntimeContext } from "../session-bootstrap";
import type {
InferenceRouteReservationAuthority,
Expand Down Expand Up @@ -67,6 +71,43 @@ function cancelRecoveryIdentity(
};
}

export function createOnboardCreatedSandboxRegistrationWithManagedLifecycle(input: {
readonly sandboxName: string;
readonly managedBootstrap: boolean;
readonly sandboxGpuEnabled: boolean;
readonly createdLifecycle: CreatedSandboxLifecycle;
readonly getRecordedRegistration: () => CreatedSandboxLifecycleRegistration;
readonly createRegistration: SandboxCreateOrchestrationRuntime["createOnboardCreatedSandboxRegistration"];
readonly registration: Omit<
Parameters<SandboxCreateOrchestrationRuntime["createOnboardCreatedSandboxRegistration"]>[0],
"createdLifecycle"
>;
}) {
let createdLifecycle = input.createdLifecycle;
if (input.managedBootstrap) {
const capture = input.sandboxGpuEnabled
? input.createdLifecycle.capture
: ({ lifecycleGeneration }: Pick<SandboxEntry, "lifecycleGeneration">) => {
const recordedRegistration = input.getRecordedRegistration();
if (lifecycleGeneration !== recordedRegistration.lifecycleGeneration) {
throw new Error(
`Cannot register sandbox '${input.sandboxName}': lifecycle setup did not preserve its generation.`,
);
}
return recordedRegistration;
};
createdLifecycle = {
...input.createdLifecycle,
capture,
revalidate: (registration) =>
input.createdLifecycle.revalidate(registration, {
allowNotReadyWithMatchingIdentity: true,
}),
};
}
return input.createRegistration({ ...input.registration, createdLifecycle });
}

/** Persist one create-attempt recovery message through the onboard session owner. */
export function persistRetainedSandboxRecoveryMessage(
input: {
Expand Down Expand Up @@ -2293,6 +2334,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche
}
return policySourcePath;
};
let managedBootstrapCreateFinished = false;
const revalidateCreatedSandboxIdentity = (
expectedIdentity: string,
operation: string,
Expand All @@ -2304,6 +2346,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche
lifecycleLiveIdentityFingerprint: expectedIdentity,
},
getSandboxRecreateObservation,
{ allowNotReadyWithMatchingIdentity: managedBootstrapCreateFinished },
);
};
const requireVerifiedPolicyGate = (): NonNullable<typeof verifiedPolicyGate> => {
Expand Down Expand Up @@ -2682,8 +2725,8 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche
await runDeferredProviderEffects(context);
}
: undefined,
create: (verifyCreatedSandbox) =>
sandboxGpuCreateFlow.runSandboxGpuCreateFlow(
create: async (verifyCreatedSandbox) => {
const created = await sandboxGpuCreateFlow.runSandboxGpuCreateFlow(
{
sandboxName,
...(resumeVerifiedCreateInput
Expand Down Expand Up @@ -2723,6 +2766,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche
terminalAgent: agentDefs.isTerminalAgent(agent),
managedBootstrap,
verifyCreatedSandboxBeforeEffects: async (identity) => {
managedBootstrapCreateFinished = managedBootstrap !== null;
await verifyCreatedSandbox(identity);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
},
revalidateVerifiedSandboxBeforeEffect: (operation) =>
Expand All @@ -2736,7 +2780,9 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche
openshellArgv,
verifyDirectSandboxGpu: createGpuVerifier,
},
),
);
return created;
},
});
};

Expand Down Expand Up @@ -2827,13 +2873,27 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche
preparedSandboxWorkload,
note,
);
const completeCreatedSandboxRegistration = createOnboardCreatedSandboxRegistration({
completion: createdSandboxCompletion,
createdLifecycle: createdSandboxLifecycle,
cleanupBuildContext,
manageDashboard,
sandboxGpuEnabled: effectiveSandboxGpuConfig.sandboxGpuEnabled,
});
// Managed bootstrap can invalidate OpenShell's cached Ready state after it
// replaces the container. Registry publication stays bound to the durable
// sandbox identity recorded before that replacement.
const completeCreatedSandboxRegistration =
createOnboardCreatedSandboxRegistrationWithManagedLifecycle({
sandboxName,
managedBootstrap: managedBootstrap !== null,
sandboxGpuEnabled: effectiveSandboxGpuConfig.sandboxGpuEnabled,
createdLifecycle: createdSandboxLifecycle,
getRecordedRegistration: () =>
requireDurableCreatedSandboxIdentity(
requireVerifiedPolicyGate().lifecycleLiveIdentityFingerprint,
),
createRegistration: createOnboardCreatedSandboxRegistration,
registration: {
completion: createdSandboxCompletion,
cleanupBuildContext,
manageDashboard,
sandboxGpuEnabled: effectiveSandboxGpuConfig.sandboxGpuEnabled,
},
});

const providerPreparationInput = {
openshellDriver: sandboxRuntimeFields.openshellDriver,
Expand Down
Loading
Loading