Skip to content

test(e2e): distinguish managed image rollback backup #8833

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
senthilr-nv merged 2 commits into from
Aug 11, 2026
Merged

test(e2e): distinguish managed image rollback backup #8833

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
92c9b05
test(e2e): distinguish managed image rollback backup
senthilr-nv Aug 11, 2026
136ad51
test(e2e): model managed rollback lifecycle
senthilr-nv Aug 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
174 changes: 127 additions & 47 deletions scripts/checks/run-managed-image-openshell-e2e.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,24 @@ export function managedImageOpenShellBasePolicyPath(agent: ManagedStartupAgent):
);
}

function commandResult(argv: readonly string[], env: NodeJS.ProcessEnv, timeout = 20_000) {
export interface ManagedImageCommandResult {
readonly error?: Error;
readonly status: number | null;
readonly stdout: string | null;
readonly stderr: string | null;
}

export type ManagedImageCommandRunner = (
argv: readonly string[],
env: NodeJS.ProcessEnv,
timeout?: number,
) => ManagedImageCommandResult;

function commandResult(
argv: readonly string[],
env: NodeJS.ProcessEnv,
timeout = 20_000,
): ManagedImageCommandResult {
const [command, ...args] = argv;
if (!command) throw new Error("command argv must not be empty");
return spawnSync(command, args, {
Expand All @@ -283,13 +300,13 @@ function commandResult(argv: readonly string[], env: NodeJS.ProcessEnv, timeout
});
}

function commandDetail(result: ReturnType<typeof commandResult>): string {
function commandDetail(result: ManagedImageCommandResult): string {
return `${result.error?.message ?? ""}\n${result.stdout ?? ""}\n${result.stderr ?? ""}`
.trim()
.slice(-8_000);
}

function isDockerNotFound(result: ReturnType<typeof commandResult>): boolean {
function isDockerNotFound(result: ManagedImageCommandResult): boolean {
return (
result.status !== 0 &&
/(?:no such (?:container|network|object)|not found)/iu.test(commandDetail(result))
Expand Down Expand Up @@ -390,25 +407,65 @@ export function managedImageOpenShellProbe(
: agent === "hermes"
? "/usr/bin/curl -fsS --max-time 5 http://127.0.0.1:8642/health >/dev/null"
: "/usr/local/bin/dcode --version >/dev/null";
const readinessLabel =
agent === "openclaw"
? "OpenClaw health endpoint"
: agent === "hermes"
? "Hermes health endpoint"
: "LangChain Deep Agents Code version command";
const probeStep = (label: string, command: string) =>
`if ! ${command}; then\n printf '%s\\n' ${JSON.stringify(
`managed-image startup probe failed: ${label}`,
)} >&2\n exit 1\nfi`;
return [
"set -eu",
`test -x ${
agent === "openclaw"
? "/usr/local/bin/openclaw"
: agent === "hermes"
? "/usr/local/bin/hermes"
: "/usr/local/bin/dcode"
}`,
`grep -F ${JSON.stringify(model)} ${JSON.stringify(managedConfigPath(agent))} >/dev/null`,
"test ! -L /run/nemoclaw/managed-startup-runtime.env",
'test "$(stat -c "%u:%g:%a" /run/nemoclaw/managed-startup-runtime.env)" = "0:0:444"',
"test ! -L /run/nemoclaw/managed-startup-complete.json",
'test "$(stat -c "%u:%g:%a" /run/nemoclaw/managed-startup-complete.json)" = "0:0:444"',
"test -s /usr/local/share/nemoclaw/corporate-ca.pem",
'test "$(stat -c "%u:%g:%a" /usr/local/share/nemoclaw/corporate-ca.pem)" = "0:0:444"',
"test -s /run/nemoclaw/managed-startup-ca-bundle.pem",
'test "$(stat -c "%u:%g:%a" /run/nemoclaw/managed-startup-ca-bundle.pem)" = "0:0:444"',
healthProbe,
"set -u",
probeStep(
`${agent} executable`,
`test -x ${
agent === "openclaw"
? "/usr/local/bin/openclaw"
: agent === "hermes"
? "/usr/local/bin/hermes"
: "/usr/local/bin/dcode"
}`,
),
probeStep(
`${agent} managed model configuration`,
`grep -F ${JSON.stringify(model)} ${JSON.stringify(managedConfigPath(agent))} >/dev/null`,
),
probeStep(
"managed runtime environment must not be a symbolic link",
"test ! -L /run/nemoclaw/managed-startup-runtime.env",
),
probeStep(
"managed runtime environment owner, group, and mode must equal 0:0:444",
'test "$(stat -c "%u:%g:%a" /run/nemoclaw/managed-startup-runtime.env)" = "0:0:444"',
),
probeStep(
"managed startup completion must not be a symbolic link",
"test ! -L /run/nemoclaw/managed-startup-complete.json",
),
probeStep(
"managed startup completion owner, group, and mode must equal 0:0:444",
'test "$(stat -c "%u:%g:%a" /run/nemoclaw/managed-startup-complete.json)" = "0:0:444"',
),
probeStep(
"corporate CA file must exist and be nonempty",
"test -s /usr/local/share/nemoclaw/corporate-ca.pem",
),
probeStep(
"corporate CA owner, group, and mode must equal 0:0:444",
'test "$(stat -c "%u:%g:%a" /usr/local/share/nemoclaw/corporate-ca.pem)" = "0:0:444"',
),
probeStep(
"managed startup CA bundle must exist and be nonempty",
"test -s /run/nemoclaw/managed-startup-ca-bundle.pem",
),
probeStep(
"managed startup CA bundle owner, group, and mode must equal 0:0:444",
'test "$(stat -c "%u:%g:%a" /run/nemoclaw/managed-startup-ca-bundle.pem)" = "0:0:444"',
),
probeStep(readinessLabel, healthProbe),
].join("\n");
}

Expand Down Expand Up @@ -631,8 +688,12 @@ function parseImmutableManifestReference(image: string): {
};
}

function resolveLocalImageContentId(image: string, env: NodeJS.ProcessEnv): string {
const inspect = commandResult(["docker", "image", "inspect", "--format", "{{.Id}}", image], env);
function resolveLocalImageContentId(
image: string,
env: NodeJS.ProcessEnv,
runCommand: ManagedImageCommandRunner = commandResult,
): string {
const inspect = runCommand(["docker", "image", "inspect", "--format", "{{.Id}}", image], env);
const contentId = String(inspect.stdout ?? "").trim();
if (inspect.status !== 0 || !/^sha256:[a-f0-9]{64}$/u.test(contentId)) {
throw new Error(
Expand All @@ -642,25 +703,31 @@ function resolveLocalImageContentId(image: string, env: NodeJS.ProcessEnv): stri
return contentId;
}

export function managedImageHarnessContainerListArgs(
sandbox: string,
includeStopped: boolean,
): string[] {
return [
"docker",
"ps",
includeStopped ? "-aq" : "-q",
"--no-trunc",
"--filter",
"label=openshell.ai/managed-by=openshell",
"--filter",
`label=openshell.ai/sandbox-name=${sandbox}`,
];
}

function exactHarnessContainerIds(
input: Inputs,
networkName: string,
env: NodeJS.ProcessEnv,
includeStopped = true,
runCommand: ManagedImageCommandRunner = commandResult,
): { candidateCount: number; exactIds: string[] } {
const expectedContentId = resolveLocalImageContentId(input.image, env);
const list = commandResult(
[
"docker",
"ps",
"-aq",
"--no-trunc",
"--filter",
"label=openshell.ai/managed-by=openshell",
"--filter",
`label=openshell.ai/sandbox-name=${input.sandbox}`,
],
env,
);
const expectedContentId = resolveLocalImageContentId(input.image, env, runCommand);
const list = runCommand(managedImageHarnessContainerListArgs(input.sandbox, includeStopped), env);
if (list.status !== 0) {
throw new Error(`could not resolve the OpenShell sandbox container: ${commandDetail(list)}`);
}
Expand All @@ -670,7 +737,7 @@ function exactHarnessContainerIds(
.filter(Boolean);
const exactIds: string[] = [];
for (const candidate of candidates) {
const inspect = commandResult(["docker", "inspect", candidate], env);
const inspect = runCommand(["docker", "inspect", candidate], env);
if (inspect.status !== 0) continue;
try {
const records = JSON.parse(String(inspect.stdout ?? "")) as Array<{
Expand All @@ -694,20 +761,38 @@ function exactHarnessContainerIds(
return { candidateCount: candidates.length, exactIds };
}

function assertExactSandboxImage(
export function assertExactSandboxImage(
input: Inputs,
networkName: string,
env: NodeJS.ProcessEnv,
runCommand: ManagedImageCommandRunner = commandResult,
): string {
const resolved = exactHarnessContainerIds(input, networkName, env);
// A managed-bootstrap transaction keeps its stopped rollback backup until
// commit. Qualify the one running replacement here; rollback cleanup below
// continues to inspect every stopped and running container.
const resolved = exactHarnessContainerIds(input, networkName, env, false, runCommand);
if (resolved.candidateCount !== 1 || resolved.exactIds.length !== 1) {
throw new Error(
`OpenShell did not launch exactly one harness-owned PR image container: found ${resolved.candidateCount} labeled and ${resolved.exactIds.length} exact`,
`OpenShell did not launch exactly one running harness-owned PR image container: found ${resolved.candidateCount} running labeled and ${resolved.exactIds.length} running exact`,
);
}
return resolved.exactIds[0] ?? "";
}

export function assertFailedBootstrapContainerCleanup(
input: Inputs,
networkName: string,
env: NodeJS.ProcessEnv,
runCommand: ManagedImageCommandRunner = commandResult,
): void {
const resolved = exactHarnessContainerIds(input, networkName, env, true, runCommand);
if (resolved.candidateCount !== 0 || resolved.exactIds.length !== 0) {
throw new Error(
`managed-bootstrap rollback retained a failed held sandbox: found ${resolved.candidateCount} labeled and ${resolved.exactIds.length} exact containers`,
);
}
}

function assertFailedSandboxAbsent(
onboard: OnboardModule,
input: Inputs,
Expand Down Expand Up @@ -928,12 +1013,7 @@ async function run<T extends ManagedImageOpenShellE2eLocalInferenceEvidence = ne
error instanceof Error &&
error.message.includes("protected-e2e-injected-bootstrap-completion-failure")
) {
const resolved = exactHarnessContainerIds(input, networkName, launch.sandboxEnv);
if (resolved.candidateCount !== 0 || resolved.exactIds.length !== 0) {
throw new Error(
`managed-bootstrap rollback retained a failed held sandbox: found ${resolved.candidateCount} labeled and ${resolved.exactIds.length} exact containers`,
);
}
assertFailedBootstrapContainerCleanup(input, networkName, launch.sandboxEnv);
assertFailedSandboxAbsent(onboard, input, launch.sandboxEnv);
failureInjectionQualified = true;
process.stdout.write(
Expand Down
Loading
Loading