Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 15 additions & 15 deletions ci/e2e-assertion-budget.json
Original file line number Diff line number Diff line change
Expand Up @@ -15,28 +15,28 @@
"testFileCount": 86,
"liveFileCount": 222,
"direct": {
"expectCalls": 1882,
"matcherAssertions": 1851,
"expectCalls": 1815,
"matcherAssertions": 1784,
"nodeAssertions": 100,
"namedAssertionHelpers": 627,
"namedAssertionHelpers": 610,
"failCalls": 8,
"throwGuards": 87,
"objectFieldAssertions": 245,
"assertionPoints": 2918,
"generatedProbeBlocks": 136,
"generatedProbeConditions": 354
"assertionPoints": 2834,
"generatedProbeBlocks": 126,
"generatedProbeConditions": 340
},
"unique": {
"expectCalls": 2374,
"matcherAssertions": 2338,
"expectCalls": 2307,
"matcherAssertions": 2271,
"nodeAssertions": 119,
"namedAssertionHelpers": 930,
"namedAssertionHelpers": 913,
"failCalls": 38,
"throwGuards": 638,
"throwGuards": 636,
"objectFieldAssertions": 348,
"assertionPoints": 4411,
"generatedProbeBlocks": 290,
"generatedProbeConditions": 975
"assertionPoints": 4325,
"generatedProbeBlocks": 280,
"generatedProbeConditions": 961
},
"fileMetricOrder": [
"directExpectCalls",
Expand Down Expand Up @@ -106,7 +106,7 @@
"test/e2e/live/openclaw-tui-chat-correlation.test.ts": [40,46,40,52,4],
"test/e2e/live/openshell-credential-generation-window.test.ts": [44,94,44,113,18],
"test/e2e/live/openshell-gateway-auth-source-contract.test.ts": [0,0,16,28,2],
"test/e2e/live/openshell-gateway-upgrade.test.ts": [77,109,77,121,17],
"test/e2e/live/openshell-gateway-upgrade.test.ts": [10,25,10,35,7],
"test/e2e/live/overlayfs-autofix.test.ts": [22,25,22,28,2],
"test/e2e/live/pi-agent-qualification.test.ts": [30,37,30,52,4],
"test/e2e/live/podman-cpu-lifecycle.test.ts": [40,67,53,102,1],
Expand All @@ -121,7 +121,7 @@
"test/e2e/live/sandbox-rlimits-connect.test.ts": [1,1,1,1,0],
"test/e2e/live/sandbox-survival.test.ts": [10,27,10,27,1],
"test/e2e/live/sessions-agents-cli.test.ts": [20,29,20,31,0],
"test/e2e/live/shields-retirement-upgrade.test.ts": [14,29,14,41,5],
"test/e2e/live/shields-retirement-upgrade.test.ts": [14,29,14,39,5],
"test/e2e/live/skill-agent.test.ts": [9,11,9,11,0],
"test/e2e/live/snapshot-commands.test.ts": [26,39,26,41,3],
"test/e2e/live/spark-express-vllm.test.ts": [35,45,47,65,4],
Expand Down
31 changes: 17 additions & 14 deletions internal/security-reviews/openclaw-2026.7.1-dependency-review.md
Original file line number Diff line number Diff line change
Expand Up @@ -673,20 +673,23 @@ follows:
- a root entrypoint starts the `gateway` user with `HOME=/sandbox`, so startup
migrations do not probe the inaccessible `/root/.openclaw` path.

Installed-base coverage is the `v0.0.89-x86_64` row in the
`openshell-gateway-upgrade` E2E matrix. It installs the immutable v0.0.89
release with OpenClaw `2026.6.10`, seeds its legacy Memory Core SQLite and
update-check state plus a durable marker in the per-agent database materialized
by the legacy CLI, then upgrades through the current installer. The row proves
the per-agent database survives intact, the global database remains healthy,
the legacy sidecar migration and `2026.7.1` startup checkpoint complete, and
the restored `apiKey: "unused"` config still receives its gateway-held
credential only at the OpenShell boundary.
This custom route supplies `COMPATIBLE_API_KEY` only to the frozen v0.0.89
install, then deliberately withholds it from the current installer so the
post-upgrade turn proves the existing gateway-held credential was reused. The
frozen runtime intentionally creates no NVIDIA auth-profile key reference; the
E2E preserves any references that do exist without inventing one for this route.
Installed-base coverage is the `openshell-gateway-upgrade-v0-0-89-x86-64` E2E
target. It installs the immutable v0.0.89
release with OpenClaw `2026.6.10`, upgrades through the current installer, and
proves that the surviving sandbox reaches `Ready`, preserves a workspace marker,
keeps the raw gateway credential out of the sandbox environment,
`openclaw.json`, and recursive `auth-profiles.json` files, and completes
authenticated agent turns before and after the upgrade.

Deterministic tests own the implementation details. Legacy `update-check.json`
cleanup and unsafe-file rejection live in
`test/agents/openclaw/openclaw-2026-7-startup-compat.test.ts`. OpenClaw's
shared and per-agent SQLite permission patches, update-check migration bypass,
and SQLite sidecar permission handling live in
`test/agents/openclaw/openclaw-shared-state-permissions-patch.test.ts`. The live
test does not assert database schemas or third-party migration checkpoints. It
verifies the pinned historical fixture inputs and current OpenShell version.
Deterministic tests own other installer and platform-selection details.

During image assembly, the shared-state repair rejects symbolic links,
non-regular entries, and multiply linked files before it changes the ownership
Expand Down
1 change: 1 addition & 0 deletions test/automation/e2e/e2e-recommendations.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,7 @@ describe("E2E recommendation normalizer", () => {
"tools/e2e/execution-coverage.mts",
"tools/e2e/gateway-runtime.mts",
"tools/e2e/onboard-timeout-contract.mts",
"tools/e2e/openshell-gateway-upgrade-fixture.mts",
"tools/e2e/selector-aliases.mts",
"tools/e2e/target-catalogue.mts",
"scripts/checks/llama-cpp-dgx-spark-qualification-paths.mts",
Expand Down
12 changes: 11 additions & 1 deletion test/e2e/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -177,13 +177,23 @@ The historical fixtures retain these version boundaries:

| Fixture | Required boundary |
| --- | --- |
| `openshell-gateway-upgrade` | Retain the historical installer commit and SHA-256 digest, sandbox image digest, and reviewed OpenClaw npm URL and SHA-512 integrity. Install the historical package before testing the candidate upgrade path. |
| `openshell-gateway-upgrade` | Retain one v0.0.89 fixture with a pinned installer commit and digest, sandbox image digest, and reviewed OpenClaw archive. Prove that the current gateway upgrade leaves its sandbox Ready, preserves a workspace marker, keeps the raw gateway credential out of the sandbox environment, `/sandbox/.openclaw/openclaw.json`, and recursive `auth-profiles.json` files below `/sandbox/.openclaw/agents`, and supports authenticated agent turns before and after the upgrade. |
| `rebuild-openclaw` | Retain the reviewed old-base build in the target. Build and create the old sandbox before testing the candidate rebuild path. |

These targets may restore the shared artifact for the candidate CLI.
They must not replace a historical installer, package, image, or version boundary with that artifact.
The gateway fixture already binds its remote historical inputs to immutable commits and cryptographic digests.
The workflow does not republish those inputs as artifacts.
Deterministic tests own installer identity, OpenShell release asset selection,
NemoClaw restore behavior, and Dockerfile patch behavior. The live target does not
assert OpenClaw database tables, migration checkpoints, or other third-party
storage details.

The retained live target owns the released-gateway upgrade and usable-survivor
boundary. Deterministic rebuild-flow tests own post-backup recreate failure,
preserved backup and registry state, recovery-journal retention, and successful
retry; stale-recovery tests own fail-closed behavior when no authoritative live
policy remains.

### Hermes Sandbox Image Artifact

Expand Down
69 changes: 22 additions & 47 deletions test/e2e/live/openshell-gateway-upgrade-helpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,18 @@
// SPDX-License-Identifier: Apache-2.0

import { shellQuote } from "../fixtures/clients/command.ts";
import { REVIEWED_GATEWAY_UPGRADE_FIXTURE } from "../../../tools/e2e/openshell-gateway-upgrade-fixture.mts";
import { reviewedOldInstallerProfile } from "./openshell-gateway-upgrade-old-installer.ts";

const NON_INTERACTIVE_INSTALLER_ARGS = ["--non-interactive", "--yes-i-accept-third-party-software"];
const GATEWAY_VOLUME_PREFIX = "openshell-cluster-nemoclaw";
const LEGACY_GATEWAY_DOCKER_NETWORK = "openshell-cluster-nemoclaw";
export const GATEWAY_UPGRADE_INSTALL_TIMEOUT_MS = 35 * 60_000;

export interface LegacyGatewayUpgradeFixture {
nemoclawRef: string;
nemoclawCommit: string;
installerSha256: string;
openShellVersion: string;
openclawVersion: string;
sandboxBaseImageRef: string;
}
Expand All @@ -28,23 +29,33 @@ export function validateLegacyGatewayUpgradeFixture(fixture: LegacyGatewayUpgrad
`NEMOCLAW_OLD_NEMOCLAW_COMMIT must be a full lowercase commit SHA; got ${fixture.nemoclawCommit}`,
);
}
if (!/^[0-9a-f]{64}$/.test(fixture.installerSha256)) {
if (
!/^[0-9a-f]{64}$/.test(fixture.installerSha256) ||
fixture.installerSha256 !== REVIEWED_GATEWAY_UPGRADE_FIXTURE.installerSha256
) {
throw new Error(
`NEMOCLAW_OLD_INSTALLER_SHA256 must be a lowercase SHA-256 digest; got ${fixture.installerSha256}`,
`NEMOCLAW_OLD_INSTALLER_SHA256 must match the reviewed descriptor's lowercase SHA-256 digest; got ${fixture.installerSha256}`,
);
}
if (!/^\d{4}\.\d{1,2}\.\d{1,2}$/.test(fixture.openclawVersion)) {
if (
!/^\d{4}\.\d{1,2}\.\d{1,2}$/.test(fixture.openclawVersion) ||
!/^\d+\.\d+\.\d+$/.test(fixture.openShellVersion) ||
fixture.openShellVersion !== REVIEWED_GATEWAY_UPGRADE_FIXTURE.openShellVersion
) {
throw new Error(
`NEMOCLAW_OLD_OPENCLAW_VERSION must use the YYYY.M.D release format; got ${fixture.openclawVersion}`,
`NEMOCLAW_OLD_OPENCLAW_VERSION and NEMOCLAW_OLD_OPENSHELL_VERSION must match the reviewed descriptor; got ${fixture.openclawVersion}/${fixture.openShellVersion}`,
);
}
reviewedOldInstallerProfile(fixture);
const sandboxBaseDigest = fixture.sandboxBaseImageRef.match(
/^[^@\s]+@sha256:([0-9a-f]{64})$/,
)?.[1];
if (!sandboxBaseDigest) {
if (
!sandboxBaseDigest ||
fixture.sandboxBaseImageRef !== REVIEWED_GATEWAY_UPGRADE_FIXTURE.sandboxBaseImageRef
) {
throw new Error(
`NEMOCLAW_OLD_SANDBOX_BASE_IMAGE_REF must be digest-pinned; got ${fixture.sandboxBaseImageRef}`,
`NEMOCLAW_OLD_SANDBOX_BASE_IMAGE_REF must match the reviewed descriptor and use a digest pin; got ${fixture.sandboxBaseImageRef}`,
);
}
return { sandboxBaseDigest };
Expand All @@ -54,11 +65,8 @@ export function oldGatewayUpgradeInstallerArgs(installer: string): string[] {
return [installer, ...NON_INTERACTIVE_INSTALLER_ARGS, "--fresh"];
}

export function currentGatewayUpgradeInstallerArgs(
installer: string,
options: { interactive?: boolean } = {},
): string[] {
return options.interactive ? [installer] : [installer, ...NON_INTERACTIVE_INSTALLER_ARGS];
export function currentGatewayUpgradeInstallerArgs(installer: string): string[] {
return [installer, ...NON_INTERACTIVE_INSTALLER_ARGS];
}

export function currentNemoclawUpgradeRef(env: NodeJS.ProcessEnv): string {
Expand All @@ -72,29 +80,13 @@ export function currentNemoclawUpgradeRef(env: NodeJS.ProcessEnv): string {
return "HEAD";
}

export function legacyGatewayUpgradeHostFirewallOptions(nemoclawRef: string): {
export function legacyGatewayUpgradeHostFirewallOptions(): {
networkName: string | undefined;
waitForNetworkMs: number;
} {
let networkName: string | undefined;
switch (nemoclawRef) {
case "v0.0.36":
// This cluster-era gateway names its bridge after the gateway; newer
// Docker gateways use the host fixture's openshell-docker default.
networkName = LEGACY_GATEWAY_DOCKER_NETWORK;
break;
case "v0.0.55":
case "v0.0.74":
case "v0.0.89":
case "v0.0.115":
networkName = undefined;
break;
default:
throw new Error(`Unsupported gateway-upgrade network fixture: ${nemoclawRef}`);
}
// The historical install creates its network after fetching and building
// its payload, so keep the parallel probe alive for the full install budget.
return { networkName, waitForNetworkMs: GATEWAY_UPGRADE_INSTALL_TIMEOUT_MS };
return { networkName: undefined, waitForNetworkMs: GATEWAY_UPGRADE_INSTALL_TIMEOUT_MS };
}

export function throwGatewayUpgradeSetupFailures(
Expand All @@ -109,23 +101,6 @@ export function throwGatewayUpgradeSetupFailures(
}
}

export function expectedLegacyRegistryMetadata(nemoclawRef: string): {
nemoclawVersion: string | undefined;
fromDockerfile: null | undefined;
} {
switch (nemoclawRef) {
case "v0.0.36":
case "v0.0.55":
return { nemoclawVersion: undefined, fromDockerfile: undefined };
case "v0.0.74":
return { nemoclawVersion: "0.0.74", fromDockerfile: null };
case "v0.0.89":
return { nemoclawVersion: "0.0.89", fromDockerfile: null };
default:
throw new Error(`Unsupported gateway-upgrade registry fixture: ${nemoclawRef}`);
}
}

export function upgradeGatewayStateCleanupScript(pidFile: string): string {
return `set -e
volume_prefix=${GATEWAY_VOLUME_PREFIX}
Expand Down
91 changes: 12 additions & 79 deletions test/e2e/live/openshell-gateway-upgrade-old-installer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,56 +2,18 @@
// SPDX-License-Identifier: Apache-2.0

import fs from "node:fs";
import { REVIEWED_GATEWAY_UPGRADE_FIXTURE } from "../../../tools/e2e/openshell-gateway-upgrade-fixture.mts";

export type ReviewedOldOpenClawArchive = Readonly<{
expectedIntegrity: string;
label: string;
packageSpec: string;
tarballUrl: string;
}>;

export type OldInstallerFixtureIdentity = Readonly<{
type ReviewedOldOpenClawArchive = typeof REVIEWED_GATEWAY_UPGRADE_FIXTURE.openClawArchive;
type OldInstallerFixtureIdentity = Readonly<{
nemoclawCommit: string;
nemoclawRef: string;
openclawVersion: string;
}>;

type ReviewedOldInstallerProfile = OldInstallerFixtureIdentity &
Readonly<{
expectedAdvisoryAuditCount: 0 | 1;
}>;

const REVIEWED_OLD_OPENCLAW_ARCHIVES: Readonly<Record<string, ReviewedOldOpenClawArchive>> =
Object.freeze({
"2026.4.24": {
expectedIntegrity:
"sha512-W6u4XeIIP4+uG4DYV9G3JeS6QNuKwfhQIej1GIoL4BdcnUFgrnB8kHYNXL3MxiHRKuhZB9OYwUMGs8jKFZR/Vg==",
label: "historical fixture OpenClaw 2026.4.24",
packageSpec: "openclaw@2026.4.24",
tarballUrl: "https://registry.npmjs.org/openclaw/-/openclaw-2026.4.24.tgz",
},
"2026.5.22": {
expectedIntegrity:
"sha512-m+zgBELGbCHjWB1IWF5WSWNPr480cMKOMff2OF72c8A0AMD4hC/9+qwYtzjYmGkETcffnB711JymlVsQnh2Tow==",
label: "historical fixture OpenClaw 2026.5.22",
packageSpec: "openclaw@2026.5.22",
tarballUrl: "https://registry.npmjs.org/openclaw/-/openclaw-2026.5.22.tgz",
},
"2026.5.27": {
expectedIntegrity:
"sha512-2N93zhdAo88KAbHt6T7KvYXf4s7XIkYXBgv1npYpn7e1Y9FvrtgtpsA38my9rtFW+70uXEojRPX5/OqnuDqJPw==",
label: "historical fixture OpenClaw 2026.5.27",
packageSpec: "openclaw@2026.5.27",
tarballUrl: "https://registry.npmjs.org/openclaw/-/openclaw-2026.5.27.tgz",
},
"2026.6.10": {
expectedIntegrity:
"sha512-LcooND2tBQw8A+kc1Ujltu3lg30bJ0w7XaeRy7eYzobb8BBdcW6DOGbwJL4vpj1vl9+gjRceOtlh5nh9OARcug==",
label: "historical fixture OpenClaw 2026.6.10",
packageSpec: "openclaw@2026.6.10",
tarballUrl: "https://registry.npmjs.org/openclaw/-/openclaw-2026.6.10.tgz",
},
});
type ReviewedOldInstallerProfile = Pick<
typeof REVIEWED_GATEWAY_UPGRADE_FIXTURE,
"expectedAdvisoryAuditCount" | "nemoclawCommit" | "nemoclawRef" | "openclawVersion"
>;

export const OLD_INSTALLER_BOOTSTRAP_NEEDLE = ' legacy_script="${source_root}/install.sh"\n';
export const OLD_INSTALLER_CLONE_NEEDLE =
Expand All @@ -60,53 +22,24 @@ export const OLD_INSTALLER_ADVISORY_AUDIT =
" npm --prefix /usr/local/lib/nemoclaw/mcporter-runtime audit --omit=dev --audit-level=low; \\\n";
export const OLD_INSTALLER_ARCHIVE_CONTEXT_PATH = "nemoclaw/src/.nemoclaw-e2e-old-openclaw.tgz";

const REVIEWED_OLD_INSTALLER_PROFILES: Readonly<Record<string, ReviewedOldInstallerProfile>> =
Object.freeze({
"v0.0.36": Object.freeze({
expectedAdvisoryAuditCount: 0,
nemoclawCommit: "3351fbdd4eb7d9b80ec471545083956327da2b10",
nemoclawRef: "v0.0.36",
openclawVersion: "2026.4.24",
}),
"v0.0.55": Object.freeze({
expectedAdvisoryAuditCount: 0,
nemoclawCommit: "95d483fe2b6569d68e59493c60f19df09a068e8f",
nemoclawRef: "v0.0.55",
openclawVersion: "2026.5.22",
}),
"v0.0.74": Object.freeze({
expectedAdvisoryAuditCount: 1,
nemoclawCommit: "3a05b54e8ec3e1d5550ec5c728de54af872bffe3",
nemoclawRef: "v0.0.74",
openclawVersion: "2026.5.27",
}),
"v0.0.89": Object.freeze({
expectedAdvisoryAuditCount: 1,
nemoclawCommit: "1143aa5cce77f3bad1b3b5588bd7fddbe438237e",
nemoclawRef: "v0.0.89",
openclawVersion: "2026.6.10",
}),
});

export function reviewedOldOpenClawArchive(version: string): ReviewedOldOpenClawArchive {
const reviewedArchive = REVIEWED_OLD_OPENCLAW_ARCHIVES[version];
if (!reviewedArchive) {
if (version !== REVIEWED_GATEWAY_UPGRADE_FIXTURE.openclawVersion) {
throw new Error(`Historical gateway upgrade OpenClaw ${version} has no reviewed archive pin`);
}
return reviewedArchive;
return REVIEWED_GATEWAY_UPGRADE_FIXTURE.openClawArchive;
}

export function reviewedOldInstallerProfile(
identity: OldInstallerFixtureIdentity,
): ReviewedOldInstallerProfile {
const profile = REVIEWED_OLD_INSTALLER_PROFILES[identity.nemoclawRef];
const profile = REVIEWED_GATEWAY_UPGRADE_FIXTURE;
if (
!profile ||
profile.nemoclawRef !== identity.nemoclawRef ||
profile.nemoclawCommit !== identity.nemoclawCommit ||
profile.openclawVersion !== identity.openclawVersion
) {
throw new Error(
`Historical gateway upgrade fixture must match an exact reviewed ref/commit/OpenClaw profile; got ${identity.nemoclawRef}/${identity.nemoclawCommit}/${identity.openclawVersion}`,
`Historical gateway upgrade fixture must match the reviewed descriptor's ref, commit, and OpenClaw version; got ${identity.nemoclawRef}/${identity.nemoclawCommit}/${identity.openclawVersion}`,
);
}
return profile;
Expand Down
Loading
Loading