Skip to content
Merged
Show file tree
Hide file tree
Changes from 16 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
96d71c5
test(e2e): consolidate priority 2 live seams
prekshivyas Jul 27, 2026
a773b52
test(e2e): guard retired selector execution
prekshivyas Jul 27, 2026
8dd2093
test(e2e): keep boundary fixtures linear
prekshivyas Jul 27, 2026
a176245
fix(e2e): preserve retired target selectors
prekshivyas Jul 27, 2026
e8b4123
perf(e2e): focus retained coverage
prekshivyas Jul 30, 2026
fbe230e
fix(e2e): retain retired selector compatibility
prekshivyas Jul 30, 2026
2bbf778
merge: resolve conflicts with main
prekshivyas Jul 30, 2026
d2b7c8f
merge(main): resolve PR #7665 conflicts
prekshivyas Jul 30, 2026
5e00ea7
test(e2e): align gateway release evidence
prekshivyas Jul 30, 2026
62bdeb9
test(e2e): retain batch upgrade coverage
prekshivyas Jul 30, 2026
c193e47
test(e2e): separate batch upgrade boundaries
prekshivyas Jul 30, 2026
4f64091
fix(e2e): keep gateway migration on the existing lane
apurvvkumaria Jul 30, 2026
d8cd881
fix(e2e): reject hosted inference step overrides
apurvvkumaria Jul 30, 2026
f37fb1d
fix(e2e): preserve selector transition jobs
prekshivyas Jul 30, 2026
7cc422d
test(e2e): bound batch rebuild calls
prekshivyas Jul 30, 2026
2b81d2a
merge(e2e): align with PR #7665 transition
prekshivyas Jul 30, 2026
f933abf
test(e2e): require retired selector cases
prekshivyas Jul 30, 2026
608a31f
merge(main): refresh PR #7665 gate state
prekshivyas Jul 30, 2026
c4a51ab
merge(e2e): refresh PR #7665 dependency
prekshivyas Jul 30, 2026
3747a78
fix(e2e): bridge retired rebuild selectors
apurvvkumaria Jul 30, 2026
5229525
merge(e2e): resolve PR #7931 conflicts after #7665
prekshivyas Jul 30, 2026
c579d39
merge(e2e): retire superseded selector bridge
prekshivyas Jul 30, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
194 changes: 7 additions & 187 deletions .github/workflows/e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -440,7 +440,7 @@ jobs:

retired-selector-compatibility:
needs: generate-matrix
if: ${{ inputs.checkout_sha != '' && (contains(format(',{0},', inputs.jobs), ',credential-migration,') || contains(format(',{0},', inputs.jobs), ',credential-sanitization,') || contains(format(',{0},', inputs.jobs), ',diagnostics,') || contains(format(',{0},', inputs.jobs), ',docs-validation,') || contains(format(',{0},', inputs.jobs), ',gateway-drift-preflight,') || contains(format(',{0},', inputs.jobs), ',gateway-health-honest,') || contains(format(',{0},', inputs.jobs), ',onboard-negative-paths,') || contains(format(',{0},', inputs.jobs), ',openshell-version-pin,') || contains(format(',{0},', inputs.jobs), ',ubuntu-repo-cli-smoke,')) }}
if: ${{ inputs.checkout_sha != '' && (contains(format(',{0},', inputs.jobs), ',credential-migration,') || contains(format(',{0},', inputs.jobs), ',credential-sanitization,') || contains(format(',{0},', inputs.jobs), ',diagnostics,') || contains(format(',{0},', inputs.jobs), ',docs-validation,') || contains(format(',{0},', inputs.jobs), ',gateway-drift-preflight,') || contains(format(',{0},', inputs.jobs), ',gateway-health-honest,') || contains(format(',{0},', inputs.jobs), ',onboard-negative-paths,') || contains(format(',{0},', inputs.jobs), ',openshell-version-pin,') || contains(format(',{0},', inputs.jobs), ',sandbox-rebuild,') || contains(format(',{0},', inputs.jobs), ',ubuntu-repo-cli-smoke,') || contains(format(',{0},', inputs.jobs), ',upgrade-stale-sandbox,') || contains(format(',{0},', inputs.targets), ',sandbox-rebuild,') || contains(format(',{0},', inputs.targets), ',upgrade-stale-sandbox,')) }}
runs-on: ubuntu-latest
timeout-minutes: 15
env:
Expand All @@ -458,6 +458,7 @@ jobs:
- name: Verify retired selector replacements
env:
JOBS: ${{ inputs.jobs }}
TARGETS: ${{ inputs.targets }}
run: npx tsx tools/e2e/retired-selector-compatibility.mts

- name: Upload retired selector compatibility evidence
Expand Down Expand Up @@ -1742,12 +1743,6 @@ jobs:
fail-fast: false
matrix:
include:
- mode: hosted
sandbox_name: e2e-hermes-inference-switch
switch_provider: nvidia-prod
switch_model: nvidia/nemotron-3-super-120b-a12b
switch_inference_api: openai-completions
switch_mock_anthropic: "0"
- mode: anthropic
sandbox_name: e2e-hermes-anthropic-inference-switch
switch_provider: compatible-anthropic-endpoint
Expand All @@ -1760,7 +1755,6 @@ jobs:
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/hermes-inference-switch/${{ matrix.mode }}
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1"
NEMOCLAW_NON_INTERACTIVE: "1"
NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
NEMOCLAW_AGENT: "hermes"
Expand Down Expand Up @@ -1809,9 +1803,6 @@ jobs:
- name: Install OpenShell CLI
run: bash scripts/install-openshell.sh
- name: Run Hermes inference switch live Vitest test
env:
NVIDIA_API_KEY: ${{ matrix.mode == 'hosted' && secrets.NVIDIA_API_KEY || '' }}
NVIDIA_INFERENCE_API_KEY: ${{ matrix.mode == 'hosted' && secrets.NVIDIA_INFERENCE_API_KEY || '' }}
run: |
set -euo pipefail
export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH"
Expand Down Expand Up @@ -3255,68 +3246,6 @@ jobs:
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh

sandbox-rebuild:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',sandbox-rebuild,') || contains(format(',{0},', inputs.targets), ',sandbox-rebuild,') }}
runs-on: ubuntu-latest
timeout-minutes: 90
env:
E2E_JOB: "1"
E2E_TARGET_ID: "sandbox-rebuild"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/sandbox-rebuild
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1"
NEMOCLAW_NON_INTERACTIVE: "1"
NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
OPENSHELL_GATEWAY: nemoclaw
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
persist-credentials: false

- *dockerhub-auth

- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75

- name: Install OpenShell
env:
NEMOCLAW_NON_INTERACTIVE: "1"
run: |
set -euo pipefail
env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u NVIDIA_INFERENCE_API_KEY -u GITHUB_TOKEN bash scripts/install-openshell.sh

- name: Run sandbox rebuild live test
env:
NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }}
run: |
set -euo pipefail
export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH"
if command -v openshell >/dev/null 2>&1; then
OPENSHELL_BIN="$(command -v openshell)"
elif [ -x "$HOME/.local/bin/openshell" ]; then
OPENSHELL_BIN="$HOME/.local/bin/openshell"
else
echo "::error::OpenShell CLI not found after install"
ls -la /usr/local/bin/openshell "$HOME/.local/bin/openshell" 2>&1 || true
exit 1
fi
export OPENSHELL_BIN
"$OPENSHELL_BIN" --version
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/sandbox-rebuild.test.ts

- name: Upload sandbox rebuild artifacts
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57

- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh

sandbox-rlimits-connect:
needs: generate-matrix
if: ${{ contains(format(',{0},', inputs.jobs), ',sandbox-rlimits-connect,') || contains(format(',{0},', inputs.targets), ',sandbox-rlimits-connect,') }}
Expand Down Expand Up @@ -3489,68 +3418,6 @@ jobs:
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh

upgrade-stale-sandbox:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',upgrade-stale-sandbox,') || contains(format(',{0},', inputs.targets), ',upgrade-stale-sandbox,') }}
runs-on: ubuntu-latest
timeout-minutes: 85
env:
E2E_JOB: "1"
E2E_TARGET_ID: "upgrade-stale-sandbox"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/upgrade-stale-sandbox
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1"
NEMOCLAW_NON_INTERACTIVE: "1"
NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
NEMOCLAW_SANDBOX_NAME: "e2e-upgrade-stale"
OPENSHELL_GATEWAY: "nemoclaw"
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
persist-credentials: false

- *dockerhub-auth

- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75

- name: Install OpenShell CLI
run: |
set -euo pipefail
env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u NVIDIA_INFERENCE_API_KEY -u GITHUB_TOKEN \
bash scripts/install-openshell.sh

- name: Run upgrade stale sandbox live Vitest test
env:
NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }}
run: |
set -euo pipefail
export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH"
if command -v openshell >/dev/null 2>&1; then
OPENSHELL_BIN="$(command -v openshell)"
elif [ -x "$HOME/.local/bin/openshell" ]; then
OPENSHELL_BIN="$HOME/.local/bin/openshell"
else
echo "::error::OpenShell CLI not found after install"
ls -la /usr/local/bin/openshell "$HOME/.local/bin/openshell" 2>&1 || true
exit 1
fi
export OPENSHELL_BIN
"$OPENSHELL_BIN" --version
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/upgrade-stale-sandbox.test.ts

- name: Upload upgrade stale sandbox artifacts
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57

- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh

double-onboard:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',double-onboard,') || contains(format(',{0},', inputs.targets), ',double-onboard,') }}
Expand Down Expand Up @@ -5054,12 +4921,6 @@ jobs:
fail-fast: false
matrix:
include:
- mode: hosted
sandbox_name: e2e-openclaw-inference-switch
switch_provider: nvidia-prod
switch_model: nvidia/nemotron-3-super-120b-a12b
switch_inference_api: openai-completions
switch_mock_anthropic: "0"
- mode: anthropic
sandbox_name: e2e-openclaw-anthropic-inference-switch
switch_provider: compatible-anthropic-endpoint
Expand All @@ -5072,7 +4933,6 @@ jobs:
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openclaw-inference-switch/${{ matrix.mode }}
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1"
NEMOCLAW_NON_INTERACTIVE: "1"
NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
NEMOCLAW_AGENT: "openclaw"
Expand All @@ -5096,13 +4956,9 @@ jobs:
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75

- name: Run OpenClaw inference switch live test
# Preserves
# the real install.sh/OpenShell/Docker/live-provider boundary while
# Preserves the real install.sh/OpenShell/Docker/managed-proxy boundary while
# Vitest owns the route, config, registry, inference.local, and agent
# assertions.
env:
NVIDIA_API_KEY: ${{ matrix.mode == 'hosted' && secrets.NVIDIA_API_KEY || '' }}
NVIDIA_INFERENCE_API_KEY: ${{ matrix.mode == 'hosted' && secrets.NVIDIA_INFERENCE_API_KEY || '' }}
run: |
set -euo pipefail
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/openclaw-inference-switch.test.ts
Expand All @@ -5128,7 +4984,7 @@ jobs:
strategy:
fail-fast: false
matrix:
agent: [openclaw, hermes]
agent: [openclaw]
env:
E2E_JOB: "1"
E2E_TARGET_ID: "bedrock-runtime-compatible-anthropic"
Expand All @@ -5143,26 +4999,6 @@ jobs:
NEMOCLAW_SANDBOX_NAME: e2e-bedrock-${{ matrix.agent }}
OPENSHELL_GATEWAY: "nemoclaw"
steps:
- id: trusted_hermes_swap
name: Provision trusted Hermes E2E swap
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && matrix.agent == 'hermes' }}
shell: /bin/bash --noprofile --norc -e -o pipefail {0}
env:
BASH_ENV: /dev/null
CHECKOUT_SHA: ${{ inputs.checkout_sha }}
DISPATCH_SHA: ${{ github.sha }}
ENV: /dev/null
EVENT_NAME: ${{ github.event_name }}
EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }}
LC_ALL: C
REF: ${{ github.ref }}
REPOSITORY: ${{ github.repository }}
RUNNER_ARCH_KIND: ${{ runner.arch }}
RUNNER_ENVIRONMENT_KIND: ${{ runner.environment }}
RUNNER_OS_KIND: ${{ runner.os }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
run: *trusted-hermes-e2e-swap

- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
Expand All @@ -5174,28 +5010,14 @@ jobs:
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75

- name: Initialize runner comparison telemetry
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' && matrix.agent == 'hermes' }}
continue-on-error: true
shell: bash
run: npx tsx tools/e2e/runner-comparison.mts initialize

- name: Run Bedrock Runtime compatible Anthropic live test
# Direct Vitest coverage for
# Preserves the
# fake Bedrock Runtime endpoint, /etc/hosts mapping, source CLI
# onboard, OpenShell adapter route, agent-specific runtime probes, and
# leak-scan contract for both OpenClaw and Hermes.
# Preserves the fake Bedrock Runtime endpoint, /etc/hosts mapping,
# source CLI onboard, OpenShell adapter route, OpenClaw runtime probe,
# and leak-scan contract.
run: |
set -euo pipefail
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts

- name: Finalize runner comparison telemetry
if: ${{ always() && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' && matrix.agent == 'hermes' }}
continue-on-error: true
shell: bash
run: npx tsx tools/e2e/runner-comparison.mts finalize

- name: Upload Bedrock Runtime compatible Anthropic artifacts
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
Expand Down Expand Up @@ -5875,11 +5697,9 @@ jobs:
rebuild-openclaw,
rebuild-hermes,
rebuild-hermes-stale-base,
sandbox-rebuild,
sandbox-rlimits-connect,
overlayfs-autofix,
state-backup-restore,
upgrade-stale-sandbox,
openshell-gateway-upgrade,
token-rotation,
messaging-compatible-endpoint,
Expand Down
29 changes: 29 additions & 0 deletions src/lib/actions/sandbox/rebuild-route-preflight.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,35 @@ describe("commitRebuildRoutePreflight", () => {
expect(state.save).toHaveBeenCalledOnce();
});

it("migrates two missing credential identities across sequential shared-route rebuilds (#7615, #7798)", () => {
const credentialEnv = "NVIDIA_INFERENCE_API_KEY";
const alpha = sandbox("alpha", "nvidia-prod");
const beta = sandbox("beta", "nvidia-prod");
const state = transactionDependencies(registry(alpha, beta));

const results = [alpha, beta].map((entry) =>
commitRebuildRoutePreflight(
{
sandboxName: entry.name,
gatewayName: "nemoclaw",
targetUpdate: targetUpdate({ ...entry, credentialEnv }),
},
state.dependencies,
),
);

expect(results).toMatchObject([
{ ok: true, receipt: { sandboxName: "alpha", migratedSandboxNames: ["beta"] } },
{ ok: true, receipt: { sandboxName: "beta", migratedSandboxNames: [] } },
]);
expect(
[alpha.name, beta.name].map(
(sandboxName) => state.persisted().sandboxes[sandboxName]?.credentialEnv,
),
).toEqual([credentialEnv, credentialEnv]);
expect(state.save).toHaveBeenCalledTimes(2);
});

it.each(
LOCAL_INFERENCE_PROVIDERS,
)("keeps credential-free local provider %s compatible (#7798)", (provider) => {
Expand Down
20 changes: 20 additions & 0 deletions src/lib/actions/upgrade-sandboxes-recovery.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -306,6 +306,26 @@ describe("upgrade-sandboxes prepared backup recovery (#6114)", () => {
);
});

it("forwards two stale shared-route sandboxes through upgrade-sandboxes --auto (#7615, #7798)", async () => {
const names = ["alpha", "beta"];
const harness = createRecoveryHarness(names, {
liveOutput: names.map((name) => `${name} Ready`).join("\n"),
staleNames: names,
});

await expect(harness.upgradeSandboxes(["--auto"])).resolves.toBeUndefined();

expect(harness.rebuildSpy).toHaveBeenNthCalledWith(1, "alpha", ["--yes"], {
recoveryManifest: undefined,
throwOnError: true,
});
expect(harness.rebuildSpy).toHaveBeenNthCalledWith(2, "beta", ["--yes"], {
recoveryManifest: undefined,
throwOnError: true,
});
expect(harness.rebuildSpy).toHaveBeenCalledTimes(2);
});

it("fails closed for a probed v0.0.55 custom image with matching backup agent version", async () => {
const probedAgentVersion = "2026.5.27";
const harness = createRecoveryHarness(["custom-box"], {
Expand Down
Loading
Loading