diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index d1df811d7a6..96f511f63eb 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -2622,6 +2622,60 @@ jobs: docker logout docker.io || true rm -rf "${DOCKER_CONFIG}" + gateway-health-honest-vitest: + needs: generate-matrix + if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',gateway-health-honest-vitest,') || contains(format(',{0},', inputs.scenarios), ',gateway-health-honest,') }} + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "gateway-health-honest" + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/gateway-health-honest + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Build CLI + run: npm run build:cli + + - name: Install OpenShell CLI + run: bash scripts/install-openshell.sh + + - name: Run gateway health honesty live test + # Migrated from test/e2e/test-gateway-health-honest.sh. It invokes + # real startGateway() with a sabotaged Docker-driver gateway binary + # and asserts the crashed process is never reported healthy. + run: | + set -euo pipefail + export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" + npx vitest run --project e2e-scenarios-live \ + test/e2e-scenario/live/gateway-health-honest.test.ts \ + --silent=false --reporter=default + + - name: Upload gateway health honesty artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-gateway-health-honest + path: e2e-artifacts/vitest/gateway-health-honest/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + channels-add-remove-vitest: needs: generate-matrix if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',channels-add-remove-vitest,') || contains(format(',{0},', inputs.scenarios), ',channels-add-remove,') }} @@ -2777,6 +2831,7 @@ jobs: issue-4434-tui-unreachable-inference-vitest, openclaw-inference-switch-vitest, bedrock-runtime-compatible-anthropic-vitest, + gateway-health-honest-vitest, channels-add-remove-vitest, ] if: ${{ always() && github.event_name == 'workflow_dispatch' }} diff --git a/test/e2e-scenario/live/gateway-health-honest.test.ts b/test/e2e-scenario/live/gateway-health-honest.test.ts new file mode 100644 index 00000000000..38ecc5bc0ca --- /dev/null +++ b/test/e2e-scenario/live/gateway-health-honest.test.ts @@ -0,0 +1,195 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Live Vitest replacement for test/e2e/test-gateway-health-honest.sh. + * + * Preserves the legacy #3111 contract by invoking the real compiled + * `startGateway()` path with a sabotaged OpenShell Docker-driver gateway + * binary that exits immediately with GLIBC-style stderr. The assertion is + * intentionally about user-visible behavior: onboarding must not print the + * misleading "Docker-driver gateway is healthy" message when the child died + * before serving a TCP probe, and it must surface a gateway-start failure. + */ + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { resultText } from "../fixtures/clients/index.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const GATEWAY_NAME = "nemoclaw-18080"; + +function gatewayStateDir(): string { + return path.join(os.homedir(), ".local", "state", "nemoclaw", "openshell-docker-gateway-18080"); +} + +function writeExecutable(file: string, content: string): void { + fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 }); + fs.writeFileSync(file, content, { encoding: "utf8", mode: 0o755 }); + fs.chmodSync(file, 0o755); +} + +test.skipIf(!shouldRunLiveE2EScenarios())( + "onboard surfaces crashed Docker-driver gateway instead of reporting healthy (#3111)", + async ({ artifacts, cleanup, host }) => { + const stateDir = gatewayStateDir(); + const sabotageBin = artifacts.pathFor("bin/openshell-gateway-sabotage"); + const gatewayLog = path.join(stateDir, "openshell-gateway.log"); + const gatewayPidFile = path.join(stateDir, "openshell-gateway.pid"); + + await artifacts.writeJson("scenario.json", { + id: "gateway-health-honest", + runner: "vitest", + boundary: "real-startGateway-openshell-docker-driver-process", + legacySource: "test/e2e/test-gateway-health-honest.sh", + contracts: [ + "startGateway() invokes a real OpenShell Docker-driver gateway child process", + "a crashed gateway binary does not log 'Docker-driver gateway is healthy'", + "startGateway() exits non-zero and surfaces a gateway-start failure", + "the gateway log proves the sabotaged GLIBC-failure binary was executed", + "no live non-zombie gateway process remains after the simulated crash", + ], + }); + + writeExecutable( + sabotageBin, + [ + "#!/usr/bin/env bash", + 'printf \'%s\\n\' "$(basename \\"$0\\"): /lib/x86_64-linux-gnu/libc.so.6: version \\`GLIBC_2.38\' not found (required by $(basename \\"$0\\"))" >&2', + 'printf \'%s\\n\' "$(basename \\"$0\\"): /lib/x86_64-linux-gnu/libc.so.6: version \\`GLIBC_2.39\' not found (required by $(basename \\"$0\\"))" >&2', + "exit 127", + "", + ].join("\n"), + ); + + fs.mkdirSync(stateDir, { recursive: true, mode: 0o700 }); + fs.rmSync(gatewayPidFile, { force: true }); + fs.rmSync(gatewayLog, { force: true }); + fs.rmSync(path.join(stateDir, "runtime-marker.json"), { force: true }); + fs.rmSync(path.join(stateDir, "openshell-gateway.toml"), { force: true }); + await host.command( + "sh", + [ + "-lc", + `command -v openshell >/dev/null 2>&1 && openshell gateway remove ${GATEWAY_NAME} || true`, + ], + { + artifactName: "pre-cleanup-openshell-gateway-remove-gateway-health-honest", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }, + ); + + cleanup.add("remove sabotaged OpenShell gateway metadata", async () => { + await host.command( + "sh", + [ + "-lc", + `command -v openshell >/dev/null 2>&1 && openshell gateway remove ${GATEWAY_NAME} || true`, + ], + { + artifactName: "cleanup-openshell-gateway-remove-gateway-health-honest", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }, + ); + }); + cleanup.add("remove sabotaged gateway runtime files", () => { + const pid = fs.existsSync(gatewayPidFile) + ? Number.parseInt(fs.readFileSync(gatewayPidFile, "utf8"), 10) + : Number.NaN; + if (Number.isInteger(pid) && pid > 0) { + try { + process.kill(pid, "SIGTERM"); + } catch { + // Best-effort: the expected child has already exited. + } + } + fs.rmSync(gatewayPidFile, { force: true }); + fs.rmSync(path.join(stateDir, "runtime-marker.json"), { force: true }); + fs.rmSync(path.join(stateDir, "openshell-gateway.toml"), { force: true }); + fs.rmSync(sabotageBin, { force: true }); + }); + + const result = await host.command( + "node", + [ + "-e", + [ + 'const { startGateway } = require("./dist/lib/onboard");', + "startGateway(null)", + " .then(() => { console.log('__onboard_startGateway_returned_successfully__'); process.exit(0); })", + " .catch((error) => { console.error('__onboard_startGateway_threw__'); console.error(error && error.stack ? error.stack : error); process.exit(3); });", + ].join("\n"), + ], + { + artifactName: "start-gateway-with-sabotaged-binary", + cwd: REPO_ROOT, + env: { + ...buildAvailabilityProbeEnv(), + NEMOCLAW_GATEWAY_PORT: "18080", + NEMOCLAW_HEALTH_POLL_COUNT: "3", + NEMOCLAW_HEALTH_POLL_INTERVAL: "1", + NEMOCLAW_OPENSHELL_GATEWAY_BIN: sabotageBin, + NEMOCLAW_OPENSHELL_GATEWAY_CONTAINER_PATCH: "0", + NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR: stateDir, + }, + timeoutMs: 60_000, + }, + ); + + const output = resultText(result); + await artifacts.writeText( + "gateway-log-tail.txt", + fs.existsSync(gatewayLog) ? fs.readFileSync(gatewayLog, "utf8") : "", + ); + + expect( + fs.existsSync(gatewayLog) ? fs.readFileSync(gatewayLog, "utf8") : "", + "sabotage binary must have been executed before health assertions are trusted", + ).toMatch(/GLIBC_2\.3(?:8|9)|openshell-gateway-sabotage/); + + expect(output).not.toContain("Docker-driver gateway is healthy"); + expect(result.exitCode, output).not.toBe(0); + expect(output).not.toContain("__onboard_startGateway_returned_successfully__"); + expect(output).toMatch( + /Docker-driver gateway failed to start|Gateway process exited with code 127|__onboard_startGateway_threw__/i, + ); + + const lingeringGateway = await host.command( + "bash", + [ + "-lc", + String.raw` +set -u +pid_file="$1" +[ -f "$pid_file" ] || exit 0 +pid="$(tr -d '[:space:]' <"$pid_file" 2>/dev/null || true)" +case "$pid" in + ""|*[!0-9]*) exit 0 ;; +esac +kill -0 "$pid" 2>/dev/null || exit 0 +state="$(ps -p "$pid" -o state= 2>/dev/null | tr -d '[:space:]')" || state="" +case "$state" in + ""|Z*) exit 0 ;; +esac +printf 'live non-zombie gateway pid remains: pid=%s state=%s\n' "$pid" "$state" >&2 +exit 1 +`, + "gateway-lingering-process-check", + gatewayPidFile, + ], + { + artifactName: "gateway-lingering-process-check", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }, + ); + expect(lingeringGateway.exitCode, resultText(lingeringGateway)).toBe(0); + }, +); diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index e6a870599f7..3ce00fe95c4 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -387,6 +387,22 @@ describe("e2e-vitest-scenarios workflow boundary", () => { selectedFreeStandingJobs: ["bedrock-runtime-compatible-anthropic-vitest"], registryScenarios: [], }); + expect( + evaluateE2eVitestWorkflowDispatchSelectors({ scenarios: "gateway-health-honest" }), + ).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["gateway-health-honest-vitest"], + registryScenarios: [], + }); + expect( + evaluateE2eVitestWorkflowDispatchSelectors({ jobs: "gateway-health-honest-vitest" }), + ).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["gateway-health-honest-vitest"], + registryScenarios: [], + }); expect( evaluateE2eVitestWorkflowDispatchSelectors({ scenarios: "channels-add-remove" }), ).toMatchObject({ diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index 6f25f04bca7..5cbe69281a3 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -3158,6 +3158,8 @@ export function validateE2eVitestScenariosWorkflowBoundary( validateBedrockRuntimeCompatibleAnthropicVitestJob(errors, jobs); + validateFreeStandingJobSelector(errors, jobs, "gateway-health-honest-vitest", "gateway-health-honest"); + validateChannelsAddRemoveVitestJob(errors, jobs); const reportToPr = asRecord(jobs["report-to-pr"]);