diff --git a/.coderabbit.yaml b/.coderabbit.yaml index 0f114e941c8..90f4a14a302 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -501,13 +501,14 @@ reviews: false-positive (detached-zombie isPidAlive + metadata-only isGatewayHealthy) regardless of OpenShell packaging choices. - **E2E test recommendation:** - - `gateway-health-honest-e2e` — false-positive health check when - the gateway binary crashes on startup (#3111) + **Regression E2E test recommendation:** + - Run `regression-e2e.yaml` with the jobs input set to + gateway-health-honest-e2e to validate false-positive health + checks when the gateway binary crashes on startup (#3111). To run selectively: ``` - gh workflow run nightly-e2e.yaml --ref -f jobs=gateway-health-honest-e2e + gh workflow run regression-e2e.yaml --ref -f 'jobs=gateway-health-honest-e2e' ``` - path: ".github/workflows/nightly-e2e.yaml" diff --git a/.github/workflows/nightly-e2e.yaml b/.github/workflows/nightly-e2e.yaml index 2d54a039a91..5adf75b2faa 100644 --- a/.github/workflows/nightly-e2e.yaml +++ b/.github/workflows/nightly-e2e.yaml @@ -22,11 +22,6 @@ # Validates real v0.0.36 curl install upgrade into # the current supported OpenShell with pre-upgrade backup, restored # agent state, and the same agent type running. -# gateway-health-honest-e2e -# Coverage guard for #3111: onboard must not log -# "Docker-driver gateway is healthy" when the gateway -# binary crashes on startup (detached-zombie + metadata- -# only health check regression). # hermes-e2e Hermes Agent E2E — install → onboard --agent hermes → health # probe → live inference. Validates the multi-agent architecture. # hermes-inference-switch-e2e @@ -72,7 +67,7 @@ on: messaging-compatible-endpoint-e2e, kimi-inference-compat-e2e, token-rotation-e2e, sandbox-survival-e2e, - openshell-gateway-upgrade-e2e, gateway-health-honest-e2e, + openshell-gateway-upgrade-e2e, issue-2478-crash-loop-recovery-e2e, hermes-e2e, hermes-inference-switch-e2e, hermes-discord-e2e, hermes-slack-e2e, sandbox-operations-e2e, inference-routing-e2e, @@ -1299,58 +1294,6 @@ jobs: /tmp/nemoclaw-e2e-openshell-gateway-compatible-mock.log if-no-files-found: ignore - # ── Gateway health-honesty E2E ────────────────────────────── - # Coverage guard for #3111. Issue #3111 reported that onboard prints - # "✓ Docker-driver gateway is healthy" on Ubuntu 22.04 even though the - # shipped openshell-gateway binary (GNU-linked against GLIBC 2.38/2.39) - # crashes immediately on a 22.04 host (GLIBC 2.35). - # - # Root cause is platform-independent: the detached child remains a - # zombie so isPidAlive() returns true, registerDockerDriverGatewayEndpoint() - # writes metadata without any TCP probe, and isGatewayHealthy() is a - # string match on openshell CLI output rather than a real health check. - # Any scenario where the gateway binary fails before serving connections - # will surface the same false-positive log on ANY Linux host — not just - # Ubuntu 22.04. - # - # This test sabotages the gateway binary with a shim that matches the - # #3111 failure mode (immediate exit with GLIBC-style stderr) and asserts - # that onboard does NOT log "healthy" and exits non-zero. - gateway-health-honest-e2e: - if: >- - github.repository == 'NVIDIA/NemoClaw' && - (github.event_name != 'workflow_dispatch' || - inputs.jobs == '' || - contains(format(',{0},', inputs.jobs), ',gateway-health-honest-e2e,')) - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - name: Checkout - uses: actions/checkout@v6 - - - name: Setup Node - uses: actions/setup-node@v6 - with: - node-version: "22" - - - name: Run gateway health-honesty E2E test - env: - GITHUB_TOKEN: ${{ github.token }} - NEMOCLAW_NON_INTERACTIVE: "1" - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" - run: bash test/e2e/test-gateway-health-honest.sh - - - name: Upload gateway health-honesty logs on failure - if: failure() - uses: actions/upload-artifact@v4 - with: - name: gateway-health-honest-logs - path: | - /tmp/nemoclaw-e2e-gateway-health-honest.log - /tmp/nemoclaw-e2e-gateway-health-honest-start.log - /tmp/nemoclaw-e2e-gateway-health-honest-process.log - if-no-files-found: ignore - # ── Hermes rebuild upgrade E2E ────────────────────────────── # Same upgrade scenario as OpenClaw but for Hermes Agent. rebuild-hermes-e2e: @@ -1924,7 +1867,6 @@ jobs: rebuild-openclaw-e2e, upgrade-stale-sandbox-e2e, openshell-gateway-upgrade-e2e, - gateway-health-honest-e2e, rebuild-hermes-e2e, rebuild-hermes-stale-base-e2e, double-onboard-e2e, @@ -2016,7 +1958,6 @@ jobs: rebuild-openclaw-e2e, upgrade-stale-sandbox-e2e, openshell-gateway-upgrade-e2e, - gateway-health-honest-e2e, rebuild-hermes-e2e, rebuild-hermes-stale-base-e2e, double-onboard-e2e, @@ -2156,7 +2097,6 @@ jobs: rebuild-openclaw-e2e, upgrade-stale-sandbox-e2e, openshell-gateway-upgrade-e2e, - gateway-health-honest-e2e, rebuild-hermes-e2e, rebuild-hermes-stale-base-e2e, double-onboard-e2e, diff --git a/.github/workflows/regression-e2e.yaml b/.github/workflows/regression-e2e.yaml index 5fe3740f422..897a2472882 100644 --- a/.github/workflows/regression-e2e.yaml +++ b/.github/workflows/regression-e2e.yaml @@ -21,7 +21,7 @@ on: jobs: description: >- Comma-separated regression job names to run (empty = all). - Valid: dashboard-remote-bind-e2e + Valid: dashboard-remote-bind-e2e,gateway-health-honest-e2e required: false type: string default: "" @@ -41,12 +41,43 @@ concurrency: cancel-in-progress: true jobs: + select_regression_jobs: + runs-on: ubuntu-latest + outputs: + dashboard: ${{ steps.select.outputs.dashboard }} + gateway: ${{ steps.select.outputs.gateway }} + steps: + - id: select + env: + JOBS: ${{ inputs.jobs }} + run: | + set -euo pipefail + normalized="$(printf '%s' "$JOBS" | tr -d '[:space:]')" + + includes_job() { + case ",${normalized}," in + *",$1,"*) return 0 ;; + *) return 1 ;; + esac + } + + if [ -z "$normalized" ] || includes_job "dashboard-remote-bind-e2e"; then + echo "dashboard=true" >> "$GITHUB_OUTPUT" + else + echo "dashboard=false" >> "$GITHUB_OUTPUT" + fi + + if [ -z "$normalized" ] || includes_job "gateway-health-honest-e2e"; then + echo "gateway=true" >> "$GITHUB_OUTPUT" + else + echo "gateway=false" >> "$GITHUB_OUTPUT" + fi + dashboard-remote-bind-e2e: + needs: select_regression_jobs if: >- github.repository == 'NVIDIA/NemoClaw' && - (github.event_name != 'workflow_dispatch' || - inputs.jobs == '' || - contains(format(',{0},', inputs.jobs), ',dashboard-remote-bind-e2e,')) + needs.select_regression_jobs.outputs.dashboard == 'true' uses: ./.github/workflows/e2e-branch-validation.yaml with: branch: ${{ github.ref_name }} @@ -55,3 +86,54 @@ jobs: use_launchable: true keep_alive: ${{ inputs.keep_alive }} secrets: inherit + + # ── Gateway health-honesty E2E ────────────────────────────── + # Coverage guard for #3111. Issue #3111 reported that onboard prints + # "✓ Docker-driver gateway is healthy" on Ubuntu 22.04 even though the + # shipped openshell-gateway binary (GNU-linked against GLIBC 2.38/2.39) + # crashes immediately on a 22.04 host (GLIBC 2.35). + # + # Root cause is platform-independent: the detached child remains a + # zombie so isPidAlive() returns true, registerDockerDriverGatewayEndpoint() + # writes metadata without any TCP probe, and isGatewayHealthy() is a + # string match on openshell CLI output rather than a real health check. + # Any scenario where the gateway binary fails before serving connections + # will surface the same false-positive log on ANY Linux host — not just + # Ubuntu 22.04. + # + # This test sabotages the gateway binary with a shim that matches the + # #3111 failure mode (immediate exit with GLIBC-style stderr) and asserts + # that onboard does NOT log "healthy" and exits non-zero. + gateway-health-honest-e2e: + needs: select_regression_jobs + if: >- + github.repository == 'NVIDIA/NemoClaw' && + needs.select_regression_jobs.outputs.gateway == 'true' + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Setup Node + uses: actions/setup-node@v6 + with: + node-version: "22" + + - name: Run gateway health-honesty E2E test + env: + GITHUB_TOKEN: ${{ github.token }} + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + run: bash test/e2e/test-gateway-health-honest.sh + + - name: Upload gateway health-honesty logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: gateway-health-honest-logs + path: | + /tmp/nemoclaw-e2e-gateway-health-honest.log + /tmp/nemoclaw-e2e-gateway-health-honest-start.log + /tmp/nemoclaw-e2e-gateway-health-honest-process.log + if-no-files-found: ignore