diff --git a/.github/workflows/nightly-e2e.yaml b/.github/workflows/nightly-e2e.yaml index c8af228b4a5..e0b8bc776e8 100644 --- a/.github/workflows/nightly-e2e.yaml +++ b/.github/workflows/nightly-e2e.yaml @@ -806,7 +806,7 @@ jobs: inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',double-onboard-e2e,')) runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 90 steps: - name: Checkout uses: actions/checkout@v6 diff --git a/test/e2e/test-double-onboard.sh b/test/e2e/test-double-onboard.sh index acf17d5f017..2721d753960 100755 --- a/test/e2e/test-double-onboard.sh +++ b/test/e2e/test-double-onboard.sh @@ -40,6 +40,33 @@ section() { } info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } +# TODO(#2562): replace shell timeout with structured timeout once unified abstraction lands + +# Per-phase timeout in seconds (20 min per onboard phase, generous for CI) +PHASE_TIMEOUT="${NEMOCLAW_E2E_PHASE_TIMEOUT:-1200}" + +# Elapsed-time helpers +phase_start_time() { date +%s; } +phase_elapsed() { + local start="$1" + local now + now="$(date +%s)" + echo $((now - start)) +} + +# Diagnostic dump — called on phase timeout or failure to aid debugging +dump_diagnostics() { + local phase_label="${1:-unknown}" + info "=== Diagnostics for ${phase_label} ===" + info "openshell status:" + openshell status 2>&1 | sed 's/^/ /' || true + info "openshell sandbox list:" + openshell sandbox list 2>&1 | sed 's/^/ /' || true + info "docker ps:" + docker ps 2>&1 | sed 's/^/ /' || true + info "=== End diagnostics ===" +} + registry_has() { local sandbox_name="$1" [ -f "$REGISTRY" ] && grep -q "$sandbox_name" "$REGISTRY" @@ -141,6 +168,7 @@ PY return 1 } +# TODO(#2562): replace shell timeout with structured timeout once unified abstraction lands run_onboard() { local sandbox_name="$1" local recreate="${2:-0}" @@ -161,7 +189,7 @@ run_onboard() { env_args+=("NEMOCLAW_RECREATE_SANDBOX=1") fi - env "${env_args[@]}" "${NEMOCLAW_CMD[@]}" onboard --non-interactive >"$log_file" 2>&1 + run_with_timeout "$PHASE_TIMEOUT" env "${env_args[@]}" "${NEMOCLAW_CMD[@]}" onboard --non-interactive >"$log_file" 2>&1 RUN_ONBOARD_EXIT=$? RUN_ONBOARD_OUTPUT="$(cat "$log_file")" rm -f "$log_file" @@ -234,14 +262,20 @@ fi section "Phase 2: First onboard ($SANDBOX_A)" info "Running successful non-interactive onboard against local compatible endpoint..." +PHASE2_START="$(phase_start_time)" run_onboard "$SANDBOX_A" output1="$RUN_ONBOARD_OUTPUT" exit1="$RUN_ONBOARD_EXIT" +info "Phase 2 elapsed: $(phase_elapsed "$PHASE2_START")s" if [ "$exit1" -eq 0 ]; then pass "First onboard completed successfully" +elif [ "$exit1" -eq 124 ]; then + fail "First onboard timed out after ${PHASE_TIMEOUT}s (exit 124)" + dump_diagnostics "Phase 2" else fail "First onboard exited $exit1 (expected 0)" + dump_diagnostics "Phase 2" fi if grep -q "Sandbox '${SANDBOX_A}' created" <<<"$output1"; then @@ -274,17 +308,23 @@ fi section "Phase 3: Second onboard ($SANDBOX_A — same name, recreate)" info "Running nemoclaw onboard with NEMOCLAW_RECREATE_SANDBOX=1..." +PHASE3_START="$(phase_start_time)" run_onboard "$SANDBOX_A" "1" output2="$RUN_ONBOARD_OUTPUT" exit2="$RUN_ONBOARD_EXIT" +info "Phase 3 elapsed: $(phase_elapsed "$PHASE3_START")s" if [ "$exit2" -eq 0 ]; then pass "Second onboard completed successfully" +elif [ "$exit2" -eq 124 ]; then + fail "Second onboard timed out after ${PHASE_TIMEOUT}s (exit 124)" + dump_diagnostics "Phase 3" else fail "Second onboard exited $exit2 (expected 0)" + dump_diagnostics "Phase 3" fi -if grep -q "Reusing existing NemoClaw gateway" <<<"$output2"; then +if grep -q "Reusing healthy NemoClaw gateway" <<<"$output2"; then pass "Healthy gateway reused on second onboard" else fail "Healthy gateway was not reused on second onboard" @@ -314,17 +354,23 @@ fi section "Phase 4: Third onboard ($SANDBOX_B — different name)" info "Running nemoclaw onboard with new sandbox name..." +PHASE4_START="$(phase_start_time)" run_onboard "$SANDBOX_B" output3="$RUN_ONBOARD_OUTPUT" exit3="$RUN_ONBOARD_EXIT" +info "Phase 4 elapsed: $(phase_elapsed "$PHASE4_START")s" if [ "$exit3" -eq 0 ]; then pass "Third onboard completed successfully" +elif [ "$exit3" -eq 124 ]; then + fail "Third onboard timed out after ${PHASE_TIMEOUT}s (exit 124)" + dump_diagnostics "Phase 4" else fail "Third onboard exited $exit3 (expected 0)" + dump_diagnostics "Phase 4" fi -if grep -q "Reusing existing NemoClaw gateway" <<<"$output3"; then +if grep -q "Reusing healthy NemoClaw gateway" <<<"$output3"; then pass "Healthy gateway reused on third onboard" else fail "Healthy gateway was not reused on third onboard" @@ -441,6 +487,13 @@ openshell sandbox delete "$SANDBOX_B" 2>/dev/null || true openshell forward stop 18789 2>/dev/null || true openshell gateway destroy -g nemoclaw 2>/dev/null || true +# Force registry reconciliation: when the gateway is in a degraded state +# (stopped in Phase 6), `nemoclaw destroy` may delete the sandbox from +# OpenShell but fail to clean its own registry entry. Running `status` for +# each sandbox triggers the stale-entry reconciliation path. +run_nemoclaw "$SANDBOX_A" status 2>/dev/null || true +run_nemoclaw "$SANDBOX_B" status 2>/dev/null || true + if openshell sandbox get "$SANDBOX_A" >/dev/null 2>&1; then fail "Sandbox '$SANDBOX_A' still exists after cleanup" else