diff --git a/test/e2e/lib/inference-switch-retry.sh b/test/e2e/lib/inference-switch-retry.sh new file mode 100755 index 00000000000..657ccc5efcd --- /dev/null +++ b/test/e2e/lib/inference-switch-retry.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Shared retry helpers for inference-switch E2Es. These tests still verify the +# final OpenShell route, sandbox config, and live inference after this helper +# returns. The --no-verify fallback is only used after verified route-setting +# attempts fail with transient upstream/network symptoms. + +is_transient_inference_set_failure() { + grep -qiE 'timed? out|timeout|ETIMEDOUT|ECONNRESET|EAI_AGAIN|ENOTFOUND|502|503|504|temporar' <<<"$1" +} + +log_inference_switch_retry_info() { + if declare -F info >/dev/null 2>&1; then + info "$1" + else + printf '\033[1;34m [info]\033[0m %s\n' "$1" + fi +} + +run_inference_set_with_retry() { + local attempts="${NEMOCLAW_SWITCH_SET_ATTEMPTS:-3}" + if ! [[ "$attempts" =~ ^[1-9][0-9]*$ ]]; then + printf 'Invalid NEMOCLAW_SWITCH_SET_ATTEMPTS=%s; expected a positive integer.\n' "$attempts" >&2 + return 2 + fi + if [ "$#" -eq 0 ]; then + printf 'run_inference_set_with_retry requires an inference set command.\n' >&2 + return 2 + fi + + local attempt rc output fallback_output + local -a command=("$@") + for ((attempt = 1; attempt <= attempts; attempt++)); do + output=$("${command[@]}" 2>&1) + rc=$? + if [ "$rc" -eq 0 ]; then + printf '%s\n' "$output" + return 0 + fi + + if ! is_transient_inference_set_failure "$output" || [ "$attempt" -ge "$attempts" ]; then + if is_transient_inference_set_failure "$output"; then + log_inference_switch_retry_info "Verified inference switch failed after ${attempts} transient attempt(s); retrying with --no-verify before live route checks..." + fallback_output=$("${command[@]}" --no-verify 2>&1) + rc=$? + printf '%s\n%s\n' "$output" "$fallback_output" + return "$rc" + fi + printf '%s\n' "$output" + return "$rc" + fi + + log_inference_switch_retry_info "Verified inference switch attempt ${attempt}/${attempts} hit a transient failure; retrying..." + sleep $((attempt * 5)) + done + + printf 'Inference switch retry loop completed without running an attempt.\n' >&2 + return 1 +} diff --git a/test/e2e/test-hermes-inference-switch.sh b/test/e2e/test-hermes-inference-switch.sh index 002bb687ea0..2ee20550a47 100755 --- a/test/e2e/test-hermes-inference-switch.sh +++ b/test/e2e/test-hermes-inference-switch.sh @@ -365,6 +365,8 @@ else fi E2E_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=test/e2e/lib/inference-switch-retry.sh +. "${E2E_DIR}/lib/inference-switch-retry.sh" SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-hermes-inference-switch}" SWITCH_PROVIDER="${NEMOCLAW_SWITCH_PROVIDER:-nvidia-prod}" SWITCH_MODEL="${NEMOCLAW_SWITCH_MODEL:-z-ai/glm-5.1}" @@ -469,7 +471,7 @@ pid_before="$(hermes_gateway_pid)" ENV_HASH_BEFORE=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sha256sum /sandbox/.hermes/.env 2>/dev/null | awk '{print $1}') || true info "Switching Hermes to ${SWITCH_PROVIDER} / ${SWITCH_MODEL} with nemohermes inference set..." -switch_output=$(nemohermes inference set --provider "$SWITCH_PROVIDER" --model "$SWITCH_MODEL" 2>&1) +switch_output=$(run_inference_set_with_retry nemohermes inference set --provider "$SWITCH_PROVIDER" --model "$SWITCH_MODEL") switch_rc=$? if [ "$switch_rc" -eq 0 ]; then pass "nemohermes inference set completed without --sandbox" diff --git a/test/e2e/test-openclaw-inference-switch.sh b/test/e2e/test-openclaw-inference-switch.sh index 4163fd83d19..8e7be41fba9 100755 --- a/test/e2e/test-openclaw-inference-switch.sh +++ b/test/e2e/test-openclaw-inference-switch.sh @@ -295,6 +295,8 @@ fi E2E_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=test/e2e/lib/openclaw-json.sh . "${E2E_DIR}/lib/openclaw-json.sh" +# shellcheck source=test/e2e/lib/inference-switch-retry.sh +. "${E2E_DIR}/lib/inference-switch-retry.sh" SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-openclaw-inference-switch}" SWITCH_PROVIDER="${NEMOCLAW_SWITCH_PROVIDER:-nvidia-prod}" SWITCH_MODEL="${NEMOCLAW_SWITCH_MODEL:-z-ai/glm-5.1}" @@ -391,7 +393,7 @@ pass "nemoclaw and openshell are on PATH" section "Phase 3: Switch inference" pid_before="$(openclaw_gateway_pid)" info "Switching ${SANDBOX_NAME} to ${SWITCH_PROVIDER} / ${SWITCH_MODEL}..." -switch_output=$(nemoclaw inference set --provider "$SWITCH_PROVIDER" --model "$SWITCH_MODEL" --sandbox "$SANDBOX_NAME" 2>&1) +switch_output=$(run_inference_set_with_retry nemoclaw inference set --provider "$SWITCH_PROVIDER" --model "$SWITCH_MODEL" --sandbox "$SANDBOX_NAME") switch_rc=$? if [ "$switch_rc" -eq 0 ]; then pass "nemoclaw inference set completed"