test(e2e): retry inference switch verification

test/e2e/lib/inference-switch-retry.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Shared retry helpers for inference-switch E2Es. These tests still verify the
+# final OpenShell route, sandbox config, and live inference after this helper
+# returns. The --no-verify fallback is only used after verified route-setting
+# attempts fail with transient upstream/network symptoms.
+
+is_transient_inference_set_failure() {
+  grep -qiE 'timed? out|timeout|ETIMEDOUT|ECONNRESET|EAI_AGAIN|ENOTFOUND|502|503|504|temporar' <<<"$1"
+}
+
+log_inference_switch_retry_info() {
+  if declare -F info >/dev/null 2>&1; then
+    info "$1"
+  else
+    printf '\033[1;34m  [info]\033[0m %s\n' "$1"
+  fi
+}
+
+run_inference_set_with_retry() {
+  local attempts="${NEMOCLAW_SWITCH_SET_ATTEMPTS:-3}"
+  if ! [[ "$attempts" =~ ^[1-9][0-9]*$ ]]; then
+    printf 'Invalid NEMOCLAW_SWITCH_SET_ATTEMPTS=%s; expected a positive integer.\n' "$attempts" >&2
+    return 2
+  fi
+  if [ "$#" -eq 0 ]; then
+    printf 'run_inference_set_with_retry requires an inference set command.\n' >&2
+    return 2
+  fi
+
+  local attempt rc output fallback_output
+  local -a command=("$@")
+  for ((attempt = 1; attempt <= attempts; attempt++)); do
+    output=$("${command[@]}" 2>&1)
+    rc=$?
+    if [ "$rc" -eq 0 ]; then
+      printf '%s\n' "$output"
+      return 0
+    fi
+
+    if ! is_transient_inference_set_failure "$output" || [ "$attempt" -ge "$attempts" ]; then
+      if is_transient_inference_set_failure "$output"; then
+        log_inference_switch_retry_info "Verified inference switch failed after ${attempts} transient attempt(s); retrying with --no-verify before live route checks..."
+        fallback_output=$("${command[@]}" --no-verify 2>&1)
+        rc=$?
+        printf '%s\n%s\n' "$output" "$fallback_output"
+        return "$rc"
+      fi
+      printf '%s\n' "$output"
+      return "$rc"
+    fi
+
+    log_inference_switch_retry_info "Verified inference switch attempt ${attempt}/${attempts} hit a transient failure; retrying..."
+    sleep $((attempt * 5))
+  done
+
+  printf 'Inference switch retry loop completed without running an attempt.\n' >&2
+  return 1
+}

test/e2e/test-hermes-inference-switch.sh

@@ -365,6 +365,8 @@ else
 fi
 
 E2E_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=test/e2e/lib/inference-switch-retry.sh
+. "${E2E_DIR}/lib/inference-switch-retry.sh"
 SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-hermes-inference-switch}"
 SWITCH_PROVIDER="${NEMOCLAW_SWITCH_PROVIDER:-nvidia-prod}"
 SWITCH_MODEL="${NEMOCLAW_SWITCH_MODEL:-z-ai/glm-5.1}"
@@ -469,7 +471,7 @@ pid_before="$(hermes_gateway_pid)"
 ENV_HASH_BEFORE=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sha256sum /sandbox/.hermes/.env 2>/dev/null | awk '{print $1}') || true
 
 info "Switching Hermes to ${SWITCH_PROVIDER} / ${SWITCH_MODEL} with nemohermes inference set..."
-switch_output=$(nemohermes inference set --provider "$SWITCH_PROVIDER" --model "$SWITCH_MODEL" 2>&1)
+switch_output=$(run_inference_set_with_retry nemohermes inference set --provider "$SWITCH_PROVIDER" --model "$SWITCH_MODEL")
 switch_rc=$?
 if [ "$switch_rc" -eq 0 ]; then
   pass "nemohermes inference set completed without --sandbox"

test/e2e/test-openclaw-inference-switch.sh

@@ -295,6 +295,8 @@ fi
 E2E_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # shellcheck source=test/e2e/lib/openclaw-json.sh
 . "${E2E_DIR}/lib/openclaw-json.sh"
+# shellcheck source=test/e2e/lib/inference-switch-retry.sh
+. "${E2E_DIR}/lib/inference-switch-retry.sh"
 SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-openclaw-inference-switch}"
 SWITCH_PROVIDER="${NEMOCLAW_SWITCH_PROVIDER:-nvidia-prod}"
 SWITCH_MODEL="${NEMOCLAW_SWITCH_MODEL:-z-ai/glm-5.1}"
@@ -391,7 +393,7 @@ pass "nemoclaw and openshell are on PATH"
 section "Phase 3: Switch inference"
 pid_before="$(openclaw_gateway_pid)"
 info "Switching ${SANDBOX_NAME} to ${SWITCH_PROVIDER} / ${SWITCH_MODEL}..."
-switch_output=$(nemoclaw inference set --provider "$SWITCH_PROVIDER" --model "$SWITCH_MODEL" --sandbox "$SANDBOX_NAME" 2>&1)
+switch_output=$(run_inference_set_with_retry nemoclaw inference set --provider "$SWITCH_PROVIDER" --model "$SWITCH_MODEL" --sandbox "$SANDBOX_NAME")
 switch_rc=$?
 if [ "$switch_rc" -eq 0 ]; then
   pass "nemoclaw inference set completed"