test(e2e): retry inference switch verification

Dockerfile.base

@@ -74,14 +74,18 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 # gosu for privilege separation (gateway vs sandbox user).
 # Install from GitHub release with checksum verification instead of
 # Debian's packaged gosu can lag upstream. Pinned to 1.19 (2025-09).
+# Binary release asset downloads can be slower than hash fetches; keep
+# longer bounded transfer timeouts while preserving checksum verification.
 # hadolint ignore=DL4006
 RUN arch="$(dpkg --print-architecture)" \
     && case "$arch" in \
         amd64) gosu_asset="gosu-amd64"; gosu_sha256="52c8749d0142edd234e9d6bd5237dff2d81e71f43537e2f4f66f75dd4b243dd0" ;; \
         arm64) gosu_asset="gosu-arm64"; gosu_sha256="3a8ef022d82c0bc4a98bcb144e77da714c25fcfa64dccc57f6aba7ae47ff1a44" ;; \
         *) echo "Unsupported architecture for gosu: $arch" >&2; exit 1 ;; \
     esac \
-    && curl -fsSL -o /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/1.19/${gosu_asset}" \
+    && curl --proto '=https' --tlsv1.2 -fsSL \
+        --retry 5 --retry-all-errors --retry-delay 2 --connect-timeout 15 --max-time 120 \
+        -o /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/1.19/${gosu_asset}" \
     && echo "${gosu_sha256}  /usr/local/bin/gosu" | sha256sum -c - \
     && chmod +x /usr/local/bin/gosu \
     && gosu --version

test/e2e/test-hermes-inference-switch.sh

@@ -44,6 +44,38 @@ section() {
 }
 info() { printf '\033[1;34m  [info]\033[0m %s\n' "$1"; }
 
+is_transient_inference_set_failure() {
+  grep -qiE 'timed? out|timeout|ETIMEDOUT|ECONNRESET|EAI_AGAIN|ENOTFOUND|502|503|504|temporar' <<<"$1"
+}
+
+run_inference_set_with_retry() {
+  local attempt rc output fallback_output
+  local attempts="${NEMOCLAW_SWITCH_SET_ATTEMPTS:-3}"
+  for attempt in $(seq 1 "$attempts"); do
+    output=$(nemohermes inference set --provider "$SWITCH_PROVIDER" --model "$SWITCH_MODEL" 2>&1)
+    rc=$?
+    if [ "$rc" -eq 0 ]; then
+      printf '%s\n' "$output"
+      return 0
+    fi
+
+    if ! is_transient_inference_set_failure "$output" || [ "$attempt" -ge "$attempts" ]; then
+      if is_transient_inference_set_failure "$output"; then
+        info "Verified Hermes inference switch failed after ${attempts} transient attempt(s); retrying with --no-verify before live route checks..."
+        fallback_output=$(nemohermes inference set --provider "$SWITCH_PROVIDER" --model "$SWITCH_MODEL" --no-verify 2>&1)
+        rc=$?
+        printf '%s\n%s\n' "$output" "$fallback_output"
+        return "$rc"
+      fi
+      printf '%s\n' "$output"
+      return "$rc"
+    fi
+
+    info "Verified Hermes inference switch attempt ${attempt}/${attempts} hit a transient failure; retrying..."
+    sleep $((attempt * 5))
+  done
+}
+
 parse_chat_content() {
   python3 -c "
 import json, sys
@@ -469,7 +501,7 @@ pid_before="$(hermes_gateway_pid)"
 ENV_HASH_BEFORE=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sha256sum /sandbox/.hermes/.env 2>/dev/null | awk '{print $1}') || true
 
 info "Switching Hermes to ${SWITCH_PROVIDER} / ${SWITCH_MODEL} with nemohermes inference set..."
-switch_output=$(nemohermes inference set --provider "$SWITCH_PROVIDER" --model "$SWITCH_MODEL" 2>&1)
+switch_output=$(run_inference_set_with_retry)
 switch_rc=$?
 if [ "$switch_rc" -eq 0 ]; then
   pass "nemohermes inference set completed without --sandbox"

test/e2e/test-openclaw-inference-switch.sh

@@ -44,6 +44,38 @@ section() {
 }
 info() { printf '\033[1;34m  [info]\033[0m %s\n' "$1"; }
 
+is_transient_inference_set_failure() {
+  grep -qiE 'timed? out|timeout|ETIMEDOUT|ECONNRESET|EAI_AGAIN|ENOTFOUND|502|503|504|temporar' <<<"$1"
+}
+
+run_inference_set_with_retry() {
+  local attempt rc output fallback_output
+  local attempts="${NEMOCLAW_SWITCH_SET_ATTEMPTS:-3}"
+  for attempt in $(seq 1 "$attempts"); do
+    output=$(nemoclaw inference set --provider "$SWITCH_PROVIDER" --model "$SWITCH_MODEL" --sandbox "$SANDBOX_NAME" 2>&1)
+    rc=$?
+    if [ "$rc" -eq 0 ]; then
+      printf '%s\n' "$output"
+      return 0
+    fi
+
+    if ! is_transient_inference_set_failure "$output" || [ "$attempt" -ge "$attempts" ]; then
+      if is_transient_inference_set_failure "$output"; then
+        info "Verified inference switch failed after ${attempts} transient attempt(s); retrying with --no-verify before live route checks..."
+        fallback_output=$(nemoclaw inference set --provider "$SWITCH_PROVIDER" --model "$SWITCH_MODEL" --sandbox "$SANDBOX_NAME" --no-verify 2>&1)
+        rc=$?
+        printf '%s\n%s\n' "$output" "$fallback_output"
+        return "$rc"
+      fi
+      printf '%s\n' "$output"
+      return "$rc"
+    fi
+
+    info "Verified inference switch attempt ${attempt}/${attempts} hit a transient failure; retrying..."
+    sleep $((attempt * 5))
+  done
+}
+
 run_with_timeout() {
   local seconds="$1"
   shift
@@ -391,7 +423,7 @@ pass "nemoclaw and openshell are on PATH"
 section "Phase 3: Switch inference"
 pid_before="$(openclaw_gateway_pid)"
 info "Switching ${SANDBOX_NAME} to ${SWITCH_PROVIDER} / ${SWITCH_MODEL}..."
-switch_output=$(nemoclaw inference set --provider "$SWITCH_PROVIDER" --model "$SWITCH_MODEL" --sandbox "$SANDBOX_NAME" 2>&1)
+switch_output=$(run_inference_set_with_retry)
 switch_rc=$?
 if [ "$switch_rc" -eq 0 ]; then
   pass "nemoclaw inference set completed"