fix(e2e): retry transient permission reads

.github/workflows/e2e.yaml

@@ -288,18 +288,74 @@ jobs:
         run: |
           set -euo pipefail
 
+          read_collaborator_permission() {
+            local maintainer="$1"
+            local attempt curl_exit failure http_status permission_file
+            permission_file="$(mktemp "${RUNNER_TEMP:-/tmp}/nemoclaw-collaborator-permission.XXXXXX")"
+
+            for attempt in 1 2 3; do
+              : >"$permission_file"
+              if http_status="$(curl --silent --proto '=https' --connect-timeout 10 --max-time 30 \
+                --output "$permission_file" --write-out "%{http_code}" \
+                --header "Authorization: Bearer ${GITHUB_TOKEN}" \
+                --header "Accept: application/vnd.github+json" \
+                --header "X-GitHub-Api-Version: 2022-11-28" \
+                "https://api.github.com/repos/${GITHUB_REPOSITORY}/collaborators/${maintainer}/permission" \
+                2>/dev/null)"; then
+                if [[ "$http_status" =~ ^2[0-9]{2}$ ]]; then
+                  if jq -e 'type == "object" and (.user.login | type == "string") and (.role_name | type == "string")' "$permission_file" >/dev/null 2>&1; then
+                    if (( attempt > 1 )); then
+                      echo "::notice::Collaborator permission read passed after retry on attempt ${attempt}/3" >&2
+                    fi
+                    cat "$permission_file"
+                    rm -f "$permission_file"
+                    return 0
+                  fi
+                  echo "::error::Collaborator permission read attempt ${attempt}/3 failed: malformed response" >&2
+                  rm -f "$permission_file"
+                  return 1
+                fi
+                if [[ "$http_status" =~ ^[0-9]{3}$ ]]; then
+                  failure="HTTP ${http_status}"
+                  case "$http_status" in
+                    408 | 429 | 5??) ;;
+                    *) echo "::error::Collaborator permission read attempt ${attempt}/3 failed: ${failure}" >&2; rm -f "$permission_file"; return 1 ;;
+                  esac
+                else
+                  echo "::error::Collaborator permission read attempt ${attempt}/3 failed: invalid HTTP status" >&2
+                  rm -f "$permission_file"
+                  return 1
+                fi
+              else
+                curl_exit=$?
+                case "$curl_exit" in
+                  5 | 6 | 7 | 16 | 18 | 28 | 35 | 52 | 55 | 56 | 92 | 95 | 96) failure="transport" ;;
+                  *) echo "::error::Collaborator permission read attempt ${attempt}/3 failed: curl exit ${curl_exit}" >&2; rm -f "$permission_file"; return 1 ;;
+                esac
+              fi
+
+              if (( attempt == 3 )); then
+                echo "::error::Collaborator permission read exhausted after attempt ${attempt}/3: ${failure}" >&2
+                rm -f "$permission_file"
+                return 1
+              fi
+              echo "::warning::Collaborator permission read attempt ${attempt}/3 failed: ${failure}; retrying" >&2
+              sleep "$attempt"
+            done
+          }
+
           require_maintainer() {
             local maintainer="$1"
             [[ "$maintainer" =~ ^[A-Za-z0-9-]{1,39}$ && "$maintainer" != -* && "$maintainer" != *- ]] || {
               echo "::error::Manual PR E2E actor is invalid" >&2
               exit 1
             }
             local permission_json
-            permission_json="$(curl --fail --silent --show-error --proto '=https' \
-              --header "Authorization: Bearer ${GITHUB_TOKEN}" \
-              --header "Accept: application/vnd.github+json" \
-              --header "X-GitHub-Api-Version: 2022-11-28" \
-              "https://api.github.com/repos/${GITHUB_REPOSITORY}/collaborators/${maintainer}/permission")"
+            permission_json="$(read_collaborator_permission "$maintainer")"
+            if [[ "$(jq -r '.user.login // ""' <<< "$permission_json" | tr '[:upper:]' '[:lower:]')" != "$(tr '[:upper:]' '[:lower:]' <<< "$maintainer")" ]]; then
+              echo "::error::Manual PR E2E permission response did not match the actor" >&2
+              exit 1
+            fi
             case "$(jq -r '.role_name // ""' <<< "$permission_json")" in
               maintain | admin) ;;
               *) echo "::error::Manual PR E2E requires a repository maintainer or administrator" >&2; exit 1 ;;
@@ -361,6 +417,62 @@ jobs:
         run: |
           set -euo pipefail
 
+          read_collaborator_permission() {
+            local administrator="$1"
+            local attempt curl_exit failure http_status permission_file
+            permission_file="$(mktemp "${RUNNER_TEMP:-/tmp}/nemoclaw-collaborator-permission.XXXXXX")"
+
+            for attempt in 1 2 3; do
+              : >"$permission_file"
+              if http_status="$(curl --silent --proto '=https' --connect-timeout 10 --max-time 30 \
+                --output "$permission_file" --write-out "%{http_code}" \
+                --header "Authorization: Bearer ${GITHUB_TOKEN}" \
+                --header "Accept: application/vnd.github+json" \
+                --header "X-GitHub-Api-Version: 2022-11-28" \
+                "https://api.github.com/repos/${GITHUB_REPOSITORY}/collaborators/${administrator}/permission" \
+                2>/dev/null)"; then
+                if [[ "$http_status" =~ ^2[0-9]{2}$ ]]; then
+                  if jq -e 'type == "object" and (.user.login | type == "string") and (.role_name | type == "string")' "$permission_file" >/dev/null 2>&1; then
+                    if (( attempt > 1 )); then
+                      echo "::notice::Collaborator permission read passed after retry on attempt ${attempt}/3" >&2
+                    fi
+                    cat "$permission_file"
+                    rm -f "$permission_file"
+                    return 0
+                  fi
+                  echo "::error::Collaborator permission read attempt ${attempt}/3 failed: malformed response" >&2
+                  rm -f "$permission_file"
+                  return 1
+                fi
+                if [[ "$http_status" =~ ^[0-9]{3}$ ]]; then
+                  failure="HTTP ${http_status}"
+                  case "$http_status" in
+                    408 | 429 | 5??) ;;
+                    *) echo "::error::Collaborator permission read attempt ${attempt}/3 failed: ${failure}" >&2; rm -f "$permission_file"; return 1 ;;
+                  esac
+                else
+                  echo "::error::Collaborator permission read attempt ${attempt}/3 failed: invalid HTTP status" >&2
+                  rm -f "$permission_file"
+                  return 1
+                fi
+              else
+                curl_exit=$?
+                case "$curl_exit" in
+                  5 | 6 | 7 | 16 | 18 | 28 | 35 | 52 | 55 | 56 | 92 | 95 | 96) failure="transport" ;;
+                  *) echo "::error::Collaborator permission read attempt ${attempt}/3 failed: curl exit ${curl_exit}" >&2; rm -f "$permission_file"; return 1 ;;
+                esac
+              fi
+
+              if (( attempt == 3 )); then
+                echo "::error::Collaborator permission read exhausted after attempt ${attempt}/3: ${failure}" >&2
+                rm -f "$permission_file"
+                return 1
+              fi
+              echo "::warning::Collaborator permission read attempt ${attempt}/3 failed: ${failure}; retrying" >&2
+              sleep "$attempt"
+            done
+          }
+
           require_admin() {
             local administrator="$1"
             if [[ ! "$administrator" =~ ^[A-Za-z0-9-]{1,39}$ || "$administrator" == -* || "$administrator" == *- ]]; then
@@ -369,11 +481,7 @@ jobs:
             fi
 
             local permission_json
-            permission_json="$(curl --fail --silent --show-error --proto '=https' \
-              --header "Authorization: Bearer ${GITHUB_TOKEN}" \
-              --header "Accept: application/vnd.github+json" \
-              --header "X-GitHub-Api-Version: 2022-11-28" \
-              "https://api.github.com/repos/${GITHUB_REPOSITORY}/collaborators/${administrator}/permission")"
+            permission_json="$(read_collaborator_permission "$administrator")"
             if [[ "$(jq -r '.user.login // ""' <<< "$permission_json" | tr '[:upper:]' '[:lower:]')" != "$(tr '[:upper:]' '[:lower:]' <<< "$administrator")" ]]; then
               echo "::error::Release qualification waiver permission response did not match the actor" >&2
               exit 1
@@ -500,6 +608,62 @@ jobs:
         run: |
           set -euo pipefail
 
+          read_collaborator_permission() {
+            local maintainer="$1"
+            local attempt curl_exit failure http_status permission_file
+            permission_file="$(mktemp "${RUNNER_TEMP:-/tmp}/nemoclaw-collaborator-permission.XXXXXX")"
+
+            for attempt in 1 2 3; do
+              : >"$permission_file"
+              if http_status="$(curl --silent --proto '=https' --connect-timeout 10 --max-time 30 \
+                --output "$permission_file" --write-out "%{http_code}" \
+                --header "Authorization: Bearer ${GITHUB_TOKEN}" \
+                --header "Accept: application/vnd.github+json" \
+                --header "X-GitHub-Api-Version: 2022-11-28" \
+                "https://api.github.com/repos/${GITHUB_REPOSITORY}/collaborators/${maintainer}/permission" \
+                2>/dev/null)"; then
+                if [[ "$http_status" =~ ^2[0-9]{2}$ ]]; then
+                  if jq -e 'type == "object" and (.user.login | type == "string") and (.role_name | type == "string")' "$permission_file" >/dev/null 2>&1; then
+                    if (( attempt > 1 )); then
+                      echo "::notice::Collaborator permission read passed after retry on attempt ${attempt}/3" >&2
+                    fi
+                    cat "$permission_file"
+                    rm -f "$permission_file"
+                    return 0
+                  fi
+                  echo "::error::Collaborator permission read attempt ${attempt}/3 failed: malformed response" >&2
+                  rm -f "$permission_file"
+                  return 1
+                fi
+                if [[ "$http_status" =~ ^[0-9]{3}$ ]]; then
+                  failure="HTTP ${http_status}"
+                  case "$http_status" in
+                    408 | 429 | 5??) ;;
+                    *) echo "::error::Collaborator permission read attempt ${attempt}/3 failed: ${failure}" >&2; rm -f "$permission_file"; return 1 ;;
+                  esac
+                else
+                  echo "::error::Collaborator permission read attempt ${attempt}/3 failed: invalid HTTP status" >&2
+                  rm -f "$permission_file"
+                  return 1
+                fi
+              else
+                curl_exit=$?
+                case "$curl_exit" in
+                  5 | 6 | 7 | 16 | 18 | 28 | 35 | 52 | 55 | 56 | 92 | 95 | 96) failure="transport" ;;
+                  *) echo "::error::Collaborator permission read attempt ${attempt}/3 failed: curl exit ${curl_exit}" >&2; rm -f "$permission_file"; return 1 ;;
+                esac
+              fi
+
+              if (( attempt == 3 )); then
+                echo "::error::Collaborator permission read exhausted after attempt ${attempt}/3: ${failure}" >&2
+                rm -f "$permission_file"
+                return 1
+              fi
+              echo "::warning::Collaborator permission read attempt ${attempt}/3 failed: ${failure}; retrying" >&2
+              sleep "$attempt"
+            done
+          }
+
           require_maintainer() {
             local maintainer="$1"
             if [[ ! "$maintainer" =~ ^[A-Za-z0-9-]{1,39}$ || "$maintainer" == -* || "$maintainer" == *- ]]; then
@@ -508,11 +672,7 @@ jobs:
             fi
 
             local permission_json
-            permission_json="$(curl --fail --silent --show-error --proto '=https' \
-              --header "Authorization: Bearer ${GITHUB_TOKEN}" \
-              --header "Accept: application/vnd.github+json" \
-              --header "X-GitHub-Api-Version: 2022-11-28" \
-              "https://api.github.com/repos/${GITHUB_REPOSITORY}/collaborators/${maintainer}/permission")"
+            permission_json="$(read_collaborator_permission "$maintainer")"
             if [[ "$(jq -r '.user.login // ""' <<< "$permission_json" | tr '[:upper:]' '[:lower:]')" != "$(tr '[:upper:]' '[:lower:]' <<< "$maintainer")" ]]; then
               echo "::error::Launchable image publication permission response did not match the actor" >&2
               exit 1

test/e2e/RETRY_INVENTORY.md

@@ -17,6 +17,7 @@ Exhaustion remains failed.
 | `hosted-runner-recovery` | Confirmed GitHub-hosted runner loss; `tools/e2e/hosted-runner-recovery.mts`, `tools/e2e/hosted-runner-loss*.mts` | Authenticated runner-allocation or internal-runner evidence that remains identical across 2 consecutive reads | 2 immediate evidence reads and at most 1 recovery request; no delay | GitHub reruns a workflow attempt | GitHub Actions | Dedicated runner-loss classifications | Source and recovery run links plus authenticated job evidence | External owner; governed by #7146, not this policy |
 | `pr-rerun-reconciliation` | PR E2E dispatch reconciliation; `tools/e2e/pr-e2e-dispatch-reconciliation.mts`, `tools/e2e/pr-e2e-retry-receipt.mts` | Trusted dispatch receipt state | Contract-defined single reconciliation | Reconciles workflow and commit identity before action | GitHub Actions | Receipt-specific terminal states | Signed workflow identity and receipt | External scope; governed by #7206 |
 | `github-publication-read` | GitHub API reads; `tools/e2e/base-image-publication.mts` | Fetch error, 408, rate limit, or 5xx | 3 attempts; Retry-After/rate-limit reset or linear delay capped at 10s | Read-only | GitHub API | Returned parsed selection on success; thrown terminal HTTP/fetch error on failure or exhaustion | Caller artifact records the returned publication selection; terminal errors identify exhausted fetch or HTTP status without response content | Eligible bounded read; existing implementation retained |
+| `trusted-controller-collaborator-permission-read` | Collaborator-permission reads for manual PR dispatch, release waiver, and Launchable publication; `.github/workflows/e2e.yaml` | Curl exit 5, 6, 7, 16, 18, 28, 35, 52, 55, 56, 92, 95, or 96; HTTP 408, 429, or 5xx | 3 attempts; linear 1s then 2s | Read-only GitHub API request | GitHub API | Transient API read versus terminal authentication, authorization, actor, or response failure | Operation name, attempt number, and sanitized failure class or HTTP status; no response body, header, or token | Eligible bounded read; HTTP 401, 403, 404, and 422, malformed responses, actor failures, and insufficient roles remain terminal; no cached permission or workflow rerun |
 | `inference-switch-ts` | Verified inference route update; `test/e2e/fixtures/inference-switch-retry.ts` | Timeout, reset, DNS/connectivity/connect error, request transport error, or exact 502/503/504 status; authentication, authorization, policy, malformed-input, and invalid-request signals take precedence | 1-10 attempts; linear 5s | Setting the same desired provider/model is idempotent | Inference provider | Shared `RetryEvidence` classifications | Every attempt classification and aggregate outcome; command artifacts remain separate and redacted | Uses `runBoundedRetry`; deterministic verification mismatches stop; no `--no-verify` exhaustion bypass |
 | `inference-switch-shell` | Verified shell inference route update; `test/e2e/lib/inference-switch-retry.sh` | Same bounded transient and terminal-precedence signatures as the TypeScript helper | 1-10 attempts; linear 5s | Setting the same desired provider/model is idempotent | Inference provider | Exit status remains failed on exhaustion | Existing command output and retry progress | Bounded compatibility helper; no `--no-verify` exhaustion bypass |
 | `provider-install-standard` | Provider validation during Brave, cron, device-auth, Hermes-switch, network-policy, and restricted onboarding | `isTransientProviderValidationFailure` allowlist only | 1 local or 3 CI attempts; linear 10s backoff | Repeats the same desired onboarding state; restricted paths destroy the prior sandbox before retry | Inference provider | Transient allowlist versus terminal install failure | Per-attempt command artifacts; restricted paths add a terminal skip artifact | Existing bounded paths; no deterministic install retry |