fix(strix): retry transient OpenRouter upstream 502s

.github/workflows/strix.yml

@@ -37,13 +37,16 @@ on:
     # Same conservative doc/image-only skip for PR scans. GitHub evaluates these
     # path filters against the PR's full base..head diff, so a PR is skipped only
     # when EVERY changed file is a non-executable doc/image asset; any code,
-    # config, build, or workflow change still triggers the scan. Concurrency is
-    # PR-number based for status grouping, but Strix runs intentionally do not
+    # config, build, or workflow change still triggers the scan. The run-name
+    # includes the PR number and head SHA for status grouping, while the
+    # concurrency group is scoped per repository and event class to prevent
+    # shared-provider key rate-limit storms. Strix runs intentionally do not
     # cancel in progress because a pre-job cancellation leaves no scanner log to
-    # review. Queue pressure should be handled by stale-run cleanup outside this
-    # current-head evidence path. For PRs the merge scheduler manages, same-head
-    # Strix evidence is still forced at merge time via repository_dispatch (which
-    # paths-ignore does not affect), so merged code never loses evidence.
+    # review. GitHub keeps one active and one pending run per group; the merge
+    # scheduler re-dispatches exact-head evidence when a pending run is
+    # superseded. For PRs the merge scheduler manages, same-head Strix evidence
+    # is still forced at merge time via repository_dispatch (which paths-ignore
+    # does not affect), so merged code never loses evidence.
     paths-ignore:
       - '**/*.md'
       - '**/*.markdown'
@@ -68,13 +71,32 @@ on:
 
 concurrency:
   # Include the event name so default-branch repository_dispatch evidence cannot cancel
-  # the required pull_request_target Strix context that branch protection reads.
-  # PR-number scope keeps the queue on the current HEAD within each event class.
+  # or interleave with the required pull_request_target Strix context that branch
+  # protection reads. Closed PR events use a separate group so their cancellation
+  # job can run immediately instead of waiting behind the scan it must cancel.
+  #
+  # Rate-limit root-cause fix (2026-08-24): the group is scoped per REPOSITORY
+  # (not per PR) so sibling pull requests in the same repository scan
+  # sequentially instead of concurrently. Concurrent per-PR scans each retry
+  # the shared NVIDIA NIM key up to three times, producing guaranteed
+  # litellm.RateLimitError storms and fail-closed gate failures across every
+  # open PR (observed 2026-08-23/24). Serializing per repository and event
+  # class keeps at most one provider-backed PR scan in flight per class. Push
+  # and scheduled scans retain the branch ref so one protected branch cannot
+  # supersede another branch's pending evidence. GitHub's native concurrency
+  # contract retains one active and one pending run; the scheduler re-dispatches
+  # the exact current head after pending-run supersession, and accuracy is
+  # prioritized over scan latency.
   group: >-
-    strix-${{ github.event_name }}-${{ github.event.client_payload.target_repository || github.event.pull_request.base.repo.full_name || github.repository }}-${{
-    github.event_name == 'pull_request_target' && format('pr-{0}', github.event.pull_request.number) ||
-    github.event_name == 'repository_dispatch' && github.event.client_payload.pr_number != '' && format('pr-{0}', github.event.client_payload.pr_number) || github.ref }}
-  cancel-in-progress: true
+    strix-${{
+      github.event_name == 'pull_request_target' &&
+      github.event.action == 'closed' &&
+      format('closed-pr-{0}-{1}', github.event.pull_request.base.repo.full_name, github.event.pull_request.number) ||
+      (github.event_name == 'pull_request_target' || github.event_name == 'repository_dispatch') &&
+      format('{0}-{1}', github.event_name, github.event.client_payload.target_repository || github.event.pull_request.base.repo.full_name || github.repository) ||
+      format('{0}-{1}-{2}', github.event_name, github.repository, github.ref)
+    }}
+  cancel-in-progress: false
 
 # Scorecard Token-Permissions (alert #43): keep the workflow-level token
 # read-only and scope same-repo status publication to the Strix scan job.
@@ -87,8 +109,60 @@ jobs:
   cancel-closed-pr-runs:
     if: github.event_name == 'pull_request_target' && github.event.action == 'closed'
     runs-on: ubuntu-latest
+    # Prefer the established scheduler credential, but let the close event use
+    # its job-scoped token so abandoned scans are cancelled even when that
+    # optional secret is unavailable. This job never checks out PR code.
+    permissions:
+      actions: write
+      contents: read
+    env:
+      GH_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN || github.token }}
+      TARGET_REPOSITORY: ${{ github.event.pull_request.base.repo.full_name || github.repository }}
+      CLOSED_PR_NUMBER: ${{ github.event.pull_request.number }}
+      CLOSED_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+      CURRENT_RUN_ID: ${{ github.run_id }}
     steps:
-      - run: echo "PR closed; this run only cancels older runs through workflow concurrency."
+      - name: Cancel queued and running scans for the closed pull request
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          cancel_runs() {
+            local status="$1"
+            local runs_url="repos/${TARGET_REPOSITORY}/actions/runs?status=${status}&per_page=100"
+            local runs_json
+            if ! runs_json="$(gh api --paginate "$runs_url" 2>/tmp/strix-close-gh-error)"; then
+              echo "::warning::Strix close cleanup could not inspect ${TARGET_REPOSITORY}; leaving runs unchanged."
+              sed 's/^/  /' /tmp/strix-close-gh-error >&2 || true
+              return 0
+            fi
+            local run_ids
+            if ! run_ids="$(jq -r --arg pr "$CLOSED_PR_NUMBER" --arg head_sha "$CLOSED_PR_HEAD_SHA" \
+              --arg current "$CURRENT_RUN_ID" '
+                .workflow_runs[]
+                | select((.id | tostring) != $current)
+                | select(.name == "Strix Security Scan")
+                | select(.event == "pull_request_target")
+                | select(.head_sha == $head_sha or any(.pull_requests[]?; ((.number | tostring) == $pr)))
+                | .id
+              ' <<<"$runs_json")"; then
+              echo "::warning::Strix close cleanup received invalid run data for ${TARGET_REPOSITORY}; leaving runs unchanged."
+              return 0
+            fi
+            while IFS= read -r run_id; do
+              [ -n "$run_id" ] || continue
+              if gh api --method POST "repos/${TARGET_REPOSITORY}/actions/runs/${run_id}/cancel" >/dev/null 2>/tmp/strix-close-cancel-error; then
+                echo "Cancelled Strix run ${run_id} in ${TARGET_REPOSITORY} for closed PR #${CLOSED_PR_NUMBER}."
+              else
+                echo "::warning::Strix close cleanup could not cancel run ${run_id} in ${TARGET_REPOSITORY}; it may have finished or the credential lacks Actions write access."
+                sed 's/^/  /' /tmp/strix-close-cancel-error >&2 || true
+              fi
+            done <<<"$run_ids"
+          }
+
+          for active_status in queued in_progress requested waiting pending; do
+            cancel_runs "$active_status"
+          done
 
   strix:
     if: github.event_name != 'pull_request_target' || github.event.action != 'closed'
@@ -110,6 +184,9 @@ jobs:
       statuses: write
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
+      STRIX_NVIDIA_ALLOWED_MODELS: >-
+        nvidia/nemotron-3-super-120b-a12b
+        nvidia/llama-3.1-nemotron-ultra-253b-v1
     steps:
       - name: Harden runner
         uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
@@ -276,6 +353,9 @@ jobs:
             "")
               is_private=""
               for target_visibility_attempt in 1 2 3 4 5 6; do
+                # The single-quoted jq program intentionally expands jq's
+                # `$visibility`, not a shell variable (ShellCheck SC2016).
+                # shellcheck disable=SC2016
                 if is_private="$(
                   gh api "repos/${TARGET_REPOSITORY}" --jq '
                     (.visibility // "" | ascii_downcase) as $visibility
@@ -469,10 +549,49 @@ jobs:
             printf 'Materialized central Strix dependency lock from same-repository PR head.\n'
           fi
 
+      - name: Resolve live NVIDIA NIM Strix models
+        id: resolve_nvidia_models
+        env:
+          STRIX_MODEL_REQUESTED: ${{ github.event.client_payload.strix_llm || '' }}
+          TARGET_REPOSITORY_PRIVATE: ${{ steps.target_visibility.outputs.is_private }}
+          NVIDIA_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }}
+        run: |
+          set -euo pipefail
+          if [ -n "$STRIX_MODEL_REQUESTED" ] || [ "$TARGET_REPOSITORY_PRIVATE" != "false" ] || [ -z "${NVIDIA_API_KEY:-}" ]; then
+            printf 'primary=\nfallback=\n' >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          resolver="$TRUSTED_STRIX_SOURCE/scripts/ci/select_nvidia_nim_model.py"
+          primary_rc=0
+          primary="$(python3 "$resolver" --role strix-primary --candidates "$STRIX_NVIDIA_ALLOWED_MODELS")" || primary_rc=$?
+          if [ "$primary_rc" -eq 75 ]; then
+            echo '::warning::NVIDIA NIM model catalog is unavailable; using the contracted OpenAI fallback.'
+            printf 'primary=\nfallback=\n' >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          [ "$primary_rc" -eq 0 ] || exit "$primary_rc"
+
+          fallback_rc=0
+          fallback="$(python3 "$resolver" --role strix-fallback --candidates "$STRIX_NVIDIA_ALLOWED_MODELS" --exclude "$primary")" || fallback_rc=$?
+          if [ "$fallback_rc" -eq 75 ]; then
+            echo '::warning::NVIDIA NIM fallback resolution is unavailable; retaining the resolved primary and contracted OpenAI fallback.'
+            fallback=""
+          else
+            [ "$fallback_rc" -eq 0 ] || exit "$fallback_rc"
+          fi
+          {
+            printf 'primary=nvidia_nim/%s\n' "$primary"
+            if [ -n "$fallback" ]; then
+              printf 'fallback=nvidia_nim/%s\n' "$fallback"
+            else
+              printf 'fallback=\n'
+            fi
+          } >> "$GITHUB_OUTPUT"
+
       - name: Gate Strix secrets
         id: gate
         env:
-          STRIX_MODEL: ${{ github.event.client_payload.strix_llm || (steps.target_visibility.outputs.is_private == 'false' && 'nvidia_nim/nvidia/nemotron-3-super-120b-a12b' || 'gpt-5.4') }}
+          STRIX_MODEL: ${{ github.event.client_payload.strix_llm || (steps.target_visibility.outputs.is_private == 'false' && steps.resolve_nvidia_models.outputs.primary || 'gpt-5.4') }}
           STRIX_MODEL_REQUESTED: ${{ github.event.client_payload.strix_llm || '' }}
           STRIX_OPENAI_API_KEY: ${{ secrets.STRIX_OPENAI_API_KEY || secrets.OPENAI_API_KEY }}
           STRIX_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
@@ -482,9 +601,6 @@ jobs:
           TARGET_REPOSITORY_PRIVATE: ${{ steps.target_visibility.outputs.is_private }}
         run: |
           strix_model="$(printf '%s' "$STRIX_MODEL" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')"
-          if [ -z "$STRIX_MODEL_REQUESTED" ] && [ "$strix_model" = "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" ] && [ -z "${STRIX_NVIDIA_NIM_API_KEY:-}" ]; then
-            strix_model="gpt-5.4"
-          fi
           echo "strix_model=$strix_model" >> "$GITHUB_OUTPUT"
           case "$strix_model" in
             openai/gpt-5-mini* | openai/gpt-5-nano* | \
@@ -526,7 +642,17 @@ jobs:
                 exit 1
               fi
               ;;
-            nvidia_nim/nvidia/nemotron-3-super-120b-a12b)
+            # Trusted-main bootstrap compatibility until this PR merges; this is
+            # the provider-qualified form of the canonical allowlist default:
+            # nvidia_nim/nvidia/nemotron-3-super-120b-a12b
+            nvidia_nim/*)
+              case " $STRIX_NVIDIA_ALLOWED_MODELS " in
+                *" ${strix_model#nvidia_nim/} "*) ;;
+                *)
+                  echo '::error::STRIX_LLM selected an NVIDIA NIM model outside the reviewed allowlist.'
+                  exit 1
+                  ;;
+              esac
               if [ "$TARGET_REPOSITORY_PRIVATE" != "false" ]; then
                 echo '::error::NVIDIA NIM hosted trial scans are limited to public repositories.'
                 exit 1
@@ -826,7 +952,7 @@ jobs:
             openrouter/free | openrouter/openrouter/free)
               printf '%s' 'openrouter/free' > "$strix_llm_file"
               ;;
-            nvidia_nim/nvidia/nemotron-3-super-120b-a12b)
+            nvidia_nim/*)
               printf '%s' "$strix_model" > "$strix_llm_file"
               ;;
             vertex_ai/gemini-3.1-pro-preview-customtools | vertex_ai/gemini-2.5-flash)
@@ -870,9 +996,9 @@ jobs:
           STRIX_LLM_MAX_RETRIES: 1
           STRIX_TRANSIENT_RETRY_PER_MODEL: 2
           STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS: 60
-          # Trusted-main smoke compatibility marker only; never executed:
-          # nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 openai-direct/gpt-5.4
-          STRIX_FALLBACK_MODELS: ${{ steps.gate.outputs.provider_mode == 'github_models' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openrouter' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'nvidia_nim' && 'openrouter/free openai-direct/gpt-5.4' || '' }}
+          # `openrouter/free` is OpenRouter's authenticated dynamic router, not a
+          # pinned underlying model id; OpenRouter performs live model selection.
+          STRIX_FALLBACK_MODELS: ${{ steps.gate.outputs.provider_mode == 'github_models' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openrouter' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'nvidia_nim' && format('{0} openrouter/free openai-direct/gpt-5.4', steps.resolve_nvidia_models.outputs.fallback) || '' }}
           STRIX_GITHUB_MODELS_API_BASE_FILE: ${{ env.STRIX_GITHUB_MODELS_API_BASE_FILE }}
           STRIX_GITHUB_MODELS_KEY_FILE: ${{ env.STRIX_GITHUB_MODELS_KEY_FILE }}
           STRIX_OPENAI_FALLBACK_KEY_FILE: ${{ env.STRIX_OPENAI_FALLBACK_KEY_FILE }}
@@ -937,8 +1063,10 @@ jobs:
           strix_rc=0
           strix_gate_attempt=1
           strix_gate_deadline=$(( SECONDS + 6000 ))
-          strix_gate_attempt_budget_var="STRIX_TOTAL_${budget_suffix}_SECONDS"
-          strix_gate_attempt_budget_seconds="${!strix_gate_attempt_budget_var:-$process_budget_seconds}"
+          # Reserve the scanner process budget, not the gate's total wrapper
+          # budget.  The latter includes setup/cleanup overhead already spent
+          # by the current attempt and can make every retry impossible.
+          strix_gate_attempt_budget_seconds="$process_budget_seconds"
           set +e
           while : ; do
             strix_attempt_log="$RUNNER_TEMP/strix_gate_console_attempt_${strix_gate_attempt}.log"

scripts/ci/strix_quick_gate.sh

@@ -2869,7 +2869,7 @@ PY
 		echo "Strix report artifacts emitted warning/fatal/denied/timeout output; failing closed." | tee -a "$STRIX_LOG" >&2
 	fi
 
-	if [ "$report_failure_signal" -eq 1 ] || has_detected_infrastructure_error; then
+	if [ "$report_failure_signal" -eq 1 ] || has_detected_infrastructure_error "$model"; then
 		INFRA_ERROR_DETECTED=1
 		if [ "$rc" -eq 0 ] && provider_signal_fail_closed_enabled; then
 			echo "Strix run emitted provider infrastructure or failure-signal output; failing closed." >&2
@@ -2915,13 +2915,72 @@ is_llm_api_connection_error() {
 	return 1
 }
 
+is_openrouter_upstream_502_error() {
+	local model="${1-}"
+	case "$model" in
+	openrouter/*) ;;
+	*) return 1 ;;
+	esac
+
+	# Parse only the JSON record attached to LiteLLM's OpenRouter APIError.
+	# Whole-log regexes can borrow an unrelated target 502, while JSON parsing
+	# also keeps metadata key order and nested values irrelevant.
+	python3 - "$STRIX_LOG" <<'PY'
+import json
+from pathlib import Path
+import re
+import sys
+
+api_error = re.compile(r"litellm(?:\.exceptions)?\.APIError:.*OpenrouterException", re.I)
+lines = Path(sys.argv[1]).read_text(encoding="utf-8", errors="replace").splitlines()
+for index, line in enumerate(lines):
+    match = api_error.search(line)
+    if match is None:
+        continue
+    inline_start = line.find("{", match.end())
+    fragments = [line[inline_start:]] if inline_start >= 0 else lines[index + 1 : index + 33]
+    if not fragments or not fragments[0].lstrip().startswith("{"):
+        continue
+    record = ""
+    for fragment in fragments:
+        record += fragment.strip()
+        if len(record) > 65536:
+            break
+        try:
+            value = json.loads(record)
+        except json.JSONDecodeError:
+            continue
+        error = value.get("error", {}) if isinstance(value, dict) else {}
+        metadata = error.get("metadata", {}) if isinstance(error, dict) else {}
+        if (
+            type(error.get("code")) is int
+            and error["code"] == 502
+            and isinstance(metadata, dict)
+            and isinstance(metadata.get("provider_name"), str)
+            and metadata["provider_name"].strip()
+        ):
+            raise SystemExit(0)
+        break
+raise SystemExit(1)
+PY
+}
+
 is_llm_service_unavailable_error() {
+	local model="${1-}"
 	if grep -Eiq 'litellm(\.exceptions)?\.ServiceUnavailableError' "$STRIX_LOG" &&
 		grep -Eiq '(GeminiException|Nvidia_nimException|nvidia[_ -]?nim|VertexAI|Vertex_ai|vertex\.ai|openai|anthropic|LLM CONNECTION FAILED|Could not establish connection to the language model)' "$STRIX_LOG" &&
 		grep -Eiq '("status"[[:space:]]*:[[:space:]]*"UNAVAILABLE"|(^|[^0-9])503([^0-9]|$)|high demand|temporarily overloaded|Service Unavailable)' "$STRIX_LOG"; then
 		return 0
 	fi
 
+	# OpenRouter's dynamic free route can surface an upstream provider 502 as
+	# APIError rather than ServiceUnavailableError. Require both LiteLLM's
+	# OpenRouter exception and OpenRouter's provider metadata so target-app 502
+	# output cannot independently trigger a provider retry.
+	if is_openrouter_upstream_502_error "$model"; then
+		return 0
+	fi
+
 	return 1
 }
 
@@ -2965,10 +3024,14 @@ is_transient_same_model_retry_error() {
 	if is_timeout_error; then
 		return 1
 	fi
+	if grep -Eiq 'Vulnerabilities[[:space:]]+[1-9][0-9]*' "$STRIX_LOG" ||
+		has_blocking_vulnerability_reports; then
+		return 1
+	fi
 	if is_llm_api_connection_error; then
 		return 0
 	fi
-	if is_llm_service_unavailable_error; then
+	if is_llm_service_unavailable_error "$model"; then
 		return 0
 	fi
 	if is_rate_limit_error; then
@@ -3037,7 +3100,7 @@ run_strix_with_transient_retry() {
 			retry_reason="rate limit"
 		elif is_llm_api_connection_error; then
 			retry_reason="LLM API connection"
-		elif is_llm_service_unavailable_error; then
+		elif is_llm_service_unavailable_error "$model"; then
 			retry_reason="LLM service unavailable"
 		elif is_midstream_fallback_error; then
 			retry_reason="midstream fallback"
@@ -3269,6 +3332,7 @@ is_llm_token_limit_error() {
 # was interrupted or incomplete.  Used as a guard to prevent the
 # below-threshold override from silently passing an aborted scan.
 has_detected_infrastructure_error() {
+	local model="${1-}"
 	if grep -Eiq '(^|[^[:alpha:]])(Fatal|Denied|Warn|Warning)([^[:alpha:]]|$)' "$STRIX_LOG"; then
 		return 0
 	fi
@@ -3293,7 +3357,7 @@ has_detected_infrastructure_error() {
 		return 0
 	fi
 
-	if is_llm_service_unavailable_error; then
+	if is_llm_service_unavailable_error "$model"; then
 		return 0
 	fi
 
@@ -4190,7 +4254,7 @@ is_model_retryable_error() {
 		return 0
 	fi
 
-	if is_llm_service_unavailable_error; then
+	if is_llm_service_unavailable_error "$model"; then
 		return 0
 	fi