fix(strix): bounded retry for typed provider outages without findings

.github/workflows/strix.yml

@@ -863,6 +863,17 @@ jobs:
           export "STRIX_PROCESS_${budget_suffix}_SECONDS=$process_budget_seconds"
           export "STRIX_TOTAL_${budget_suffix}_SECONDS=5700"
 
+          # Recognized signals that the LLM backend was unavailable / starved.
+          # Defined before the gate loop so the bounded retry decision below
+          # can classify outcomes without duplicating the patterns later.
+          backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|Error during penetration test: loginAsGuest failed after [0-9]+ attempts: curl exit 7: curl: \(7\) Failed to connect to 127\.0\.0\.1 port 48080'
+          model_behavior_error_signal='(^|[^A-Za-z0-9_])(agents|pydantic_ai|strix)(\.[A-Za-z_][A-Za-z0-9_]*)*\.ModelBehaviorError([^A-Za-z0-9_]|$)'
+          # Any evidence that a vulnerability was actually reported. Its presence
+          # forces a hard failure so real findings are NEVER downgraded. Keep the
+          # severity branch anchored away from identifiers so environment lines
+          # such as STRIX_FAIL_ON_MIN_SEVERITY do not look like findings.
+          reported_vulnerability_signal='Vulnerabilities[[:space:]]+[1-9]|(^|[^A-Za-z0-9_])severity[[:space:]]*:'
+
           # Capture the gate exit code plus its console output. The gate returns
           # exit 1 both for genuine blocking vulnerabilities AND for
           # LLM-backend-unavailable outcomes (GitHub Models "Too many requests"
@@ -871,11 +882,50 @@ jobs:
           # could not complete a scan. Provider failure is typed infrastructure
           # evidence, but remains non-passing because no authoritative complete
           # vulnerability result exists.
+          #
+          # A typed provider outage with no reported vulnerability finding is
+          # retried with bounded linear backoff inside this step so transient
+          # provider failures do not fail the required check on the first
+          # attempt. Genuine findings, configuration failures, and unexpected
+          # exit codes never retry; the deadline keeps every path inside the
+          # deterministic 120-minute job budget, and all-terminal outcomes
+          # remain fail-closed.
           strix_run_log="$RUNNER_TEMP/strix_gate_console.log"
           strix_rc=0
+          strix_gate_attempt=1
+          strix_gate_deadline=$(( SECONDS + 6000 ))
           set +e
-          bash "$TRUSTED_STRIX_GATE" 2>&1 | tee "$strix_run_log"
-          strix_rc="${PIPESTATUS[0]}"
+          while : ; do
+            : > "$strix_run_log"
+            bash "$TRUSTED_STRIX_GATE" 2>&1 | tee "$strix_run_log"
+            strix_rc="${PIPESTATUS[0]}"
+            if [ "$strix_rc" -eq 0 ]; then
+              break
+            fi
+            # Only exit-code 1 scan failures can be infrastructure outcomes.
+            if [ "$strix_rc" -ne 1 ]; then
+              break
+            fi
+            # A reported vulnerability is authoritative evidence: never retry
+            # and never risk downgrading it.
+            if grep -Eiq "$reported_vulnerability_signal" "$strix_run_log"; then
+              break
+            fi
+            # Retry only recognized provider-outage / model-behavior classes.
+            if ! grep -Eiq "$backend_unavailable_signal" "$strix_run_log" \
+              && ! grep -Eq "$model_behavior_error_signal" "$strix_run_log"; then
+              break
+            fi
+            remaining_seconds=$(( strix_gate_deadline - SECONDS ))
+            if [ "$strix_gate_attempt" -ge 3 ] || [ "$remaining_seconds" -lt 600 ]; then
+              echo "Provider-unavailable Strix attempt ${strix_gate_attempt} reached the bounded retry limit or the remaining job time budget (${remaining_seconds}s) is too small to retry; failing closed." >&2
+              break
+            fi
+            backoff_seconds=$(( strix_gate_attempt * 90 ))
+            echo "Strix provider outage on attempt ${strix_gate_attempt}; retrying after ${backoff_seconds}s backoff." >&2
+            sleep "$backoff_seconds"
+            strix_gate_attempt=$(( strix_gate_attempt + 1 ))
+          done
           set -e
 
           if [ "$strix_rc" -eq 0 ]; then
@@ -889,15 +939,6 @@ jobs:
             exit "$strix_rc"
           fi
 
-          # Recognized signals that the LLM backend was unavailable / starved.
-          backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|Error during penetration test: loginAsGuest failed after [0-9]+ attempts: curl exit 7: curl: \(7\) Failed to connect to 127\.0\.0\.1 port 48080'
-          model_behavior_error_signal='(^|[^A-Za-z0-9_])(agents|pydantic_ai|strix)(\.[A-Za-z_][A-Za-z0-9_]*)*\.ModelBehaviorError([^A-Za-z0-9_]|$)'
-          # Any evidence that a vulnerability was actually reported. Its presence
-          # forces a hard failure so real findings are NEVER downgraded. Keep the
-          # severity branch anchored away from identifiers so environment lines
-          # such as STRIX_FAIL_ON_MIN_SEVERITY do not look like findings.
-          reported_vulnerability_signal='Vulnerabilities[[:space:]]+[1-9]|(^|[^A-Za-z0-9_])severity[[:space:]]*:'
-
           # An earlier out-of-scope/below-threshold finding may already have
           # been exempted by the trusted gate. Classify a later provider
           # outage from the tail after the last continuation marker, but keep

tests/test_required_workflow_queue_contract.py

@@ -505,7 +505,7 @@ def test_nvidia_nim_defaults_preserve_existing_fallbacks_without_secret(
     assert strix.returncode == 0, strix.stderr
     assert {
         "provider_mode=openai_direct",
-        "strix_model=gpt-5.6-luna",
+        "strix_model=gpt-5.4",
     } <= set(strix_output.read_text().splitlines())
     assert (
         "STRIX_MODEL: ${{ steps.gate.outputs.strix_model }}"

tests/test_strix_backend_unavailable_after_exempted_finding.py

@@ -63,9 +63,12 @@ def _extract_neutralization_block(workflow: str) -> str:
     stale logic.
     """
 
-    start_marker = (
-        "          # Recognized signals that the LLM backend was unavailable"
-    )
+    # The classification block starts at the neutralization-scope assignment
+    # and runs to the terminal failure exit. The gate-execution retry loop and
+    # the raw signal definitions live outside this region; the signal values
+    # are injected by _run_gate_tail so the extracted decision logic stays the
+    # single tested authority.
+    start_marker = '          strix_neutralization_scope_log="$strix_run_log"'
     terminal_failure_marker = (
         '          echo "Strix reported security findings or failed for a '
         'non-backend reason; failing the required check'
@@ -77,6 +80,23 @@ def _extract_neutralization_block(workflow: str) -> str:
     return workflow[start:end]
 
 
+def _extract_signal_definitions(workflow: str) -> str:
+    """Return the canonical backend-outage / finding-signal definitions.
+
+    Bounded by the same unique anchors used in production so the injected
+    patterns cannot drift from the ones the gate itself classifies with.
+    """
+
+    start_marker = (
+        "          # Recognized signals that the LLM backend was unavailable"
+    )
+    end_marker = "reported_vulnerability_signal="
+    start = workflow.index(start_marker)
+    end = workflow.index(end_marker, start)
+    end = workflow.index("\n", end) + 1
+    return workflow[start:end]
+
+
 def _run_gate_tail(log_text: str) -> int:
     """Execute the extracted block against a synthetic log; return its exit code.
 
@@ -85,6 +105,7 @@ def _run_gate_tail(log_text: str) -> int:
     """
 
     workflow = STRIX_WORKFLOW.read_text(encoding="utf-8")
+    signals = _extract_signal_definitions(workflow)
     block = _extract_neutralization_block(workflow)
     with tempfile.TemporaryDirectory(prefix="strix-tail-scope-") as temp_dir:
         strix_run_log = Path(temp_dir) / "strix_gate_console.log"
@@ -94,6 +115,7 @@ def _run_gate_tail(log_text: str) -> int:
                 "set -uo pipefail",
                 'strix_run_log="$1"',
                 "strix_rc=1",
+                signals,
                 block,
             )
         )

tests/test_strix_nvidia_nim_not_found_fallback.py

@@ -192,7 +192,7 @@ def test_workflow_uses_available_free_first_nvidia_plan(self) -> None:
         workflow = STRIX_WORKFLOW.read_text(encoding="utf-8")
         default_expression = (
             "steps.target_visibility.outputs.is_private == 'false' && "
-            f"'{DEFAULT_NVIDIA_MODEL}' || 'gpt-5.6-luna'"
+            f"'{DEFAULT_NVIDIA_MODEL}' || 'gpt-5.4'"
         )
         self.assertIn(default_expression, workflow)
         self.assertIn(
@@ -202,7 +202,7 @@ def test_workflow_uses_available_free_first_nvidia_plan(self) -> None:
         )
         self.assertIn(
             "steps.gate.outputs.provider_mode == 'nvidia_nim' && "
-            f"'{FREE_NVIDIA_FALLBACK} openai-direct/gpt-5.6-luna'",
+            f"'{FREE_NVIDIA_FALLBACK} openai-direct/gpt-5.4'",
             workflow,
         )