fix(strix): scope neutral-skip to the log tail after an exemption

.github/workflows/strix.yml

@@ -866,19 +866,36 @@ jobs:
           fi
 
           # Recognized signals that the LLM backend was unavailable / starved.
-          backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404'
+          backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|Error code:[[:space:]]*410|github_models_retirement_brownout|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404'
           # Any evidence that a vulnerability was actually reported. Its presence
           # forces a hard failure so real findings are NEVER downgraded. Keep the
           # severity branch anchored away from identifiers so environment lines
           # such as STRIX_FAIL_ON_MIN_SEVERITY do not look like findings.
           reported_vulnerability_signal='Vulnerabilities[[:space:]]+[1-9]|(^|[^A-Za-z0-9_])severity[[:space:]]*:'
 
+          # The gate may already have exempted an earlier, out-of-scope
+          # finding (unchanged-file evidence, or below the configured minimum
+          # severity) and logged "allowing pipeline continuation" before
+          # moving on to a later, independent model attempt. That earlier
+          # finding's own "Vulnerabilities N" / "severity:" text must not
+          # poison the backend-unavailable check for a later, unrelated
+          # provider outage. Scope the neutral-skip decision to the log tail
+          # after the LAST such continuation marker (the full log when no
+          # exemption occurred), so an unresolved vulnerability anywhere in
+          # that scope still fails closed.
+          strix_neutralization_scope_log="$strix_run_log"
+          if grep -Fq 'allowing pipeline continuation' "$strix_run_log"; then
+            strix_neutralization_scope_log="$RUNNER_TEMP/strix_gate_console_tail.log"
+            awk '/allowing pipeline continuation/{buf=""; next} {buf=buf $0 "\n"} END{printf "%s", buf}' \
+              "$strix_run_log" > "$strix_neutralization_scope_log"
+          fi
+
           # Neutral skip only when ALL hold: a backend-unavailability signal is
-          # present and no vulnerability was reported anywhere. This preserves
-          # real security gating while keeping uncontrollable provider outages
-          # from blocking current-head merge progress.
-          if grep -Eiq "$backend_unavailable_signal" "$strix_run_log" \
-            && ! grep -Eiq "$reported_vulnerability_signal" "$strix_run_log"; then
+          # present and no vulnerability was reported in the relevant scope.
+          # This preserves real security gating while keeping uncontrollable
+          # provider outages from blocking current-head merge progress.
+          if grep -Eiq "$backend_unavailable_signal" "$strix_neutralization_scope_log" \
+            && ! grep -Eiq "$reported_vulnerability_signal" "$strix_neutralization_scope_log"; then
             echo "::warning title=Strix backend unavailable::Strix could not complete because its LLM backend was unavailable (rate limit / token cap / connection or warm-up failure) before producing a vulnerability report. Treating as a neutral skip so an infrastructure outage does not block merges; genuine findings still fail the check. See the strix-reports artifact and the run log."
             exit 0
           fi

tests/test_required_workflow_queue_contract.py

@@ -1255,7 +1255,8 @@ def test_strix_provider_outage_without_findings_is_neutralized() -> None:
     assert "before producing a vulnerability report" in workflow
     assert "genuine findings still fail the check" in workflow
     assert (
-        '&& ! grep -Eiq "$reported_vulnerability_signal" "$strix_run_log"' in workflow
+        '&& ! grep -Eiq "$reported_vulnerability_signal" '
+        '"$strix_neutralization_scope_log"' in workflow
     )
 
 

tests/test_strix_backend_unavailable_after_exempted_finding.py

@@ -0,0 +1,154 @@
+"""Regression contract for backend-outage neutral-skip after an exempted finding.
+
+The Strix required check's console log can legitimately contain an
+already-exempted vulnerability (out-of-scope unchanged-file evidence, or one
+below the configured minimum severity) *and* a later, unrelated provider
+outage in the same run: the gate first prints "... allowing pipeline
+continuation." for the exempted finding, then a later fallback-model attempt
+fails for an infrastructure reason (for example GitHub Models' scheduled
+retirement brownout, HTTP 410 code `github_models_retirement_brownout`).
+
+Before this fix, the workflow's outer neutral-skip decision grepped the whole
+combined log for `reported_vulnerability_signal`, so the earlier -- already
+exempted -- finding's own "Vulnerabilities N" / "severity:" text permanently
+disqualified the neutral skip, turning a pure CI-infrastructure outage into a
+required-check failure that blocks merges. The fix scopes that decision to
+the log tail after the last "allowing pipeline continuation" marker. This
+test extracts the actual bash block from the workflow (not a reimplementation)
+and executes it against synthetic logs shaped like the real PR #392 run.
+"""
+
+from __future__ import annotations
+
+import re
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+
+
+REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
+STRIX_WORKFLOW = REPOSITORY_ROOT / ".github" / "workflows" / "strix.yml"
+
+# An exempted finding: Strix reported a real vulnerability, but the gate
+# itself already decided it is out of scope (unchanged files) and continued.
+EXEMPTED_FINDING_AND_CONTINUATION = (
+    "Vulnerability Report\n"
+    "Severity: CRITICAL\n"
+    "Vulnerabilities 1\n"
+    "CRITICAL: 1\n"
+    "Strix findings are limited to unchanged files in this pull request; "
+    "allowing pipeline continuation.\n"
+)
+
+# The exact GitHub Models scheduled-retirement brownout observed in PR #392's
+# required Strix check (run 32530198775, job 96920534139).
+GITHUB_MODELS_BROWNOUT = (
+    "LLM CONNECTION FAILED\n"
+    "Could not establish connection to the language model.\n"
+    "Error: Error code: 410 - {'error': {'code': "
+    "'github_models_retirement_brownout', 'message': 'GitHub Models is "
+    "temporarily unavailable as part of a scheduled retirement brownout.'}}\n"
+    "Strix run failed for model 'github_models/openai/o3' after 5s "
+    "(exit code 1).\n"
+    "Configured model and fallback models were unavailable.\n"
+)
+
+
+def _extract_neutralization_block(workflow: str) -> str:
+    """Return the gate's neutral-skip decision block, verbatim from the yml.
+
+    Bounded by two unique anchors already present in the workflow so a future
+    unrelated edit to this step fails the test instead of silently testing
+    stale logic.
+    """
+
+    start_marker = (
+        "          # Recognized signals that the LLM backend was unavailable"
+    )
+    end_marker = '          exit "$strix_rc"\n'
+    start = workflow.index(start_marker)
+    end = workflow.index(end_marker, start) + len(end_marker)
+    return workflow[start:end]
+
+
+def _run_gate_tail(log_text: str) -> int:
+    """Execute the extracted block against a synthetic log; return its exit code.
+
+    0 means the run neutral-skips (CI-infrastructure outage, not a finding).
+    Any other code means the block falls through to the hard failure branch,
+    matching the real workflow's `exit "$strix_rc"`.
+    """
+
+    workflow = STRIX_WORKFLOW.read_text(encoding="utf-8")
+    block = _extract_neutralization_block(workflow)
+    with tempfile.TemporaryDirectory(prefix="strix-tail-scope-") as temp_dir:
+        strix_run_log = Path(temp_dir) / "strix_gate_console.log"
+        strix_run_log.write_text(log_text, encoding="utf-8")
+        script = "\n".join(
+            (
+                "set -uo pipefail",
+                'strix_run_log="$1"',
+                "strix_rc=1",
+                block,
+            )
+        )
+        completed = subprocess.run(
+            [
+                "bash",
+                "-c",
+                script,
+                "strix-tail-scope",
+                str(strix_run_log),
+            ],
+            check=False,
+            capture_output=True,
+            text=True,
+            env={"RUNNER_TEMP": temp_dir, "PATH": "/usr/bin:/bin"},
+        )
+    return completed.returncode
+
+
+class StrixBackendUnavailableAfterExemptedFindingTests(unittest.TestCase):
+    """Protect the PR #392-shaped scenario without weakening the real gate."""
+
+    def test_workflow_defines_the_tail_scoping_step(self) -> None:
+        """Keep the fix's shape present so a future edit cannot drop it silently."""
+
+        workflow = STRIX_WORKFLOW.read_text(encoding="utf-8")
+        self.assertIn("strix_neutralization_scope_log", workflow)
+        self.assertIn("allowing pipeline continuation", workflow)
+        self.assertIn("github_models_retirement_brownout", workflow)
+        self.assertIn("Error code:[[:space:]]*410", workflow)
+
+    def test_neutralizes_brownout_after_an_already_exempted_finding(self) -> None:
+        """The PR #392 shape: exempted finding, then an unrelated 410 brownout."""
+
+        log = EXEMPTED_FINDING_AND_CONTINUATION + GITHUB_MODELS_BROWNOUT
+        self.assertEqual(_run_gate_tail(log), 0)
+
+    def test_still_fails_closed_on_a_finding_reported_after_continuation(self) -> None:
+        """A real finding surfacing *after* the continuation marker still blocks."""
+
+        log = (
+            EXEMPTED_FINDING_AND_CONTINUATION
+            + "Vulnerability Report\nSeverity: CRITICAL\nVulnerabilities 1\n"
+        )
+        self.assertNotEqual(_run_gate_tail(log), 0)
+
+    def test_still_fails_closed_with_no_continuation_marker_at_all(self) -> None:
+        """Preserve prior behavior: a bare unresolved finding still blocks."""
+
+        log = "Vulnerability Report\nSeverity: CRITICAL\nVulnerabilities 1\n"
+        self.assertNotEqual(_run_gate_tail(log), 0)
+
+    def test_still_neutralizes_a_bare_backend_outage_with_no_finding_at_all(
+        self,
+    ) -> None:
+        """Preserve prior behavior: a pure outage with no finding still skips."""
+
+        self.assertEqual(_run_gate_tail(GITHUB_MODELS_BROWNOUT), 0)
+
+
+if __name__ == "__main__":
+    unittest.main()