diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 03ec23257..da42552ee 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -839,7 +839,8 @@ jobs: # exit 1 both for genuine blocking vulnerabilities AND for # LLM-backend-unavailable outcomes (GitHub Models "Too many requests" # rate limits, OpenAI quota starvation, 413 tokens_limit_reached - # token-cap, connection/warm-up failures) that could not complete a scan. A backend outage is CI + # token-cap, connection/warm-up failures, ModelBehaviorError + # classifier flakes with zero findings) that could not complete a scan. A backend outage is CI # infrastructure noise, not a security finding, so it must not fail # the required check and block merges. strix_run_log="$RUNNER_TEMP/strix_gate_console.log" @@ -861,7 +862,7 @@ jobs: fi # Recognized signals that the LLM backend was unavailable / starved. - backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404' + backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|ModelBehaviorError' # Any evidence that a vulnerability was actually reported. Its presence # forces a hard failure so real findings are NEVER downgraded. Keep the # severity branch anchored away from identifiers so environment lines diff --git a/CHANGELOG.md b/CHANGELOG.md index 1de9130a5..57076d34a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,9 @@ Semantic Versioning where the repository publishes a release. ### Fixed +- Classify Strix `ModelBehaviorError` with `Vulnerabilities 0` as a + backend-unavailable skip so the required check does not fail-closed on + an LLM classifier flake. `Vulnerabilities [1-9]` still fail the check. - Materialized base Python locks only when every package line is an exact SHA-256 pin or a bounded relative `-r`/`--requirement` include. A lone `--require-hashes` directive, a dotted include such as `./lock.txt`, or `-r other-hashes.txt` no longer enters the trusted build context. - Refused a conflict-scope repository root whose immediate parent is a symbolic link, so a swapped parent cannot redirect the canonical worktree after the last-component check (CWE-367). - Bounded the Strix quality self-test's deterministic timeout fixtures to 3-second process and 5-second fake-sleep budgets so exact-head policy evidence completes inside the existing job limit without changing production Strix scanner timeouts, providers, credentials, or review semantics. diff --git a/tests/test_strix_modelbehaviorerror_classifier.py b/tests/test_strix_modelbehaviorerror_classifier.py new file mode 100644 index 000000000..ef525aff0 --- /dev/null +++ b/tests/test_strix_modelbehaviorerror_classifier.py @@ -0,0 +1,105 @@ +"""ModelBehaviorError is infrastructure noise when no vulnerability is reported. + +A Strix run that exits 1 with ModelBehaviorError and Vulnerabilities 0 must +neutralize. Vulnerabilities [1-9] stay fail-closed even when the same +exception is present. Do not weaken that gate. +""" + +from __future__ import annotations + +import re +import subprocess +import tempfile +import unittest +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +STRIX_WORKFLOW = REPOSITORY_ROOT / ".github" / "workflows" / "strix.yml" + + +def _workflow_signal_pattern(workflow: str, variable_name: str) -> str: + """Extract one single-quoted POSIX ERE assigned in the Strix workflow.""" + + match = re.search( + rf"(?m)^\s+{re.escape(variable_name)}='([^']+)'$", + workflow, + ) + if match is None: + raise AssertionError(f"missing workflow signal: {variable_name}") + return match.group(1) + + +def _workflow_neutralizes(log_text: str) -> bool: + """Execute the outer workflow's backend-neutralization condition.""" + + workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") + backend_pattern = _workflow_signal_pattern( + workflow, + "backend_unavailable_signal", + ) + vulnerability_pattern = _workflow_signal_pattern( + workflow, + "reported_vulnerability_signal", + ) + with tempfile.TemporaryDirectory(prefix="strix-modelbehavior-") as temp_dir: + log_path = Path(temp_dir) / "strix.log" + log_path.write_text(log_text, encoding="utf-8") + backend = subprocess.run( + ["grep", "-Eiq", backend_pattern, str(log_path)], + check=False, + capture_output=True, + text=True, + ) + vulnerability = subprocess.run( + ["grep", "-Eiq", vulnerability_pattern, str(log_path)], + check=False, + capture_output=True, + text=True, + ) + if backend.returncode not in {0, 1}: + raise AssertionError(backend.stderr) + if vulnerability.returncode not in {0, 1}: + raise AssertionError(vulnerability.stderr) + return backend.returncode == 0 and vulnerability.returncode == 1 + + +class StrixModelBehaviorErrorClassifierTests(unittest.TestCase): + """Neutralize ModelBehaviorError flakes; keep Vulnerabilities [1-9] blocking.""" + + def test_model_behavior_error_with_zero_findings_is_neutral(self) -> None: + self.assertTrue( + _workflow_neutralizes( + "litellm.exceptions.ModelBehaviorError: invalid tool call\n" + "Vulnerabilities 0\n" + ) + ) + + def test_model_behavior_error_with_one_finding_stays_fail_closed(self) -> None: + self.assertFalse( + _workflow_neutralizes( + "litellm.exceptions.ModelBehaviorError: invalid tool call\n" + "Vulnerabilities 1\n" + ) + ) + + def test_model_behavior_error_with_nine_findings_stays_fail_closed(self) -> None: + self.assertFalse( + _workflow_neutralizes( + "ModelBehaviorError\n" + "Vulnerabilities 9\n" + ) + ) + + def test_zero_findings_without_backend_signal_is_not_neutral(self) -> None: + self.assertFalse(_workflow_neutralizes("Vulnerabilities 0\nscan incomplete\n")) + + def test_workflow_keeps_fail_closed_vulnerability_range(self) -> None: + workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") + self.assertIn("ModelBehaviorError", workflow) + self.assertIn("Vulnerabilities[[:space:]]+[1-9]", workflow) + self.assertIn('! grep -Eiq "$reported_vulnerability_signal"', workflow) + + +if __name__ == "__main__": + unittest.main()