Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CHANGELOG.d/20260912-strix-optional-web-search-warning.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
### Fixed

- Keep completed Strix scans valid when their report contains only the exact
upstream warning for missing optional `EXA_API_KEY`, `PERPLEXITY_API_KEY`, or
combined web-search credentials. Unknown warnings remain fail-closed, and raw
report artifacts remain unchanged for audit evidence.
8 changes: 8 additions & 0 deletions scripts/ci/strix_quick_gate.sh
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,13 @@ known_scanner_warning = re.compile(
r"^(?:│ MODEL QUALITY WARNING\s+│|"
r"Warning: You are sending unauthenticated requests to the HF Hub\.)"
)
known_optional_search_warning = re.compile(
r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ WARNING "
r"[^ ]+ - strix\.tools\.web_search\.tool: "
r"web_search invoked without "
r"(?:EXA_API_KEY|PERPLEXITY_API_KEY|EXA_API_KEY or PERPLEXITY_API_KEY) "
r"configured$"
)


def iter_report_logs(root: Path):
Expand Down Expand Up @@ -231,6 +238,7 @@ for log_path in iter_report_logs(root):
for line in lines
if not known_internal_warning.match(line)
and not known_scanner_warning.match(line)
and not known_optional_search_warning.match(line)
]
if filtered != lines:
log_path.write_text("".join(filtered), encoding="utf-8")
Expand Down
108 changes: 108 additions & 0 deletions tests/test_strix_optional_web_search_warning.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
"""Regression for Strix's optional web-search capability warning.

Strix documents PERPLEXITY_API_KEY as optional and its web-search tool returns a
sanitized ``success: false`` result instructing the agent to proceed when the
key is absent. A completed scan must therefore not be reclassified as provider
unavailability solely because that exact warning appears in ``strix.log``.
Unknown warnings remain fail-closed.
"""

from __future__ import annotations

import re
import subprocess
import tempfile
import unittest
from pathlib import Path


REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
STRIX_GATE = REPOSITORY_ROOT / "scripts" / "ci" / "strix_quick_gate.sh"

OPTIONAL_SEARCH_KEY_LABELS = (
"EXA_API_KEY",
"PERPLEXITY_API_KEY",
"EXA_API_KEY or PERPLEXITY_API_KEY",
)
UNKNOWN_SEARCH_WARNING = (
"2026-09-12 12:30:48.087 WARNING strix-pr-scope-vjsusm_b6ed - "
"strix.tools.web_search.tool: provider returned malformed search evidence\n"
)
COMPLETION_LINE = (
"2026-09-12 12:45:56.390 INFO strix-pr-scope-vjsusm_b6ed - "
"strix.core.runner: Strix scan strix-pr-scope-vjsusm_b6ed done\n"
)


def _function_block(source: str, function_name: str) -> str:
match = re.search(
rf"(?ms)^{re.escape(function_name)}\(\) \{{\n.*?^\}}\n",
source,
)
if match is None:
raise AssertionError(f"missing Bash function: {function_name}")
return match.group(0)


def _sanitize_then_signal(log_text: str) -> tuple[str, bool]:
gate_source = STRIX_GATE.read_text(encoding="utf-8")
blocks = [
_function_block(gate_source, name)
for name in (
"sanitize_known_strix_report_warnings",
"has_strix_report_failure_signal",
)
]
with tempfile.TemporaryDirectory(prefix="strix-optional-search-") as temp_dir:
report_root = Path(temp_dir) / "run"
report_root.mkdir()
log_path = report_root / "strix.log"
log_path.write_text(log_text, encoding="utf-8")
script = "\n".join(
(
"set -uo pipefail",
'STRIX_REPORTS_DIR="/nonexistent/strix-reports"',
*blocks,
'sanitize_known_strix_report_warnings "$1"',
'if has_strix_report_failure_signal "$1"; then echo signal=1; else echo signal=0; fi',
)
)
completed = subprocess.run(
["bash", "-c", script, "strix-optional-search", str(report_root)],
check=False,
capture_output=True,
text=True,
)
remaining = log_path.read_text(encoding="utf-8")
if completed.returncode != 0:
raise AssertionError(f"rc={completed.returncode}\n{completed.stderr}")
return remaining, "signal=1" in completed.stdout


class StrixOptionalWebSearchWarningTests(unittest.TestCase):
def test_missing_optional_search_keys_do_not_fail_completed_scan(self) -> None:
for search_key_label in OPTIONAL_SEARCH_KEY_LABELS:
with self.subTest(search_key_label=search_key_label):
optional_search_warning = (
"2026-09-12 12:30:48.087 WARNING strix-pr-scope-vjsusm_b6ed - "
"strix.tools.web_search.tool: web_search invoked without "
f"{search_key_label} configured\n"
)
remaining, signal = _sanitize_then_signal(
optional_search_warning + COMPLETION_LINE
)
self.assertNotIn(
f"web_search invoked without {search_key_label} configured",
remaining,
)
self.assertIn("Strix scan strix-pr-scope-vjsusm_b6ed done", remaining)
self.assertFalse(signal)

def test_unknown_web_search_warning_remains_fail_closed(self) -> None:
remaining, signal = _sanitize_then_signal(UNKNOWN_SEARCH_WARNING + COMPLETION_LINE)
self.assertIn("provider returned malformed search evidence", remaining)
self.assertTrue(signal)


if __name__ == "__main__":
unittest.main()
Loading