diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 36ec3e5f8..c8b74267f 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -3,10 +3,10 @@ # automatic model fallback, transient-error retry, and severity-based # pass/fail decisions. # -# STRIX_LOG is a per-attempt temp file consumed only by -# is_transient_same_model_retry_error(); cumulative report dirs in -# STRIX_REPORTS_DIR are never overwritten. Refer to ARCHITECTURE.md -# for the 3-tier timeout classification hierarchy. +# RAW_STRIX_LOG is the immutable per-attempt console evidence. STRIX_LOG points +# to it while the scanner runs, then to a private sanitized classification copy. +# Cumulative report dirs in STRIX_REPORTS_DIR are never overwritten. Refer to +# ARCHITECTURE.md for the 3-tier timeout classification hierarchy. set -euo pipefail SCRIPT_DIR="$({ CDPATH='' && cd -P -- "$(dirname -- "$0")" && pwd -P; })" @@ -25,7 +25,9 @@ RAW_SCAN_MODE="${STRIX_SCAN_MODE:-quick}" SCAN_MODE="" ARTIFACT_REPORTS_DIR="$REPO_ROOT/strix_runs" STRIX_RUNTIME_DIR="$(mktemp -d /tmp/strix-runtime.XXXXXX)" -STRIX_LOG="$STRIX_RUNTIME_DIR/strix.log" +RAW_STRIX_LOG="$STRIX_RUNTIME_DIR/strix.log" +STRIX_CLASSIFICATION_LOG="$STRIX_RUNTIME_DIR/strix-classification.log" +STRIX_LOG="$RAW_STRIX_LOG" ACTIVE_REPORTS_DIR="$STRIX_RUNTIME_DIR/reports" ATTEMPT_LOGS_DIR="$STRIX_RUNTIME_DIR/gate-attempts" STRIX_SCAN_WORKING_DIR="$STRIX_RUNTIME_DIR/scan-cwd" @@ -128,8 +130,8 @@ publish_artifact_reports() { if [ -d "$ATTEMPT_LOGS_DIR" ] && [ ! -L "$ATTEMPT_LOGS_DIR" ]; then cp -R -- "$ATTEMPT_LOGS_DIR" "$ARTIFACT_REPORTS_DIR/gate-attempts" fi - if [ -f "$STRIX_LOG" ] && [ ! -L "$STRIX_LOG" ]; then - cp -- "$STRIX_LOG" "$ARTIFACT_REPORTS_DIR/gate-last-attempt.log" + if [ -f "$RAW_STRIX_LOG" ] && [ ! -L "$RAW_STRIX_LOG" ]; then + cp -- "$RAW_STRIX_LOG" "$ARTIFACT_REPORTS_DIR/gate-last-attempt.log" fi # Relative scanner output is copied into ACTIVE_REPORTS_DIR immediately # after each attempt and sanitized before this publication trap runs. @@ -172,6 +174,12 @@ known_internal_warning = re.compile( r"|ended a turn without a lifecycle tool call \(interactive=False\)" r"); forcing tool continuation \(\d+/\d+\): " ) +# Strix prints this exact box-content heading at startup whenever the +# configured model is not on its own hardcoded "recommended frontier +# model" list. Remove only that one cosmetic line: deleting the whole box +# could hide a real provider failure emitted beside the heading. +model_quality_heading = re.compile(r"│[ \t]*MODEL QUALITY WARNING[ \t]*│") +ansi_csi = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]") def iter_report_logs(root: Path): @@ -191,16 +199,55 @@ def iter_report_logs(root: Path): for log_path in iter_report_logs(root): try: - lines = log_path.read_text(encoding="utf-8").splitlines(keepends=True) + text = log_path.read_text(encoding="utf-8") except UnicodeDecodeError: continue - filtered = [line for line in lines if not known_internal_warning.match(line)] - if filtered != lines: - log_path.write_text("".join(filtered), encoding="utf-8") + original = text + lines = text.splitlines(keepends=True) + filtered = [ + line + for line in lines + if not model_quality_heading.fullmatch(ansi_csi.sub("", line.rstrip("\r\n"))) + and not known_internal_warning.match(line) + ] + text = "".join(filtered) + if text != original: + log_path.write_text(text, encoding="utf-8") PY done } +# Strips the same benign MODEL QUALITY WARNING startup heading (see above) +# from the private classification copy so has_detected_infrastructure_error +# does not mistake it for a real provider/infrastructure failure signal. +sanitize_strix_console_log() { + local log_path="$1" + if [ -z "$log_path" ] || [ ! -f "$log_path" ] || [ -L "$log_path" ]; then + return 0 + fi + python3 - "$log_path" <<'PY' +from pathlib import Path +import re +import sys + +log_path = Path(sys.argv[1]) +model_quality_heading = re.compile(r"│[ \t]*MODEL QUALITY WARNING[ \t]*│") +ansi_csi = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]") + +try: + text = log_path.read_text(encoding="utf-8") +except UnicodeDecodeError: + raise SystemExit(0) +sanitized = "".join( + line + for line in text.splitlines(keepends=True) + if not model_quality_heading.fullmatch(ansi_csi.sub("", line.rstrip("\r\n"))) +) +if sanitized != text: + log_path.write_text(sanitized, encoding="utf-8") +PY +} + has_strix_report_failure_signal() { local report_root local report_log @@ -256,7 +303,7 @@ has_strix_report_provider_failure_signal() { # shellcheck disable=SC2317,SC2329 # invoked from EXIT/INT/TERM trap cleanup_runtime() { publish_artifact_reports || true - rm -f "$STRIX_LOG" + rm -f "$RAW_STRIX_LOG" "$STRIX_CLASSIFICATION_LOG" rm -rf "$STRIX_RUNTIME_DIR" local scope_dir for scope_dir in "${PULL_REQUEST_SCOPE_DIRS[@]}"; do @@ -2512,6 +2559,10 @@ run_strix_once() { local resolved_target_path local timeout_seconds="$STRIX_PROCESS_TIMEOUT_SECONDS" local total_budget_limited_timeout=0 + # Every invocation writes an untouched raw console transcript. Classifiers + # switch to a sanitized private copy only after the raw attempt is archived. + STRIX_LOG="$RAW_STRIX_LOG" + rm -f -- "$STRIX_CLASSIFICATION_LOG" if [ "$RUN_START_EPOCH" -le 0 ]; then RUN_START_EPOCH="$(date +%s)" fi @@ -2790,6 +2841,9 @@ PY fi fi preserve_attempt_log "$model" "$rc" + cp -- "$RAW_STRIX_LOG" "$STRIX_CLASSIFICATION_LOG" + STRIX_LOG="$STRIX_CLASSIFICATION_LOG" + sanitize_strix_console_log "$STRIX_LOG" sanitize_known_strix_report_warnings "$ACTIVE_REPORTS_DIR" "${resolved_target_path%/}/strix_runs" local report_failure_signal=0 diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 945eb3fb3..5a357794e 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -4542,6 +4542,93 @@ EOS echo "scan ok but unknown report warning remains" exit 0 ;; + report-model-quality-warning-preserves-prior-failure) + mkdir -p "$STRIX_REPORTS_DIR/fake-model-quality-prior-failure" + cat >"$STRIX_REPORTS_DIR/fake-model-quality-prior-failure/strix.log" <<'EOS' +╭─ STRIX ──────────────────────────────────────────────────────────────────────╮ +│ Provider WARNING: report evidence is incomplete │ +╰──────────────────────────────────────────────────────────────────────────────╯ +╭─ STRIX ──────────────────────────────────────────────────────────────────────╮ +│ MODEL QUALITY WARNING │ +│ This cosmetic banner alone is not a provider failure. │ +╰──────────────────────────────────────────────────────────────────────────────╯ +EOS + echo "scan returned zero findings with an incomplete provider report" + exit 0 + ;; + report-model-quality-warning-preserves-same-box-failure) + mkdir -p "$STRIX_REPORTS_DIR/fake-model-quality-same-box-failure" + cat >"$STRIX_REPORTS_DIR/fake-model-quality-same-box-failure/strix.log" <<'EOS' +╭─ STRIX ──────────────────────────────────────────────────────────────────────╮ +│ MODEL QUALITY WARNING │ +│ Provider WARNING: report evidence is incomplete │ +╰──────────────────────────────────────────────────────────────────────────────╯ +EOS + echo "scan returned zero findings with incomplete evidence in the banner box" + exit 0 + ;; + console-model-quality-warning-banner-sanitized) + # Reproduces Strix's own startup banner, printed to the console + # (not a report artifact) whenever the configured model is not on + # its hardcoded "recommended frontier model" list. It must be + # sanitized out of $STRIX_LOG before has_detected_infrastructure_error + # runs, or a completely clean 0-vulnerability scan on a non-listed + # model is misclassified as a provider infrastructure failure. + cat <<'EOS' +╭─ STRIX ──────────────────────────────────────────────────────────────────────╮ +│ │ +│ MODEL QUALITY WARNING │ +│ │ +│ 'vertex_ai/cosmetic-banner-sanitized' is not a │ +│ recommended frontier model for Strix. │ +│ │ +│ You can continue, but weaker models may miss vulnerabilities or produce │ +│ lower-quality findings. │ +│ │ +╰──────────────────────────────────────────────────────────────────────────────╯ +╭─ STRIX ──────────────────────────────────────────────────────────────────────╮ +│ │ +│ Penetration test completed │ +│ │ +│ Vulnerabilities 0 │ +│ │ +╰──────────────────────────────────────────────────────────────────────────────╯ +EOS + exit 0 + ;; + console-model-quality-warning-preserves-prior-failure) + # A real provider failure may be emitted in an earlier Strix box. + # Sanitizing the later cosmetic model-quality banner must not consume + # that preceding box or the failure signal between box boundaries. + cat <<'EOS' +╭─ STRIX ──────────────────────────────────────────────────────────────────────╮ +│ Provider WARNING: backend returned incomplete scan evidence │ +╰──────────────────────────────────────────────────────────────────────────────╯ +╭─ STRIX ──────────────────────────────────────────────────────────────────────╮ +│ MODEL QUALITY WARNING │ +│ 'vertex_ai/cosmetic-banner-prior-failure' is not a │ +│ recommended frontier model for Strix. │ +╰──────────────────────────────────────────────────────────────────────────────╯ +╭─ STRIX ──────────────────────────────────────────────────────────────────────╮ +│ Penetration test completed │ +│ Vulnerabilities 0 │ +╰──────────────────────────────────────────────────────────────────────────────╯ +EOS + exit 0 + ;; + console-model-quality-warning-preserves-same-box-failure) + cat <<'EOS' +╭─ STRIX ──────────────────────────────────────────────────────────────────────╮ +│ MODEL QUALITY WARNING │ +│ Provider WARNING: backend returned incomplete scan evidence │ +╰──────────────────────────────────────────────────────────────────────────────╯ +╭─ STRIX ──────────────────────────────────────────────────────────────────────╮ +│ Penetration test completed │ +│ Vulnerabilities 0 │ +╰──────────────────────────────────────────────────────────────────────────────╯ +EOS + exit 0 + ;; bare-timeout-with-provider-marker) # Emit bare "Connection timed out" alongside a provider marker so # is_timeout_error() matches the Tier 3 branch gated on @@ -5846,6 +5933,19 @@ PY "scenario=$scenario keeps non-warning Strix report evidence" fi + if [ "$scenario" = "console-model-quality-warning-banner-sanitized" ]; then + assert_file_contains \ + "$repo_root_dir/strix_runs/gate-last-attempt.log" \ + "MODEL QUALITY WARNING" \ + "scenario=$scenario preserves the raw last-attempt console artifact" + local raw_attempt_log="" + raw_attempt_log="$(find "$repo_root_dir/strix_runs/gate-attempts" -type f -name '*.log' -print -quit 2>/dev/null || true)" + assert_file_contains \ + "$raw_attempt_log" \ + "MODEL QUALITY WARNING" \ + "scenario=$scenario preserves the raw per-attempt console artifact" + fi + if [ "$scenario" = "github-models-primary-ratelimit-fallback-success" ]; then assert_file_contains \ "$output_log" \ @@ -6368,6 +6468,56 @@ run_filtered_gate_case_if_requested() { "vertex_ai/report-known-internal-warning-sanitized" \ "" ;; + report-model-quality-warning-preserves-prior-failure) + run_gate_case "$STRIX_TEST_CASE_FILTER" \ + "vertex_ai/report-model-quality-warning-preserves-prior-failure" \ + "" \ + "1" \ + "Strix report artifacts emitted warning/fatal/denied/timeout output; failing closed." \ + "1" \ + "vertex_ai/report-model-quality-warning-preserves-prior-failure" \ + "" + ;; + report-model-quality-warning-preserves-same-box-failure) + run_gate_case "$STRIX_TEST_CASE_FILTER" \ + "vertex_ai/report-model-quality-warning-preserves-same-box-failure" \ + "" \ + "1" \ + "Strix report artifacts emitted warning/fatal/denied/timeout output; failing closed." \ + "1" \ + "vertex_ai/report-model-quality-warning-preserves-same-box-failure" \ + "" + ;; + console-model-quality-warning-banner-sanitized) + run_gate_case "$STRIX_TEST_CASE_FILTER" \ + "vertex_ai/cosmetic-banner-sanitized" \ + "" \ + "0" \ + "Strix run succeeded for model 'vertex_ai/cosmetic-banner-sanitized'" \ + "1" \ + "vertex_ai/cosmetic-banner-sanitized" \ + "" + ;; + console-model-quality-warning-preserves-prior-failure) + run_gate_case "$STRIX_TEST_CASE_FILTER" \ + "vertex_ai/cosmetic-banner-prior-failure" \ + "" \ + "1" \ + "Strix run emitted provider infrastructure or failure-signal output; failing closed." \ + "1" \ + "vertex_ai/cosmetic-banner-prior-failure" \ + "" + ;; + console-model-quality-warning-preserves-same-box-failure) + run_gate_case "$STRIX_TEST_CASE_FILTER" \ + "vertex_ai/cosmetic-banner-same-box-failure" \ + "" \ + "1" \ + "Strix run emitted provider infrastructure or failure-signal output; failing closed." \ + "1" \ + "vertex_ai/cosmetic-banner-same-box-failure" \ + "" + ;; provider-fatal-success-signal | provider-warning-success-signal) run_gate_case "$STRIX_TEST_CASE_FILTER" \ "vertex_ai/$STRIX_TEST_CASE_FILTER" \ @@ -10434,6 +10584,156 @@ run_gate_case "report-known-internal-warning-sanitized" \ "" \ "1" +run_gate_case "console-model-quality-warning-banner-sanitized" \ + "vertex_ai/cosmetic-banner-sanitized" \ + "" \ + "0" \ + "Strix run succeeded for model 'vertex_ai/cosmetic-banner-sanitized'" \ + "1" \ + "vertex_ai/cosmetic-banner-sanitized" \ + "" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "__SAME_AS_FALLBACK_MODELS__" \ + "" \ + "1" + +run_gate_case "console-model-quality-warning-preserves-prior-failure" \ + "vertex_ai/cosmetic-banner-prior-failure" \ + "" \ + "1" \ + "Strix run emitted provider infrastructure or failure-signal output; failing closed." \ + "1" \ + "vertex_ai/cosmetic-banner-prior-failure" \ + "" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "__SAME_AS_FALLBACK_MODELS__" \ + "" \ + "1" + +run_gate_case "console-model-quality-warning-preserves-same-box-failure" \ + "vertex_ai/cosmetic-banner-same-box-failure" \ + "" \ + "1" \ + "Strix run emitted provider infrastructure or failure-signal output; failing closed." \ + "1" \ + "vertex_ai/cosmetic-banner-same-box-failure" \ + "" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "__SAME_AS_FALLBACK_MODELS__" \ + "" \ + "1" + +run_gate_case "report-model-quality-warning-preserves-prior-failure" \ + "vertex_ai/report-model-quality-warning-preserves-prior-failure" \ + "" \ + "1" \ + "Strix report artifacts emitted warning/fatal/denied/timeout output; failing closed." \ + "1" \ + "vertex_ai/report-model-quality-warning-preserves-prior-failure" \ + "" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "__SAME_AS_FALLBACK_MODELS__" \ + "" \ + "1" + +run_gate_case "report-model-quality-warning-preserves-same-box-failure" \ + "vertex_ai/report-model-quality-warning-preserves-same-box-failure" \ + "" \ + "1" \ + "Strix report artifacts emitted warning/fatal/denied/timeout output; failing closed." \ + "1" \ + "vertex_ai/report-model-quality-warning-preserves-same-box-failure" \ + "" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "__SAME_AS_FALLBACK_MODELS__" \ + "" \ + "1" + run_gate_case "report-known-internal-warning-variant-sanitized" \ "vertex_ai/report-known-internal-warning-variant-sanitized" \ "" \