feat: implement issue #286 — Compliance: check-suite-prefs-unreadable

scripts/apply-repo-settings.sh

@@ -61,6 +61,34 @@ check_setting() {
   fi
 }
 
+# Read back the check-suite preferences PATCH response and confirm the Claude
+# app auto-trigger setting is present and disabled. Reports readability so a
+# `check-suite-prefs-unreadable` finding can be cleared with confidence.
+verify_check_suite_pref() {
+  local file="$1"
+  local setting
+  setting=$(
+    python3 - "$file" <<'PY' 2>/dev/null || true
+import json, sys
+try:
+    with open(sys.argv[1]) as f:
+        d = json.load(f)
+    checks = d.get("preferences", {}).get("auto_trigger_checks")
+    if not isinstance(checks, list):
+        raise ValueError
+    by_app = {c.get("app_id"): c.get("setting") for c in checks if isinstance(c, dict)}
+    print(str(by_app.get(1236702, "missing")).lower())
+except Exception:
+    print("unreadable")
+PY
+  )
+  if [ "$setting" = "false" ]; then
+    echo "  [OK] check-suite preferences readable; auto-trigger for app 1236702 is disabled"
+  else
+    echo "  [WARN] check-suite preferences could not be read back (got: ${setting:-unreadable})"
+  fi
+}
+
 check_setting "secret_scanning"
 check_setting "secret_scanning_push_protection"
 check_setting "secret_scanning_non_provider_patterns"
@@ -81,8 +109,15 @@ check_setting "dependabot_security_updates"
 
 echo "Disabling check-suite auto-trigger for Claude app (id: 1236702)..."
 
+# The PATCH response echoes the resulting preferences object. Capture it so we
+# can read back and confirm the auto-trigger setting was applied — this is the
+# same state the compliance audit reports as `check-suite-prefs-unreadable`
+# when its token cannot read the preferences.
+CS_RESPONSE_FILE="$(mktemp)"
+trap 'rm -f "$CS_RESPONSE_FILE"' EXIT
+
 if gh api -X PATCH "repos/$REPO/check-suites/preferences" \
-  --input - >/dev/null 2>&1 <<'JSON'; then
+  --input - >"$CS_RESPONSE_FILE" 2>/dev/null <<'JSON'; then
 {
   "auto_trigger_checks": [
     {
@@ -93,6 +128,7 @@ if gh api -X PATCH "repos/$REPO/check-suites/preferences" \
 }
 JSON
   echo "  [OK] check-suite auto-trigger disabled for Claude app (1236702)"
+  verify_check_suite_pref "$CS_RESPONSE_FILE"
 else
   echo "  [WARN] check-suite preferences require a fine-grained PAT (Administration: write + Checks: write) or GitHub App token."
   echo "         Re-run with: GH_PAT=<fine-grained-pat> bash scripts/apply-repo-settings.sh"

scripts/tests/apply-repo-settings.bats

@@ -33,7 +33,13 @@ if [[ " $* " == *" --input - "* ]]; then
   done
   url_slug=$(printf '%s' "$url_arg" | tr -cs 'a-zA-Z0-9_-' '_' | head -c 60)
   cat > "$PAYLOAD_DIR/${url_slug}.json"
-  echo '{}'
+  if [[ "$url_arg" == *check-suites/preferences* ]]; then
+    # Echo the applied preferences the way the real check-suites/preferences
+    # PATCH endpoint does, so the script can read back and verify them.
+    printf '{"preferences":{"auto_trigger_checks":[{"app_id":1236702,"setting":false}]},"repository":{"id":1,"name":"test-repo"}}'
+  else
+    echo '{}'
+  fi
 elif [[ "$*" == *--jq* && "$*" == *security_and_analysis* ]]; then
   printf '{"secret_scanning":{"status":"enabled"},"secret_scanning_push_protection":{"status":"enabled"},"secret_scanning_non_provider_patterns":{"status":"enabled"},"secret_scanning_ai_detection":{"status":"enabled"},"dependabot_security_updates":{"status":"enabled"}}'
 else
@@ -148,3 +154,56 @@ PY
   [ "$status" -eq 0 ]
   [[ "$output" == *"secret_scanning_non_provider_patterns"* ]]
 }
+
+# ── Check-suite preferences tests (compliance: issue #286) ────────────────────
+
+@test "check-suite PATCH payload disables auto-trigger for Claude app 1236702" {
+  run bash "$SCRIPT"
+  [ "$status" -eq 0 ]
+
+  payload_file=$(grep -rl '"auto_trigger_checks"' "$PAYLOAD_DIR" 2>/dev/null | head -1)
+  [ -n "$payload_file" ] || {
+    echo "No payload file captured containing auto_trigger_checks"
+    return 1
+  }
+
+  run python3 - "$payload_file" << 'PY'
+import json, sys
+with open(sys.argv[1]) as f:
+    d = json.load(f)
+checks = {c["app_id"]: c["setting"] for c in d["auto_trigger_checks"]}
+assert checks.get(1236702) is False, f"expected app 1236702 disabled, got {checks}"
+print("ok")
+PY
+  [ "$status" -eq 0 ]
+  [[ "$output" == "ok" ]]
+}
+
+@test "script reads back and reports check-suite preferences as readable" {
+  run bash "$SCRIPT"
+  [ "$status" -eq 0 ]
+  [[ "$output" == *"check-suite preferences readable"* ]]
+}
+
+@test "script warns when check-suite preferences cannot be read back" {
+  # Override the mock so the check-suites PATCH returns an empty body,
+  # simulating the unreadable-preferences condition from the finding.
+  cat > "$MOCK_BIN/gh" << 'MOCK'
+#!/usr/bin/env bash
+set -euo pipefail
+echo "$*" >> "$CALLS_FILE"
+if [[ " $* " == *" --input - "* ]]; then
+  cat >/dev/null
+  echo '{}'
+elif [[ "$*" == *--jq* && "$*" == *security_and_analysis* ]]; then
+  printf '{"secret_scanning":{"status":"enabled"},"secret_scanning_push_protection":{"status":"enabled"},"secret_scanning_non_provider_patterns":{"status":"enabled"},"secret_scanning_ai_detection":{"status":"enabled"},"dependabot_security_updates":{"status":"enabled"}}'
+else
+  echo '{}'
+fi
+MOCK
+  chmod +x "$MOCK_BIN/gh"
+
+  run bash "$SCRIPT"
+  [ "$status" -eq 0 ]
+  [[ "$output" == *"check-suite preferences could not be read back"* ]]
+}