diff --git a/.github/workflows/audit-central-ruleset.yml b/.github/workflows/audit-central-ruleset.yml index bf24e36c7c..2b72f21aab 100644 --- a/.github/workflows/audit-central-ruleset.yml +++ b/.github/workflows/audit-central-ruleset.yml @@ -105,6 +105,18 @@ jobs: python3 scripts/ci/audit_central_required_workflows.py --stacked "$stacked_ruleset_json" - name: Audit organization CodeQL coverage + # Runs even when the ruleset step above failed. Those two audits share a + # job but not a subject: the ruleset step exits 1 on owner-configured + # governance drift, and on 2026-09-06 it did exactly that ("exactly two + # approving reviews are not required", "last-push approval protection is + # disabled"), which silently took this CodeQL coverage detector down with + # it -- every run since 2026-09-04 failed there and never reached this + # step. This step builds its own repository list into its own temp file + # and the step above exports nothing to GITHUB_ENV or GITHUB_OUTPUT, so + # it has no data dependency to lose. The job still fails overall; what + # changes is that a coverage gap is reported instead of hidden behind an + # unrelated failure. + if: always() env: ORG_LOGIN: ContextualWisdomLab ORG_WIDE_CREDENTIAL_AVAILABLE: ${{ secrets.PR_REVIEW_MERGE_TOKEN != '' || secrets.OPENCODE_APPROVE_TOKEN != '' }} @@ -165,13 +177,21 @@ jobs: printf '[]\n' >"$coverage_json" while IFS=$'\t' read -r repository archived; do default_setup_state=null + # `state` alone is not coverage: a repository can report + # "configured" with an empty `languages` list, which scans nothing + # and produces no analyses (measured 2026-09-07 on life-os, aFIPC + # and inkspan). Collect both fields so the audit can tell those + # apart from a setup that actually covers a language. + default_setup_languages=null if [ "$archived" != "true" ]; then - default_setup_state_json="$RUNNER_TEMP/codeql-default-setup-${repository//[^A-Za-z0-9_.-]/_}.json" - if gh api "repos/${ORG_LOGIN}/${repository}/code-scanning/default-setup" --jq .state \ - >"$default_setup_state_json" 2>/dev/null; then - default_setup_state=$(jq -R '.' "$default_setup_state_json") + default_setup_json="$RUNNER_TEMP/codeql-default-setup-${repository//[^A-Za-z0-9_.-]/_}.json" + if gh api "repos/${ORG_LOGIN}/${repository}/code-scanning/default-setup" \ + >"$default_setup_json" 2>/dev/null; then + default_setup_state=$(jq '.state // null' "$default_setup_json") + default_setup_languages=$(jq '.languages // []' "$default_setup_json") else default_setup_state=null + default_setup_languages=null fi fi @@ -187,12 +207,13 @@ jobs: fi fi - echo "CODEQL_COVERAGE repository=${repository} archived=${archived} default_setup_state=${default_setup_state} latest_codeql_analysis=${latest_codeql_analysis}" + echo "CODEQL_COVERAGE repository=${repository} archived=${archived} default_setup_state=${default_setup_state} default_setup_languages=${default_setup_languages} latest_codeql_analysis=${latest_codeql_analysis}" jq --arg name "$repository" \ --argjson archived "$archived" \ --argjson default_setup_state "$default_setup_state" \ + --argjson default_setup_languages "$default_setup_languages" \ --argjson latest_codeql_analysis "$latest_codeql_analysis" \ - '. + [{name: $name, archived: $archived, default_setup_state: $default_setup_state, latest_codeql_analysis: $latest_codeql_analysis}]' \ + '. + [{name: $name, archived: $archived, default_setup_state: $default_setup_state, default_setup_languages: $default_setup_languages, latest_codeql_analysis: $latest_codeql_analysis}]' \ "$coverage_json" >"${coverage_json}.next" mv "${coverage_json}.next" "$coverage_json" done < <(jq -r '.[] | [.name, (.archived | tostring)] | @tsv' "$repositories_json") diff --git a/scripts/ci/audit_org_codeql_coverage.py b/scripts/ci/audit_org_codeql_coverage.py index f9fb2eaf17..9051c67568 100644 --- a/scripts/ci/audit_org_codeql_coverage.py +++ b/scripts/ci/audit_org_codeql_coverage.py @@ -66,6 +66,27 @@ def _is_analysis_fresh_and_successful( return parsed >= now - timedelta(days=CODEQL_ANALYSIS_FRESHNESS_DAYS) +def _default_setup_scans_a_language(repository: dict[str, Any]) -> bool: + """Return True when default-setup is configured AND has languages enabled. + + ``state == "configured"`` alone is not coverage. Measured 2026-09-07: + ``life-os``, ``aFIPC`` and ``inkspan`` all report ``configured`` with an + **empty** ``languages`` list and no ``schedule``; ``life-os`` has zero CodeQL + analyses of any language as a result, while still satisfying the + configured-state check this function replaces. A default setup with nothing + enabled is a commitment to scan nothing. + + A missing ``default_setup_languages`` key fails closed rather than falling + back to the state alone, which would silently restore that gap. The audit + workflow collects the field in the same change that introduced this check, + so the key is absent only when the payload predates them both. + """ + if repository.get("default_setup_state") != "configured": + return False + languages = repository.get("default_setup_languages") + return isinstance(languages, list) and bool(languages) + + def repositories_without_codeql( repositories: list[dict[str, Any]], now: datetime | None = None ) -> list[dict[str, Any]]: @@ -88,8 +109,10 @@ def repositories_without_codeql( # CodeQL going forward (like a scheduled cron guarantee), not a # one-time historical scan that can go stale -- so it does not need # the same freshness check as latest_codeql_analysis below. Do not - # "fix" this into requiring a completed scan. - has_default_setup = repository.get("default_setup_state") == "configured" + # "fix" this into requiring a completed scan. It does need the + # commitment to cover at least one language: see + # _default_setup_scans_a_language. + has_default_setup = _default_setup_scans_a_language(repository) has_fresh_analysis = _is_analysis_fresh_and_successful( repository.get("latest_codeql_analysis"), current ) @@ -98,13 +121,30 @@ def repositories_without_codeql( return uncovered +def _coverage_gap_reason(repository: dict[str, Any]) -> str: + """Return the gap description that tells the operator what to change. + + "Default setup is on but scans nothing" and "there is no coverage at all" + need different fixes -- enable languages on the existing setup, versus set + coverage up -- so they are reported as different sentences. + """ + if repository.get("default_setup_state") == "configured": + return ( + f"{repository.get('name')} has CodeQL default-setup configured with no " + "languages enabled, so it scans nothing and produces no analyses" + ) + return ( + f"{repository.get('name')} has no CodeQL coverage from any source " + "(no default-setup, no recent analysis)" + ) + + def audit_codeql_coverage( repositories: list[dict[str, Any]], now: datetime | None = None ) -> list[str]: """Return one human-readable error per repository with zero CodeQL coverage.""" return [ - f"{repository.get('name')} has no CodeQL coverage from any source " - "(no default-setup, no recent analysis)" + _coverage_gap_reason(repository) for repository in repositories_without_codeql(repositories, now) ] @@ -137,6 +177,20 @@ def main(argv: list[str] | None = None) -> int: print(f"ERROR: unable to load repository JSON: {exc}", file=sys.stderr) return 2 + if not repositories: + # An empty payload is not a clean organization: it is an audit that + # examined nothing, and "PASS: all 0 repositories have real CodeQL + # coverage" reads as success. The calling workflow already refuses an + # enumeration missing its known-private sentinel repositories, which + # an empty list would also fail -- this closes the same hole for any + # other entry point, because the script is directly runnable against a + # JSON path or stdin. + print( + "ERROR: repository payload is empty, so this run audited nothing", + file=sys.stderr, + ) + return 2 + errors = audit_codeql_coverage(repositories) if errors: for error in errors: diff --git a/tests/test_audit_org_codeql_coverage.py b/tests/test_audit_org_codeql_coverage.py index ccd2cd9c42..20d623aca5 100644 --- a/tests/test_audit_org_codeql_coverage.py +++ b/tests/test_audit_org_codeql_coverage.py @@ -14,6 +14,23 @@ def covered_by_default_setup(name: str) -> dict: "name": name, "archived": False, "default_setup_state": "configured", + "default_setup_languages": ["actions", "python"], + "latest_codeql_analysis": None, + } + + +def default_setup_scanning_nothing(name: str) -> dict: + """Return a repository whose default-setup is on but has no languages enabled. + + The live shape measured on 2026-09-07 for ``life-os``, ``aFIPC`` and + ``inkspan``: ``state`` is ``configured``, ``languages`` is empty and + ``schedule`` is null. ``life-os`` had zero CodeQL analyses of any language. + """ + return { + "name": name, + "archived": False, + "default_setup_state": "configured", + "default_setup_languages": [], "latest_codeql_analysis": None, } @@ -89,6 +106,60 @@ def test_default_setup_alone_counts_as_coverage() -> None: assert audit.audit_codeql_coverage(repositories, now=NOW) == [] +def test_default_setup_with_no_languages_enabled_is_not_coverage() -> None: + """A setup that scans nothing must not satisfy the configured-state check. + + Measured 2026-09-07: ``life-os`` reports ``configured`` with an empty + ``languages`` list and has zero CodeQL analyses of any language, while + ``codeql-pr.yml`` still runs on every pull request head. Before this check + the audit passed it on the state alone. + """ + repositories = [default_setup_scanning_nothing("life-os")] + + assert audit.audit_codeql_coverage(repositories, now=NOW) == [ + "life-os has CodeQL default-setup configured with no languages enabled, " + "so it scans nothing and produces no analyses" + ] + + +def test_default_setup_scanning_nothing_still_passes_on_a_fresh_analysis() -> None: + """The empty-language setup is only a gap when nothing else covers the repo. + + ``aFIPC`` and ``inkspan`` both report the empty-language shape yet receive + analyses from a repository-local ``codeql.yml``, so flagging them would be a + false alarm. + """ + repository = default_setup_scanning_nothing("aFIPC") + repository["latest_codeql_analysis"] = covered_by_recent_analysis("aFIPC")[ + "latest_codeql_analysis" + ] + + assert audit.audit_codeql_coverage([repository], now=NOW) == [] + + +def test_payload_without_the_languages_key_fails_closed() -> None: + """A payload predating the workflow change must not pass on state alone. + + Falling back to ``default_setup_state`` when the key is missing would + silently restore the gap this check exists to close. + """ + repository = covered_by_default_setup("PolicyWeave") + del repository["default_setup_languages"] + + assert audit.audit_codeql_coverage([repository], now=NOW) == [ + "PolicyWeave has CodeQL default-setup configured with no languages " + "enabled, so it scans nothing and produces no analyses" + ] + + +def test_non_list_languages_value_fails_closed() -> None: + """A malformed ``languages`` value is not evidence that anything is scanned.""" + repository = covered_by_default_setup("PolicyWeave") + repository["default_setup_languages"] = "python" + + assert len(audit.audit_codeql_coverage([repository], now=NOW)) == 1 + + def test_recent_analysis_alone_counts_as_coverage() -> None: repositories = [covered_by_recent_analysis("TEPP")] @@ -269,3 +340,22 @@ def test_parse_args_defaults_to_none() -> None: args = audit.parse_args([]) assert args.repositories_json is None + + +def test_main_refuses_an_empty_payload_instead_of_passing_vacuously( + monkeypatch, capsys +) -> None: + """An audit that examined nothing must not print PASS. + + ``audit_codeql_coverage([])`` returning no gaps is correct -- there are no + repositories to have gaps. What is wrong is ``main`` turning that into + "PASS: all 0 repositories have real CodeQL coverage" and exiting 0, which + is the same vacuous-pass shape as a default setup that is configured with + no languages enabled. + """ + monkeypatch.setattr("sys.stdin", StringIO("[]")) + + assert audit.main([]) == 2 + captured = capsys.readouterr() + assert "audited nothing" in captured.err + assert "PASS" not in captured.out diff --git a/tests/test_central_required_workflow_ruleset_audit.py b/tests/test_central_required_workflow_ruleset_audit.py index 77bbf53305..cec0d2aead 100644 --- a/tests/test_central_required_workflow_ruleset_audit.py +++ b/tests/test_central_required_workflow_ruleset_audit.py @@ -517,13 +517,21 @@ def test_audit_organization_codeql_coverage_step_has_freshness_and_credential_gu " exit 1\n" " fi" ) in workflow + # This pinned `--jq .state` until 2026-09-07. What it protects is that the + # audit reads default-setup per repository, not that it reads only the + # state: `state == "configured"` with an empty `languages` list scans + # nothing and produces no analyses (live on life-os, aFIPC and inkspan), + # so the step now fetches the whole object and extracts both fields. + assert 'repos/${ORG_LOGIN}/${repository}/code-scanning/default-setup"' in workflow + assert """default_setup_state=$(jq '.state // null' "$default_setup_json")""" in workflow assert ( - 'repos/${ORG_LOGIN}/${repository}/code-scanning/default-setup" --jq .state' + """default_setup_languages=$(jq '.languages // []' "$default_setup_json")""" in workflow ) + assert "default_setup_languages: $default_setup_languages" in workflow assert ( 'if [ "$archived" != "true" ]; then\n' - ' default_setup_state_json="$RUNNER_TEMP/codeql-default-setup-' + ' default_setup_json="$RUNNER_TEMP/codeql-default-setup-' '${repository//[^A-Za-z0-9_.-]/_}.json"' ) in workflow assert ( @@ -540,6 +548,33 @@ def test_audit_organization_codeql_coverage_step_has_freshness_and_credential_gu assert "python3 scripts/ci/audit_org_codeql_coverage.py" in workflow +def test_codeql_coverage_audit_survives_a_ruleset_drift_failure() -> None: + """An owner-configured ruleset drift must not disable the coverage detector. + + Both audits live in one job, and the ruleset step exits 1 on governance + drift. It did on 2026-09-06 ("exactly two approving reviews are not + required", "last-push approval protection is disabled"), so every run since + 2026-09-04 failed before reaching the CodeQL coverage step. The subjects are + unrelated and the coverage step has no data dependency on the one above it, + so it is guarded by ``if: always()``. + + The bootstrap steps below it are deliberately *not* given the same guard: + they open pull requests, and running a mutation after an unexplained + upstream failure is a different decision from running a read-only detector. + """ + workflow = (REPO_ROOT / ".github/workflows/audit-central-ruleset.yml").read_text( + encoding="utf-8" + ) + coverage_step = workflow.split("- name: Audit organization CodeQL coverage\n", 1)[1] + before_next_step = coverage_step.split(" - name: ", 1)[0] + + assert "\n if: always()\n" in before_next_step + bootstrap_step = workflow.split( + "- name: Create missing CodeQL setup pull requests\n", 1 + )[1].split(" - name: ", 1)[0] + assert "if: always()" not in bootstrap_step + + def test_codeql_gap_bootstrap_uses_trusted_opencode_identity_without_pr_head_execution() -> None: """Backlog item 38 stays on trusted main and treats installation tokens as opaque.""" workflow = (REPO_ROOT / ".github/workflows/audit-central-ruleset.yml").read_text(