From c30556abb2e3deac190e469a95e7461deb7b4e0b Mon Sep 17 00:00:00 2001 From: David Beihl Date: Sun, 13 Sep 2026 17:03:16 -0400 Subject: [PATCH 1/5] feat(bin): add verified Dependabot alert counts --- bin/fm-alert-count.py | 261 +++++++++++++++++++++++++++++++++++ bin/fm-test-run.sh | 1 + docs/scripts.md | 1 + tests/fm-alert-count.test.sh | 116 ++++++++++++++++ 4 files changed, 379 insertions(+) create mode 100755 bin/fm-alert-count.py create mode 100755 tests/fm-alert-count.test.sh diff --git a/bin/fm-alert-count.py b/bin/fm-alert-count.py new file mode 100755 index 00000000000..b404cbe41b9 --- /dev/null +++ b/bin/fm-alert-count.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 +"""Count live Dependabot alerts that an npm package-lock branch proves it remediates. + +Usage: + fm-alert-count.py + +The command reads the live open Dependabot alert list for origin's GitHub +repository through gh-axi, then evaluates each alert's package-lock manifest at +the supplied base and head refs. +It is deliberately fail-closed: an unsupported manifest, a non-semver version, +or an advisory without a non-major patched version is reported as not checked, +never counted as remediated. +Output is silent only when the live default-branch alert list is empty. +""" + +from __future__ import annotations + +import base64 +import json +import re +import subprocess +import sys +from collections import defaultdict +from pathlib import PurePosixPath + + +REPO_RE = re.compile(r"(?:git@github\.com:|https://github\.com/)([^/\s]+)/([^/\s]+?)(?:\.git)?$") +SEMVER_RE = re.compile(r"^v?(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?(?:\+.*)?$") + + +class CheckError(Exception): + pass + + +def run(*args: str) -> str: + completed = subprocess.run(args, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if completed.returncode: + detail = completed.stderr.strip() or completed.stdout.strip() or "command failed" + raise CheckError(f"{' '.join(args[:2])}: {detail}") + return completed.stdout + + +def repository() -> str: + remote = run("git", "remote", "get-url", "origin").strip() + match = REPO_RE.fullmatch(remote) + if not match: + raise CheckError("origin is not a GitHub repository, so the live alert list was not checked") + return f"{match.group(1)}/{match.group(2)}" + + +def gh_axi_bodies(output: str) -> list[str]: + """Extract gh-axi's scalar response bodies without parsing presentation YAML.""" + bodies: list[str] = [] + for line in output.splitlines(): + match = re.fullmatch(r"\s*body:\s*['\"]?([A-Za-z0-9+/=]+)['\"]?\s*", line) + if match: + bodies.append(match.group(1)) + if not bodies: + raise CheckError("gh-axi returned no readable alert data") + return bodies + + +def live_alerts(repo: str) -> list[dict[str, object]]: + query = "[.[] | {number, dependency, security_vulnerability}] | @base64" + output = run( + "gh-axi", + "api", + f"/repos/{repo}/dependabot/alerts?state=open&per_page=100", + "--paginate", + "--jq", + query, + "--full", + ) + alerts: list[dict[str, object]] = [] + for body in gh_axi_bodies(output): + try: + page = json.loads(base64.b64decode(body, validate=True)) + except (ValueError, json.JSONDecodeError) as exc: + raise CheckError(f"gh-axi returned malformed alert data: {exc}") from exc + if not isinstance(page, list) or not all(isinstance(alert, dict) for alert in page): + raise CheckError("gh-axi returned an unexpected alert list") + alerts.extend(page) + return alerts + + +def git_file(ref: str, path: str) -> bytes: + completed = subprocess.run(("git", "show", f"{ref}:{path}"), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if completed.returncode: + raise CheckError(f"{path} is unavailable at {ref}") + return completed.stdout + + +def package_name_from_path(path: str) -> str | None: + parts = PurePosixPath(path).parts + indexes = [index for index, part in enumerate(parts) if part == "node_modules"] + if not indexes: + return None + start = indexes[-1] + 1 + if start >= len(parts): + return None + if parts[start].startswith("@") and start + 1 < len(parts): + return f"{parts[start]}/{parts[start + 1]}" + return parts[start] + + +def lock_versions(raw: bytes) -> dict[str, list[str]]: + try: + lock = json.loads(raw) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise CheckError(f"package-lock.json is not valid JSON: {exc}") from exc + versions: dict[str, list[str]] = defaultdict(list) + packages = lock.get("packages") + if isinstance(packages, dict): + for path, item in packages.items(): + if not isinstance(path, str) or not isinstance(item, dict): + continue + package = package_name_from_path(path) + version = item.get("version") + if package and isinstance(version, str): + versions[package].append(version) + return versions + + def visit(dependencies: object) -> None: + if not isinstance(dependencies, dict): + return + for package, item in dependencies.items(): + if not isinstance(package, str) or not isinstance(item, dict): + continue + version = item.get("version") + if isinstance(version, str): + versions[package].append(version) + visit(item.get("dependencies")) + + visit(lock.get("dependencies")) + return versions + + +def semver(version: str) -> tuple[int, int, int, int] | None: + match = SEMVER_RE.fullmatch(version) + if not match: + return None + major, minor, patch, prerelease = match.groups() + return int(major), int(minor), int(patch), 0 if prerelease else 1 + + +def version_state(versions: list[str], patched: str) -> str: + patch_version = semver(patched) + parsed = [semver(version) for version in versions] + if patch_version is None or any(version is None for version in parsed): + return "unknown" + return "safe" if all(version >= patch_version for version in parsed if version is not None) else "vulnerable" + + +def requires_major_upgrade(versions: list[str], patched: str) -> bool: + patch_version = semver(patched) + parsed = [semver(version) for version in versions] + return bool( + patch_version + and parsed + and all(version is not None and version[0] < patch_version[0] for version in parsed) + ) + + +def alert_fields(alert: dict[str, object]) -> tuple[int, str, str, str | None] | None: + number = alert.get("number") + dependency = alert.get("dependency") + vulnerability = alert.get("security_vulnerability") + if not isinstance(number, int) or not isinstance(dependency, dict) or not isinstance(vulnerability, dict): + return None + package = dependency.get("package") + manifest = dependency.get("manifest_path") + first_patched = vulnerability.get("first_patched_version") + if not isinstance(package, dict) or not isinstance(package.get("name"), str) or not isinstance(manifest, str): + return None + patched = first_patched.get("identifier") if isinstance(first_patched, dict) else None + return number, package["name"], manifest.lstrip("/"), patched if isinstance(patched, str) else None + + +def main(argv: list[str]) -> int: + if len(argv) != 3 or argv[1] in {"-h", "--help"}: + print(__doc__.strip(), file=sys.stderr) + return 2 + base, head = argv[1:] + try: + repo = repository() + alerts = live_alerts(repo) + except CheckError as exc: + print(f"NOT CHECKED: {exc}") + return 1 + if not alerts: + return 0 + + remediated: list[str] = [] + excluded: list[str] = [] + unchecked: list[str] = [] + by_package: dict[str, list[int]] = defaultdict(list) + caches: dict[tuple[str, str], dict[str, list[str]]] = {} + + for raw_alert in alerts: + fields = alert_fields(raw_alert) + if fields is None: + unchecked.append("malformed live alert record") + continue + number, package, manifest, patched = fields + by_package[package].append(number) + label = f"#{number} {package} ({manifest})" + if PurePosixPath(manifest).name != "package-lock.json": + unchecked.append(f"{label}: unsupported manifest") + continue + if patched is None: + excluded.append(f"{label}: major-only advisory, no non-major patched version to verify") + continue + try: + for ref in (base, head): + key = (ref, manifest) + if key not in caches: + caches[key] = lock_versions(git_file(ref, manifest)) + base_versions = caches[(base, manifest)].get(package, []) + head_versions = caches[(head, manifest)].get(package, []) + except CheckError as exc: + unchecked.append(f"{label}: {exc}") + continue + base_state = version_state(base_versions, patched) if base_versions else "safe" + head_state = version_state(head_versions, patched) if head_versions else "safe" + if base_state == "unknown" or head_state == "unknown": + unchecked.append(f"{label}: non-semver package-lock version") + elif requires_major_upgrade(base_versions, patched): + excluded.append(f"{label}: major-only advisory, patch requires {patched}") + elif base_state == "safe": + excluded.append(f"{label}: already resolved on {base}") + elif head_state == "safe": + remediated.append(label) + else: + excluded.append(f"{label}: {head} does not reach a patched version") + + print("OPEN DEFAULT-BRANCH ADVISORIES BY PACKAGE:") + for package, numbers in sorted(by_package.items()): + identifiers = ", ".join(f"#{number}" for number in sorted(numbers)) + print(f"- {package}: {len(numbers)} ({identifiers})") + print("PER-ADVISORY VERDICTS:") + for item in sorted(remediated + excluded + unchecked): + print(f"- {item}") + print(f"VERIFIED BRANCH REMEDIATION COUNT: {len(remediated)} ({', '.join(remediated) or 'none'})") + print( + f"DEFAULT-BRANCH CAVEAT: {len(remediated)} verified branch remediation(s) close none now. " + "Dependabot advisories attach to the default branch and close only after release to it." + ) + print("EXCLUDED FROM THE VERIFIED COUNT:") + for item in excluded: + print(f"- {item}") + if unchecked: + print("NOT CHECKED:") + for item in unchecked: + print(f"- {item}") + return 1 + print("NOT CHECKED: none") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index e69855f3875..37ea9c50f92 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -271,6 +271,7 @@ cpu_count() { family_for_basename() { case "$1" in fm-arm-pretool-check.test.sh|fm-ask-user-authority.test.sh|\ + fm-alert-count.test.sh|\ fm-bearings-board.test.sh|\ fm-brief.test.sh|fm-vendor-auth-probe.test.sh|\ fm-calm-pi-extension.test.sh|fm-cd-pretool-check.test.sh|\ diff --git a/docs/scripts.md b/docs/scripts.md index 09a27089aae..7d25d3a511c 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -41,6 +41,7 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-herdr-ci-cleanup.sh` | Snapshot and tear down only job-owned `fm-lab-*` sessions in the Herdr CI lane | | `fm-test-run.sh` | Behavior-test runner: selection, portable lanes, bounded concurrency, budgets, coverage guard, timing/JSON; refuses to execute in the repository primary checkout when `FM_TASK_ID` marks a task worker | | `fm-test-isolation-proof.sh` | Concurrent isolation harness and portable candidate set owner | +| `fm-alert-count.py` | Count live Dependabot alerts with per-advisory identifiers and fail-closed branch-remediation exclusions | | `fm-ensure-agents-md.sh` | Ensure a project's real `AGENTS.md`, its `CLAUDE.md` `@AGENTS.md` pointer, and self-governance guidance (explicit project mark documented in the helper's header and help) | | `fm-guard.sh` | Warn on primary-checkout tangles, main-session pending wakes, and unhealthy supervision | | `fm-primary-scope-lib.sh` | Shared marker-or-plain-checkout primary-home predicate for tracked hooks | diff --git a/tests/fm-alert-count.test.sh b/tests/fm-alert-count.test.sh new file mode 100755 index 00000000000..406a834b520 --- /dev/null +++ b/tests/fm-alert-count.test.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# tests/fm-alert-count.test.sh - executable behavior coverage for live advisory accounting. +set -euo pipefail + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +make_repository() { + local dir=$1 + mkdir -p "$dir/repo" "$dir/fakebin" + git init -q "$dir/repo" + git -C "$dir/repo" config user.email 'alerts-test@example.invalid' + git -C "$dir/repo" config user.name 'alerts test' + git -C "$dir/repo" remote add origin https://github.com/acme/widget.git + cat >"$dir/repo/package-lock.json" <<'JSON' +{"lockfileVersion":3,"packages":{"node_modules/foo":{"version":"1.0.0"},"node_modules/already":{"version":"2.0.0"},"node_modules/rejected":{"version":"1.0.0"},"node_modules/major":{"version":"1.0.0"}}} +JSON + git -C "$dir/repo" add package-lock.json + git -C "$dir/repo" commit -qm base + git -C "$dir/repo" branch integration + git -C "$dir/repo" checkout -qb security + perl -0pi -e 's/"node_modules\/foo":\{"version":"1\.0\.0"\}/"node_modules\/foo":{"version":"1.0.1"}/' "$dir/repo/package-lock.json" + git -C "$dir/repo" commit -am head -q +} + +write_gh_axi() { + local file=$1 payload=$2 + cat >"$file" <&2; exit 1 ;; +esac +EOF + chmod +x "$file" +} + +test_verified_remediation_and_exclusions_are_explicit() { + local dir payload out rc + dir=$(fm_test_tmproot fm-alert-count) + make_repository "$dir" + payload=$(printf '%s' '[ + {"number":101,"dependency":{"package":{"name":"foo"},"manifest_path":"package-lock.json"},"security_vulnerability":{"first_patched_version":{"identifier":"1.0.1"}}}, + {"number":102,"dependency":{"package":{"name":"already"},"manifest_path":"package-lock.json"},"security_vulnerability":{"first_patched_version":{"identifier":"2.0.0"}}}, + {"number":103,"dependency":{"package":{"name":"rejected"},"manifest_path":"package-lock.json"},"security_vulnerability":{"first_patched_version":{"identifier":"1.0.1"}}}, + {"number":104,"dependency":{"package":{"name":"major"},"manifest_path":"package-lock.json"},"security_vulnerability":{"first_patched_version":{"identifier":"2.0.0"}}} + ]' | base64 | tr -d '\n') + write_gh_axi "$dir/fakebin/gh-axi" "$payload" + set +e + out=$(cd "$dir/repo" && PATH="$dir/fakebin:$PATH" "$ROOT/bin/fm-alert-count.py" integration security) + rc=$? + set -e + [ "$rc" -eq 0 ] || fail "complete evidence should succeed: $out" + printf '%s\n' "$out" | grep -F 'foo: 1 (#101)' >/dev/null || fail "missing per-package count: $out" + printf '%s\n' "$out" | grep -F '#101 foo (package-lock.json)' >/dev/null || fail "missing advisory identifier: $out" + printf '%s\n' "$out" | grep -F 'VERIFIED BRANCH REMEDIATION COUNT: 1' >/dev/null || fail "wrong verified count: $out" + printf '%s\n' "$out" | grep -F '#102 already (package-lock.json): already resolved on integration' >/dev/null || fail "missing already-resolved exclusion: $out" + printf '%s\n' "$out" | grep -F '#103 rejected (package-lock.json): security does not reach a patched version' >/dev/null || fail "missing rejected exclusion: $out" + printf '%s\n' "$out" | grep -F '#104 major (package-lock.json): major-only advisory' >/dev/null || fail "missing major-only exclusion: $out" + printf '%s\n' "$out" | grep -F 'close none now' >/dev/null || fail "missing default-branch caveat: $out" + printf '%s\n' "$out" | grep -Fx 'NOT CHECKED: none' >/dev/null || fail "missing checked-scope statement: $out" + pass "alert count prints identifiers, exclusions, and the default-branch caveat" +} + +test_empty_live_list_is_silent() { + local dir out + dir=$(fm_test_tmproot fm-alert-count-empty) + make_repository "$dir" + write_gh_axi "$dir/fakebin/gh-axi" "W10=" + out=$(cd "$dir/repo" && PATH="$dir/fakebin:$PATH" "$ROOT/bin/fm-alert-count.py" integration security) + [ -z "$out" ] || fail "an empty live alert list must be silent: $out" + pass "alert count is silent when the default branch has no open alerts" +} + +test_live_list_failure_is_not_a_count() { + local dir out rc + dir=$(fm_test_tmproot fm-alert-count-failure) + make_repository "$dir" + cat >"$dir/fakebin/gh-axi" <<'EOF' +#!/usr/bin/env bash +printf 'error: insufficient permissions\n' >&2 +exit 1 +EOF + chmod +x "$dir/fakebin/gh-axi" + set +e + out=$(cd "$dir/repo" && PATH="$dir/fakebin:$PATH" "$ROOT/bin/fm-alert-count.py" integration security) + rc=$? + set -e + [ "$rc" -ne 0 ] || fail "an inaccessible live list must fail" + printf '%s\n' "$out" | grep -F 'NOT CHECKED:' >/dev/null || fail "failure claimed a count: $out" + pass "alert count reports an inaccessible live list as not checked" +} + +test_unsupported_manifest_is_explicitly_not_checked() { + local dir payload out rc + dir=$(fm_test_tmproot fm-alert-count-unsupported) + make_repository "$dir" + payload=$(printf '%s' '[{"number":201,"dependency":{"package":{"name":"java-lib"},"manifest_path":"pom.xml"},"security_vulnerability":{"first_patched_version":{"identifier":"1.0.1"}}}]' | base64 | tr -d '\n') + write_gh_axi "$dir/fakebin/gh-axi" "$payload" + set +e + out=$(cd "$dir/repo" && PATH="$dir/fakebin:$PATH" "$ROOT/bin/fm-alert-count.py" integration security) + rc=$? + set -e + [ "$rc" -ne 0 ] || fail "an unsupported manifest must not produce a verified count" + printf '%s\n' "$out" | grep -F '#201 java-lib (pom.xml): unsupported manifest' >/dev/null || fail "missing unsupported manifest evidence: $out" + printf '%s\n' "$out" | grep -F 'NOT CHECKED:' >/dev/null || fail "unsupported scope was not declared: $out" + pass "alert count refuses unsupported manifests instead of guessing" +} + +test_verified_remediation_and_exclusions_are_explicit +test_empty_live_list_is_silent +test_live_list_failure_is_not_a_count +test_unsupported_manifest_is_explicitly_not_checked From 1c982af92cdf70563fbf9df90c839d4728a20285 Mon Sep 17 00:00:00 2001 From: David Beihl Date: Sun, 13 Sep 2026 17:37:49 -0400 Subject: [PATCH 2/5] no-mistakes(review): Verify alerts against every advisory range across pages --- bin/fm-alert-count.py | 187 +++++++++++++++++++---------------- tests/fm-alert-count.test.sh | 101 ++++++++++++------- 2 files changed, 168 insertions(+), 120 deletions(-) diff --git a/bin/fm-alert-count.py b/bin/fm-alert-count.py index b404cbe41b9..04a11d520e5 100755 --- a/bin/fm-alert-count.py +++ b/bin/fm-alert-count.py @@ -6,10 +6,13 @@ The command reads the live open Dependabot alert list for origin's GitHub repository through gh-axi, then evaluates each alert's package-lock manifest at -the supplied base and head refs. -It is deliberately fail-closed: an unsupported manifest, a non-semver version, -or an advisory without a non-major patched version is reported as not checked, -never counted as remediated. +the supplied base and head refs against every npm vulnerable range the advisory +publishes for that package. +An alert counts as remediated only when base holds a copy inside a vulnerable +range and head holds none. It is deliberately fail-closed: an unsupported +manifest, a lockfile without a packages object, a non-semver or prerelease +installed version, or an unparseable vulnerable range is reported as not +checked, never counted as remediated. Output is silent only when the live default-branch alert list is empty. """ @@ -17,6 +20,7 @@ import base64 import json +import operator import re import subprocess import sys @@ -26,6 +30,11 @@ REPO_RE = re.compile(r"(?:git@github\.com:|https://github\.com/)([^/\s]+)/([^/\s]+?)(?:\.git)?$") SEMVER_RE = re.compile(r"^v?(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?(?:\+.*)?$") +RANGE_RE = re.compile(r"(<=|>=|<|>|=)\s*(\S+)") +BASE64_PAGE = r"[A-Za-z0-9+/=]+" +BODY_RE = re.compile(rf'^\s*body: (?:"({BASE64_PAGE}(?:\\n{BASE64_PAGE})*)"|({BASE64_PAGE}))\s*$', re.MULTILINE) +OPERATORS = {"<": operator.lt, "<=": operator.le, ">": operator.gt, ">=": operator.ge, "=": operator.eq} +RELEASE = ((2,),) class CheckError(Exception): @@ -48,20 +57,18 @@ def repository() -> str: return f"{match.group(1)}/{match.group(2)}" -def gh_axi_bodies(output: str) -> list[str]: - """Extract gh-axi's scalar response bodies without parsing presentation YAML.""" - bodies: list[str] = [] - for line in output.splitlines(): - match = re.fullmatch(r"\s*body:\s*['\"]?([A-Za-z0-9+/=]+)['\"]?\s*", line) - if match: - bodies.append(match.group(1)) - if not bodies: +def gh_axi_pages(output: str) -> list[str]: + """Extract the per-page base64 lines from gh-axi's TOON body scalar without parsing presentation YAML.""" + match = BODY_RE.search(output) + if not match: raise CheckError("gh-axi returned no readable alert data") - return bodies + return (match.group(1) or match.group(2)).split("\\n") def live_alerts(repo: str) -> list[dict[str, object]]: - query = "[.[] | {number, dependency, security_vulnerability}] | @base64" + query = ( + "[.[] | {number, dependency, security_advisory: {vulnerabilities: .security_advisory.vulnerabilities}}] | @base64" + ) output = run( "gh-axi", "api", @@ -72,7 +79,7 @@ def live_alerts(repo: str) -> list[dict[str, object]]: "--full", ) alerts: list[dict[str, object]] = [] - for body in gh_axi_bodies(output): + for body in gh_axi_pages(output): try: page = json.loads(base64.b64decode(body, validate=True)) except (ValueError, json.JSONDecodeError) as exc: @@ -108,72 +115,91 @@ def lock_versions(raw: bytes) -> dict[str, list[str]]: lock = json.loads(raw) except (UnicodeDecodeError, json.JSONDecodeError) as exc: raise CheckError(f"package-lock.json is not valid JSON: {exc}") from exc + packages = lock.get("packages") if isinstance(lock, dict) else None + if not isinstance(packages, dict): + raise CheckError("package-lock.json has no packages object") versions: dict[str, list[str]] = defaultdict(list) - packages = lock.get("packages") - if isinstance(packages, dict): - for path, item in packages.items(): - if not isinstance(path, str) or not isinstance(item, dict): - continue - package = package_name_from_path(path) - version = item.get("version") - if package and isinstance(version, str): - versions[package].append(version) - return versions - - def visit(dependencies: object) -> None: - if not isinstance(dependencies, dict): - return - for package, item in dependencies.items(): - if not isinstance(package, str) or not isinstance(item, dict): - continue - version = item.get("version") - if isinstance(version, str): - versions[package].append(version) - visit(item.get("dependencies")) - - visit(lock.get("dependencies")) + for path, item in packages.items(): + if not isinstance(path, str) or not isinstance(item, dict): + continue + package = package_name_from_path(path) + version = item.get("version") + if package and isinstance(version, str): + versions[package].append(version) return versions -def semver(version: str) -> tuple[int, int, int, int] | None: +def semver(version: str) -> tuple[int, int, int, tuple] | None: match = SEMVER_RE.fullmatch(version) if not match: return None major, minor, patch, prerelease = match.groups() - return int(major), int(minor), int(patch), 0 if prerelease else 1 + tag = tuple((0, int(part)) if part.isdigit() else (1, part) for part in prerelease.split(".")) if prerelease else RELEASE + return int(major), int(minor), int(patch), tag -def version_state(versions: list[str], patched: str) -> str: - patch_version = semver(patched) - parsed = [semver(version) for version in versions] - if patch_version is None or any(version is None for version in parsed): - return "unknown" - return "safe" if all(version >= patch_version for version in parsed if version is not None) else "vulnerable" - - -def requires_major_upgrade(versions: list[str], patched: str) -> bool: - patch_version = semver(patched) - parsed = [semver(version) for version in versions] - return bool( - patch_version - and parsed - and all(version is not None and version[0] < patch_version[0] for version in parsed) - ) - - -def alert_fields(alert: dict[str, object]) -> tuple[int, str, str, str | None] | None: +def parse_range(text: object) -> list[tuple[str, tuple]] | None: + if not isinstance(text, str): + return None + bounds = [] + for part in text.split(","): + match = RANGE_RE.fullmatch(part.strip()) + bound = semver(match.group(2)) if match else None + if bound is None: + return None + bounds.append((match.group(1), bound)) + return bounds + + +def vulnerable_ranges(package: str, vulnerabilities: list[object]) -> list[tuple[list[tuple[str, tuple]], str | None]]: + ranges = [] + for vulnerability in vulnerabilities: + if not isinstance(vulnerability, dict) or not isinstance(vulnerability.get("package"), dict): + raise CheckError("malformed advisory vulnerability") + affected = vulnerability["package"] + if str(affected.get("ecosystem")).lower() != "npm" or affected.get("name") != package: + continue + text = vulnerability.get("vulnerable_version_range") + bounds = parse_range(text) + first_patched = vulnerability.get("first_patched_version") + patched = first_patched.get("identifier") if isinstance(first_patched, dict) else None + if bounds is None or (first_patched is not None and not (isinstance(patched, str) and semver(patched))): + raise CheckError(f"unparseable advisory range {text!r}") + ranges.append((bounds, patched)) + if not ranges: + raise CheckError("advisory publishes no npm vulnerable range") + return ranges + + +def vulnerable_copies(versions: list[str], ranges: list[tuple[list[tuple[str, tuple]], str | None]]) -> list[tuple[tuple, str | None]]: + copies = [] + for version in versions: + parsed = semver(version) + if parsed is None or parsed[3] != RELEASE: + raise CheckError(f"non-semver or prerelease package-lock version {version}") + copies.extend( + (parsed, patched) for bounds, patched in ranges if all(OPERATORS[op](parsed, bound) for op, bound in bounds) + ) + return copies + + +def alert_fields(alert: dict[str, object]) -> tuple[int, str, str, list[object]] | None: number = alert.get("number") dependency = alert.get("dependency") - vulnerability = alert.get("security_vulnerability") - if not isinstance(number, int) or not isinstance(dependency, dict) or not isinstance(vulnerability, dict): + advisory = alert.get("security_advisory") + if not isinstance(number, int) or not isinstance(dependency, dict) or not isinstance(advisory, dict): return None package = dependency.get("package") manifest = dependency.get("manifest_path") - first_patched = vulnerability.get("first_patched_version") - if not isinstance(package, dict) or not isinstance(package.get("name"), str) or not isinstance(manifest, str): + vulnerabilities = advisory.get("vulnerabilities") + if ( + not isinstance(package, dict) + or not isinstance(package.get("name"), str) + or not isinstance(manifest, str) + or not isinstance(vulnerabilities, list) + ): return None - patched = first_patched.get("identifier") if isinstance(first_patched, dict) else None - return number, package["name"], manifest.lstrip("/"), patched if isinstance(patched, str) else None + return number, package["name"], manifest.lstrip("/"), vulnerabilities def main(argv: list[str]) -> int: @@ -193,7 +219,6 @@ def main(argv: list[str]) -> int: remediated: list[str] = [] excluded: list[str] = [] unchecked: list[str] = [] - by_package: dict[str, list[int]] = defaultdict(list) caches: dict[tuple[str, str], dict[str, list[str]]] = {} for raw_alert in alerts: @@ -201,42 +226,34 @@ def main(argv: list[str]) -> int: if fields is None: unchecked.append("malformed live alert record") continue - number, package, manifest, patched = fields - by_package[package].append(number) + number, package, manifest, vulnerabilities = fields label = f"#{number} {package} ({manifest})" if PurePosixPath(manifest).name != "package-lock.json": unchecked.append(f"{label}: unsupported manifest") continue - if patched is None: - excluded.append(f"{label}: major-only advisory, no non-major patched version to verify") - continue try: + ranges = vulnerable_ranges(package, vulnerabilities) for ref in (base, head): key = (ref, manifest) if key not in caches: caches[key] = lock_versions(git_file(ref, manifest)) - base_versions = caches[(base, manifest)].get(package, []) - head_versions = caches[(head, manifest)].get(package, []) + base_copies = vulnerable_copies(caches[(base, manifest)].get(package, []), ranges) + head_copies = vulnerable_copies(caches[(head, manifest)].get(package, []), ranges) except CheckError as exc: unchecked.append(f"{label}: {exc}") continue - base_state = version_state(base_versions, patched) if base_versions else "safe" - head_state = version_state(head_versions, patched) if head_versions else "safe" - if base_state == "unknown" or head_state == "unknown": - unchecked.append(f"{label}: non-semver package-lock version") - elif requires_major_upgrade(base_versions, patched): - excluded.append(f"{label}: major-only advisory, patch requires {patched}") - elif base_state == "safe": + patches = {patched for _, patched in head_copies} + if not base_copies: excluded.append(f"{label}: already resolved on {base}") - elif head_state == "safe": + elif not head_copies: remediated.append(label) + elif None in patches: + excluded.append(f"{label}: no patched version published") + elif all(semver(patched)[0] > version[0] for version, patched in head_copies): + excluded.append(f"{label}: rejected major, patch requires {', '.join(sorted(patches))}") else: excluded.append(f"{label}: {head} does not reach a patched version") - print("OPEN DEFAULT-BRANCH ADVISORIES BY PACKAGE:") - for package, numbers in sorted(by_package.items()): - identifiers = ", ".join(f"#{number}" for number in sorted(numbers)) - print(f"- {package}: {len(numbers)} ({identifiers})") print("PER-ADVISORY VERDICTS:") for item in sorted(remediated + excluded + unchecked): print(f"- {item}") diff --git a/tests/fm-alert-count.test.sh b/tests/fm-alert-count.test.sh index 406a834b520..59727cfdf2f 100755 --- a/tests/fm-alert-count.test.sh +++ b/tests/fm-alert-count.test.sh @@ -6,21 +6,19 @@ set -euo pipefail . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" make_repository() { - local dir=$1 + local dir=$1 base_lock=$2 head_lock=$3 mkdir -p "$dir/repo" "$dir/fakebin" git init -q "$dir/repo" git -C "$dir/repo" config user.email 'alerts-test@example.invalid' git -C "$dir/repo" config user.name 'alerts test' git -C "$dir/repo" remote add origin https://github.com/acme/widget.git - cat >"$dir/repo/package-lock.json" <<'JSON' -{"lockfileVersion":3,"packages":{"node_modules/foo":{"version":"1.0.0"},"node_modules/already":{"version":"2.0.0"},"node_modules/rejected":{"version":"1.0.0"},"node_modules/major":{"version":"1.0.0"}}} -JSON - git -C "$dir/repo" add package-lock.json + printf '%s\n' "$base_lock" >"$dir/repo/package-lock.json" + git -C "$dir/repo" add -A git -C "$dir/repo" commit -qm base git -C "$dir/repo" branch integration git -C "$dir/repo" checkout -qb security - perl -0pi -e 's/"node_modules\/foo":\{"version":"1\.0\.0"\}/"node_modules\/foo":{"version":"1.0.1"}/' "$dir/repo/package-lock.json" - git -C "$dir/repo" commit -am head -q + printf '%s\n' "$head_lock" >"$dir/repo/package-lock.json" + git -C "$dir/repo" commit -qam head --allow-empty } write_gh_axi() { @@ -38,37 +36,58 @@ EOF chmod +x "$file" } +# alert [extra advisory vulnerability JSON] +alert() { + local number=$1 package=$2 manifest=$3 range=$4 patched=$5 extra=${6:-} vulnerability + vulnerability=$(printf '{"package":{"ecosystem":"npm","name":"%s"},"vulnerable_version_range":"%s","first_patched_version":%s}' "$package" "$range" "$patched") + printf '{"number":%s,"dependency":{"package":{"ecosystem":"npm","name":"%s"},"manifest_path":"%s"},"security_vulnerability":%s,"security_advisory":{"vulnerabilities":[%s%s]}}' \ + "$number" "$package" "$manifest" "$vulnerability" "$vulnerability" "${extra:+,$extra}" +} + +page() { + local IFS=, + printf '[%s]' "$*" | base64 | tr -d '\n' +} + test_verified_remediation_and_exclusions_are_explicit() { - local dir payload out rc + local dir base_lock head_lock semver_7 first second out rc dir=$(fm_test_tmproot fm-alert-count) - make_repository "$dir" - payload=$(printf '%s' '[ - {"number":101,"dependency":{"package":{"name":"foo"},"manifest_path":"package-lock.json"},"security_vulnerability":{"first_patched_version":{"identifier":"1.0.1"}}}, - {"number":102,"dependency":{"package":{"name":"already"},"manifest_path":"package-lock.json"},"security_vulnerability":{"first_patched_version":{"identifier":"2.0.0"}}}, - {"number":103,"dependency":{"package":{"name":"rejected"},"manifest_path":"package-lock.json"},"security_vulnerability":{"first_patched_version":{"identifier":"1.0.1"}}}, - {"number":104,"dependency":{"package":{"name":"major"},"manifest_path":"package-lock.json"},"security_vulnerability":{"first_patched_version":{"identifier":"2.0.0"}}} - ]' | base64 | tr -d '\n') - write_gh_axi "$dir/fakebin/gh-axi" "$payload" + base_lock='{"lockfileVersion":3,"packages":{"node_modules/foo":{"version":"1.0.0"},"node_modules/already":{"version":"2.0.0"},"node_modules/rejected":{"version":"1.0.0"},"node_modules/major":{"version":"1.0.0"},"node_modules/taken":{"version":"1.0.0"},"node_modules/semver":{"version":"5.7.1"},"node_modules/unpatched":{"version":"3.0.0"}}}' + head_lock='{"lockfileVersion":3,"packages":{"node_modules/foo":{"version":"1.0.1"},"node_modules/already":{"version":"2.0.0"},"node_modules/rejected":{"version":"1.0.0"},"node_modules/major":{"version":"1.0.0"},"node_modules/taken":{"version":"2.0.0"},"node_modules/semver":{"version":"7.0.0"},"node_modules/unpatched":{"version":"3.0.0"}}}' + make_repository "$dir" "$base_lock" "$head_lock" + semver_7='{"package":{"ecosystem":"npm","name":"semver"},"vulnerable_version_range":">= 7.0.0, < 7.5.2","first_patched_version":{"identifier":"7.5.2"}}' + first=$(page \ + "$(alert 101 foo package-lock.json '< 1.0.1' '{"identifier":"1.0.1"}')" \ + "$(alert 102 already package-lock.json '< 2.0.0' '{"identifier":"2.0.0"}')" \ + "$(alert 103 rejected package-lock.json '< 1.0.1' '{"identifier":"1.0.1"}')" \ + "$(alert 104 major package-lock.json '< 2.0.0' '{"identifier":"2.0.0"}')") + second=$(page \ + "$(alert 105 taken package-lock.json '< 2.0.0' '{"identifier":"2.0.0"}')" \ + "$(alert 106 semver package-lock.json '>= 2.0.0-alpha, < 5.7.2' '{"identifier":"5.7.2"}' "$semver_7")" \ + "$(alert 107 unpatched package-lock.json '<= 3.0.0' null)") + write_gh_axi "$dir/fakebin/gh-axi" "\"$first\\n$second\"" set +e out=$(cd "$dir/repo" && PATH="$dir/fakebin:$PATH" "$ROOT/bin/fm-alert-count.py" integration security) rc=$? set -e [ "$rc" -eq 0 ] || fail "complete evidence should succeed: $out" - printf '%s\n' "$out" | grep -F 'foo: 1 (#101)' >/dev/null || fail "missing per-package count: $out" - printf '%s\n' "$out" | grep -F '#101 foo (package-lock.json)' >/dev/null || fail "missing advisory identifier: $out" - printf '%s\n' "$out" | grep -F 'VERIFIED BRANCH REMEDIATION COUNT: 1' >/dev/null || fail "wrong verified count: $out" + printf '%s\n' "$out" | grep -F 'BY PACKAGE' >/dev/null && fail "per-package tally must not be printed: $out" + printf '%s\n' "$out" | grep -Fx 'VERIFIED BRANCH REMEDIATION COUNT: 2 (#101 foo (package-lock.json), #105 taken (package-lock.json))' >/dev/null || fail "wrong verified count: $out" printf '%s\n' "$out" | grep -F '#102 already (package-lock.json): already resolved on integration' >/dev/null || fail "missing already-resolved exclusion: $out" - printf '%s\n' "$out" | grep -F '#103 rejected (package-lock.json): security does not reach a patched version' >/dev/null || fail "missing rejected exclusion: $out" - printf '%s\n' "$out" | grep -F '#104 major (package-lock.json): major-only advisory' >/dev/null || fail "missing major-only exclusion: $out" + printf '%s\n' "$out" | grep -F '#103 rejected (package-lock.json): security does not reach a patched version' >/dev/null || fail "missing unpatched-branch exclusion: $out" + printf '%s\n' "$out" | grep -F '#104 major (package-lock.json): rejected major, patch requires 2.0.0' >/dev/null || fail "missing rejected-major exclusion: $out" + printf '%s\n' "$out" | grep -F '#106 semver (package-lock.json): security does not reach a patched version' >/dev/null || fail "a version inside another advisory range was counted: $out" + printf '%s\n' "$out" | grep -F '#107 unpatched (package-lock.json): no patched version published' >/dev/null || fail "missing no-patch exclusion: $out" printf '%s\n' "$out" | grep -F 'close none now' >/dev/null || fail "missing default-branch caveat: $out" printf '%s\n' "$out" | grep -Fx 'NOT CHECKED: none' >/dev/null || fail "missing checked-scope statement: $out" - pass "alert count prints identifiers, exclusions, and the default-branch caveat" + pass "alert count checks every advisory range across pages and labels each exclusion" } test_empty_live_list_is_silent() { - local dir out + local dir lock out dir=$(fm_test_tmproot fm-alert-count-empty) - make_repository "$dir" + lock='{"lockfileVersion":3,"packages":{}}' + make_repository "$dir" "$lock" "$lock" write_gh_axi "$dir/fakebin/gh-axi" "W10=" out=$(cd "$dir/repo" && PATH="$dir/fakebin:$PATH" "$ROOT/bin/fm-alert-count.py" integration security) [ -z "$out" ] || fail "an empty live alert list must be silent: $out" @@ -76,9 +95,10 @@ test_empty_live_list_is_silent() { } test_live_list_failure_is_not_a_count() { - local dir out rc + local dir lock out rc dir=$(fm_test_tmproot fm-alert-count-failure) - make_repository "$dir" + lock='{"lockfileVersion":3,"packages":{}}' + make_repository "$dir" "$lock" "$lock" cat >"$dir/fakebin/gh-axi" <<'EOF' #!/usr/bin/env bash printf 'error: insufficient permissions\n' >&2 @@ -94,23 +114,34 @@ EOF pass "alert count reports an inaccessible live list as not checked" } -test_unsupported_manifest_is_explicitly_not_checked() { - local dir payload out rc - dir=$(fm_test_tmproot fm-alert-count-unsupported) - make_repository "$dir" - payload=$(printf '%s' '[{"number":201,"dependency":{"package":{"name":"java-lib"},"manifest_path":"pom.xml"},"security_vulnerability":{"first_patched_version":{"identifier":"1.0.1"}}}]' | base64 | tr -d '\n') +test_unverifiable_evidence_is_explicitly_not_checked() { + local dir base_lock head_lock payload out rc + dir=$(fm_test_tmproot fm-alert-count-unverifiable) + mkdir -p "$dir/repo/legacy" + printf '%s\n' '{"lockfileVersion":1,"dependencies":{"old":{"version":"1.0.0"}}}' >"$dir/repo/legacy/package-lock.json" + base_lock='{"lockfileVersion":3,"packages":{"node_modules/pre":{"version":"1.0.0"},"node_modules/ranged":{"version":"1.0.0"}}}' + head_lock='{"lockfileVersion":3,"packages":{"node_modules/pre":{"version":"1.0.1-beta.1"},"node_modules/ranged":{"version":"1.0.1"}}}' + make_repository "$dir" "$base_lock" "$head_lock" + payload=$(page \ + "$(alert 201 java-lib pom.xml '< 1.0.1' '{"identifier":"1.0.1"}')" \ + "$(alert 202 pre package-lock.json '< 1.0.1' '{"identifier":"1.0.1"}')" \ + "$(alert 203 ranged package-lock.json '^1.0.0' '{"identifier":"1.0.1"}')" \ + "$(alert 204 old legacy/package-lock.json '< 1.0.1' '{"identifier":"1.0.1"}')") write_gh_axi "$dir/fakebin/gh-axi" "$payload" set +e out=$(cd "$dir/repo" && PATH="$dir/fakebin:$PATH" "$ROOT/bin/fm-alert-count.py" integration security) rc=$? set -e - [ "$rc" -ne 0 ] || fail "an unsupported manifest must not produce a verified count" + [ "$rc" -ne 0 ] || fail "unverifiable evidence must not produce a complete count: $out" + printf '%s\n' "$out" | grep -F 'VERIFIED BRANCH REMEDIATION COUNT: 0' >/dev/null || fail "unverifiable evidence was counted: $out" printf '%s\n' "$out" | grep -F '#201 java-lib (pom.xml): unsupported manifest' >/dev/null || fail "missing unsupported manifest evidence: $out" - printf '%s\n' "$out" | grep -F 'NOT CHECKED:' >/dev/null || fail "unsupported scope was not declared: $out" - pass "alert count refuses unsupported manifests instead of guessing" + printf '%s\n' "$out" | grep -F '#202 pre (package-lock.json): non-semver or prerelease package-lock version 1.0.1-beta.1' >/dev/null || fail "missing prerelease evidence: $out" + printf '%s\n' "$out" | grep -F "#203 ranged (package-lock.json): unparseable advisory range '^1.0.0'" >/dev/null || fail "missing unparseable range evidence: $out" + printf '%s\n' "$out" | grep -F '#204 old (legacy/package-lock.json): package-lock.json has no packages object' >/dev/null || fail "missing packages-object evidence: $out" + pass "alert count refuses evidence it cannot verify instead of guessing" } test_verified_remediation_and_exclusions_are_explicit test_empty_live_list_is_silent test_live_list_failure_is_not_a_count -test_unsupported_manifest_is_explicitly_not_checked +test_unverifiable_evidence_is_explicitly_not_checked From d34a6df194cfd8b65a2027a7a9ae32e506c881d7 Mon Sep 17 00:00:00 2001 From: David Beihl Date: Sun, 13 Sep 2026 17:56:11 -0400 Subject: [PATCH 3/5] no-mistakes(review): Check alias copies and partial bounds in alert counts --- bin/fm-alert-count.py | 28 ++++++++++++++++++++-------- tests/fm-alert-count.test.sh | 21 ++++++++++++++++----- 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/bin/fm-alert-count.py b/bin/fm-alert-count.py index 04a11d520e5..d2eb55e1cbe 100755 --- a/bin/fm-alert-count.py +++ b/bin/fm-alert-count.py @@ -123,9 +123,10 @@ def lock_versions(raw: bytes) -> dict[str, list[str]]: if not isinstance(path, str) or not isinstance(item, dict): continue package = package_name_from_path(path) + name = item.get("name") version = item.get("version") if package and isinstance(version, str): - versions[package].append(version) + versions[name if isinstance(name, str) else package].append(version) return versions @@ -138,14 +139,20 @@ def semver(version: str) -> tuple[int, int, int, tuple] | None: return int(major), int(minor), int(patch), tag +def padded(version: str) -> str: + parts = version.split(".") + return ".".join(parts + ["0"] * (3 - len(parts))) if len(parts) < 3 and all(part.isdigit() for part in parts) else version + + def parse_range(text: object) -> list[tuple[str, tuple]] | None: if not isinstance(text, str): return None bounds = [] for part in text.split(","): match = RANGE_RE.fullmatch(part.strip()) - bound = semver(match.group(2)) if match else None - if bound is None: + full = padded(match.group(2)) if match else "" + bound = semver(full) + if bound is None or (full != match.group(2) and match.group(1) in {"<=", "="}): return None bounds.append((match.group(1), bound)) return bounds @@ -161,10 +168,12 @@ def vulnerable_ranges(package: str, vulnerabilities: list[object]) -> list[tuple continue text = vulnerability.get("vulnerable_version_range") bounds = parse_range(text) + if bounds is None: + raise CheckError(f"unparseable advisory range {text!r}") first_patched = vulnerability.get("first_patched_version") patched = first_patched.get("identifier") if isinstance(first_patched, dict) else None - if bounds is None or (first_patched is not None and not (isinstance(patched, str) and semver(patched))): - raise CheckError(f"unparseable advisory range {text!r}") + if first_patched is not None and not (isinstance(patched, str) and semver(padded(patched))): + raise CheckError(f"unparseable first patched version {patched!r}") ranges.append((bounds, patched)) if not ranges: raise CheckError("advisory publishes no npm vulnerable range") @@ -224,7 +233,8 @@ def main(argv: list[str]) -> int: for raw_alert in alerts: fields = alert_fields(raw_alert) if fields is None: - unchecked.append("malformed live alert record") + number = raw_alert.get("number") + unchecked.append(f"#{number} malformed live alert record" if isinstance(number, int) else "malformed live alert record") continue number, package, manifest, vulnerabilities = fields label = f"#{number} {package} ({manifest})" @@ -243,13 +253,15 @@ def main(argv: list[str]) -> int: unchecked.append(f"{label}: {exc}") continue patches = {patched for _, patched in head_copies} - if not base_copies: + if not base_copies and not head_copies: excluded.append(f"{label}: already resolved on {base}") + elif not base_copies: + excluded.append(f"{label}: {head} reintroduces a vulnerable copy") elif not head_copies: remediated.append(label) elif None in patches: excluded.append(f"{label}: no patched version published") - elif all(semver(patched)[0] > version[0] for version, patched in head_copies): + elif all(semver(padded(patched))[0] > version[0] for version, patched in head_copies): excluded.append(f"{label}: rejected major, patch requires {', '.join(sorted(patches))}") else: excluded.append(f"{label}: {head} does not reach a patched version") diff --git a/tests/fm-alert-count.test.sh b/tests/fm-alert-count.test.sh index 59727cfdf2f..25efe221337 100755 --- a/tests/fm-alert-count.test.sh +++ b/tests/fm-alert-count.test.sh @@ -52,8 +52,8 @@ page() { test_verified_remediation_and_exclusions_are_explicit() { local dir base_lock head_lock semver_7 first second out rc dir=$(fm_test_tmproot fm-alert-count) - base_lock='{"lockfileVersion":3,"packages":{"node_modules/foo":{"version":"1.0.0"},"node_modules/already":{"version":"2.0.0"},"node_modules/rejected":{"version":"1.0.0"},"node_modules/major":{"version":"1.0.0"},"node_modules/taken":{"version":"1.0.0"},"node_modules/semver":{"version":"5.7.1"},"node_modules/unpatched":{"version":"3.0.0"}}}' - head_lock='{"lockfileVersion":3,"packages":{"node_modules/foo":{"version":"1.0.1"},"node_modules/already":{"version":"2.0.0"},"node_modules/rejected":{"version":"1.0.0"},"node_modules/major":{"version":"1.0.0"},"node_modules/taken":{"version":"2.0.0"},"node_modules/semver":{"version":"7.0.0"},"node_modules/unpatched":{"version":"3.0.0"}}}' + base_lock='{"lockfileVersion":3,"packages":{"node_modules/foo":{"version":"1.0.0"},"node_modules/already":{"version":"2.0.0"},"node_modules/rejected":{"version":"1.0.0"},"node_modules/major":{"version":"1.0.0"},"node_modules/taken":{"version":"1.0.0"},"node_modules/semver":{"version":"5.7.1"},"node_modules/unpatched":{"version":"3.0.0"},"node_modules/aliased":{"version":"1.0.0"},"node_modules/aliased-cjs":{"name":"aliased","version":"1.0.0"},"node_modules/malware":{"version":"1.0.0"},"node_modules/partial":{"version":"7.9.0"},"node_modules/returned":{"version":"1.0.1"}}}' + head_lock='{"lockfileVersion":3,"packages":{"node_modules/foo":{"version":"1.0.1"},"node_modules/already":{"version":"2.0.0"},"node_modules/rejected":{"version":"1.0.0"},"node_modules/major":{"version":"1.0.0"},"node_modules/taken":{"version":"2.0.0"},"node_modules/semver":{"version":"7.0.0"},"node_modules/unpatched":{"version":"3.0.0"},"node_modules/aliased":{"version":"1.0.1"},"node_modules/aliased-cjs":{"name":"aliased","version":"1.0.0"},"node_modules/malware":{"version":"1.0.0"},"node_modules/partial":{"version":"8.0.0"},"node_modules/returned":{"version":"1.0.0"}}}' make_repository "$dir" "$base_lock" "$head_lock" semver_7='{"package":{"ecosystem":"npm","name":"semver"},"vulnerable_version_range":">= 7.0.0, < 7.5.2","first_patched_version":{"identifier":"7.5.2"}}' first=$(page \ @@ -64,7 +64,11 @@ test_verified_remediation_and_exclusions_are_explicit() { second=$(page \ "$(alert 105 taken package-lock.json '< 2.0.0' '{"identifier":"2.0.0"}')" \ "$(alert 106 semver package-lock.json '>= 2.0.0-alpha, < 5.7.2' '{"identifier":"5.7.2"}' "$semver_7")" \ - "$(alert 107 unpatched package-lock.json '<= 3.0.0' null)") + "$(alert 107 unpatched package-lock.json '<= 3.0.0' null)" \ + "$(alert 108 aliased package-lock.json '< 1.0.1' '{"identifier":"1.0.1"}')" \ + "$(alert 109 malware package-lock.json '> 0' null)" \ + "$(alert 110 partial package-lock.json '>= 7.0, < 8.0' '{"identifier":"8.0"}')" \ + "$(alert 111 returned package-lock.json '< 1.0.1' '{"identifier":"1.0.1"}')") write_gh_axi "$dir/fakebin/gh-axi" "\"$first\\n$second\"" set +e out=$(cd "$dir/repo" && PATH="$dir/fakebin:$PATH" "$ROOT/bin/fm-alert-count.py" integration security) @@ -72,7 +76,10 @@ test_verified_remediation_and_exclusions_are_explicit() { set -e [ "$rc" -eq 0 ] || fail "complete evidence should succeed: $out" printf '%s\n' "$out" | grep -F 'BY PACKAGE' >/dev/null && fail "per-package tally must not be printed: $out" - printf '%s\n' "$out" | grep -Fx 'VERIFIED BRANCH REMEDIATION COUNT: 2 (#101 foo (package-lock.json), #105 taken (package-lock.json))' >/dev/null || fail "wrong verified count: $out" + printf '%s\n' "$out" | grep -Fx 'VERIFIED BRANCH REMEDIATION COUNT: 3 (#101 foo (package-lock.json), #105 taken (package-lock.json), #110 partial (package-lock.json))' >/dev/null || fail "wrong verified count: $out" + printf '%s\n' "$out" | grep -F '#108 aliased (package-lock.json): security does not reach a patched version' >/dev/null || fail "an aliased vulnerable copy was ignored: $out" + printf '%s\n' "$out" | grep -F '#109 malware (package-lock.json): no patched version published' >/dev/null || fail "missing zero-bound no-patch exclusion: $out" + printf '%s\n' "$out" | grep -F '#111 returned (package-lock.json): security reintroduces a vulnerable copy' >/dev/null || fail "a reintroduced copy was labeled resolved: $out" printf '%s\n' "$out" | grep -F '#102 already (package-lock.json): already resolved on integration' >/dev/null || fail "missing already-resolved exclusion: $out" printf '%s\n' "$out" | grep -F '#103 rejected (package-lock.json): security does not reach a patched version' >/dev/null || fail "missing unpatched-branch exclusion: $out" printf '%s\n' "$out" | grep -F '#104 major (package-lock.json): rejected major, patch requires 2.0.0' >/dev/null || fail "missing rejected-major exclusion: $out" @@ -126,7 +133,9 @@ test_unverifiable_evidence_is_explicitly_not_checked() { "$(alert 201 java-lib pom.xml '< 1.0.1' '{"identifier":"1.0.1"}')" \ "$(alert 202 pre package-lock.json '< 1.0.1' '{"identifier":"1.0.1"}')" \ "$(alert 203 ranged package-lock.json '^1.0.0' '{"identifier":"1.0.1"}')" \ - "$(alert 204 old legacy/package-lock.json '< 1.0.1' '{"identifier":"1.0.1"}')") + "$(alert 204 old legacy/package-lock.json '< 1.0.1' '{"identifier":"1.0.1"}')" \ + "$(alert 205 ranged package-lock.json '<= 1.3' '{"identifier":"1.4.0"}')" \ + '{"number":206,"dependency":{"package":{"ecosystem":"npm","name":"ranged"},"manifest_path":"package-lock.json"}}') write_gh_axi "$dir/fakebin/gh-axi" "$payload" set +e out=$(cd "$dir/repo" && PATH="$dir/fakebin:$PATH" "$ROOT/bin/fm-alert-count.py" integration security) @@ -138,6 +147,8 @@ test_unverifiable_evidence_is_explicitly_not_checked() { printf '%s\n' "$out" | grep -F '#202 pre (package-lock.json): non-semver or prerelease package-lock version 1.0.1-beta.1' >/dev/null || fail "missing prerelease evidence: $out" printf '%s\n' "$out" | grep -F "#203 ranged (package-lock.json): unparseable advisory range '^1.0.0'" >/dev/null || fail "missing unparseable range evidence: $out" printf '%s\n' "$out" | grep -F '#204 old (legacy/package-lock.json): package-lock.json has no packages object' >/dev/null || fail "missing packages-object evidence: $out" + printf '%s\n' "$out" | grep -F "#205 ranged (package-lock.json): unparseable advisory range '<= 1.3'" >/dev/null || fail "an ambiguous partial bound was accepted: $out" + printf '%s\n' "$out" | grep -F '#206 malformed live alert record' >/dev/null || fail "malformed record lost its identifier: $out" pass "alert count refuses evidence it cannot verify instead of guessing" } From f2f2d7001ba62b47125ffaa6c56e2eea04eb81e0 Mon Sep 17 00:00:00 2001 From: David Beihl Date: Sun, 13 Sep 2026 18:14:53 -0400 Subject: [PATCH 4/5] no-mistakes(review): Reject non-zero partial bounds and drop duplicate verdicts --- bin/fm-alert-count.py | 26 ++++++++++++-------------- tests/fm-alert-count.test.sh | 18 +++++++++++------- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/bin/fm-alert-count.py b/bin/fm-alert-count.py index d2eb55e1cbe..c47cc5a19d8 100755 --- a/bin/fm-alert-count.py +++ b/bin/fm-alert-count.py @@ -139,22 +139,23 @@ def semver(version: str) -> tuple[int, int, int, tuple] | None: return int(major), int(minor), int(patch), tag -def padded(version: str) -> str: - parts = version.split(".") - return ".".join(parts + ["0"] * (3 - len(parts))) if len(parts) < 3 and all(part.isdigit() for part in parts) else version - - def parse_range(text: object) -> list[tuple[str, tuple]] | None: if not isinstance(text, str): return None bounds = [] for part in text.split(","): match = RANGE_RE.fullmatch(part.strip()) - full = padded(match.group(2)) if match else "" - bound = semver(full) - if bound is None or (full != match.group(2) and match.group(1) in {"<=", "="}): + if not match: return None - bounds.append((match.group(1), bound)) + op, version = match.groups() + if version.count(".") < 2: + if op not in {">", ">="} or any(component != "0" for component in version.split(".")): + return None + version = "0.0.0" + bound = semver(version) + if bound is None: + return None + bounds.append((op, bound)) return bounds @@ -172,7 +173,7 @@ def vulnerable_ranges(package: str, vulnerabilities: list[object]) -> list[tuple raise CheckError(f"unparseable advisory range {text!r}") first_patched = vulnerability.get("first_patched_version") patched = first_patched.get("identifier") if isinstance(first_patched, dict) else None - if first_patched is not None and not (isinstance(patched, str) and semver(padded(patched))): + if first_patched is not None and not (isinstance(patched, str) and semver(patched)): raise CheckError(f"unparseable first patched version {patched!r}") ranges.append((bounds, patched)) if not ranges: @@ -261,14 +262,11 @@ def main(argv: list[str]) -> int: remediated.append(label) elif None in patches: excluded.append(f"{label}: no patched version published") - elif all(semver(padded(patched))[0] > version[0] for version, patched in head_copies): + elif all(semver(patched)[0] > version[0] for version, patched in head_copies): excluded.append(f"{label}: rejected major, patch requires {', '.join(sorted(patches))}") else: excluded.append(f"{label}: {head} does not reach a patched version") - print("PER-ADVISORY VERDICTS:") - for item in sorted(remediated + excluded + unchecked): - print(f"- {item}") print(f"VERIFIED BRANCH REMEDIATION COUNT: {len(remediated)} ({', '.join(remediated) or 'none'})") print( f"DEFAULT-BRANCH CAVEAT: {len(remediated)} verified branch remediation(s) close none now. " diff --git a/tests/fm-alert-count.test.sh b/tests/fm-alert-count.test.sh index 25efe221337..e62ee3c8155 100755 --- a/tests/fm-alert-count.test.sh +++ b/tests/fm-alert-count.test.sh @@ -52,8 +52,8 @@ page() { test_verified_remediation_and_exclusions_are_explicit() { local dir base_lock head_lock semver_7 first second out rc dir=$(fm_test_tmproot fm-alert-count) - base_lock='{"lockfileVersion":3,"packages":{"node_modules/foo":{"version":"1.0.0"},"node_modules/already":{"version":"2.0.0"},"node_modules/rejected":{"version":"1.0.0"},"node_modules/major":{"version":"1.0.0"},"node_modules/taken":{"version":"1.0.0"},"node_modules/semver":{"version":"5.7.1"},"node_modules/unpatched":{"version":"3.0.0"},"node_modules/aliased":{"version":"1.0.0"},"node_modules/aliased-cjs":{"name":"aliased","version":"1.0.0"},"node_modules/malware":{"version":"1.0.0"},"node_modules/partial":{"version":"7.9.0"},"node_modules/returned":{"version":"1.0.1"}}}' - head_lock='{"lockfileVersion":3,"packages":{"node_modules/foo":{"version":"1.0.1"},"node_modules/already":{"version":"2.0.0"},"node_modules/rejected":{"version":"1.0.0"},"node_modules/major":{"version":"1.0.0"},"node_modules/taken":{"version":"2.0.0"},"node_modules/semver":{"version":"7.0.0"},"node_modules/unpatched":{"version":"3.0.0"},"node_modules/aliased":{"version":"1.0.1"},"node_modules/aliased-cjs":{"name":"aliased","version":"1.0.0"},"node_modules/malware":{"version":"1.0.0"},"node_modules/partial":{"version":"8.0.0"},"node_modules/returned":{"version":"1.0.0"}}}' + base_lock='{"lockfileVersion":3,"packages":{"node_modules/foo":{"version":"1.0.0"},"node_modules/already":{"version":"2.0.0"},"node_modules/rejected":{"version":"1.0.0"},"node_modules/major":{"version":"1.0.0"},"node_modules/taken":{"version":"1.0.0"},"node_modules/semver":{"version":"5.7.1"},"node_modules/unpatched":{"version":"3.0.0"},"node_modules/aliased":{"version":"1.0.0"},"node_modules/aliased-cjs":{"name":"aliased","version":"1.0.0"},"node_modules/malware":{"version":"1.0.0"},"node_modules/returned":{"version":"1.0.1"}}}' + head_lock='{"lockfileVersion":3,"packages":{"node_modules/foo":{"version":"1.0.1"},"node_modules/already":{"version":"2.0.0"},"node_modules/rejected":{"version":"1.0.0"},"node_modules/major":{"version":"1.0.0"},"node_modules/taken":{"version":"2.0.0"},"node_modules/semver":{"version":"7.0.0"},"node_modules/unpatched":{"version":"3.0.0"},"node_modules/aliased":{"version":"1.0.1"},"node_modules/aliased-cjs":{"name":"aliased","version":"1.0.0"},"node_modules/malware":{"version":"1.0.0"},"node_modules/returned":{"version":"1.0.0"}}}' make_repository "$dir" "$base_lock" "$head_lock" semver_7='{"package":{"ecosystem":"npm","name":"semver"},"vulnerable_version_range":">= 7.0.0, < 7.5.2","first_patched_version":{"identifier":"7.5.2"}}' first=$(page \ @@ -67,7 +67,6 @@ test_verified_remediation_and_exclusions_are_explicit() { "$(alert 107 unpatched package-lock.json '<= 3.0.0' null)" \ "$(alert 108 aliased package-lock.json '< 1.0.1' '{"identifier":"1.0.1"}')" \ "$(alert 109 malware package-lock.json '> 0' null)" \ - "$(alert 110 partial package-lock.json '>= 7.0, < 8.0' '{"identifier":"8.0"}')" \ "$(alert 111 returned package-lock.json '< 1.0.1' '{"identifier":"1.0.1"}')") write_gh_axi "$dir/fakebin/gh-axi" "\"$first\\n$second\"" set +e @@ -76,7 +75,8 @@ test_verified_remediation_and_exclusions_are_explicit() { set -e [ "$rc" -eq 0 ] || fail "complete evidence should succeed: $out" printf '%s\n' "$out" | grep -F 'BY PACKAGE' >/dev/null && fail "per-package tally must not be printed: $out" - printf '%s\n' "$out" | grep -Fx 'VERIFIED BRANCH REMEDIATION COUNT: 3 (#101 foo (package-lock.json), #105 taken (package-lock.json), #110 partial (package-lock.json))' >/dev/null || fail "wrong verified count: $out" + printf '%s\n' "$out" | grep -F 'PER-ADVISORY' >/dev/null && fail "duplicate verdict section must not be printed: $out" + printf '%s\n' "$out" | grep -Fx 'VERIFIED BRANCH REMEDIATION COUNT: 2 (#101 foo (package-lock.json), #105 taken (package-lock.json))' >/dev/null || fail "wrong verified count: $out" printf '%s\n' "$out" | grep -F '#108 aliased (package-lock.json): security does not reach a patched version' >/dev/null || fail "an aliased vulnerable copy was ignored: $out" printf '%s\n' "$out" | grep -F '#109 malware (package-lock.json): no patched version published' >/dev/null || fail "missing zero-bound no-patch exclusion: $out" printf '%s\n' "$out" | grep -F '#111 returned (package-lock.json): security reintroduces a vulnerable copy' >/dev/null || fail "a reintroduced copy was labeled resolved: $out" @@ -126,8 +126,8 @@ test_unverifiable_evidence_is_explicitly_not_checked() { dir=$(fm_test_tmproot fm-alert-count-unverifiable) mkdir -p "$dir/repo/legacy" printf '%s\n' '{"lockfileVersion":1,"dependencies":{"old":{"version":"1.0.0"}}}' >"$dir/repo/legacy/package-lock.json" - base_lock='{"lockfileVersion":3,"packages":{"node_modules/pre":{"version":"1.0.0"},"node_modules/ranged":{"version":"1.0.0"}}}' - head_lock='{"lockfileVersion":3,"packages":{"node_modules/pre":{"version":"1.0.1-beta.1"},"node_modules/ranged":{"version":"1.0.1"}}}' + base_lock='{"lockfileVersion":3,"packages":{"node_modules/pre":{"version":"1.0.0"},"node_modules/ranged":{"version":"1.0.0"},"node_modules/partial":{"version":"7.9.0"},"node_modules/above":{"version":"8.0.5"}}}' + head_lock='{"lockfileVersion":3,"packages":{"node_modules/pre":{"version":"1.0.1-beta.1"},"node_modules/ranged":{"version":"1.0.1"},"node_modules/partial":{"version":"8.0.0"},"node_modules/above":{"version":"9.0.0"}}}' make_repository "$dir" "$base_lock" "$head_lock" payload=$(page \ "$(alert 201 java-lib pom.xml '< 1.0.1' '{"identifier":"1.0.1"}')" \ @@ -135,7 +135,9 @@ test_unverifiable_evidence_is_explicitly_not_checked() { "$(alert 203 ranged package-lock.json '^1.0.0' '{"identifier":"1.0.1"}')" \ "$(alert 204 old legacy/package-lock.json '< 1.0.1' '{"identifier":"1.0.1"}')" \ "$(alert 205 ranged package-lock.json '<= 1.3' '{"identifier":"1.4.0"}')" \ - '{"number":206,"dependency":{"package":{"ecosystem":"npm","name":"ranged"},"manifest_path":"package-lock.json"}}') + '{"number":206,"dependency":{"package":{"ecosystem":"npm","name":"ranged"},"manifest_path":"package-lock.json"}}' \ + "$(alert 207 partial package-lock.json '>= 7.0, < 8.0' '{"identifier":"8.0.0"}')" \ + "$(alert 208 above package-lock.json '> 8.0, < 9.0' '{"identifier":"9.0.0"}')") write_gh_axi "$dir/fakebin/gh-axi" "$payload" set +e out=$(cd "$dir/repo" && PATH="$dir/fakebin:$PATH" "$ROOT/bin/fm-alert-count.py" integration security) @@ -149,6 +151,8 @@ test_unverifiable_evidence_is_explicitly_not_checked() { printf '%s\n' "$out" | grep -F '#204 old (legacy/package-lock.json): package-lock.json has no packages object' >/dev/null || fail "missing packages-object evidence: $out" printf '%s\n' "$out" | grep -F "#205 ranged (package-lock.json): unparseable advisory range '<= 1.3'" >/dev/null || fail "an ambiguous partial bound was accepted: $out" printf '%s\n' "$out" | grep -F '#206 malformed live alert record' >/dev/null || fail "malformed record lost its identifier: $out" + printf '%s\n' "$out" | grep -F "#207 partial (package-lock.json): unparseable advisory range '>= 7.0, < 8.0'" >/dev/null || fail "a non-zero partial bound was accepted: $out" + printf '%s\n' "$out" | grep -F "#208 above (package-lock.json): unparseable advisory range '> 8.0, < 9.0'" >/dev/null || fail "a non-zero partial lower bound was accepted: $out" pass "alert count refuses evidence it cannot verify instead of guessing" } From c39a22e2636d47dc01d5f25d7501fe485c2309ec Mon Sep 17 00:00:00 2001 From: David Beihl Date: Sun, 13 Sep 2026 18:37:17 -0400 Subject: [PATCH 5/5] no-mistakes(document): Relabel alert-count toolbelt row as per-alert counting --- docs/scripts.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/scripts.md b/docs/scripts.md index 7d25d3a511c..edf11384afd 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -41,7 +41,7 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-herdr-ci-cleanup.sh` | Snapshot and tear down only job-owned `fm-lab-*` sessions in the Herdr CI lane | | `fm-test-run.sh` | Behavior-test runner: selection, portable lanes, bounded concurrency, budgets, coverage guard, timing/JSON; refuses to execute in the repository primary checkout when `FM_TASK_ID` marks a task worker | | `fm-test-isolation-proof.sh` | Concurrent isolation harness and portable candidate set owner | -| `fm-alert-count.py` | Count live Dependabot alerts with per-advisory identifiers and fail-closed branch-remediation exclusions | +| `fm-alert-count.py` | Count the live Dependabot alerts an npm package-lock branch verifiably remediates, with per-alert exclusions and fail-closed NOT CHECKED rows | | `fm-ensure-agents-md.sh` | Ensure a project's real `AGENTS.md`, its `CLAUDE.md` `@AGENTS.md` pointer, and self-governance guidance (explicit project mark documented in the helper's header and help) | | `fm-guard.sh` | Warn on primary-checkout tangles, main-session pending wakes, and unhealthy supervision | | `fm-primary-scope-lib.sh` | Shared marker-or-plain-checkout primary-home predicate for tracked hooks |