From 4d99b9fd89ce385e54f2e2a7027416cc92eac9a0 Mon Sep 17 00:00:00 2001 From: "Jason Zhou (Engrg-Hardware 1)" Date: Wed, 16 Sep 2026 17:56:18 -0700 Subject: [PATCH 1/2] ci: complete nightly license matrix and verify security handoff Signed-off-by: Jason Zhou (Engrg-Hardware 1) --- .github/workflows/ci.yml | 1 + .github/workflows/fast-ci.yml | 2 +- .github/workflows/nightly-ci.yml | 10 ++- docs/ci.md | 31 ++++++- scripts/check_python_licenses.py | 3 + tests/test_ci_workflow_contracts.py | 123 +++++++++++++++++++++++++++- 6 files changed, 162 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a204687030..08c232cbd0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -662,6 +662,7 @@ jobs: - name: Run repository and package contracts if: matrix.shard.suite == 'contracts' run: | + python -m pip install pip-licenses==5.5.5 python scripts/check_application_test_inventory.py --output application-test-inventory.json python -m pytest -q \ python/aisimulate/tests/unit/cli/test_aisimulate_compatibility.py \ diff --git a/.github/workflows/fast-ci.yml b/.github/workflows/fast-ci.yml index 16c1b3e75d..09f08d8f65 100644 --- a/.github/workflows/fast-ci.yml +++ b/.github/workflows/fast-ci.yml @@ -60,7 +60,7 @@ jobs: test -n "${EXPECTED_SHA}" test "${TARGET_SHA}" = "${GITHUB_SHA}" fi - - run: python -m pip install pyyaml==6.0.3 pytest==8.4.2 markdown-it-py==4.2.0 + - run: python -m pip install pyyaml==6.0.3 pytest==8.4.2 markdown-it-py==4.2.0 pip-licenses==5.5.5 - name: Check source and packaged legal files run: | python scripts/check_copyright.py diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index ea0c0bb47e..505ed79fa1 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -130,7 +130,7 @@ jobs: echo "run-attempt=${GITHUB_RUN_ATTEMPT}" >> "${GITHUB_OUTPUT}" python-compliance: - name: Python Dependency Licenses (${{ matrix.arch }}) + name: Python Dependency Licenses (${{ matrix.arch }}, ${{ matrix.python-version }}) # No release environment or build artifacts are available to dependency # build hooks. Both architectures must pass before staging can start. needs: changes-guard @@ -139,6 +139,7 @@ jobs: fail-fast: false matrix: arch: [amd64, arm64] + python-version: ["3.11", "3.12", "3.13"] runs-on: prod-aisimulate-default-${{ matrix.arch }}-v1 timeout-minutes: 15 container: @@ -153,19 +154,20 @@ jobs: persist-credentials: false - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 with: - python-version: "3.12" + python-version: ${{ matrix.python-version }} - name: Check Python dependency licenses env: MATRIX_ARCH: ${{ matrix.arch }} + PYTHON_SERIES: ${{ matrix.python-version }} run: | python -m venv "${RUNNER_TEMP}/venv-licenses" python scripts/check_python_licenses.py \ --python "${RUNNER_TEMP}/venv-licenses/bin/python" \ - --inventory "${RUNNER_TEMP}/python-inventory/${MATRIX_ARCH}-3.12.csv" + --inventory "${RUNNER_TEMP}/python-inventory/${MATRIX_ARCH}-${PYTHON_SERIES}.csv" - name: Preserve the dependency inventory as data uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: nightly-python-inventory-${{ matrix.arch }} + name: nightly-python-inventory-${{ matrix.arch }}-${{ matrix.python-version }} path: ${{ runner.temp }}/python-inventory/*.csv if-no-files-found: error retention-days: 30 diff --git a/docs/ci.md b/docs/ci.md index a7b58c5646..d39002dc13 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -82,7 +82,8 @@ flowchart TD ReleaseQualified --> AllReleases["Require every release to succeed"] end - Security -.->|Main branch nightly CI succeeds| Pages["GitHub Pages
Select and validate branch snapshots
Build site from trusted main"] + MainQualified -.->|Main branch nightly CI succeeds| Pages["GitHub Pages
Select and validate branch snapshots
Build site from trusted main"] + Evidence -.->|Main branch nightly CI succeeds| Pages AllReleases -.->|Release branch nightly CI succeeds| Pages Pages --> Deploy["Deploy support matrix pages"] ``` @@ -94,6 +95,8 @@ starts after the staged-wheel smoke tests succeed. Release qualification process branches sequentially, with up to 20 shard jobs within each release. Each release uses the commit SHA recorded at the start of the run, even if its branch receives new commits while the checks are running. +Pages requires a successful completed nightly; the optional GitLab security +job is skipped when disabled and must succeed when enabled. The release path runs **only FPE support-matrix qualification**. Its scheduler discovers branches and calls the per-release helper, whose jobs prepare the @@ -327,6 +330,12 @@ checksums/provenance are retained as `nightly-dist-` GitHub artifacts before runtime dependencies execute, preserving the accuracy and installation consumer contract. +Python license evidence covers the installed audit environment, including +the runtime dependency closure and audit tools such as `pip` and `pip-licenses`. +Package names are not exempted through a hard-coded ignore list. Detailed +license failures remain suppressed in public job logs; reproduce with +`pip-licenses --with-system` in the affected environment. + Two kinds of validation then run: - **Wheel smoke tests:** fresh installations on amd64/arm64, each tested with @@ -352,6 +361,26 @@ reviewer protections must be configured to enforce it. FPE output is retained as workflow artifacts; publishing dashboard pages is a separate Pages workflow that consumes successful nightlies. +The GitLab request contract was checked against release-automation revision +`1aaaeae1f4a29345085b68bb460d09ee47cc4bb0`, specifically the root +`.gitlab-ci.yml` variable forwarding and `projects/aisimulate.yml` consumer. +The endpoint comes from `GITLAB_PIPELINE_URL`; the authenticated multipart +request selects `ref=main` and forwards these fields: + +| Variable | Nightly value | +| --- | --- | +| `PROJECT` | `aisimulate` | +| `PIPELINE_TYPE` / `RELEASE_TYPE` | `security` / `nightly` | +| `NIGHTLY_TAG` | `nightly-YYYYMMDD-` | +| `WHEEL_VERSION` | Exact stamped wheel version | +| `GITHUB_RUN_ID` / `COMMIT_SHA` | Producing GitHub run and full source commit | +| `SLACK_THREAD_TS` / `SLACK_CHANNEL_ID` | Notification thread and channel, optionally empty | +| `DRY_RUN` | `false` | + +The workflow-contract test executes the real trigger script against a fake +HTTP client, including missing credentials and HTTP errors. It validates the +request boundary; it does not certify GitLab scan or publication outcomes. + [Release branch nightly CI](../.github/workflows/release-nightly-ci.yml) separately discovers every `release/` branch each day, including new releases and days when their source is unchanged. From trusted `main`, it records each release diff --git a/scripts/check_python_licenses.py b/scripts/check_python_licenses.py index 7187b91df1..d64f6beb0c 100644 --- a/scripts/check_python_licenses.py +++ b/scripts/check_python_licenses.py @@ -37,7 +37,10 @@ def check_licenses(python: str, inventory: Path | None = None) -> int: # pip-licenses otherwise excludes prettytable and wcwidth even though # both are also AISimulate runtime dependencies. + # Audit the installed environment, including tooling; blanket exclusions + # must not silently exempt a future runtime dependency with the same name. command = [python, "-m", "piplicenses", "--with-system"] + # Preserve the release policy: detailed findings stay out of public CI logs. result = subprocess.run( [*command, "--allow-only", ALLOWED_LICENSES], stdout=subprocess.DEVNULL, diff --git a/tests/test_ci_workflow_contracts.py b/tests/test_ci_workflow_contracts.py index 84fb20512e..3dead8eece 100644 --- a/tests/test_ci_workflow_contracts.py +++ b/tests/test_ci_workflow_contracts.py @@ -8,6 +8,7 @@ import base64 import csv import hashlib +import importlib.metadata import io import json import os @@ -1875,7 +1876,7 @@ def test_nightly_dependency_execution_cannot_modify_staged_artifacts_or_inherit_ assert set(evidence["needs"]) == {"build-artifacts", "python-compliance"} -def _nightly_license_report(tmp_path, crates, prior=None, lookup_error=None): +def _nightly_license_report(tmp_path, crates, prior=None, lookup_error=None, workspace_members=()): inventories = tmp_path / "python" inventories.mkdir(exist_ok=True) # Same runtime dependency in multiple Python/architecture inventories must @@ -1885,10 +1886,11 @@ def _nightly_license_report(tmp_path, crates, prior=None, lookup_error=None): (tmp_path / "cargo-metadata.json").write_text( json.dumps( { + "workspace_members": list(workspace_members), "packages": [ {"id": f"getrandom@{version}", "name": "getrandom", "version": version, "license": spdx} for version, spdx in crates - ] + ], } ) ) @@ -1961,6 +1963,16 @@ def test_nightly_license_baseline_warns_only_on_lookup_failure(tmp_path, capsys, assert ("::warning::prior-artifact lookup failed" in capsys.readouterr().out) == (lookup_error is not None) +def test_nightly_license_inventory_excludes_workspace_packages(tmp_path): + rows, _ = _nightly_license_report(tmp_path, [("0.2.17", "MIT")], workspace_members=["getrandom@0.2.17"]) + assert all(row["dependency_type"] != "crate" for row in rows) + + +def test_nightly_license_inventory_rejects_conflicting_metadata(tmp_path): + with pytest.raises(SystemExit, match="conflicting license metadata"): + _nightly_license_report(tmp_path, [("0.2.17", "MIT"), ("0.2.17", "GPL-3.0-only")]) + + @pytest.mark.parametrize( "conclusion,expected", [("failure", True), ("timed_out", True), ("success", False), ("skipped", False)] ) @@ -2015,6 +2027,46 @@ def test_python_compliance_workflows_use_the_same_policy(): assert "--allow-only" not in commands +def test_nightly_license_matrix_covers_the_smoked_python_versions(): + jobs = _workflow("nightly-ci.yml")["jobs"] + compliance = jobs["python-compliance"] + matrix = compliance["strategy"]["matrix"] + assert set(matrix["python-version"]) == set(jobs["build-artifacts"]["env"]["PYTHON_SERIES_LIST"].split()) + assert set(matrix["arch"]) == {entry["arch"] for entry in jobs["build-artifacts"]["strategy"]["matrix"]["include"]} + setup = next(step for step in compliance["steps"] if "setup-python@" in step.get("uses", "")) + assert setup["with"]["python-version"] == "${{ matrix.python-version }}" + upload = next(step for step in compliance["steps"] if "upload-artifact@" in step.get("uses", "")) + assert upload["with"]["name"] == "nightly-python-inventory-${{ matrix.arch }}-${{ matrix.python-version }}" + + +@pytest.mark.parametrize("spdx,allowed", [("MIT", True), ("GPL-3.0-only", False)]) +def test_real_pip_licenses_enforces_the_policy(tmp_path, spdx, allowed): + assert importlib.metadata.version("pip-licenses") == "5.5.5" + metadata = tmp_path / "license_policy_fixture-1.0.dist-info" + metadata.mkdir() + (metadata / "METADATA").write_text( + f"Metadata-Version: 2.1\nName: license-policy-fixture\nVersion: 1.0\nLicense: {spdx}\n" + ) + result = subprocess.run( + [ + sys.executable, + "-m", + "piplicenses", + "--with-system", + "--packages", + "license-policy-fixture", + "--allow-only", + python_licenses.ALLOWED_LICENSES, + ], + cwd=tmp_path, + env={**os.environ, "PYTHONPATH": str(tmp_path)}, + text=True, + capture_output=True, + ) + assert (result.returncode == 0) == allowed, result.stdout + result.stderr + assert "license-policy-fixture" in result.stdout + result.stderr + + @pytest.mark.parametrize("license_result", [0, 1]) def test_python_license_gate_checks_target_environment_before_export(tmp_path, monkeypatch, capsys, license_result): manifest = tmp_path / "pyproject.toml" @@ -2155,3 +2207,70 @@ def verify(): wheel.write_bytes(b"fixture wheel") env["EXPECTED_SHA"] = "c" * 40 assert verify().returncode != 0 + + +@pytest.mark.parametrize("failure", [None, "missing-token", "http-error"]) +def test_gitlab_security_trigger_matches_the_verified_consumer_contract(tmp_path, failure): + # Interface verified against release-automation commit + # 1aaaeae1f4a29345085b68bb460d09ee47cc4bb0: .gitlab-ci.yml forwarding + # and projects/aisimulate.yml. This is an independent request fixture, + # not a copy of the external pipeline implementation. + capture = tmp_path / "request.json" + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + curl = fake_bin / "curl" + curl.write_text( + f"#!{sys.executable}\n" + "import json, os, sys\n" + "from pathlib import Path\n" + "Path(os.environ['CAPTURE_REQUEST']).write_text(json.dumps(sys.argv[1:]))\n" + "sys.exit(int(os.environ['CURL_EXIT_CODE']))\n" + ) + curl.chmod(0o755) + step = next( + step + for step in _workflow("nightly-ci.yml")["jobs"]["trigger-gitlab-security"]["steps"] + if step.get("name") == "Trigger internal security scan" + ) + endpoint = "https://gitlab.invalid/api/v4/projects/123/trigger/pipeline" + result = subprocess.run( + ["bash", "-e", "-c", step["run"]], + cwd=tmp_path, + capture_output=True, + text=True, + env={ + **os.environ, + "PATH": f"{fake_bin}:{os.environ['PATH']}", + "CAPTURE_REQUEST": str(capture), + "CURL_EXIT_CODE": "22" if failure == "http-error" else "0", + "GITLAB_TRIGGER_TOKEN": "" if failure == "missing-token" else "fixture-token", + "GITLAB_PIPELINE_URL": endpoint, + "WHEEL_VERSION": "0.12.0.dev20260917", + "GH_RUN_ID": "123456", + "GH_SHA": "a" * 40, + "SLACK_THREAD_TS": "1234567890.123456", + "SLACK_CHANNEL_ID": "fixture-channel", + }, + ) + assert (result.returncode == 0) == (failure is None), result.stdout + result.stderr + if failure == "missing-token": + assert not capture.exists() + return + args = json.loads(capture.read_text()) + assert args[-1] == endpoint + assert "--fail" in args + fields = dict(args[i + 1].split("=", 1) for i, arg in enumerate(args) if arg == "-F") + assert fields == { + "token": "fixture-token", + "ref": "main", + "variables[PROJECT]": "aisimulate", + "variables[PIPELINE_TYPE]": "security", + "variables[RELEASE_TYPE]": "nightly", + "variables[NIGHTLY_TAG]": "nightly-20260917-aaaaaaa", + "variables[WHEEL_VERSION]": "0.12.0.dev20260917", + "variables[GITHUB_RUN_ID]": "123456", + "variables[COMMIT_SHA]": "a" * 40, + "variables[SLACK_THREAD_TS]": "1234567890.123456", + "variables[SLACK_CHANNEL_ID]": "fixture-channel", + "variables[DRY_RUN]": "false", + } From 45f6c0c360a0f689479d564eedf07974e3796409 Mon Sep 17 00:00:00 2001 From: "Jason Zhou (Engrg-Hardware 1)" Date: Wed, 16 Sep 2026 18:00:03 -0700 Subject: [PATCH 2/2] fix(ci): close license evidence streams under strict warnings Signed-off-by: Jason Zhou (Engrg-Hardware 1) --- .github/workflows/fast-ci.yml | 2 +- .github/workflows/nightly-ci.yml | 20 ++++++++++++-------- tests/test_ci_workflow_contracts.py | 11 +++++++++-- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/.github/workflows/fast-ci.yml b/.github/workflows/fast-ci.yml index 09f08d8f65..764947a603 100644 --- a/.github/workflows/fast-ci.yml +++ b/.github/workflows/fast-ci.yml @@ -96,7 +96,7 @@ jobs: python -m pytest -c /dev/null tests/test_ci_workflow_contracts.py tests/test_ci_qualification.py tests/test_release_fpe.py -q \ -p no:cacheprovider \ --override-ini="addopts=" \ - --override-ini="filterwarnings=" + --override-ini="filterwarnings=error" python-static: name: Python Static Checks diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 505ed79fa1..1f1114db4a 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -645,7 +645,8 @@ jobs: "version": r["Version"], "spdx_license": r["License"]}) if not rows: raise SystemExit("missing Python dependency inventories") - meta = json.load(open(f"{outdir}/cargo-metadata.json")) + with open(f"{outdir}/cargo-metadata.json") as f: + meta = json.load(f) workspace = set(meta.get("workspace_members") or []) for p in meta["packages"]: if p.get("id") in workspace: @@ -678,23 +679,26 @@ jobs: "Authorization": f"Bearer {token}", "Accept": "application/vnd.github+json"}) return urllib.request.urlopen(req, timeout=30) - runs = json.load(get( + with get( f"https://api.github.com/repos/{repo}/actions/workflows/" - "nightly-ci.yml/runs?branch=main&status=success&per_page=10")) + "nightly-ci.yml/runs?branch=main&status=success&per_page=10") as response: + runs = json.load(response) for run in runs.get("workflow_runs", []): if str(run["id"]) == current: continue - arts = json.load(get(run["artifacts_url"])) + with get(run["artifacts_url"]) as response: + arts = json.load(response) for a in arts.get("artifacts", []): if "license" not in a["name"] or a.get("expired"): continue - data = get(a["archive_download_url"]).read() + with get(a["archive_download_url"]) as response: + data = response.read() with zipfile.ZipFile(io.BytesIO(data)) as zf: for n in zf.namelist(): if os.path.basename(n) == "deps.csv": - reader = csv.DictReader( - io.TextIOWrapper(zf.open(n))) - return {identity(r): r for r in reader} + with io.TextIOWrapper(zf.open(n)) as stream: + reader = csv.DictReader(stream) + return {identity(r): r for r in reader} return None try: diff --git a/tests/test_ci_workflow_contracts.py b/tests/test_ci_workflow_contracts.py index 3dead8eece..f4b2e32b09 100644 --- a/tests/test_ci_workflow_contracts.py +++ b/tests/test_ci_workflow_contracts.py @@ -1902,6 +1902,8 @@ def _nightly_license_report(tmp_path, crates, prior=None, lookup_error=None, wor with zipfile.ZipFile(archive, "w") as zipped: zipped.writestr("deps.csv", prior_csv.getvalue()) + responses = [] + def urlopen(request, timeout): assert timeout == 30 if lookup_error is not None: @@ -1915,10 +1917,14 @@ def urlopen(request, timeout): elif request.full_url == "https://fixture/artifacts": payload = {"artifacts": [{"name": "license-artifacts", "archive_download_url": "https://fixture/archive"}]} elif request.full_url == "https://fixture/archive": - return io.BytesIO(archive.getvalue()) + response = io.BytesIO(archive.getvalue()) + responses.append(response) + return response else: raise AssertionError(request.full_url) - return io.BytesIO(json.dumps(payload).encode()) + response = io.BytesIO(json.dumps(payload).encode()) + responses.append(response) + return response step = next( s @@ -1932,6 +1938,7 @@ def urlopen(request, timeout): patch("urllib.request.urlopen", side_effect=urlopen), ): exec(compile(source, "nightly-ci-evidence", "exec"), {}) + assert all(response.closed for response in responses) with (tmp_path / "deps.csv").open() as inventory, (tmp_path / "deps-diff.csv").open() as difference: return list(csv.DictReader(inventory)), list(csv.DictReader(difference))