Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -662,6 +662,7 @@ jobs:
- name: Run repository and package contracts
if: matrix.shard.suite == 'contracts'
run: |
python -m pip install pip-licenses==5.5.5
python scripts/check_application_test_inventory.py --output application-test-inventory.json
python -m pytest -q \
python/aisimulate/tests/unit/cli/test_aisimulate_compatibility.py \
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/fast-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ jobs:
test -n "${EXPECTED_SHA}"
test "${TARGET_SHA}" = "${GITHUB_SHA}"
fi
- run: python -m pip install pyyaml==6.0.3 pytest==8.4.2 markdown-it-py==4.2.0
- run: python -m pip install pyyaml==6.0.3 pytest==8.4.2 markdown-it-py==4.2.0 pip-licenses==5.5.5
- name: Check source and packaged legal files
run: |
python scripts/check_copyright.py
Expand Down Expand Up @@ -96,7 +96,7 @@ jobs:
python -m pytest -c /dev/null tests/test_ci_workflow_contracts.py tests/test_ci_qualification.py tests/test_release_fpe.py -q \
-p no:cacheprovider \
--override-ini="addopts=" \
--override-ini="filterwarnings="
--override-ini="filterwarnings=error"

python-static:
name: Python Static Checks
Expand Down
30 changes: 18 additions & 12 deletions .github/workflows/nightly-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ jobs:
echo "run-attempt=${GITHUB_RUN_ATTEMPT}" >> "${GITHUB_OUTPUT}"

python-compliance:
name: Python Dependency Licenses (${{ matrix.arch }})
name: Python Dependency Licenses (${{ matrix.arch }}, ${{ matrix.python-version }})
# No release environment or build artifacts are available to dependency
# build hooks. Both architectures must pass before staging can start.
needs: changes-guard
Expand All @@ -139,6 +139,7 @@ jobs:
fail-fast: false
matrix:
arch: [amd64, arm64]
python-version: ["3.11", "3.12", "3.13"]
runs-on: prod-aisimulate-default-${{ matrix.arch }}-v1
timeout-minutes: 15
container:
Expand All @@ -153,19 +154,20 @@ jobs:
persist-credentials: false
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
with:
python-version: "3.12"
python-version: ${{ matrix.python-version }}
- name: Check Python dependency licenses
env:
MATRIX_ARCH: ${{ matrix.arch }}
PYTHON_SERIES: ${{ matrix.python-version }}
run: |
python -m venv "${RUNNER_TEMP}/venv-licenses"
python scripts/check_python_licenses.py \
--python "${RUNNER_TEMP}/venv-licenses/bin/python" \
--inventory "${RUNNER_TEMP}/python-inventory/${MATRIX_ARCH}-3.12.csv"
--inventory "${RUNNER_TEMP}/python-inventory/${MATRIX_ARCH}-${PYTHON_SERIES}.csv"
- name: Preserve the dependency inventory as data
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: nightly-python-inventory-${{ matrix.arch }}
name: nightly-python-inventory-${{ matrix.arch }}-${{ matrix.python-version }}
path: ${{ runner.temp }}/python-inventory/*.csv
if-no-files-found: error
retention-days: 30
Expand Down Expand Up @@ -643,7 +645,8 @@ jobs:
"version": r["Version"], "spdx_license": r["License"]})
if not rows:
raise SystemExit("missing Python dependency inventories")
meta = json.load(open(f"{outdir}/cargo-metadata.json"))
with open(f"{outdir}/cargo-metadata.json") as f:
meta = json.load(f)
workspace = set(meta.get("workspace_members") or [])
for p in meta["packages"]:
if p.get("id") in workspace:
Expand Down Expand Up @@ -676,23 +679,26 @@ jobs:
"Authorization": f"Bearer {token}",
"Accept": "application/vnd.github+json"})
return urllib.request.urlopen(req, timeout=30)
runs = json.load(get(
with get(
f"https://api.github.com/repos/{repo}/actions/workflows/"
"nightly-ci.yml/runs?branch=main&status=success&per_page=10"))
"nightly-ci.yml/runs?branch=main&status=success&per_page=10") as response:
runs = json.load(response)
for run in runs.get("workflow_runs", []):
if str(run["id"]) == current:
continue
arts = json.load(get(run["artifacts_url"]))
with get(run["artifacts_url"]) as response:
arts = json.load(response)
for a in arts.get("artifacts", []):
if "license" not in a["name"] or a.get("expired"):
continue
data = get(a["archive_download_url"]).read()
with get(a["archive_download_url"]) as response:
data = response.read()
with zipfile.ZipFile(io.BytesIO(data)) as zf:
for n in zf.namelist():
if os.path.basename(n) == "deps.csv":
reader = csv.DictReader(
io.TextIOWrapper(zf.open(n)))
return {identity(r): r for r in reader}
with io.TextIOWrapper(zf.open(n)) as stream:
reader = csv.DictReader(stream)
return {identity(r): r for r in reader}
return None

try:
Expand Down
31 changes: 30 additions & 1 deletion docs/ci.md
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,8 @@ flowchart TD
ReleaseQualified --> AllReleases["Require every release to succeed"]
end

Security -.->|Main branch nightly CI succeeds| Pages["GitHub Pages<br/>Select and validate branch snapshots<br/>Build site from trusted main"]
MainQualified -.->|Main branch nightly CI succeeds| Pages["GitHub Pages<br/>Select and validate branch snapshots<br/>Build site from trusted main"]
Evidence -.->|Main branch nightly CI succeeds| Pages
AllReleases -.->|Release branch nightly CI succeeds| Pages
Pages --> Deploy["Deploy support matrix pages"]
```
Expand All @@ -94,6 +95,8 @@ starts after the staged-wheel smoke tests succeed. Release qualification process
branches sequentially, with up to 20 shard jobs within each release.
Each release uses the commit SHA recorded at the start of the run, even if its
branch receives new commits while the checks are running.
Pages requires a successful completed nightly; the optional GitLab security
job is skipped when disabled and must succeed when enabled.

The release path runs **only FPE support-matrix qualification**. Its scheduler
discovers branches and calls the per-release helper, whose jobs prepare the
Expand Down Expand Up @@ -327,6 +330,12 @@ checksums/provenance are retained as `nightly-dist-<arch>` GitHub artifacts
before runtime dependencies execute, preserving the accuracy and installation
consumer contract.

Python license evidence covers the installed audit environment, including
the runtime dependency closure and audit tools such as `pip` and `pip-licenses`.
Package names are not exempted through a hard-coded ignore list. Detailed
license failures remain suppressed in public job logs; reproduce with
`pip-licenses --with-system` in the affected environment.

Two kinds of validation then run:

- **Wheel smoke tests:** fresh installations on amd64/arm64, each tested with
Expand All @@ -352,6 +361,26 @@ reviewer protections must be configured to enforce it. FPE output is retained
as workflow artifacts; publishing dashboard pages is a separate Pages workflow
that consumes successful nightlies.

The GitLab request contract was checked against release-automation revision
`1aaaeae1f4a29345085b68bb460d09ee47cc4bb0`, specifically the root
`.gitlab-ci.yml` variable forwarding and `projects/aisimulate.yml` consumer.
The endpoint comes from `GITLAB_PIPELINE_URL`; the authenticated multipart
request selects `ref=main` and forwards these fields:

| Variable | Nightly value |
| --- | --- |
| `PROJECT` | `aisimulate` |
| `PIPELINE_TYPE` / `RELEASE_TYPE` | `security` / `nightly` |
| `NIGHTLY_TAG` | `nightly-YYYYMMDD-<first-seven-commit-characters>` |
| `WHEEL_VERSION` | Exact stamped wheel version |
| `GITHUB_RUN_ID` / `COMMIT_SHA` | Producing GitHub run and full source commit |
| `SLACK_THREAD_TS` / `SLACK_CHANNEL_ID` | Notification thread and channel, optionally empty |
| `DRY_RUN` | `false` |

The workflow-contract test executes the real trigger script against a fake
HTTP client, including missing credentials and HTTP errors. It validates the
request boundary; it does not certify GitLab scan or publication outcomes.

[Release branch nightly CI](../.github/workflows/release-nightly-ci.yml) separately
discovers every `release/<version>` branch each day, including new releases and
days when their source is unchanged. From trusted `main`, it records each release
Expand Down
3 changes: 3 additions & 0 deletions scripts/check_python_licenses.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,10 @@ def check_licenses(python: str, inventory: Path | None = None) -> int:

# pip-licenses otherwise excludes prettytable and wcwidth even though
# both are also AISimulate runtime dependencies.
# Audit the installed environment, including tooling; blanket exclusions
# must not silently exempt a future runtime dependency with the same name.
command = [python, "-m", "piplicenses", "--with-system"]
# Preserve the release policy: detailed findings stay out of public CI logs.
result = subprocess.run(
[*command, "--allow-only", ALLOWED_LICENSES],
stdout=subprocess.DEVNULL,
Expand Down
134 changes: 130 additions & 4 deletions tests/test_ci_workflow_contracts.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import base64
import csv
import hashlib
import importlib.metadata
import io
import json
import os
Expand Down Expand Up @@ -1875,7 +1876,7 @@ def test_nightly_dependency_execution_cannot_modify_staged_artifacts_or_inherit_
assert set(evidence["needs"]) == {"build-artifacts", "python-compliance"}


def _nightly_license_report(tmp_path, crates, prior=None, lookup_error=None):
def _nightly_license_report(tmp_path, crates, prior=None, lookup_error=None, workspace_members=()):
inventories = tmp_path / "python"
inventories.mkdir(exist_ok=True)
# Same runtime dependency in multiple Python/architecture inventories must
Expand All @@ -1885,10 +1886,11 @@ def _nightly_license_report(tmp_path, crates, prior=None, lookup_error=None):
(tmp_path / "cargo-metadata.json").write_text(
json.dumps(
{
"workspace_members": list(workspace_members),
"packages": [
{"id": f"getrandom@{version}", "name": "getrandom", "version": version, "license": spdx}
for version, spdx in crates
]
],
}
)
)
Expand All @@ -1900,6 +1902,8 @@ def _nightly_license_report(tmp_path, crates, prior=None, lookup_error=None):
with zipfile.ZipFile(archive, "w") as zipped:
zipped.writestr("deps.csv", prior_csv.getvalue())

responses = []

def urlopen(request, timeout):
assert timeout == 30
if lookup_error is not None:
Expand All @@ -1913,10 +1917,14 @@ def urlopen(request, timeout):
elif request.full_url == "https://fixture/artifacts":
payload = {"artifacts": [{"name": "license-artifacts", "archive_download_url": "https://fixture/archive"}]}
elif request.full_url == "https://fixture/archive":
return io.BytesIO(archive.getvalue())
response = io.BytesIO(archive.getvalue())
responses.append(response)
return response
else:
raise AssertionError(request.full_url)
return io.BytesIO(json.dumps(payload).encode())
response = io.BytesIO(json.dumps(payload).encode())
responses.append(response)
return response

step = next(
s
Expand All @@ -1930,6 +1938,7 @@ def urlopen(request, timeout):
patch("urllib.request.urlopen", side_effect=urlopen),
):
exec(compile(source, "nightly-ci-evidence", "exec"), {})
assert all(response.closed for response in responses)
with (tmp_path / "deps.csv").open() as inventory, (tmp_path / "deps-diff.csv").open() as difference:
return list(csv.DictReader(inventory)), list(csv.DictReader(difference))

Expand Down Expand Up @@ -1961,6 +1970,16 @@ def test_nightly_license_baseline_warns_only_on_lookup_failure(tmp_path, capsys,
assert ("::warning::prior-artifact lookup failed" in capsys.readouterr().out) == (lookup_error is not None)


def test_nightly_license_inventory_excludes_workspace_packages(tmp_path):
rows, _ = _nightly_license_report(tmp_path, [("0.2.17", "MIT")], workspace_members=["getrandom@0.2.17"])
assert all(row["dependency_type"] != "crate" for row in rows)


def test_nightly_license_inventory_rejects_conflicting_metadata(tmp_path):
with pytest.raises(SystemExit, match="conflicting license metadata"):
_nightly_license_report(tmp_path, [("0.2.17", "MIT"), ("0.2.17", "GPL-3.0-only")])


@pytest.mark.parametrize(
"conclusion,expected", [("failure", True), ("timed_out", True), ("success", False), ("skipped", False)]
)
Expand Down Expand Up @@ -2015,6 +2034,46 @@ def test_python_compliance_workflows_use_the_same_policy():
assert "--allow-only" not in commands


def test_nightly_license_matrix_covers_the_smoked_python_versions():
jobs = _workflow("nightly-ci.yml")["jobs"]
compliance = jobs["python-compliance"]
matrix = compliance["strategy"]["matrix"]
assert set(matrix["python-version"]) == set(jobs["build-artifacts"]["env"]["PYTHON_SERIES_LIST"].split())
assert set(matrix["arch"]) == {entry["arch"] for entry in jobs["build-artifacts"]["strategy"]["matrix"]["include"]}
setup = next(step for step in compliance["steps"] if "setup-python@" in step.get("uses", ""))
assert setup["with"]["python-version"] == "${{ matrix.python-version }}"
upload = next(step for step in compliance["steps"] if "upload-artifact@" in step.get("uses", ""))
assert upload["with"]["name"] == "nightly-python-inventory-${{ matrix.arch }}-${{ matrix.python-version }}"


@pytest.mark.parametrize("spdx,allowed", [("MIT", True), ("GPL-3.0-only", False)])
def test_real_pip_licenses_enforces_the_policy(tmp_path, spdx, allowed):
assert importlib.metadata.version("pip-licenses") == "5.5.5"
metadata = tmp_path / "license_policy_fixture-1.0.dist-info"
metadata.mkdir()
(metadata / "METADATA").write_text(
f"Metadata-Version: 2.1\nName: license-policy-fixture\nVersion: 1.0\nLicense: {spdx}\n"
)
result = subprocess.run(
[
sys.executable,
"-m",
"piplicenses",
"--with-system",
"--packages",
"license-policy-fixture",
"--allow-only",
python_licenses.ALLOWED_LICENSES,
],
cwd=tmp_path,
env={**os.environ, "PYTHONPATH": str(tmp_path)},
text=True,
capture_output=True,
)
assert (result.returncode == 0) == allowed, result.stdout + result.stderr
assert "license-policy-fixture" in result.stdout + result.stderr


@pytest.mark.parametrize("license_result", [0, 1])
def test_python_license_gate_checks_target_environment_before_export(tmp_path, monkeypatch, capsys, license_result):
manifest = tmp_path / "pyproject.toml"
Expand Down Expand Up @@ -2155,3 +2214,70 @@ def verify():
wheel.write_bytes(b"fixture wheel")
env["EXPECTED_SHA"] = "c" * 40
assert verify().returncode != 0


@pytest.mark.parametrize("failure", [None, "missing-token", "http-error"])
def test_gitlab_security_trigger_matches_the_verified_consumer_contract(tmp_path, failure):
# Interface verified against release-automation commit
# 1aaaeae1f4a29345085b68bb460d09ee47cc4bb0: .gitlab-ci.yml forwarding
# and projects/aisimulate.yml. This is an independent request fixture,
# not a copy of the external pipeline implementation.
capture = tmp_path / "request.json"
fake_bin = tmp_path / "bin"
fake_bin.mkdir()
curl = fake_bin / "curl"
curl.write_text(
f"#!{sys.executable}\n"
"import json, os, sys\n"
"from pathlib import Path\n"
"Path(os.environ['CAPTURE_REQUEST']).write_text(json.dumps(sys.argv[1:]))\n"
"sys.exit(int(os.environ['CURL_EXIT_CODE']))\n"
)
curl.chmod(0o755)
step = next(
step
for step in _workflow("nightly-ci.yml")["jobs"]["trigger-gitlab-security"]["steps"]
if step.get("name") == "Trigger internal security scan"
)
endpoint = "https://gitlab.invalid/api/v4/projects/123/trigger/pipeline"
result = subprocess.run(
["bash", "-e", "-c", step["run"]],
cwd=tmp_path,
capture_output=True,
text=True,
env={
**os.environ,
"PATH": f"{fake_bin}:{os.environ['PATH']}",
"CAPTURE_REQUEST": str(capture),
"CURL_EXIT_CODE": "22" if failure == "http-error" else "0",
"GITLAB_TRIGGER_TOKEN": "" if failure == "missing-token" else "fixture-token",
"GITLAB_PIPELINE_URL": endpoint,
"WHEEL_VERSION": "0.12.0.dev20260917",
"GH_RUN_ID": "123456",
"GH_SHA": "a" * 40,
"SLACK_THREAD_TS": "1234567890.123456",
"SLACK_CHANNEL_ID": "fixture-channel",
},
)
assert (result.returncode == 0) == (failure is None), result.stdout + result.stderr
if failure == "missing-token":
assert not capture.exists()
return
args = json.loads(capture.read_text())
assert args[-1] == endpoint
assert "--fail" in args
fields = dict(args[i + 1].split("=", 1) for i, arg in enumerate(args) if arg == "-F")
assert fields == {
"token": "fixture-token",
"ref": "main",
"variables[PROJECT]": "aisimulate",
"variables[PIPELINE_TYPE]": "security",
"variables[RELEASE_TYPE]": "nightly",
"variables[NIGHTLY_TAG]": "nightly-20260917-aaaaaaa",
"variables[WHEEL_VERSION]": "0.12.0.dev20260917",
"variables[GITHUB_RUN_ID]": "123456",
"variables[COMMIT_SHA]": "a" * 40,
"variables[SLACK_THREAD_TS]": "1234567890.123456",
"variables[SLACK_CHANNEL_ID]": "fixture-channel",
"variables[DRY_RUN]": "false",
}
Loading