Skip to content
Draft
5 changes: 3 additions & 2 deletions scripts/build_release_artifacts.py
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ def main() -> None:
global ROOT, VERSION, EXPECTED_PYTHON_PROJECTS, EXPECTED_CRATE
parser = argparse.ArgumentParser()
parser.add_argument("--root", type=Path, help="Release source checkout")
parser.add_argument("--output-dir", type=Path, default=ROOT / "dist")
parser.add_argument("--output-dir", type=Path)
parser.add_argument("--check-only", action="store_true")
args = parser.parse_args()

Expand All @@ -183,7 +183,8 @@ def main() -> None:

py_version, crate_version = check_manifests()
if not args.check_only:
build(args.output_dir.resolve(), py_version, crate_version)
output_dir = args.output_dir if args.output_dir is not None else ROOT / "dist"
build(output_dir.resolve(), py_version, crate_version)


if __name__ == "__main__":
Expand Down
45 changes: 4 additions & 41 deletions tests/test_ci_workflow_contracts.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
import os
import re
import shlex
import shutil
import subprocess
import sys
import zipfile
Expand Down Expand Up @@ -2676,50 +2675,14 @@ def test_manual_publish_rejects_wrong_ref_disabled_trigger_and_cancellation():
assert not _nightly_condition("trigger-gitlab-security", **{**approved, **override})


def _nightly_version(created, number):
script = next(
step["with"]["script"]
for step in _workflow("nightly-ci.yml")["jobs"]["changes-guard"]["steps"]
if step.get("id") == "version"
)
program = """
const run = JSON.parse(process.argv[1]);
const output = {};
const core = {setOutput: (key, value) => { output[key] = value; }};
const github = {rest: {actions: {getWorkflowRun: async () => ({data: run})}}};
const context = {repo: {owner: 'fixture', repo: 'fixture'}, runId: 123};
const AsyncFunction = Object.getPrototypeOf(async function(){}).constructor;
new AsyncFunction('github', 'context', 'core', process.argv[2])(github, context, core)
.then(() => console.log(JSON.stringify(output)))
.catch(error => { console.error(error.message); process.exitCode = 1; });
"""
return subprocess.run(
[shutil.which("node"), "-e", program, json.dumps({"created_at": created, "run_number": number}), script],
text=True,
capture_output=True,
)


def test_nightly_versions_are_unique_date_ordered_and_stable_across_retries():
def test_nightly_versions_preserve_pep440_ordering():
from packaging.version import Version

versions = []
for date, number in [
("2026-09-17T23:59:59Z", 1234),
("2026-09-17T23:59:59Z", 1235),
("2026-09-18T00:00:00Z", 1236),
]:
result = _nightly_version(date, number)
assert result.returncode == 0, result.stderr
value = json.loads(result.stdout)
assert value["dev-date"] == date[:10].replace("-", "")
versions.append(value["dev-version"])
assert json.loads(_nightly_version(date, number).stdout) == value
assert versions == ["202609170000001234", "202609170000001235", "202609180000001236"]
# The Node workflow suite verifies these outputs from the actual script;
# this Python-only shard checks the package consumer's version ordering.
versions = ["202609170000001234", "202609170000001235", "202609180000001236"]
assert Version("0.12.0.dev20260917") < Version("0.12.0.dev" + versions[0])
assert [Version("0.12.0.dev" + v) for v in versions] == sorted(Version("0.12.0.dev" + v) for v in versions)
Comment thread
jasonqinzhou marked this conversation as resolved.
Resolved
for number in (0, -1, 10000000000, "invalid"):
assert _nightly_version("2026-09-17T00:00:00Z", number).returncode != 0


@pytest.mark.parametrize("suffix", [".dev20260917", ".dev202609170000001234"])
Expand Down
66 changes: 62 additions & 4 deletions tests/test_e2e_accuracy_workflow.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,70 @@ const source = workflow.match(/ script: \|\n((?:(?: .*)?\n)+
assert.match(source, /core\.setOutput\('matrix', JSON\.stringify\(\{include: entries\}\)\);\s*$/);
const sha = "a".repeat(40);
const nightlyWorkflow = readFileSync(new URL("../.github/workflows/nightly-ci.yml", import.meta.url), "utf8");
function nightlyScript(id) {
const step = nightlyWorkflow.split("\n - ").find(step => step.includes(`\n id: ${id}\n`));
assert.ok(step, `Missing nightly step: ${id}`);
return step.match(/ script: \|\n((?:(?: .*)?\n)+)/)[1];
function nightlyScript(id, workflow = nightlyWorkflow) {
Comment thread
jasonqinzhou marked this conversation as resolved.
Resolved
const step = workflow.split(` id: ${id}\n`)[1]?.split(/\n - /)[0];
assert.ok(step, `missing nightly step ${id}`);
const script = step.match(/ script: \|\n((?:(?: .*)?\n)+)/);
assert.ok(script, `missing script for nightly step ${id}`);
Comment thread
jasonqinzhou marked this conversation as resolved.
Resolved
return script[1];
}

test("nightly script lookup rejects missing IDs", () => {
assert.throws(() => nightlyScript("unknown-step"), /missing nightly step unknown-step/);
});

test("nightly script lookup cannot borrow a later step's script", () => {
const fixture = ` - name: Missing script
id: missing
run: echo fixture
- name: Different step
id: different
with:
script: |
throw new Error('wrong step');
`;
assert.throws(() => nightlyScript("missing", fixture), /missing script for nightly step missing/);
});

async function nightlyVersion(created, number, event = "schedule", attempt = 1) {
const outputs = {};
const sandbox = vm.createContext({
context: { repo: { owner: "ai-dynamo", repo: "aisimulate" }, runId: 123,
eventName: event, runAttempt: attempt },
core: { setOutput: (name, value) => { outputs[name] = value; } },
github: { rest: { actions: { getWorkflowRun: async args => {
assert.equal(args.owner, "ai-dynamo");
assert.equal(args.repo, "aisimulate");
assert.equal(args.run_id, 123);
return { data: { created_at: created, run_number: number } };
} } } },
});
await vm.runInContext(`(async () => { ${nightlyScript("version")} })()`, sandbox);
return outputs;
}

test("nightly versions are unique, date ordered, and stable across retries", async () => {
for (const event of ["schedule", "workflow_dispatch"]) {
for (const [created, number, expected] of [
["2026-09-17T23:59:59Z", 1234, "202609170000001234"],
["2026-09-17T23:59:59Z", 1235, "202609170000001235"],
["2026-09-18T00:00:00Z", 1236, "202609180000001236"],
]) {
const outputs = await nightlyVersion(created, number, event);
assert.deepEqual(outputs, { "dev-date": created.slice(0, 10).replaceAll("-", ""),
"dev-version": expected });
assert.deepEqual(await nightlyVersion(created, number, event, 2), outputs);
}
}
});

test("nightly versions reject invalid run metadata", async () => {
for (const number of [0, -1, 10000000000, "invalid"]) {
await assert.rejects(nightlyVersion("2026-09-17T00:00:00Z", number), /invalid nightly/);
}
await assert.rejects(nightlyVersion("invalid", 1234), /invalid nightly/);
});

async function nightlyTarget({ event = "workflow_dispatch", ref = "refs/heads/main", requested = sha,
branches = ["main", "release/0.12.0", "feature/test"], statuses = { main: "ahead" }, apiError } = {}) {
const outputs = {};
Expand Down
43 changes: 43 additions & 0 deletions tests/test_release_fpe.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
"""Release package isolation and producer-to-publisher contract tests."""

import csv
import hashlib
import importlib.util
import io
import json
Expand Down Expand Up @@ -256,3 +257,45 @@ def test_current_harness_keeps_selected_source_and_tooling_identities_distinct(t
"source_sha": SHA,
"tooling_sha": TOOLING,
}


@pytest.mark.parametrize("branch", ["main", "release/0.12.0"])
def test_record_wheel_normalizes_workflow_refs_in_provenance(tmp_path, monkeypatch, branch):
wheel = tmp_path / "aisimulate-fixture.whl"
wheel.write_bytes(b"release wheel")
monkeypatch.setenv("FPE_SOURCE_SHA", SHA)
monkeypatch.setenv("FPE_TOOLING_SHA", TOOLING)
monkeypatch.setenv("FPE_BRANCH", f"refs/heads/{branch}")
monkeypatch.setenv("GITHUB_OUTPUT", str(tmp_path / "output"))
with (
patch.object(RELEASE.sys, "argv", ["run_release_fpe.py", "record-wheel", "--wheel-dir", str(tmp_path)]),
patch.object(RELEASE, "revision", side_effect=[SHA, TOOLING]),
patch.object(RELEASE.subprocess, "run"),
):
RELEASE.main()
assert json.loads((tmp_path / "provenance.json").read_text()) == {
"schema_version": 1,
"source_branch": branch,
"source_sha": SHA,
"tooling_sha": TOOLING,
"wheel_sha256": hashlib.sha256(wheel.read_bytes()).hexdigest(),
}


@pytest.mark.parametrize("explicit_output", [False, True])
def test_artifact_builder_defaults_to_selected_source_output(tmp_path, explicit_output):
builder = load_script("build_release_artifacts")
source = tmp_path / "historical-source"
output = tmp_path / "custom-output" if explicit_output else source / "dist"
args = ["build_release_artifacts.py", "--root", str(source)]
if explicit_output:
args.extend(["--output-dir", str(output)])
with (
patch.object(builder.sys, "argv", args),
patch.object(builder.subprocess, "check_output", return_value='[project]\nversion = "0.11.0"\n'),
patch.object(builder, "check_manifests", return_value=("0.11.0", "0.11.0")),
patch.object(builder, "build") as build,
):
builder.main()
assert source.resolve() == builder.ROOT
build.assert_called_once_with(output.resolve(), "0.11.0", "0.11.0")
Loading