diff --git a/.github/scripts/_lib.py b/.github/scripts/_lib.py new file mode 100644 index 000000000..d005ff8ea --- /dev/null +++ b/.github/scripts/_lib.py @@ -0,0 +1,204 @@ +"""Shared helpers for .github/scripts/* CLI tools.""" +from __future__ import annotations + +import json +import re +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +SemverKey = tuple[tuple[int, ...], int, str] +BumpKind = Literal["patch", "minor", "major"] +ManifestKind = Literal["cargo", "node", "python"] + + +def parse_semver(v: str) -> SemverKey: + """Returns a tuple suitable for lexicographic compare. + + Shape: (core_tuple, 1 if stable else 0, pre_suffix). + The middle int makes stable strictly greater than any pre-release at the + same core (1.2.3 > 1.2.3-rc.1). The trailing string lexically orders + multiple pre-releases at the same core (rc.1 < rc.2). + """ + # Strip build metadata (semver 2.0.0 §10: ignored for precedence). + v_nobuild, _, _ = v.partition("+") + core, _, pre = v_nobuild.partition("-") + parts = [int(x) for x in core.split(".")] + while len(parts) < 3: + parts.append(0) + return (tuple(parts), 0 if pre else 1, pre) + + +def bump(current: str, kind: BumpKind) -> str: + """Returns `current` with the requested component incremented. + + Pre-release suffixes are stripped (bumping past a pre-release yields the + next stable). Short versions are padded to three components. + """ + core, _, _pre = current.partition("-") + parts = [int(x) for x in core.split(".")] + while len(parts) < 3: + parts.append(0) + major, minor, patch = parts[0], parts[1], parts[2] + if kind == "major": + major, minor, patch = major + 1, 0, 0 + elif kind == "minor": + minor, patch = minor + 1, 0 + elif kind == "patch": + patch += 1 + else: + raise ValueError(f"unknown bump kind: {kind!r}") + return f"{major}.{minor}.{patch}" + + +def detect_kind(manifest_path: Path) -> ManifestKind: + """Identifies a manifest file by its filename.""" + name = manifest_path.name + if name == "Cargo.toml": + return "cargo" + if name == "package.json": + return "node" + if name == "pyproject.toml": + return "python" + raise ValueError(f"unsupported manifest filename: {name!r}") + + +def _read_toml_section_version(text: str, section: str) -> str: + """Read `version = "X"` inside a top-level TOML section like `[package]`. + + Subsections like `[package.metadata]` correctly exit the section because + matching is strict equality on the bracket-stripped header. + """ + in_section = False + for line in text.splitlines(): + s = line.strip() + if s.startswith("["): + in_section = (s == section) + continue + if in_section: + m = re.match(r'^version\s*=\s*"([^"]+)"', s) + if m: + return m.group(1) + raise ValueError(f"no version field in {section} section") + + +def _write_toml_section_version(path: Path, section: str, new_version: str) -> None: + """Replace `version = "X"` inside a top-level TOML section (first match).""" + text = path.read_text() + out: list[str] = [] + in_section = False + replaced = False + for line in text.splitlines(): + s = line.strip() + if s.startswith("["): + in_section = (s == section) + if in_section and not replaced and re.match(r'^version\s*=\s*"[^"]+"', s): + line = re.sub(r'^version\s*=\s*"[^"]+"', f'version = "{new_version}"', line) + replaced = True + out.append(line) + if not replaced: + raise ValueError(f"could not find version in {section}") + trailing = "\n" if text.endswith("\n") else "" + path.write_text("\n".join(out) + trailing) + + +def _read_node_version(text: str) -> str: + data = json.loads(text) + if "version" not in data: + raise ValueError("no version key in package.json") + return str(data["version"]) + + +def _write_node_version(path: Path, new_version: str) -> None: + data = json.loads(path.read_text()) + data["version"] = new_version + path.write_text(json.dumps(data, indent=2) + "\n") + + +def read_version(manifest_path: Path) -> str: + """Reads the manifest's package version. Dispatches on filename.""" + kind = detect_kind(manifest_path) + text = manifest_path.read_text() + if kind == "cargo": + return _read_toml_section_version(text, "[package]") + if kind == "node": + return _read_node_version(text) + if kind == "python": + return _read_toml_section_version(text, "[project]") + raise ValueError(f"unhandled manifest kind: {kind}") + + +def write_version(manifest_path: Path, new_version: str) -> None: + """Writes a new version to the manifest. Dispatches on filename.""" + kind = detect_kind(manifest_path) + if kind == "cargo": + _write_toml_section_version(manifest_path, "[package]", new_version) + return + if kind == "node": + _write_node_version(manifest_path, new_version) + return + if kind == "python": + _write_toml_section_version(manifest_path, "[project]", new_version) + return + raise ValueError(f"unhandled manifest kind: {kind}") + + +@dataclass(frozen=True) +class WorkerManifest: + """Parsed view of a `/iii.worker.yaml` file.""" + + name: str + language: str | None + deploy: str | None + manifest: str | None + bin: str | None + raw: dict[str, object] + + +def read_iii_worker_yaml(worker_dir: Path) -> WorkerManifest: + """Loads `/iii.worker.yaml` and returns a `WorkerManifest`.""" + import yaml # imported here so `_lib` is usable without pyyaml at import time + + path = worker_dir / "iii.worker.yaml" + if not path.exists(): + raise FileNotFoundError(f"{path} does not exist") + raw = yaml.safe_load(path.read_text()) or {} + if not isinstance(raw, dict): + raise ValueError(f"{path}: expected a mapping at top level") + name = raw.get("name") or worker_dir.name + return WorkerManifest( + name=str(name), + language=raw.get("language"), + deploy=raw.get("deploy"), + manifest=raw.get("manifest"), + bin=raw.get("bin"), + raw=raw, + ) + + +def read_tag_annotation(tag: str) -> dict[str, str]: + """Parses 'key: value' lines from an annotated tag's body. + + Returns {} on lightweight tags, missing tags, or git failures. Lines that + start with `#` (subject lines like the release name) or are blank are + skipped. Only top-level lines containing a single `:` separator count. + """ + try: + msg = subprocess.check_output( + ["git", "tag", "-l", "--format=%(contents)", tag], + text=True, + stderr=subprocess.DEVNULL, + ) + except subprocess.CalledProcessError: + return {} + out: dict[str, str] = {} + for line in msg.splitlines(): + s = line.strip() + if not s or s.startswith("#"): + continue + if ":" not in s: + continue + k, _, v = s.partition(":") + out[k.strip()] = v.strip() + return out diff --git a/.github/scripts/collect_worker_interface.py b/.github/scripts/collect_worker_interface.py index 21adf3879..5b4cb3b04 100644 --- a/.github/scripts/collect_worker_interface.py +++ b/.github/scripts/collect_worker_interface.py @@ -67,12 +67,43 @@ def collect_trigger_types() -> dict[str, object] | None: def main() -> int: parser = argparse.ArgumentParser() - parser.add_argument("--worker", required=True) + parser.add_argument("--worker", required=False) parser.add_argument("--out", default="worker-interface.json") parser.add_argument("--wait-seconds", type=int, default=0) parser.add_argument("--trigger-types-baseline", default="") + parser.add_argument( + "--assert-non-empty", + action="store_true", + help="after writing --out, assert payload['functions'] is non-empty", + ) + parser.add_argument( + "--assert-file", + help="path to an already-written interface JSON to assert; bypasses collection", + ) args = parser.parse_args() + # Standalone assertion mode — bypass collection entirely. + if args.assert_file: + if not args.assert_non_empty: + print("--assert-file requires --assert-non-empty", file=sys.stderr) + return 1 + try: + data = json.loads(pathlib.Path(args.assert_file).read_text()) + except Exception as e: + print(f"could not read {args.assert_file}: {e}", file=sys.stderr) + return 1 + if not data.get("functions"): + print( + f"::error::no worker functions in {args.assert_file} (empty)", + file=sys.stderr, + ) + return 1 + return 0 + + if not args.worker: + print("--worker is required unless --assert-file is given", file=sys.stderr) + return 1 + baseline_json = None if args.trigger_types_baseline: baseline_path = pathlib.Path(args.trigger_types_baseline) @@ -92,6 +123,14 @@ def main() -> int: ) pathlib.Path(args.out).write_text(json.dumps(interface, indent=2) + "\n", encoding="utf-8") print(json.dumps(interface, indent=2)) + + if args.assert_non_empty and not args.assert_file: + with open(args.out) as f: + data = json.load(f) + if not data.get("functions"): + print(f"::error::no worker functions in {args.out} (empty)", file=sys.stderr) + return 1 + return 0 diff --git a/.github/scripts/discover_changed_workers.py b/.github/scripts/discover_changed_workers.py new file mode 100644 index 000000000..1ece5e534 --- /dev/null +++ b/.github/scripts/discover_changed_workers.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Discover worker folders that changed between two git refs. + +Outputs a JSON object to stdout AND (if $GITHUB_OUTPUT is set) writes keys: + changed_workers (alias `all`) : all workers with any change + source_changed : workers whose change wasn't only metadata + rust / node / python : language buckets (subset of changed_workers) + vscode_changed : bool, did iii-lsp-vscode/ change + any : bool, any worker or vscode change +""" +from __future__ import annotations + +import argparse +import fnmatch +import json +import os +import pathlib +import subprocess +import sys + +# Files inside a worker dir that DON'T count as a real source change. If the +# only files a worker touched match these globs, version-bump + tests/ gates +# in pr-checks downgrade to notices. +METADATA_GLOBS = ( + "iii.worker.yaml", + "README.md", + "AGENTS.md", + "AGENTS-*.md", + "Cargo.lock", + "Cargo.toml", +) + +# Top-level dirs we never treat as workers, regardless of contents. +IGNORE_DIRS = {".git", ".github", "registry", "target", "node_modules"} + +# Special non-worker dir tracked separately for the vscode-changed gate. +VSCODE_DIR = "iii-lsp-vscode" + +# Source changes in this worker fan out: every in-repo dep listed in its +# iii.worker.yaml joins the changed-workers matrix so the rust lint+test +# job exercises them against the new shared crate. Deps are deliberately +# NOT added to source_changed — version-bump / README / tests/ gates only +# apply to workers the PR author actually edited. Bundle-release handles +# downstream version bumps at tag time. +FANOUT_PARENT = "harness" + + +def is_metadata(rel: str) -> bool: + return any(fnmatch.fnmatch(rel, g) for g in METADATA_GLOBS) + + +def list_worker_dirs(repo_root: pathlib.Path) -> set[str]: + return { + p.name + for p in repo_root.iterdir() + if p.is_dir() + and not p.name.startswith(".") + and p.name not in IGNORE_DIRS + and (p / "iii.worker.yaml").exists() + } + + +def changed_files(base: str, head: str) -> list[str]: + try: + out = subprocess.check_output( + ["git", "diff", "--name-only", f"{base}...{head}"], text=True + ) + except subprocess.CalledProcessError: + out = subprocess.check_output( + ["git", "diff", "--name-only", "HEAD~1...HEAD"], text=True + ) + return out.splitlines() + + +def language_of(worker_dir: pathlib.Path) -> str | None: + meta = worker_dir / "iii.worker.yaml" + if not meta.exists(): + return None + for line in meta.read_text().splitlines(): + s = line.strip() + if s.startswith("language:"): + return s.split(":", 1)[1].strip() + return None + + +def fanout_dependents(repo_root: pathlib.Path, workers: set[str]) -> list[str]: + """In-repo workers listed as deps in FANOUT_PARENT/iii.worker.yaml. + + Missing manifest or unreadable yaml → empty list (no fan-out). + """ + meta = repo_root / FANOUT_PARENT / "iii.worker.yaml" + if not meta.exists(): + return [] + try: + import yaml # type: ignore[import-not-found] + data = yaml.safe_load(meta.read_text()) or {} + except Exception: # noqa: BLE001 + return [] + if not isinstance(data, dict): + return [] + deps = data.get("dependencies") + if not isinstance(deps, dict): + return [] + return sorted(d for d in deps if d in workers and d != FANOUT_PARENT) + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser() + p.add_argument("--base", required=True, help="base git ref") + p.add_argument("--head", default="HEAD", help="head git ref (default: HEAD)") + args = p.parse_args(argv) + + repo_root = pathlib.Path(".").resolve() + workers = list_worker_dirs(repo_root) + files = changed_files(args.base, args.head) + + touched: dict[str, list[str]] = {} + vscode_changed = False + for f in files: + parts = f.split("/", 1) + if len(parts) < 2: + continue + top, rel = parts[0], parts[1] + if top == VSCODE_DIR: + vscode_changed = True + continue + if top in workers: + touched.setdefault(top, []).append(rel) + + forced: set[str] = set() + parent_rels = touched.get(FANOUT_PARENT, []) + if any(not is_metadata(rel) for rel in parent_rels): + forced.update(fanout_dependents(repo_root, workers)) + + changed = sorted(set(touched.keys()) | forced) + source_changed = sorted( + w for w, rels in touched.items() + if any(not is_metadata(rel) for rel in rels) + ) + by_language: dict[str, list[str]] = {"rust": [], "node": [], "python": []} + for w in changed: + lang = language_of(repo_root / w) + if lang in by_language: + by_language[lang].append(w) + else: + print(f"::warning::{w} has unknown language={lang}", file=sys.stderr) + + payload = { + "changed_workers": changed, + "source_changed": source_changed, + "by_language": by_language, + "vscode_changed": vscode_changed, + } + print(json.dumps(payload)) + + gh_out = os.environ.get("GITHUB_OUTPUT") + if gh_out: + any_change = bool(changed) or vscode_changed + with open(gh_out, "a") as f: + f.write(f"changed_workers={json.dumps(changed)}\n") + # Back-compat alias used by ci.yml downstream matrix jobs. + f.write(f"all={json.dumps(changed)}\n") + f.write(f"source_changed={json.dumps(source_changed)}\n") + for lang in ("rust", "node", "python"): + f.write(f"{lang}={json.dumps(by_language[lang])}\n") + f.write(f"vscode_changed={'true' if vscode_changed else 'false'}\n") + f.write(f"any={'true' if any_change else 'false'}\n") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/manifest_version.py b/.github/scripts/manifest_version.py new file mode 100644 index 000000000..7bffa623e --- /dev/null +++ b/.github/scripts/manifest_version.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""CLI for reading/bumping/verifying language manifests. + +Subcommands: + read print the manifest's version to stdout + bump --kind ... bump the version in-place + verify --expected V + assert the file's version equals V + deploy-mode print the interface-collection mode + +Exit codes: 0 on success, 1 on parse / IO / mismatch failure. +""" +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +# Make _lib importable when run as a script. +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import _lib # noqa: E402 + + +def cmd_read(args: argparse.Namespace) -> int: + path = Path(args.manifest) + try: + print(_lib.read_version(path)) + except (FileNotFoundError, ValueError) as e: + print(f"error: {e}", file=sys.stderr) + return 1 + return 0 + + +def cmd_bump(args: argparse.Namespace) -> int: + path = Path(args.manifest) + try: + current = _lib.read_version(path) + new = _lib.bump(current, args.kind) + _lib.write_version(path, new) + except (FileNotFoundError, ValueError) as e: + print(f"error: {e}", file=sys.stderr) + return 1 + print(new) + return 0 + + +def cmd_verify(args: argparse.Namespace) -> int: + path = Path(args.manifest) + try: + actual = _lib.read_version(path) + except (FileNotFoundError, ValueError) as e: + print(f"error: {e}", file=sys.stderr) + return 1 + if actual != args.expected: + print( + f"version mismatch in {path}: expected {args.expected}, got {actual}", + file=sys.stderr, + ) + return 1 + return 0 + + +def cmd_deploy_mode(args: argparse.Namespace) -> int: + """Mirrors the historic _publish-registry.yml heredoc. + + Decides how _publish-registry should start a local copy of the worker + for interface collection. Output is one of: + release-binary -> download from GitHub Release + iii-add -> `iii worker add ./` (worker is self-bootstrapping) + cargo-run -> `cargo run` from source + unsupported -> can't collect locally + """ + worker_dir = Path(args.worker_dir) + try: + m = _lib.read_iii_worker_yaml(worker_dir) + except (FileNotFoundError, ValueError) as e: + print(f"error: {e}", file=sys.stderr) + return 1 + + raw = m.raw + scripts = raw.get("scripts") or {} + runtime = raw.get("runtime") or {} + has_scripts_start = bool(str(scripts.get("start") or "").strip()) + has_runtime = bool(runtime.get("kind") or runtime.get("language")) + + if m.deploy == "binary": + print("release-binary") + elif has_scripts_start or has_runtime: + print("iii-add") + elif (m.language or "").lower() == "rust": + print("cargo-run") + else: + print("unsupported") + return 0 + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(prog="manifest_version.py") + sub = p.add_subparsers(dest="cmd", required=True) + + p_read = sub.add_parser("read", help="print the manifest version") + p_read.add_argument("manifest", help="path to Cargo.toml/package.json/pyproject.toml") + p_read.set_defaults(func=cmd_read) + + p_bump = sub.add_parser("bump", help="bump the manifest version in place") + p_bump.add_argument("manifest") + p_bump.add_argument("--kind", choices=["patch", "minor", "major"], required=True) + p_bump.set_defaults(func=cmd_bump) + + p_verify = sub.add_parser("verify", help="assert the manifest version equals --expected") + p_verify.add_argument("manifest") + p_verify.add_argument("--expected", required=True) + p_verify.set_defaults(func=cmd_verify) + + p_dm = sub.add_parser("deploy-mode", help="print the interface-collection mode") + p_dm.add_argument("worker_dir") + p_dm.set_defaults(func=cmd_deploy_mode) + + args = p.parse_args(argv) + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/parse_release_tag.py b/.github/scripts/parse_release_tag.py new file mode 100644 index 000000000..b7d67f74b --- /dev/null +++ b/.github/scripts/parse_release_tag.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +"""Parse a release tag and emit setup outputs for release.yml. + +Writes 10 keys to $GITHUB_OUTPUT: + tag, worker, version, deploy, language, + bin, manifest, registry_tag, is_prerelease, dry_run +""" +from __future__ import annotations + +import argparse +import os +import pathlib +import re +import sys + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) +import _lib # noqa: E402 + + +TAG_RE = re.compile(r"^([a-z0-9][a-z0-9_-]*)/v(.+)$") +DRY_RUN_RE = re.compile(r"-dry-run\.\d+$") +PRERELEASE_RE = re.compile(r"-[a-z]+\.\d+$") + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser() + p.add_argument("raw_tag") + args = p.parse_args(argv) + + raw = args.raw_tag.strip() + m = TAG_RE.match(raw) + if not m: + print(f"::error::Invalid tag shape: {raw}", file=sys.stderr) + return 1 + worker, version = m.group(1), m.group(2) + + if DRY_RUN_RE.search(version): + dry_run, is_pre = "true", "true" + elif PRERELEASE_RE.search(version): + dry_run, is_pre = "false", "true" + else: + dry_run, is_pre = "false", "false" + + worker_dir = pathlib.Path(worker) + try: + wm = _lib.read_iii_worker_yaml(worker_dir) + except FileNotFoundError as e: + print(f"::error::{e}", file=sys.stderr) + return 1 + + if wm.deploy not in ("binary", "image"): + print( + f"::error::{worker}: deploy must be binary|image (got {wm.deploy!r})", + file=sys.stderr, + ) + return 1 + + registry_tag = _lib.read_tag_annotation(raw).get("registry-tag", "latest") or "latest" + + # `targets` is an optional iii.worker.yaml field that can be either a + # list (`- aarch64-apple-darwin\n- x86_64-unknown-linux-gnu`) or a comma + # string. Normalise to a single comma-joined string for the workflow. + targets_raw = wm.raw.get("targets") + if isinstance(targets_raw, list): + targets = ",".join(str(t).strip() for t in targets_raw if str(t).strip()) + elif isinstance(targets_raw, str): + targets = targets_raw.strip() + else: + targets = "" + + pairs = [ + ("tag", raw), + ("worker", worker), + ("version", version), + ("deploy", wm.deploy), + ("language", wm.language or ""), + ("bin", wm.bin or wm.name), + ("manifest", wm.manifest or ""), + ("registry_tag", registry_tag), + ("is_prerelease", is_pre), + ("dry_run", dry_run), + ("targets", targets), + ] + + gh_out = os.environ.get("GITHUB_OUTPUT") + if gh_out: + with open(gh_out, "a") as f: + for k, v in pairs: + f.write(f"{k}={v}\n") + + print( + f"::notice::release {worker} v{version} deploy={wm.deploy} " + f"registry-tag={registry_tag}" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/tests/__init__.py b/.github/scripts/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/.github/scripts/tests/_test_helpers.py b/.github/scripts/tests/_test_helpers.py new file mode 100644 index 000000000..a26025263 --- /dev/null +++ b/.github/scripts/tests/_test_helpers.py @@ -0,0 +1,16 @@ +"""Small shared constants/helpers for the .github/scripts/tests/ suite. + +Kept separate from conftest.py because pytest treats conftest as a fixture +module — re-imports of its top-level symbols from sibling test files are +fragile in unusual collection paths. Putting plain Python constants here +keeps both conftest and tests reading from one source. +""" +from __future__ import annotations + +import os + +GIT_HERMETIC_ENV = { + **os.environ, + "GIT_CONFIG_GLOBAL": "/dev/null", + "GIT_CONFIG_SYSTEM": "/dev/null", +} diff --git a/.github/scripts/tests/conftest.py b/.github/scripts/tests/conftest.py new file mode 100644 index 000000000..ad084f9ad --- /dev/null +++ b/.github/scripts/tests/conftest.py @@ -0,0 +1,99 @@ +"""Shared fixtures for .github/scripts/ test suite.""" +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import pytest + +# Make `.github/scripts/*.py` importable as top-level modules, and ensure +# the tests dir itself is on sys.path so sibling helper modules like +# `_test_helpers` are importable from both conftest and test_*.py. +SCRIPTS_DIR = Path(__file__).resolve().parents[1] +TESTS_DIR = Path(__file__).resolve().parent +for _p in (SCRIPTS_DIR, TESTS_DIR): + if str(_p) not in sys.path: + sys.path.insert(0, str(_p)) + +# Reuse this env in every fixture that shells out to `git`. Neutralising +# global/system config keeps `git tag -a` and `git commit` working on +# developer machines that have `commit.gpgsign=true` or other surprises in +# global config. Source-of-truth lives in `_test_helpers` so test modules +# can import the same constant without re-importing conftest. +from _test_helpers import GIT_HERMETIC_ENV # noqa: E402 + + +@pytest.fixture +def cargo_manifest(tmp_path: Path) -> Path: + p = tmp_path / "Cargo.toml" + p.write_text( + '[package]\n' + 'name = "smoke"\n' + 'version = "0.1.0"\n' + 'edition = "2021"\n' + ) + return p + + +@pytest.fixture +def package_json_manifest(tmp_path: Path) -> Path: + p = tmp_path / "package.json" + p.write_text('{\n "name": "smoke",\n "version": "0.1.0"\n}\n') + return p + + +@pytest.fixture +def pyproject_manifest(tmp_path: Path) -> Path: + p = tmp_path / "pyproject.toml" + p.write_text( + '[project]\n' + 'name = "smoke"\n' + 'version = "0.1.0"\n' + ) + return p + + +@pytest.fixture +def iii_worker_yaml_dir(tmp_path: Path) -> Path: + """Returns a tmp dir containing a minimal iii.worker.yaml (rust binary).""" + (tmp_path / "iii.worker.yaml").write_text( + 'iii: v1\n' + 'name: smoke\n' + 'language: rust\n' + 'deploy: binary\n' + 'manifest: Cargo.toml\n' + 'bin: smoke-bin\n' + 'description: smoke test worker\n' + ) + return tmp_path + + +@pytest.fixture +def github_output(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Captures writes to $GITHUB_OUTPUT into a tmp file the test can read.""" + f = tmp_path / "github_output" + f.touch() + monkeypatch.setenv("GITHUB_OUTPUT", str(f)) + return f + + +@pytest.fixture +def tmp_git_repo_with_tag(tmp_path: Path) -> Path: + """Initialises a tmp git repo, makes one commit, creates an annotated tag + `smoke/v0.1.0` with a multi-line body containing `registry-tag: next`. + Used to test `read_tag_annotation` without monkeypatching subprocess.""" + subprocess.run(["git", "init", "-q", "-b", "main"], cwd=tmp_path, check=True, env=GIT_HERMETIC_ENV) + subprocess.run(["git", "config", "user.email", "test@example.com"], cwd=tmp_path, check=True, env=GIT_HERMETIC_ENV) + subprocess.run(["git", "config", "user.name", "Test"], cwd=tmp_path, check=True, env=GIT_HERMETIC_ENV) + (tmp_path / "README.md").write_text("hello\n") + subprocess.run(["git", "add", "."], cwd=tmp_path, check=True, env=GIT_HERMETIC_ENV) + subprocess.run(["git", "commit", "-q", "-m", "init"], cwd=tmp_path, check=True, env=GIT_HERMETIC_ENV) + subprocess.run( + ["git", "tag", "-a", "smoke/v0.1.0", "-m", + "Release smoke/v0.1.0\n\nregistry-tag: next\nother: value\n"], + cwd=tmp_path, + check=True, + env=GIT_HERMETIC_ENV, + ) + return tmp_path diff --git a/.github/scripts/tests/test_collect_assert_non_empty.py b/.github/scripts/tests/test_collect_assert_non_empty.py new file mode 100644 index 000000000..76bfffa14 --- /dev/null +++ b/.github/scripts/tests/test_collect_assert_non_empty.py @@ -0,0 +1,46 @@ +"""Tests for the --assert-non-empty flag on collect_worker_interface.py.""" +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parents[1] / "collect_worker_interface.py" + + +def write_payload(path: Path, *, functions: list) -> None: + path.write_text(json.dumps({"functions": functions, "triggers": []})) + + +class TestAssertNonEmpty: + def test_passes_when_functions_non_empty(self, tmp_path): + out = tmp_path / "interface.json" + write_payload(out, functions=[{"id": "smoke::ping"}]) + r = subprocess.run( + [sys.executable, str(SCRIPT), + "--assert-non-empty", "--assert-file", str(out)], + capture_output=True, text=True, + ) + assert r.returncode == 0, r.stderr + + def test_fails_when_functions_empty(self, tmp_path): + out = tmp_path / "interface.json" + write_payload(out, functions=[]) + r = subprocess.run( + [sys.executable, str(SCRIPT), + "--assert-non-empty", "--assert-file", str(out)], + capture_output=True, text=True, + ) + assert r.returncode != 0 + assert "empty" in (r.stderr + r.stdout).lower() + + def test_fails_when_functions_key_missing(self, tmp_path): + out = tmp_path / "interface.json" + out.write_text("{}") + r = subprocess.run( + [sys.executable, str(SCRIPT), + "--assert-non-empty", "--assert-file", str(out)], + capture_output=True, text=True, + ) + assert r.returncode != 0 diff --git a/.github/scripts/tests/test_discover_changed_workers.py b/.github/scripts/tests/test_discover_changed_workers.py new file mode 100644 index 000000000..fc1af197f --- /dev/null +++ b/.github/scripts/tests/test_discover_changed_workers.py @@ -0,0 +1,205 @@ +"""Tests for .github/scripts/discover_changed_workers.py.""" +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +from _test_helpers import GIT_HERMETIC_ENV + +SCRIPT = Path(__file__).resolve().parents[1] / "discover_changed_workers.py" + + +def make_repo_with_workers(tmp_path: Path) -> Path: + """Initialise a tmp git repo with three workers (rust binary, node image, + python image) and one non-worker dir. Returns the repo path.""" + def run(*args, **kwargs): + return subprocess.run(args, cwd=tmp_path, check=True, env=GIT_HERMETIC_ENV, **kwargs) + run("git", "init", "-q", "-b", "main") + run("git", "config", "user.email", "t@e.com") + run("git", "config", "user.name", "T") + # Worker A — rust binary + (tmp_path / "worker-a").mkdir() + (tmp_path / "worker-a" / "iii.worker.yaml").write_text( + 'iii: v1\nname: worker-a\nlanguage: rust\ndeploy: binary\nmanifest: Cargo.toml\n' + ) + (tmp_path / "worker-a" / "Cargo.toml").write_text( + '[package]\nname = "worker-a"\nversion = "0.1.0"\n' + ) + # Worker B — node image + (tmp_path / "worker-b").mkdir() + (tmp_path / "worker-b" / "iii.worker.yaml").write_text( + 'iii: v1\nname: worker-b\nlanguage: node\ndeploy: image\nmanifest: package.json\n' + ) + (tmp_path / "worker-b" / "package.json").write_text('{"name":"worker-b","version":"0.1.0"}') + # Worker C — python image + (tmp_path / "worker-c").mkdir() + (tmp_path / "worker-c" / "iii.worker.yaml").write_text( + 'iii: v1\nname: worker-c\nlanguage: python\ndeploy: image\nmanifest: pyproject.toml\n' + ) + (tmp_path / "worker-c" / "pyproject.toml").write_text( + '[project]\nname = "worker-c"\nversion = "0.1.0"\n' + ) + # Non-worker dir + (tmp_path / "docs").mkdir() + (tmp_path / "docs" / "README.md").write_text("# hello\n") + run("git", "add", ".") + run("git", "commit", "-q", "-m", "init") + return tmp_path + + +def run_script(repo: Path, base: str, head: str = "HEAD") -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(SCRIPT), "--base", base, "--head", head], + capture_output=True, + text=True, + cwd=repo, + env=GIT_HERMETIC_ENV, + ) + + +class TestDiscoverChangedWorkers: + def test_single_worker_source_change(self, tmp_path): + repo = make_repo_with_workers(tmp_path) + (repo / "worker-a" / "src.rs").write_text("// hi\n") + subprocess.run(["git", "add", "."], cwd=repo, check=True, env=GIT_HERMETIC_ENV) + subprocess.run(["git", "commit", "-q", "-m", "edit a"], cwd=repo, check=True, env=GIT_HERMETIC_ENV) + r = run_script(repo, "main~1") + assert r.returncode == 0, r.stderr + data = json.loads(r.stdout) + assert data["changed_workers"] == ["worker-a"] + assert data["source_changed"] == ["worker-a"] + assert data["by_language"]["rust"] == ["worker-a"] + + def test_multi_worker_change(self, tmp_path): + repo = make_repo_with_workers(tmp_path) + (repo / "worker-a" / "src.rs").write_text("// hi\n") + (repo / "worker-c" / "x.py").write_text("# hi\n") + subprocess.run(["git", "add", "."], cwd=repo, check=True, env=GIT_HERMETIC_ENV) + subprocess.run(["git", "commit", "-q", "-m", "edit a+c"], cwd=repo, check=True, env=GIT_HERMETIC_ENV) + r = run_script(repo, "main~1") + assert r.returncode == 0 + data = json.loads(r.stdout) + assert sorted(data["changed_workers"]) == ["worker-a", "worker-c"] + assert sorted(data["by_language"]["rust"]) == ["worker-a"] + assert sorted(data["by_language"]["python"]) == ["worker-c"] + + def test_metadata_only_change_not_source_changed(self, tmp_path): + repo = make_repo_with_workers(tmp_path) + (repo / "worker-a" / "README.md").write_text("# a\n") + subprocess.run(["git", "add", "."], cwd=repo, check=True, env=GIT_HERMETIC_ENV) + subprocess.run(["git", "commit", "-q", "-m", "docs"], cwd=repo, check=True, env=GIT_HERMETIC_ENV) + r = run_script(repo, "main~1") + assert r.returncode == 0 + data = json.loads(r.stdout) + assert data["changed_workers"] == ["worker-a"] + assert data["source_changed"] == [] + + def test_non_worker_change_ignored(self, tmp_path): + repo = make_repo_with_workers(tmp_path) + (repo / "docs" / "more.md").write_text("# more\n") + subprocess.run(["git", "add", "."], cwd=repo, check=True, env=GIT_HERMETIC_ENV) + subprocess.run(["git", "commit", "-q", "-m", "doc"], cwd=repo, check=True, env=GIT_HERMETIC_ENV) + r = run_script(repo, "main~1") + assert r.returncode == 0 + data = json.loads(r.stdout) + assert data["changed_workers"] == [] + + +def make_repo_with_harness(tmp_path: Path) -> Path: + """Tmp repo: harness + two in-repo deps + one external dep + one unrelated worker.""" + def run(*args): + return subprocess.run(args, cwd=tmp_path, check=True, env=GIT_HERMETIC_ENV) + run("git", "init", "-q", "-b", "main") + run("git", "config", "user.email", "t@e.com") + run("git", "config", "user.name", "T") + (tmp_path / "harness").mkdir() + (tmp_path / "harness" / "iii.worker.yaml").write_text( + 'iii: v1\nname: harness\nlanguage: rust\ndeploy: binary\nmanifest: Cargo.toml\n' + 'dependencies:\n' + ' dep-rust: "^0.1.0"\n' + ' dep-node: "^0.1.0"\n' + ' external-dep: "^0.1.0"\n' + ) + (tmp_path / "harness" / "Cargo.toml").write_text( + '[package]\nname = "harness"\nversion = "0.1.0"\n' + ) + (tmp_path / "dep-rust").mkdir() + (tmp_path / "dep-rust" / "iii.worker.yaml").write_text( + 'iii: v1\nname: dep-rust\nlanguage: rust\ndeploy: binary\nmanifest: Cargo.toml\n' + ) + (tmp_path / "dep-rust" / "Cargo.toml").write_text( + '[package]\nname = "dep-rust"\nversion = "0.1.0"\n' + ) + (tmp_path / "dep-node").mkdir() + (tmp_path / "dep-node" / "iii.worker.yaml").write_text( + 'iii: v1\nname: dep-node\nlanguage: node\ndeploy: image\nmanifest: package.json\n' + ) + (tmp_path / "dep-node" / "package.json").write_text('{"name":"dep-node","version":"0.1.0"}') + (tmp_path / "unrelated").mkdir() + (tmp_path / "unrelated" / "iii.worker.yaml").write_text( + 'iii: v1\nname: unrelated\nlanguage: python\ndeploy: image\nmanifest: pyproject.toml\n' + ) + (tmp_path / "unrelated" / "pyproject.toml").write_text( + '[project]\nname = "unrelated"\nversion = "0.1.0"\n' + ) + run("git", "add", ".") + run("git", "commit", "-q", "-m", "init") + return tmp_path + + +class TestHarnessFanOut: + def test_harness_source_change_fans_out_to_in_repo_deps(self, tmp_path): + repo = make_repo_with_harness(tmp_path) + (repo / "harness" / "lib.rs").write_text("// breaking change\n") + subprocess.run(["git", "add", "."], cwd=repo, check=True, env=GIT_HERMETIC_ENV) + subprocess.run(["git", "commit", "-q", "-m", "harness edit"], cwd=repo, check=True, env=GIT_HERMETIC_ENV) + r = run_script(repo, "main~1") + assert r.returncode == 0, r.stderr + data = json.loads(r.stdout) + assert sorted(data["changed_workers"]) == ["dep-node", "dep-rust", "harness"] + assert "unrelated" not in data["changed_workers"] + # External dep listed in harness deps but absent from repo is skipped. + assert "external-dep" not in data["changed_workers"] + + def test_fanned_out_deps_excluded_from_source_changed(self, tmp_path): + """Fan-out enters deps into the matrix (so lint+test runs against the + new harness) but must NOT mark them source_changed. Otherwise the PR + gate in validate_worker.py would force the author to bump every dep's + Cargo.toml — version bumps belong to bundle release, not PR CI.""" + repo = make_repo_with_harness(tmp_path) + (repo / "harness" / "lib.rs").write_text("// edit\n") + subprocess.run(["git", "add", "."], cwd=repo, check=True, env=GIT_HERMETIC_ENV) + subprocess.run(["git", "commit", "-q", "-m", "harness edit"], cwd=repo, check=True, env=GIT_HERMETIC_ENV) + r = run_script(repo, "main~1") + assert r.returncode == 0, r.stderr + data = json.loads(r.stdout) + assert data["source_changed"] == ["harness"] + assert "dep-rust" not in data["source_changed"] + assert "dep-node" not in data["source_changed"] + + def test_harness_metadata_change_does_not_fan_out(self, tmp_path): + repo = make_repo_with_harness(tmp_path) + (repo / "harness" / "README.md").write_text("# harness\n") + subprocess.run(["git", "add", "."], cwd=repo, check=True, env=GIT_HERMETIC_ENV) + subprocess.run(["git", "commit", "-q", "-m", "harness docs"], cwd=repo, check=True, env=GIT_HERMETIC_ENV) + r = run_script(repo, "main~1") + assert r.returncode == 0, r.stderr + data = json.loads(r.stdout) + assert data["changed_workers"] == ["harness"] + assert data["source_changed"] == [] + + def test_fanned_out_deps_bucketed_by_language(self, tmp_path): + repo = make_repo_with_harness(tmp_path) + (repo / "harness" / "lib.rs").write_text("// edit\n") + subprocess.run(["git", "add", "."], cwd=repo, check=True, env=GIT_HERMETIC_ENV) + subprocess.run(["git", "commit", "-q", "-m", "harness edit"], cwd=repo, check=True, env=GIT_HERMETIC_ENV) + r = run_script(repo, "main~1") + assert r.returncode == 0, r.stderr + data = json.loads(r.stdout) + assert sorted(data["by_language"]["rust"]) == ["dep-rust", "harness"] + assert data["by_language"]["node"] == ["dep-node"] + assert data["by_language"]["python"] == [] diff --git a/.github/scripts/tests/test_lib.py b/.github/scripts/tests/test_lib.py new file mode 100644 index 000000000..f295e054d --- /dev/null +++ b/.github/scripts/tests/test_lib.py @@ -0,0 +1,227 @@ +"""Tests for .github/scripts/_lib.py.""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +import _lib + + +class TestParseSemver: + def test_stable_three_part(self): + assert _lib.parse_semver("1.2.3") == ((1, 2, 3), 1, "") + + def test_stable_pads_to_three(self): + assert _lib.parse_semver("1.2") == ((1, 2, 0), 1, "") + + def test_prerelease_strips_core_keeps_suffix(self): + assert _lib.parse_semver("1.2.3-rc.1") == ((1, 2, 3), 0, "rc.1") + + def test_stable_greater_than_prerelease_same_core(self): + # 1.2.3 must rank above 1.2.3-rc.1 (the audit bug). + assert _lib.parse_semver("1.2.3") > _lib.parse_semver("1.2.3-rc.1") + + def test_prerelease_greater_core_beats_stable(self): + # 1.2.4-rc.1 must rank above 1.2.3 (newer core wins). + assert _lib.parse_semver("1.2.4-rc.1") > _lib.parse_semver("1.2.3") + + def test_two_prereleases_sort_lexicographically(self): + assert _lib.parse_semver("1.2.3-rc.1") < _lib.parse_semver("1.2.3-rc.2") + + def test_build_metadata_is_ignored(self): + # SemVer 2.0.0 §10: build metadata after `+` MUST NOT affect precedence. + assert _lib.parse_semver("1.0.0+build.5") == ((1, 0, 0), 1, "") + assert _lib.parse_semver("1.0.0+a") == _lib.parse_semver("1.0.0+b") + + +class TestBump: + def test_patch(self): + assert _lib.bump("1.2.3", "patch") == "1.2.4" + + def test_minor_resets_patch(self): + assert _lib.bump("1.2.3", "minor") == "1.3.0" + + def test_major_resets_minor_and_patch(self): + assert _lib.bump("1.2.3", "major") == "2.0.0" + + def test_strips_prerelease_suffix(self): + # `bump("1.2.3-rc.1", "patch")` should yield "1.2.4" — bumping a + # pre-release means moving past the stable release at the same core. + assert _lib.bump("1.2.3-rc.1", "patch") == "1.2.4" + + def test_pads_short_version(self): + assert _lib.bump("1", "patch") == "1.0.1" + + +class TestDetectKind: + def test_cargo(self, tmp_path): + assert _lib.detect_kind(tmp_path / "Cargo.toml") == "cargo" + + def test_node(self, tmp_path): + assert _lib.detect_kind(tmp_path / "package.json") == "node" + + def test_python(self, tmp_path): + assert _lib.detect_kind(tmp_path / "pyproject.toml") == "python" + + def test_unknown_raises(self, tmp_path): + with pytest.raises(ValueError): + _lib.detect_kind(tmp_path / "Makefile") + + +class TestReadVersionCargo: + def test_reads_first_package_version(self, cargo_manifest): + assert _lib.read_version(cargo_manifest) == "0.1.0" + + def test_ignores_dependency_versions(self, tmp_path): + p = tmp_path / "Cargo.toml" + p.write_text( + '[package]\nname = "x"\nversion = "1.0.0"\n' + '[dependencies]\nfoo = { version = "2.0.0" }\n' + ) + assert _lib.read_version(p) == "1.0.0" + + def test_missing_version_raises(self, tmp_path): + p = tmp_path / "Cargo.toml" + p.write_text('[package]\nname = "x"\n') + with pytest.raises(ValueError): + _lib.read_version(p) + + +class TestWriteVersionCargo: + def test_writes_and_round_trips(self, cargo_manifest): + _lib.write_version(cargo_manifest, "9.9.9") + assert _lib.read_version(cargo_manifest) == "9.9.9" + + def test_preserves_other_lines(self, cargo_manifest): + _lib.write_version(cargo_manifest, "9.9.9") + text = cargo_manifest.read_text() + assert 'name = "smoke"' in text + assert 'edition = "2021"' in text + + +class TestReadVersionNode: + def test_reads_version(self, package_json_manifest): + assert _lib.read_version(package_json_manifest) == "0.1.0" + + def test_missing_version_raises(self, tmp_path): + p = tmp_path / "package.json" + p.write_text('{"name": "x"}') + with pytest.raises(ValueError): + _lib.read_version(p) + + +class TestWriteVersionNode: + def test_writes_and_round_trips(self, package_json_manifest): + _lib.write_version(package_json_manifest, "9.9.9") + assert _lib.read_version(package_json_manifest) == "9.9.9" + + def test_preserves_other_keys(self, tmp_path): + import json + p = tmp_path / "package.json" + p.write_text('{\n "name": "smoke",\n "version": "0.1.0",\n "private": true\n}\n') + _lib.write_version(p, "9.9.9") + data = json.loads(p.read_text()) + assert data == {"name": "smoke", "version": "9.9.9", "private": True} + + +class TestReadVersionPython: + def test_reads_project_version(self, pyproject_manifest): + assert _lib.read_version(pyproject_manifest) == "0.1.0" + + def test_ignores_non_project_sections(self, tmp_path): + p = tmp_path / "pyproject.toml" + p.write_text( + '[build-system]\nrequires = ["hatchling"]\n' + '[project]\nname = "x"\nversion = "1.0.0"\n' + '[tool.foo]\nversion = "9.9.9"\n' + ) + assert _lib.read_version(p) == "1.0.0" + + def test_missing_version_raises(self, tmp_path): + p = tmp_path / "pyproject.toml" + p.write_text('[project]\nname = "x"\n') + with pytest.raises(ValueError): + _lib.read_version(p) + + +class TestWriteVersionPython: + def test_writes_and_round_trips(self, pyproject_manifest): + _lib.write_version(pyproject_manifest, "9.9.9") + assert _lib.read_version(pyproject_manifest) == "9.9.9" + + def test_preserves_trailing_newline(self, tmp_path): + p = tmp_path / "pyproject.toml" + p.write_text('[project]\nname = "x"\nversion = "1.0.0"\n') + _lib.write_version(p, "9.9.9") + assert p.read_text().endswith("\n") + + def test_only_replaces_project_version(self, tmp_path): + p = tmp_path / "pyproject.toml" + p.write_text( + '[project]\nname = "x"\nversion = "1.0.0"\n' + '[tool.foo]\nversion = "0.0.1"\n' + ) + _lib.write_version(p, "9.9.9") + text = p.read_text() + assert 'version = "9.9.9"' in text + assert 'version = "0.0.1"' in text # untouched + + +class TestReadIIIWorkerYaml: + def test_well_formed_rust_binary(self, iii_worker_yaml_dir): + m = _lib.read_iii_worker_yaml(iii_worker_yaml_dir) + assert m.name == "smoke" + assert m.language == "rust" + assert m.deploy == "binary" + assert m.manifest == "Cargo.toml" + assert m.bin == "smoke-bin" + assert isinstance(m.raw, dict) + + def test_missing_file_raises(self, tmp_path): + with pytest.raises(FileNotFoundError): + _lib.read_iii_worker_yaml(tmp_path) + + def test_falls_back_name_to_folder_name(self, tmp_path): + (tmp_path / "iii.worker.yaml").write_text( + 'iii: v1\nlanguage: node\ndeploy: image\nmanifest: package.json\n' + ) + m = _lib.read_iii_worker_yaml(tmp_path) + assert m.name == tmp_path.name + assert m.language == "node" + assert m.bin is None + + def test_is_frozen(self, iii_worker_yaml_dir): + m = _lib.read_iii_worker_yaml(iii_worker_yaml_dir) + with pytest.raises(Exception): # FrozenInstanceError + m.name = "other" # type: ignore[misc] + + +class TestReadTagAnnotation: + def test_parses_key_value_lines(self, tmp_git_repo_with_tag, monkeypatch): + monkeypatch.chdir(tmp_git_repo_with_tag) + ann = _lib.read_tag_annotation("smoke/v0.1.0") + assert ann["registry-tag"] == "next" + assert ann["other"] == "value" + + def test_missing_tag_returns_empty_dict(self, tmp_git_repo_with_tag, monkeypatch): + monkeypatch.chdir(tmp_git_repo_with_tag) + ann = _lib.read_tag_annotation("does/not/exist") + assert ann == {} + + def test_skips_comments_and_blank_lines(self, tmp_path, monkeypatch): + import subprocess + from _test_helpers import GIT_HERMETIC_ENV + subprocess.run(["git", "init", "-q", "-b", "main"], cwd=tmp_path, check=True, env=GIT_HERMETIC_ENV) + subprocess.run(["git", "config", "user.email", "t@e.com"], cwd=tmp_path, check=True, env=GIT_HERMETIC_ENV) + subprocess.run(["git", "config", "user.name", "T"], cwd=tmp_path, check=True, env=GIT_HERMETIC_ENV) + (tmp_path / "x").write_text("x") + subprocess.run(["git", "add", "."], cwd=tmp_path, check=True, env=GIT_HERMETIC_ENV) + subprocess.run(["git", "commit", "-q", "-m", "init"], cwd=tmp_path, check=True, env=GIT_HERMETIC_ENV) + subprocess.run( + ["git", "tag", "-a", "v0", "-m", "# comment\nkey: val\n\nother: more\n"], + cwd=tmp_path, check=True, env=GIT_HERMETIC_ENV, + ) + monkeypatch.chdir(tmp_path) + ann = _lib.read_tag_annotation("v0") + assert ann == {"key": "val", "other": "more"} diff --git a/.github/scripts/tests/test_manifest_version.py b/.github/scripts/tests/test_manifest_version.py new file mode 100644 index 000000000..6abfadbb7 --- /dev/null +++ b/.github/scripts/tests/test_manifest_version.py @@ -0,0 +1,121 @@ +"""Tests for .github/scripts/manifest_version.py.""" +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parents[1] / "manifest_version.py" + + +def run_script(*args: str) -> subprocess.CompletedProcess[str]: + """Run manifest_version.py with arguments; capture stdout/stderr/exit.""" + return subprocess.run( + [sys.executable, str(SCRIPT), *args], + capture_output=True, + text=True, + ) + + +class TestReadSubcommand: + def test_read_cargo(self, cargo_manifest): + r = run_script("read", str(cargo_manifest)) + assert r.returncode == 0 + assert r.stdout.strip() == "0.1.0" + + def test_read_node(self, package_json_manifest): + r = run_script("read", str(package_json_manifest)) + assert r.returncode == 0 + assert r.stdout.strip() == "0.1.0" + + def test_read_python(self, pyproject_manifest): + r = run_script("read", str(pyproject_manifest)) + assert r.returncode == 0 + assert r.stdout.strip() == "0.1.0" + + def test_read_missing_file(self, tmp_path): + r = run_script("read", str(tmp_path / "nope.toml")) + assert r.returncode != 0 + + def test_read_unsupported_manifest(self, tmp_path): + p = tmp_path / "Makefile" + p.write_text("# nope") + r = run_script("read", str(p)) + assert r.returncode != 0 + assert "unsupported" in (r.stderr + r.stdout).lower() + + +class TestBumpSubcommand: + def test_bump_cargo_patch(self, cargo_manifest): + r = run_script("bump", str(cargo_manifest), "--kind", "patch") + assert r.returncode == 0 + assert r.stdout.strip() == "0.1.1" + # File was actually written. + r2 = run_script("read", str(cargo_manifest)) + assert r2.stdout.strip() == "0.1.1" + + def test_bump_node_minor(self, package_json_manifest): + r = run_script("bump", str(package_json_manifest), "--kind", "minor") + assert r.returncode == 0 + assert r.stdout.strip() == "0.2.0" + + def test_bump_python_major(self, pyproject_manifest): + r = run_script("bump", str(pyproject_manifest), "--kind", "major") + assert r.returncode == 0 + assert r.stdout.strip() == "1.0.0" + + def test_bump_rejects_unknown_kind(self, cargo_manifest): + r = run_script("bump", str(cargo_manifest), "--kind", "weird") + assert r.returncode != 0 + + +class TestVerifySubcommand: + def test_verify_match(self, cargo_manifest): + r = run_script("verify", str(cargo_manifest), "--expected", "0.1.0") + assert r.returncode == 0 + + def test_verify_mismatch(self, cargo_manifest): + r = run_script("verify", str(cargo_manifest), "--expected", "9.9.9") + assert r.returncode != 0 + assert "0.1.0" in (r.stderr + r.stdout) + assert "9.9.9" in (r.stderr + r.stdout) + + +class TestDeployModeSubcommand: + def test_binary_yields_release_binary(self, tmp_path): + (tmp_path / "iii.worker.yaml").write_text( + 'iii: v1\nname: x\nlanguage: rust\ndeploy: binary\nmanifest: Cargo.toml\n' + ) + r = run_script("deploy-mode", str(tmp_path)) + assert r.returncode == 0 + assert r.stdout.strip() == "release-binary" + + def test_image_with_runtime_yields_iii_add(self, tmp_path): + (tmp_path / "iii.worker.yaml").write_text( + 'iii: v1\nname: x\nlanguage: node\ndeploy: image\nmanifest: package.json\n' + 'runtime:\n kind: node\n' + ) + r = run_script("deploy-mode", str(tmp_path)) + assert r.returncode == 0 + assert r.stdout.strip() == "iii-add" + + def test_image_with_scripts_start_yields_iii_add(self, tmp_path): + (tmp_path / "iii.worker.yaml").write_text( + 'iii: v1\nname: x\nlanguage: python\ndeploy: image\nmanifest: pyproject.toml\n' + 'scripts:\n start: python -m smoke\n' + ) + r = run_script("deploy-mode", str(tmp_path)) + assert r.returncode == 0 + assert r.stdout.strip() == "iii-add" + + def test_rust_no_runtime_yields_cargo_run(self, tmp_path): + (tmp_path / "iii.worker.yaml").write_text( + 'iii: v1\nname: x\nlanguage: rust\ndeploy: image\nmanifest: Cargo.toml\n' + ) + r = run_script("deploy-mode", str(tmp_path)) + assert r.returncode == 0 + assert r.stdout.strip() == "cargo-run" + + def test_missing_iii_worker_yaml(self, tmp_path): + r = run_script("deploy-mode", str(tmp_path)) + assert r.returncode != 0 diff --git a/.github/scripts/tests/test_parse_release_tag.py b/.github/scripts/tests/test_parse_release_tag.py new file mode 100644 index 000000000..6a6ccc79f --- /dev/null +++ b/.github/scripts/tests/test_parse_release_tag.py @@ -0,0 +1,116 @@ +"""Tests for .github/scripts/parse_release_tag.py.""" +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from _test_helpers import GIT_HERMETIC_ENV + +SCRIPT = Path(__file__).resolve().parents[1] / "parse_release_tag.py" + + +def make_repo_with_tagged_worker(tmp_path, tag, version, deploy="binary", + registry_tag_line="registry-tag: latest"): + """Init a tmp repo with one worker + an annotated tag matching `tag`.""" + def run(*args): + return subprocess.run(args, cwd=tmp_path, check=True, env=GIT_HERMETIC_ENV) + run("git", "init", "-q", "-b", "main") + run("git", "config", "user.email", "t@e.com") + run("git", "config", "user.name", "T") + worker = tag.split("/")[0] + w = tmp_path / worker + w.mkdir() + (w / "iii.worker.yaml").write_text( + f'iii: v1\nname: {worker}\nlanguage: rust\ndeploy: {deploy}\n' + f'manifest: Cargo.toml\nbin: {worker}-bin\n' + ) + (w / "Cargo.toml").write_text(f'[package]\nname = "{worker}"\nversion = "{version}"\n') + run("git", "add", ".") + run("git", "commit", "-q", "-m", "init") + body = f"Release {tag}\n\n{registry_tag_line}\n" + run("git", "tag", "-a", tag, "-m", body) + return tmp_path + + +def run_script(repo, raw_tag, github_output_path): + env = {**GIT_HERMETIC_ENV, "GITHUB_OUTPUT": str(github_output_path)} + return subprocess.run( + [sys.executable, str(SCRIPT), raw_tag], + capture_output=True, text=True, cwd=repo, env=env, + ) + + +def parse_outputs(path): + out = {} + for line in path.read_text().splitlines(): + if "=" in line: + k, _, v = line.partition("=") + out[k] = v + return out + + +class TestParseReleaseTag: + def test_stable_binary_tag(self, tmp_path): + repo = make_repo_with_tagged_worker(tmp_path, "smoke/v1.2.3", "1.2.3") + out_path = tmp_path / "gh_output" + out_path.touch() + r = run_script(repo, "smoke/v1.2.3", out_path) + assert r.returncode == 0, r.stderr + out = parse_outputs(out_path) + assert out["tag"] == "smoke/v1.2.3" + assert out["worker"] == "smoke" + assert out["version"] == "1.2.3" + assert out["deploy"] == "binary" + assert out["bin"] == "smoke-bin" + assert out["registry_tag"] == "latest" + assert out["is_prerelease"] == "false" + assert out["dry_run"] == "false" + + def test_prerelease_sets_is_prerelease(self, tmp_path): + repo = make_repo_with_tagged_worker(tmp_path, "smoke/v1.2.3-rc.1", "1.2.3-rc.1", + registry_tag_line="registry-tag: next") + out_path = tmp_path / "gh_output" + out_path.touch() + r = run_script(repo, "smoke/v1.2.3-rc.1", out_path) + assert r.returncode == 0 + out = parse_outputs(out_path) + assert out["is_prerelease"] == "true" + assert out["dry_run"] == "false" + assert out["registry_tag"] == "next" + + def test_dry_run_tag(self, tmp_path): + repo = make_repo_with_tagged_worker(tmp_path, "smoke/v9.9.9-dry-run.1", "9.9.9-dry-run.1") + out_path = tmp_path / "gh_output" + out_path.touch() + r = run_script(repo, "smoke/v9.9.9-dry-run.1", out_path) + assert r.returncode == 0 + out = parse_outputs(out_path) + assert out["dry_run"] == "true" + assert out["is_prerelease"] == "true" + + def test_image_deploy(self, tmp_path): + repo = make_repo_with_tagged_worker(tmp_path, "smoke/v1.0.0", "1.0.0", deploy="image") + out_path = tmp_path / "gh_output" + out_path.touch() + r = run_script(repo, "smoke/v1.0.0", out_path) + assert r.returncode == 0 + out = parse_outputs(out_path) + assert out["deploy"] == "image" + + def test_malformed_tag_fails(self, tmp_path): + repo = make_repo_with_tagged_worker(tmp_path, "smoke/v1.0.0", "1.0.0") + out_path = tmp_path / "gh_output" + out_path.touch() + r = run_script(repo, "not-a-tag", out_path) + assert r.returncode != 0 + + def test_missing_iii_worker_yaml_fails(self, tmp_path): + repo = make_repo_with_tagged_worker(tmp_path, "smoke/v1.0.0", "1.0.0") + out_path = tmp_path / "gh_output" + out_path.touch() + r = run_script(repo, "missing/v1.0.0", out_path) + assert r.returncode != 0 diff --git a/.github/scripts/tests/test_validate_worker.py b/.github/scripts/tests/test_validate_worker.py new file mode 100644 index 000000000..58824d430 --- /dev/null +++ b/.github/scripts/tests/test_validate_worker.py @@ -0,0 +1,128 @@ +"""Tests for .github/scripts/validate_worker.py.""" +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +from _test_helpers import GIT_HERMETIC_ENV + +SCRIPT = Path(__file__).resolve().parents[1] / "validate_worker.py" + + +def make_worker(tmp_path: Path, name: str, version: str = "0.1.0", + language: str = "rust", deploy: str = "binary", + manifest_name: str = "Cargo.toml", + tests_dir: bool = True) -> Path: + """Returns the repo-root path; worker lives at //.""" + w = tmp_path / name + w.mkdir() + (w / "README.md").write_text(f"# {name}\nhello\n") + (w / "iii.worker.yaml").write_text( + f'iii: v1\nname: {name}\nlanguage: {language}\n' + f'deploy: {deploy}\nmanifest: {manifest_name}\n' + ) + if manifest_name == "Cargo.toml": + (w / manifest_name).write_text(f'[package]\nname = "{name}"\nversion = "{version}"\n') + elif manifest_name == "package.json": + (w / manifest_name).write_text(f'{{"name":"{name}","version":"{version}"}}') + elif manifest_name == "pyproject.toml": + (w / manifest_name).write_text(f'[project]\nname = "{name}"\nversion = "{version}"\n') + if tests_dir: + (w / "tests").mkdir() + (w / "tests" / "smoke.rs").write_text("// stub\n") + return tmp_path + + +def run_script(repo: Path, worker: str, base_ref: str = "main", + source_changed: list[str] | None = None) -> subprocess.CompletedProcess[str]: + src = json.dumps(source_changed or []) + return subprocess.run( + [sys.executable, str(SCRIPT), + "--worker", worker, "--base-ref", base_ref, "--source-changed", src], + capture_output=True, + text=True, + cwd=repo, + env=GIT_HERMETIC_ENV, + ) + + +def init_git(repo: Path) -> None: + def run(*args): + return subprocess.run(args, cwd=repo, check=True, env=GIT_HERMETIC_ENV) + run("git", "init", "-q", "-b", "main") + run("git", "config", "user.email", "t@e.com") + run("git", "config", "user.name", "T") + run("git", "add", ".") + run("git", "commit", "-q", "-m", "init") + + +class TestValidateWorker: + def test_happy_path_passes(self, tmp_path): + repo = make_worker(tmp_path, "smoke") + init_git(repo) + r = run_script(repo, "smoke", source_changed=["smoke"]) + assert r.returncode == 0, r.stderr + + def test_missing_readme_fails_in_strict_mode(self, tmp_path): + repo = make_worker(tmp_path, "smoke") + (repo / "smoke" / "README.md").unlink() + init_git(repo) + r = run_script(repo, "smoke", source_changed=["smoke"]) + assert r.returncode != 0 + assert "README" in r.stdout + r.stderr + + def test_missing_readme_passes_in_metadata_only_mode(self, tmp_path): + repo = make_worker(tmp_path, "smoke") + (repo / "smoke" / "README.md").unlink() + init_git(repo) + # source_changed=[] means metadata-only — README requirement is a notice. + r = run_script(repo, "smoke", source_changed=[]) + assert r.returncode == 0 + + def test_missing_tests_dir_fails_strict(self, tmp_path): + repo = make_worker(tmp_path, "smoke", tests_dir=False) + init_git(repo) + r = run_script(repo, "smoke", source_changed=["smoke"]) + assert r.returncode != 0 + assert "tests" in r.stdout + r.stderr + + def test_wrong_deploy_value_fails(self, tmp_path): + repo = make_worker(tmp_path, "smoke", deploy="binary") + init_git(repo) + meta = repo / "smoke" / "iii.worker.yaml" + meta.write_text(meta.read_text().replace("deploy: binary", "deploy: weird")) + subprocess.run(["git", "add", "."], cwd=repo, check=True, env=GIT_HERMETIC_ENV) + subprocess.run(["git", "commit", "-q", "-m", "bad"], cwd=repo, check=True, env=GIT_HERMETIC_ENV) + r = run_script(repo, "smoke", source_changed=["smoke"]) + assert r.returncode != 0 + assert "deploy" in r.stdout + r.stderr + + def test_missing_name_field_fails(self, tmp_path): + repo = make_worker(tmp_path, "smoke") + meta = repo / "smoke" / "iii.worker.yaml" + # Strip the `name:` line — _lib falls back to folder name, so the + # downstream m.name != worker check passes trivially. The gate must + # still reject manifests missing the required key. + meta.write_text("\n".join( + line for line in meta.read_text().splitlines() + if not line.startswith("name:") + ) + "\n") + init_git(repo) + r = run_script(repo, "smoke", source_changed=["smoke"]) + assert r.returncode != 0, r.stdout + r.stderr + assert "name" in r.stdout + r.stderr + + def test_version_must_be_strictly_greater_than_base(self, tmp_path): + repo = make_worker(tmp_path, "smoke", version="1.0.0") + init_git(repo) + # Touch source so worker is source_changed, but don't bump version. + (repo / "smoke" / "src.rs").write_text("// touch\n") + subprocess.run(["git", "add", "."], cwd=repo, check=True, env=GIT_HERMETIC_ENV) + subprocess.run(["git", "commit", "-q", "-m", "touch"], cwd=repo, check=True, env=GIT_HERMETIC_ENV) + r = run_script(repo, "smoke", base_ref="main~1", source_changed=["smoke"]) + assert r.returncode != 0 + assert "version" in r.stdout + r.stderr diff --git a/.github/scripts/validate_worker.py b/.github/scripts/validate_worker.py new file mode 100644 index 000000000..3683fe7c3 --- /dev/null +++ b/.github/scripts/validate_worker.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""Per-worker pr-checks validation. + +Enforces: + 1. README.md exists and is non-empty. + 2. iii.worker.yaml parses and has required fields + valid enum values. + 3. The manifest version on this ref is strictly greater than on --base-ref. + 4. tests/ exists and is non-empty. + +If `--worker` is not in `--source-changed`, requirements 1, 3, and 4 are +downgraded to GitHub Actions notices instead of hard errors. +""" +from __future__ import annotations + +import argparse +import json +import pathlib +import subprocess +import sys +import tempfile + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) +import _lib # noqa: E402 + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser() + p.add_argument("--worker", required=True, help="worker folder name") + p.add_argument("--base-ref", required=True, help="base branch ref, e.g. 'main'") + p.add_argument( + "--source-changed", required=True, + help="JSON array of workers that had non-metadata source changes", + ) + args = p.parse_args(argv) + + worker = args.worker + source_changed = set(json.loads(args.source_changed)) + strict = worker in source_changed + root = pathlib.Path(worker) + errs: list[str] = [] + + def hard(msg: str) -> None: + errs.append(msg) + + def soft(msg: str) -> None: + if strict: + errs.append(msg) + else: + print(f"::notice::{msg} (skipped: {worker} only changed metadata)") + + # 1. README.md present and non-empty + readme = root / "README.md" + if not readme.exists(): + soft(f"{worker}/README.md is missing") + elif readme.stat().st_size == 0: + soft(f"{worker}/README.md is empty") + + # 2. iii.worker.yaml — always strict + m = None + try: + m = _lib.read_iii_worker_yaml(root) + except FileNotFoundError: + hard(f"{worker}/iii.worker.yaml is missing") + except ValueError as e: + hard(f"{worker}/iii.worker.yaml: {e}") + + if m is not None: + # Use raw dict, not WorkerManifest attrs: _lib silently fills `name` + # with the folder name when the yaml omits it, so getattr(m, "name") + # would hide a missing key. + for key in ("name", "language", "deploy", "manifest"): + if not m.raw.get(key): + hard(f"{worker}/iii.worker.yaml is missing key: {key}") + if m.name != worker: + hard(f"{worker}/iii.worker.yaml name={m.name!r} does not match folder") + if m.deploy not in ("binary", "image"): + hard(f"{worker}/iii.worker.yaml deploy must be 'binary' or 'image'") + if m.language not in ("rust", "node", "python"): + hard( + f"{worker}/iii.worker.yaml language must be 'rust' | 'node' | 'python'" + ) + + # 3. Manifest version > base + if m is not None and m.manifest: + manifest_path = root / m.manifest + if not manifest_path.exists(): + hard(f"{worker}/{m.manifest} not found") + else: + try: + pr_ver = _lib.read_version(manifest_path) + except (ValueError, FileNotFoundError) as e: + hard(f"could not read version from {worker}/{m.manifest}: {e}") + pr_ver = None + if pr_ver is not None: + try: + base_blob = subprocess.check_output( + ["git", "show", f"{args.base_ref}:{worker}/{m.manifest}"], + text=True, + stderr=subprocess.DEVNULL, + ) + except subprocess.CalledProcessError: + base_blob = None + # Only enforce when base resolves to a commit distinct from + # HEAD. With a single-commit repo (e.g. brand-new branch on + # this PR), base == HEAD and "strictly greater" is impossible. + try: + base_sha = subprocess.check_output( + ["git", "rev-parse", args.base_ref], + text=True, + stderr=subprocess.DEVNULL, + ).strip() + head_sha = subprocess.check_output( + ["git", "rev-parse", "HEAD"], + text=True, + stderr=subprocess.DEVNULL, + ).strip() + except subprocess.CalledProcessError: + base_sha = head_sha = "" + if base_blob is not None and base_sha != head_sha: + with tempfile.TemporaryDirectory() as td: + tmp = pathlib.Path(td) / m.manifest + tmp.write_text(base_blob) + try: + base_ver = _lib.read_version(tmp) + except (ValueError, FileNotFoundError): + base_ver = None + if base_ver is not None and _lib.parse_semver(pr_ver) <= _lib.parse_semver(base_ver): + soft( + f"{worker}/{m.manifest} version {pr_ver} is not greater " + f"than base {base_ver}" + ) + + # 4. tests/ exists and is non-empty + tests_dir = root / "tests" + if not tests_dir.exists(): + soft(f"{worker}/tests/ is missing") + elif not any(tests_dir.iterdir()): + soft(f"{worker}/tests/ is empty") + + for e in errs: + print(f"::error::{e}") + return 1 if errs else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/_publish-registry.yml b/.github/workflows/_publish-registry.yml index 00bf60d02..9e4a8f556 100644 --- a/.github/workflows/_publish-registry.yml +++ b/.github/workflows/_publish-registry.yml @@ -111,30 +111,7 @@ jobs: REPO_URL: ${{ format('https://github.com/{0}', github.repository) }} run: | set -euo pipefail - mode=$(python3 - <<'PY' - import os - import yaml - - worker = os.environ["WORKER"] - deploy = os.environ.get("DEPLOY", "") - manifest_path = f"{worker}/iii.worker.yaml" - manifest = yaml.safe_load(open(manifest_path, encoding="utf-8").read()) or {} - scripts = manifest.get("scripts") or {} - runtime = manifest.get("runtime") or {} - has_scripts_start = bool(str(scripts.get("start") or "").strip()) - has_runtime = bool(runtime.get("kind") or runtime.get("language")) - language = str(manifest.get("language") or "").strip().lower() - - if deploy == "binary": - print("release-binary") - elif has_scripts_start or has_runtime: - print("iii-add") - elif language == "rust": - print("cargo-run") - else: - print("unsupported") - PY - ) + mode=$(python3 .github/scripts/manifest_version.py deploy-mode "$WORKER") case "$mode" in iii-add) @@ -209,12 +186,9 @@ jobs: - name: Assert worker interface was collected run: | - python3 - <<'PY' - import json - data = json.load(open("worker-interface.json")) - if not data.get("functions"): - raise SystemExit("no worker functions were collected") - PY + set -euo pipefail + python3 .github/scripts/collect_worker_interface.py \ + --assert-non-empty --assert-file worker-interface.json - name: Resolve binary artifacts if: inputs.deploy == 'binary' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b6797369..e8decaf1d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,17 @@ env: CARGO_TERM_COLOR: always jobs: + scripts-tests: + name: .github/scripts tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - run: pip install --quiet pytest pyyaml + - run: pytest .github/scripts/tests/ -v + # ────────────────────────────────────────────────────────────── # Discover: enumerate changed workers, read their iii.worker.yaml, # bucket them per language for the matrix jobs below. @@ -45,97 +56,9 @@ jobs: env: BASE: ${{ steps.base.outputs.ref }} run: | - python3 - <<'PY' - import json, os, pathlib, subprocess, sys - - base = os.environ["BASE"] - try: - changed = subprocess.check_output( - ["git", "diff", "--name-only", f"{base}...HEAD"], text=True - ).splitlines() - except subprocess.CalledProcessError: - changed = subprocess.check_output( - ["git", "diff", "--name-only", "HEAD~1...HEAD"], text=True - ).splitlines() - - repo_root = pathlib.Path(".").resolve() - ignore = {".git", ".github", "registry", "target", "node_modules"} - worker_dirs = sorted( - p.name - for p in repo_root.iterdir() - if p.is_dir() and not p.name.startswith(".") and p.name not in ignore - and (p / "iii.worker.yaml").exists() - ) - - # Files inside a worker dir that don't count as a "source" change. - # If a worker only touched these, version-bump and tests/ gates are - # downgraded to notices in pr-checks. - import fnmatch - metadata_globs = ( - "iii.worker.yaml", - "README.md", - "AGENTS.md", - "AGENTS-*.md", - "Cargo.lock", - "Cargo.toml", - ) - - def is_metadata(rel: str) -> bool: - return any(fnmatch.fnmatch(rel, g) for g in metadata_globs) - - changed_workers = set() - worker_files: dict[str, list[str]] = {} - vscode_changed = False - for f in changed: - parts = f.split("/", 1) - if len(parts) < 2: - continue - top, rel = parts[0], parts[1] - if top == "iii-lsp-vscode": - vscode_changed = True - continue - if top in worker_dirs: - changed_workers.add(top) - worker_files.setdefault(top, []).append(rel) - - source_changed = sorted( - w for w in changed_workers - if any(not is_metadata(rel) for rel in worker_files.get(w, [])) - ) - - rust, node, python = [], [], [] - for w in sorted(changed_workers): - meta_path = pathlib.Path(w) / "iii.worker.yaml" - lang = None - for line in meta_path.read_text().splitlines(): - s = line.strip() - if s.startswith("language:"): - lang = s.split(":", 1)[1].strip() - break - if lang == "rust": - rust.append(w) - elif lang == "node": - node.append(w) - elif lang == "python": - python.append(w) - else: - print(f"::warning::{w} has unknown language={lang}") - - out = open(os.environ["GITHUB_OUTPUT"], "a") - out.write(f"rust={json.dumps(rust)}\n") - out.write(f"node={json.dumps(node)}\n") - out.write(f"python={json.dumps(python)}\n") - out.write(f"all={json.dumps(sorted(changed_workers))}\n") - out.write(f"source_changed={json.dumps(source_changed)}\n") - out.write(f"vscode_changed={'true' if vscode_changed else 'false'}\n") - out.write(f"any={'true' if (changed_workers or vscode_changed) else 'false'}\n") - out.close() - - print( - f"::notice::changed rust={rust} node={node} python={python} " - f"vscode={vscode_changed} source_changed={source_changed}" - ) - PY + set -euo pipefail + pip install --quiet pyyaml + python3 .github/scripts/discover_changed_workers.py --base "$BASE" # ────────────────────────────────────────────────────────────── # PR gates: readme present, manifest version > main, iii.worker.yaml valid. @@ -160,134 +83,18 @@ jobs: BASE_REF: ${{ github.event.pull_request.base.ref || github.event.repository.default_branch || 'main' }} run: git fetch --no-tags --depth=1 origin "$BASE_REF" - - name: Install pyyaml - run: pip install --quiet pyyaml - - name: Validate worker env: WORKER: ${{ matrix.worker }} BASE_REF: ${{ github.event.pull_request.base.ref || github.event.repository.default_branch || 'main' }} SOURCE_CHANGED: ${{ needs.discover.outputs.source_changed }} run: | - python3 - <<'PY' - import json, os, pathlib, re, subprocess, sys, yaml - - worker = os.environ["WORKER"] - base = os.environ["BASE_REF"] - source_changed = set(json.loads(os.environ.get("SOURCE_CHANGED") or "[]")) - strict = worker in source_changed - root = pathlib.Path(worker) - errs = [] - - def soft(msg: str) -> None: - # Strict for workers with real source changes; notice otherwise. - if strict: - errs.append(msg) - else: - print(f"::notice::{msg} (skipped: {worker} only changed metadata)") - - # 1. README.md present and non-empty - readme = root / "README.md" - if not readme.exists(): - errs.append(f"{worker}/README.md is missing") - elif readme.stat().st_size == 0: - errs.append(f"{worker}/README.md is empty") - - # 2. iii.worker.yaml parses and has required fields - meta_path = root / "iii.worker.yaml" - if not meta_path.exists(): - errs.append(f"{worker}/iii.worker.yaml is missing") - meta = {} - else: - meta = yaml.safe_load(meta_path.read_text()) or {} - for key in ("name", "language", "deploy", "manifest"): - if not meta.get(key): - errs.append(f"{worker}/iii.worker.yaml is missing key: {key}") - if meta.get("name") and meta["name"] != worker: - errs.append( - f"{worker}/iii.worker.yaml name={meta['name']!r} does not match folder" - ) - if meta.get("deploy") not in ("binary", "image"): - errs.append( - f"{worker}/iii.worker.yaml deploy must be 'binary' or 'image'" - ) - if meta.get("language") not in ("rust", "node", "python"): - errs.append( - f"{worker}/iii.worker.yaml language must be 'rust' | 'node' | 'python'" - ) - - # 3. manifest version on PR > version on base - def read_version(text: str, kind: str) -> str | None: - if kind == "Cargo.toml": - for line in text.splitlines(): - m = re.match(r'^version\s*=\s*"([^"]+)"', line.strip()) - if m: - return m.group(1) - elif kind == "package.json": - import json - return json.loads(text).get("version") - elif kind == "pyproject.toml": - for line in text.splitlines(): - m = re.match(r'^version\s*=\s*"([^"]+)"', line.strip()) - if m: - return m.group(1) - return None - - def parse_semver(v: str) -> tuple: - # Strip prerelease/build for ordering: ..[-pre] - core, _, pre = v.partition("-") - parts = [int(x) for x in core.split(".")] - while len(parts) < 3: - parts.append(0) - # No pre = greater than any pre at the same core - return (tuple(parts), 1 if not pre else 0, pre) - - manifest_name = meta.get("manifest") - if manifest_name: - manifest_path = root / manifest_name - if not manifest_path.exists(): - errs.append(f"{worker}/{manifest_name} not found") - else: - pr_ver = read_version(manifest_path.read_text(), manifest_name) - if not pr_ver: - errs.append(f"could not read version from {worker}/{manifest_name}") - else: - try: - base_blob = subprocess.check_output( - ["git", "show", f"origin/{base}:{worker}/{manifest_name}"], - text=True, - stderr=subprocess.DEVNULL, - ) - base_ver = read_version(base_blob, manifest_name) - except subprocess.CalledProcessError: - base_ver = None # New worker on this PR - - if base_ver is None: - print( - f"::notice::{worker}: new worker on this PR " - f"(no base version), pr={pr_ver}" - ) - elif parse_semver(pr_ver) < parse_semver(base_ver): - soft( - f"{worker}/{manifest_name} version must be no less than base: " - f"pr={pr_ver} base={base_ver}" - ) - else: - print( - f"::notice::{worker}: version {base_ver} -> {pr_ver}" - ) - - # 4. tests dir present and non-empty - tests_dir = root / "tests" - if not tests_dir.exists() or not any(tests_dir.iterdir()): - soft(f"{worker}/tests/ is missing or empty") - - if errs: - for e in errs: - print(f"::error::{e}") - sys.exit(1) - print(f"::notice::{worker}: all PR checks passed") - PY + set -euo pipefail + pip install --quiet pyyaml + python3 .github/scripts/validate_worker.py \ + --worker "$WORKER" \ + --base-ref "origin/$BASE_REF" \ + --source-changed "$SOURCE_CHANGED" # ────────────────────────────────────────────────────────────── # Rust per-worker lint + test diff --git a/.github/workflows/create-tag.yml b/.github/workflows/create-tag.yml index 8f24f2ae1..7aef73358 100644 --- a/.github/workflows/create-tag.yml +++ b/.github/workflows/create-tag.yml @@ -93,15 +93,15 @@ jobs: exit 1 fi - - name: Discover manifest type + - name: Discover manifest id: meta env: WORKER: ${{ inputs.worker }} run: | - # iii-lsp-vscode does not have iii.worker.yaml (special VS Code extension flow) + set -euo pipefail + # iii-lsp-vscode has no iii.worker.yaml (special VS Code extension flow). if [[ "$WORKER" == "iii-lsp-vscode" ]]; then echo "manifest=package.json" >> "$GITHUB_OUTPUT" - echo "manifest_type=node" >> "$GITHUB_OUTPUT" exit 0 fi if [[ ! -f "$WORKER/iii.worker.yaml" ]]; then @@ -114,134 +114,32 @@ jobs: exit 1 fi echo "manifest=$MANIFEST" >> "$GITHUB_OUTPUT" - case "$MANIFEST" in - Cargo.toml) echo "manifest_type=cargo" >> "$GITHUB_OUTPUT" ;; - package.json) echo "manifest_type=node" >> "$GITHUB_OUTPUT" ;; - pyproject.toml) echo "manifest_type=python" >> "$GITHUB_OUTPUT" ;; - *) echo "::error::Unsupported manifest: $MANIFEST"; exit 1 ;; - esac - - name: Calculate next version + - name: Calculate and write next version id: versions env: WORKER: ${{ inputs.worker }} BUMP: ${{ inputs.bump }} - MANIFEST_TYPE: ${{ steps.meta.outputs.manifest_type }} MANIFEST: ${{ steps.meta.outputs.manifest }} run: | - read_version() { - case "$MANIFEST_TYPE" in - cargo) - grep '^version = ' "$1" | head -n1 | cut -d'"' -f2 - ;; - node) - jq -r '.version' "$1" - ;; - python) - # PEP 621 [project] version = "..." - python3 -c " - import re, sys - text = open(sys.argv[1]).read() - in_proj = False - for line in text.splitlines(): - s = line.strip() - if s.startswith('['): - in_proj = (s == '[project]') - continue - if in_proj: - m = re.match(r'version\s*=\s*\"([^\"]+)\"', s) - if m: - print(m.group(1)) - sys.exit(0) - " "$1" - ;; - esac - } - - bump_version() { - local current="$1" - local base="${current%%-*}" - IFS='.' read -r major minor patch <<< "$base" - case "$BUMP" in - major) major=$((major + 1)); minor=0; patch=0 ;; - minor) minor=$((minor + 1)); patch=0 ;; - patch) patch=$((patch + 1)) ;; - esac - echo "${major}.${minor}.${patch}" - } - - current=$(read_version "$WORKER/$MANIFEST") - if [[ -z "$current" ]]; then - echo "::error::Could not read version from $WORKER/$MANIFEST" - exit 1 - fi - new_ver=$(bump_version "$current") + set -euo pipefail + current=$(python3 .github/scripts/manifest_version.py read "$WORKER/$MANIFEST") + new_ver=$(python3 .github/scripts/manifest_version.py bump "$WORKER/$MANIFEST" --kind "$BUMP") echo "current=$current" >> "$GITHUB_OUTPUT" echo "version=$new_ver" >> "$GITHUB_OUTPUT" echo "tag=${WORKER}/v${new_ver}" >> "$GITHUB_OUTPUT" echo "::notice::${WORKER}: ${current} -> ${new_ver}" - - name: Update manifest - env: - WORKER: ${{ inputs.worker }} - MANIFEST_TYPE: ${{ steps.meta.outputs.manifest_type }} - MANIFEST: ${{ steps.meta.outputs.manifest }} - NEW_VERSION: ${{ steps.versions.outputs.version }} - run: | - path="$WORKER/$MANIFEST" - case "$MANIFEST_TYPE" in - cargo) - sed -i "0,/^version = \".*\"/{s/^version = \".*\"/version = \"${NEW_VERSION}\"/}" "$path" - ;; - node) - jq --arg v "$NEW_VERSION" '.version = $v' "$path" > "${path}.tmp" - mv "${path}.tmp" "$path" - ;; - python) - python3 - <