diff --git a/.github/workflows/unsloth-pin-preflight.yml b/.github/workflows/unsloth-pin-preflight.yml index 9e2f4d406df9..3f3318904e8e 100644 --- a/.github/workflows/unsloth-pin-preflight.yml +++ b/.github/workflows/unsloth-pin-preflight.yml @@ -167,6 +167,33 @@ jobs: if ! python3 ../scripts/unsloth/merge_checks.py --root . ; then PROBLEMS="${PROBLEMS}- the merged tree builds, but \`scripts/unsloth/merge_checks.py\` found a resolution that is silently wrong. See the run log for file and line.\n" fi + + # The other half of that question. merge_checks.py asks whether the + # tree contains something wrong; this asks whether it still contains + # what each pin carries. A pin that has rotted into a no-op, or an + # arch registration a resolution quietly dropped, is invisible to + # every other check here and to the compiler. + if ! python3 ../scripts/unsloth/pin_contract.py --root . --base "$BASE" \ + --pr-set ../scripts/unsloth/pr-set.json --report "${RUNNER_TEMP}/pin_contract.json" ; then + PROBLEMS="${PROBLEMS}- the merged tree is missing code a pin carries. See the run log for the pin and file.\n" + fi + NOTES="$(jq -r '.notices[]?' "${RUNNER_TEMP}/pin_contract.json" 2>/dev/null || true)" + if [ -n "$NOTES" ]; then + PROBLEMS="${PROBLEMS}- pins upstream has taken over, safe to delete from \`pr-set.json\`:\n\n\`\`\`\n${NOTES}\n\`\`\`\n" + fi + + # A clean merge is not a compiling tree. On 09-03 ggml-org#27754 + # merged with no conflicts at all and did not compile: upstream had + # added a parameter to build_attn_mha and the pin's new + # build_attn_sparse still called the old signature. Nothing above + # can see that. CPU only and the `llama` target only, which is where + # that translation unit lives; 59s cold at -j4 with no ccache. + if ! cmake -B "${RUNNER_TEMP}/gate" -DCMAKE_BUILD_TYPE=Release \ + -DGGML_CUDA=OFF -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_SERVER=OFF \ + -DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_TOOLS=OFF -DLLAMA_CURL=OFF > /dev/null \ + || ! cmake --build "${RUNNER_TEMP}/gate" --target llama -j "$(nproc)" ; then + PROBLEMS="${PROBLEMS}- the pins merge cleanly and the merged tree does not compile. See the run log for the file and line; this is the failure that only shows up in the CUDA leg once the nightly has fanned out.\n" + fi fi if [ -z "$PROBLEMS" ]; then diff --git a/.github/workflows/unsloth-pr-set-lint.yml b/.github/workflows/unsloth-pr-set-lint.yml index 06ab4f9813ee..c4c5b1771a56 100644 --- a/.github/workflows/unsloth-pr-set-lint.yml +++ b/.github/workflows/unsloth-pr-set-lint.yml @@ -119,7 +119,8 @@ jobs: scripts/unsloth/test_pin_merge.py \ scripts/unsloth/test_merge_checks.py \ scripts/unsloth/test_sync_deletes.py \ - scripts/unsloth/test_carry_vintage.py; do + scripts/unsloth/test_carry_vintage.py \ + scripts/unsloth/test_pin_contract.py; do echo "::group::$t" python3 "$t" || fail=1 echo "::endgroup::" diff --git a/.github/workflows/unsloth-prebuilt.yml b/.github/workflows/unsloth-prebuilt.yml index 8ef65ec3f357..834ff5449093 100644 --- a/.github/workflows/unsloth-prebuilt.yml +++ b/.github/workflows/unsloth-prebuilt.yml @@ -282,12 +282,10 @@ jobs: # .github/workflows, which upstream history routinely does). if [ "$EXISTS" != "true" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ]; then git remote add upstream https://github.com/ggml-org/llama.cpp.git - # Checking out the upstream base replaces this working tree with - # upstream's, which has no scripts/unsloth/. Keep the resolver - # somewhere the checkout cannot take away. - ADDITIVE_MERGE="${RUNNER_TEMP}/additive_merge.py" - cp scripts/unsloth/additive_merge.py "$ADDITIVE_MERGE" - cp scripts/unsloth/merge_checks.py "${RUNNER_TEMP}/merge_checks.py" + # The upstream checkout below takes scripts/unsloth/ away. Copy the + # whole dir out, not file by file: see the note above the step. + cp -r scripts/unsloth "${RUNNER_TEMP}/us" + ADDITIVE_MERGE="${RUNNER_TEMP}/us/additive_merge.py" if [ "$(jq length <<<"$PRS")" != 0 ]; then # Merges need a merge-base, so unshallow first. if [ "$(git rev-parse --is-shallow-repository)" = "true" ]; then @@ -448,15 +446,53 @@ jobs: # A bad pin resolution can still build fine, so it must be caught before the source artifact ships. See merge_checks.py. # Its own step, not more script in `resolve`: GitHub caps one workflow string at 21000 chars and that step is near it. See check_workflow_scalars.py. + # That is also why `resolve` copies all of scripts/unsloth/ to ${RUNNER_TEMP}/us in one line rather than one cp per script: every check added + # here would otherwise cost another line inside the capped block, and going over silently disables the whole workflow. - name: Check the merged tree for silently wrong resolutions if: ${{ env.MERGED_PINS == '1' }} run: | set -euo pipefail - if ! python3 "${RUNNER_TEMP}/merge_checks.py" --root . ; then + if ! python3 "${RUNNER_TEMP}/us/merge_checks.py" --root . ; then echo "::error::the pinned PRs merged, but merge_checks.py found a resolution that is silently wrong; see the log for file and line" >&2 exit 1 fi + # merge_checks.py asserts the ABSENCE of two known-bad shapes. This asserts the PRESENCE of what each pin carries, which is a different question and + # the one that goes unanswered when a pin rots into a no-op or a resolution quietly drops an arch registration. Free, so it runs before the compile gate. + - name: Check every pin still contributes what it carries + if: ${{ env.MERGED_PINS == '1' }} + # Through env, never interpolated into the script: `prs` carries PR + # titles, which are third-party text, and `${{ }}` pastes them into the + # shell source before bash ever sees it. + env: + PRS: ${{ steps.r.outputs.prs }} + BASE: ${{ steps.r.outputs.base }} + run: | + set -euo pipefail + if ! python3 "${RUNNER_TEMP}/us/pin_contract.py" --root . --base "$BASE" \ + --prs-json "$PRS" --report "${RUNNER_TEMP}/pin_contract.json" ; then + echo "::error::the pinned PRs merged, but the merged tree is missing code a pin carries; see the log for the pin and file" >&2 + exit 1 + fi + + # The gap this closes, observed 09-03: ggml-org#27754 merged with zero conflicts and did not compile, because upstream had added a parameter to + # build_attn_mha and the pin's new build_attn_sparse still called the old signature. Nothing before this point can see that, and without it the release + # dies in the CUDA leg after the 38-job fan-out. CPU only: a cold `llama` build took 59s at -j4 with no ccache, against 20-60 minutes for a CUDA build. + # mtmd is in the gate because `llama` alone is not enough: observed 09-04, ggml-org#25731 built `llama` clean while tools/mtmd did not compile at all, + # upstream having made mtmd_image_preprocessor::preprocess const while the pin's Inkling subclass stayed non-const, so it overrode nothing and the + # vision and audio towers were abstract. Every vision pin lands in mtmd, so a gate that skips it cannot see the whole class. + - name: Compile gate (CPU, llama and mtmd targets) + if: ${{ env.MERGED_PINS == '1' }} + run: | + set -euo pipefail + cmake -B "${RUNNER_TEMP}/gate" -DCMAKE_BUILD_TYPE=Release \ + -DGGML_CUDA=OFF -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_SERVER=OFF \ + -DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_TOOLS=ON -DLLAMA_CURL=OFF > /dev/null + if ! cmake --build "${RUNNER_TEMP}/gate" --target llama mtmd -j "$(nproc)" ; then + echo "::error::the pinned PRs merged cleanly and the merged tree does not compile; fix or drop the pin rather than letting the build matrix find this" >&2 + exit 1 + fi + # The stamped source tree (every build): every build child extracts this # instead of cloning, and assemble ships it as the release's source-tarball # asset, so a source build reproduces the same fingerprinted binary. Only diff --git a/scripts/unsloth/pin_contract.py b/scripts/unsloth/pin_contract.py new file mode 100644 index 000000000000..bf07d58408e3 --- /dev/null +++ b/scripts/unsloth/pin_contract.py @@ -0,0 +1,361 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +"""Assert the merged tree still contains what each pin carries. + +The nightly proves the pins MERGED. That is not the same as proving they are in +the release, and the difference has cost us three outages: + + * ggml-org#28133 was squash-merged upstream. The pinned commit stopped being + an ancestor of the base tag, so the merge was not a no-op -- it re-applied + code the base already had. It happened to conflict, which is the only + reason anybody noticed. A pin in that state that merges quietly ships + nothing and nothing says so. + * an additive resolution can keep the wrong side, or a later pin can land on + top of an earlier one, and the arch registration the pin exists for is + simply not in the tree any more. It still compiles. + * a pin can rot into contributing nothing at all while its entry stays in + pr-set.json for weeks. + +So: derive from each pin's OWN diff what it puts in the tree, then check the +merged tree still has it. Nothing to maintain -- the expectation comes out of +the commit, so a repin regenerates it. + +Four assertions per pin, cheapest first: + + symbols every LLM_ARCH_/GGML_OP_/PROJECTOR_TYPE_/... name the pin + introduces, in each file it introduces it to. Per FILE, not per + tree: LLM_ARCH_INKLING surviving in llama-arch.h while its arm was + dropped from llama-model.cpp is exactly the failure being looked + for, and a tree-wide grep passes it. + files every file the pin adds still exists. + lines every non-comment code line the pin adds is still in that file. + Catches a resolution that ate a hunk without touching a symbol. + redundancy + a pin whose added lines the BASE TAG already has is work upstream + took. Reported, never fatal -- upstream landing a feature overnight + must not stop that night's release. + +What this CANNOT do, stated plainly so nobody reads more into a pass than is +there: the contract is re-derived from the pin, so it can only ever prove the +MERGE did not lose something. A regression inside the pin itself regenerates a +smaller contract that passes. Proving a feature works is feature_matrix.py's +job, and it needs a build. +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from collections import defaultdict +from pathlib import Path + +PIN_RE = re.compile( + r"^https://github\.com/([^/]+)/llama\.cpp/pull/(\d+)/commits/([0-9a-f]{40})/?$" +) + +# Identifier families that name a FEATURE. Deliberately not "every new symbol": +# a helper function renamed by a later upstream commit is not a lost feature, +# but a missing LLM_ARCH_ entry always is. These are the tables that decide +# whether an architecture, an op, a projector or a quant type exists at all. +SYMBOL_FAMILIES = ( + "LLM_ARCH_", "LLM_TENSOR_", "LLM_KV_", "LLM_TYPE_", + "PROJECTOR_TYPE_", "GGML_OP_", "GGML_TYPE_", "LLAMA_FTYPE_", +) +SYMBOL_RE = re.compile(r"\b(?:" + "|".join(SYMBOL_FAMILIES) + r")[A-Z0-9_]+\b") + +# The subset that names a whole feature rather than one of its tensors. Used +# only to keep --emit readable; the check itself uses all of SYMBOL_FAMILIES. +HEADLINE = ("LLM_ARCH_", "GGML_OP_", "GGML_TYPE_", "PROJECTOR_TYPE_", "LLAMA_FTYPE_") + +# A line worth tracking for survival. Comments and short punctuation drift with +# every reformat and would make the check noise; a substantial code line does +# not move on its own. +TRIVIAL_RE = re.compile(r"^\s*(?://|/\*|\*|\*/|#\s|$)") +MIN_LINE = 12 + +# Comments are stripped before anything is read off a line. A pin that merely +# NAMES an arch in a comment has not registered it, and holding the comment's +# wording as a contract fails the moment upstream rewords it. Observed on +# unslothai#70, whose comment mentions GGML_OP_SSM_SCAN to explain why it does +# NOT use it. +COMMENT_RE = re.compile(r"//.*$|/\*.*?\*/|(? str: + r = subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True) + if check and r.returncode != 0: + raise RuntimeError(f"git {' '.join(args[:3])}...: {r.stderr.strip()[:300]}") + return r.stdout + + +def blob(rev: str, path: str, cwd: Path) -> str | None: + """The tree entry as "mode oid", or None if the rev has no such path. + + Lifted from carry_vintage.py, mode included for the same reason: a change + that only chmods a file it otherwise took verbatim has identical content, + and an oid-only comparison would call that "contributed nothing". + """ + r = subprocess.run(["git", "ls-tree", "--full-tree", "-z", rev, "--", path], + cwd=cwd, capture_output=True, text=True) + if r.returncode != 0 or not r.stdout.strip(): + return None + mode, _type, oid = r.stdout.split("\0")[0].split("\t", 1)[0].split() + return f"{mode} {oid}" + + +def load_effective(prs_json: str) -> list[dict]: + """The pin list the resolve step actually merged. + + Not the same as pr-set.json: resolve drops an optional pin once its PR is + no longer open, and re-reading the file would then check a pin that is not + in the tree and report it missing. The step already has the effective list + as an output, so take it rather than recomputing the filter here and + getting it subtly different. + """ + return [{"url": p.get("url", ""), "src": p["repo"].split("/")[0], + "num": int(p["number"]), "sha": p["sha"], "required": True} + for p in json.loads(prs_json)] + + +def load_pins(pr_set: Path) -> list[dict]: + data = json.loads(pr_set.read_text()) + pins = [] + for entry in data["prs"]: + url = entry if isinstance(entry, str) else entry["url"] + m = PIN_RE.match(url) + if not m: + raise SystemExit(f"malformed pin: {url}") + pins.append({"url": url, "src": m.group(1), "num": int(m.group(2)), + "sha": m.group(3), + "required": True if isinstance(entry, str) + else entry.get("required", True)}) + return pins + + +def derive(pin: dict, base: str, cwd: Path) -> dict: + """What this pin puts in the tree, read off its own diff against the base. + + The fork point is merge-base(pin, base), not the pin's parent: a pin that + has already had the base merged into it (which repin.py and every carry + branch produce) would otherwise look like it contributed all of upstream. + """ + fork = git(["merge-base", pin["sha"], base], cwd).strip() + diff = git(["diff", "--no-renames", fork, pin["sha"]], cwd) + + symbols: dict[str, set[str]] = defaultdict(set) + lines: dict[str, list[str]] = defaultdict(list) + cur = None + for ln in diff.split("\n"): + if ln.startswith("+++ b/"): + cur = ln[6:] + elif ln.startswith("+++ "): + cur = None # /dev/null: a deletion + elif cur and ln.startswith("+") and not ln.startswith("+++"): + text = ln[1:] + stripped = text.strip() + if len(stripped) >= MIN_LINE and not TRIVIAL_RE.match(stripped): + lines[cur].append(stripped) + code = COMMENT_RE.sub("", text).strip() + if code: + symbols[cur].update(SYMBOL_RE.findall(code)) + + # Only symbols the base does not ALREADY have in that file are evidence of + # this pin. Upstream naming an arch in a file the pin also touches is not + # something the pin is owed. + new_symbols: dict[str, list[str]] = {} + for path, names in symbols.items(): + fresh = sorted(n for n in names + if n not in git(["show", f"{base}:{path}"], cwd, check=False)) + if fresh: + new_symbols[path] = fresh + + status = git(["diff", "--name-status", "--no-renames", fork, pin["sha"]], cwd) + added, owned = [], [] + for ln in status.split("\n"): + if not ln.strip(): + continue + code, path = ln.split("\t", 1) + owned.append(path) + if code.startswith("A"): + added.append(path) + + return { + "fork": fork, + "symbols": new_symbols, + "added_files": added, + "owned_paths": owned, + "lines": {p: v for p, v in lines.items() + if not p.endswith(SKIP_SUFFIXES)}, + } + + +def redundancy(contract: dict, base: str, cwd: Path) -> tuple[int, int]: + """How much of what this pin adds the base tag already has. + + This is the pr-set.json retirement rule, mechanised: "delete the entry once + a base tag carries the work". Upstream almost always SQUASHES, so the + pinned commit never becomes an ancestor and no ancestry test will ever say + the work landed; comparing the text is the only thing that can. + + Measured on the real set at b10775, the separation is not close: the pin + that upstream had already absorbed (ggml-org#28133) scored 99%, and the + highest live pin scored 33%. + """ + total = hit = 0 + for path, wanted in contract["lines"].items(): + text = git(["show", f"{base}:{path}"], cwd, check=False) + total += len(wanted) + hit += sum(1 for w in wanted if w in text) + return hit, total + + +def check(pin: dict, contract: dict, root: Path, base: str, cwd: Path, + threshold: float) -> list[str]: + problems = [] + + for path, names in sorted(contract["symbols"].items()): + target = root / path + text = target.read_text(errors="replace") if target.is_file() else "" + for name in names: + if name not in text: + problems.append( + f"{name} is missing from {path}; the pin adds it there and " + "the merged tree does not have it") + + for path in contract["added_files"]: + if not (root / path).exists(): + problems.append(f"{path} is added by the pin and missing from the merged tree") + + for path, wanted in sorted(contract["lines"].items()): + target = root / path + if not target.is_file(): + continue # already reported, or a deletion + text = target.read_text(errors="replace") + lost = [w for w in wanted if w not in text] + if not lost: + continue + kept = len(wanted) - len(lost) + ratio = kept / len(wanted) + if ratio < threshold: + problems.append( + f"{path} kept {kept}/{len(wanted)} of the lines this pin adds " + f"({ratio:.0%}); first missing: {lost[0][:90]}") + + return problems + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--root", default=".", help="the merged tree to check") + src = ap.add_mutually_exclusive_group(required=True) + src.add_argument("--pr-set", help="scripts/unsloth/pr-set.json") + src.add_argument("--prs-json", help="the resolve step's `prs` output: the pins it " + "actually merged, optional ones already dropped") + ap.add_argument("--base", required=True, help="upstream base tag the mix was built on") + ap.add_argument("--git-dir", help="repo the pin commits are reachable from " + "(default: --root)") + ap.add_argument("--threshold", type=float, default=1.0, + help="fraction of a pin's added lines that must survive per file") + ap.add_argument("--redundant-at", type=float, default=0.95, + help="report a pin whose added lines the base tag already has " + "at this fraction or more (never fatal)") + ap.add_argument("--report", help="write a JSON report here") + ap.add_argument("--emit", action="store_true", + help="print the derived contracts and check nothing") + args = ap.parse_args() + + root = Path(args.root).resolve() + cwd = Path(args.git_dir).resolve() if args.git_dir else root + pins = (load_pins(Path(args.pr_set)) if args.pr_set + else load_effective(args.prs_json)) + + report: dict = {"base": args.base, "ok": False, "pins": [], "notices": []} + failed = 0 + notices: list[str] = [] + + for pin in pins: + name = f"{pin['src']}#{pin['num']}" + try: + contract = derive(pin, args.base, cwd) + except RuntimeError as e: + report["pins"].append({"pin": name, "sha": pin["sha"], "problems": [str(e)]}) + print(f"ERROR {name}: {e}", file=sys.stderr) + failed += 1 + continue + + entry = { + "pin": name, + "sha": pin["sha"], + "fork": contract["fork"], + "symbols": contract["symbols"], + "added_files": contract["added_files"], + "line_count": sum(len(v) for v in contract["lines"].values()), + "problems": [], + } + + if args.emit: + report["pins"].append(entry) + # Only the families that NAME a feature are printed. Every symbol + # is still checked; a new file legitimately contributes a hundred + # LLM_TENSOR_ names and listing them buries the one that matters. + sym = sorted({s for v in contract["symbols"].values() for s in v + if s.startswith(HEADLINE)}) + print(f"{name:>18} {entry['line_count']:>5} lines, " + f"{len(contract['added_files'])} new files, symbols: " + f"{', '.join(sym) if sym else '-'}") + continue + + problems = check(pin, contract, root, args.base, cwd, args.threshold) + hit, total = redundancy(contract, args.base, cwd) + entry["problems"] = problems + entry["redundant_lines"] = [hit, total] + + if total and hit / total >= args.redundant_at: + note = (f"the base tag already has {hit}/{total} ({hit / total:.0%}) of the " + "lines this pin adds; upstream has taken this work and the entry " + "should be deleted from pr-set.json") + entry["notices"] = [note] + notices.append(f"{name}: {note}") + + report["pins"].append(entry) + if problems: + failed += 1 + print(f"FAIL {name}", file=sys.stderr) + for p in problems: + print(f" {p}", file=sys.stderr) + else: + print(f"ok {name}: {len(contract['symbols'])} file(s) with new symbols, " + f"{entry['line_count']} line(s) accounted for") + + report["ok"] = failed == 0 or args.emit + report["notices"] = notices + if args.report: + Path(args.report).write_text(json.dumps(report, indent=2)) + if args.emit: + return 0 + + # Notices after the verdict lines, never mixed into them: "upstream took + # this, drop the entry" is housekeeping and must not read as a failure. + for n in notices: + print(f"note {n}") + if failed: + print(f"\n{failed} pin(s) are not intact in the merged tree", file=sys.stderr) + return 1 + print(f"\nall {len(pins)} pins are intact in the merged tree" + + (f", {len(notices)} can be retired" if notices else "")) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/unsloth/test_pin_contract.py b/scripts/unsloth/test_pin_contract.py new file mode 100644 index 000000000000..3c7b58d62d54 --- /dev/null +++ b/scripts/unsloth/test_pin_contract.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +"""Tests for pin_contract.py. Run: python3 scripts/unsloth/test_pin_contract.py + +Every case builds a real repository with a real base tag, a real pin branch and +a real merge, then damages the merged tree the way a bad resolution damages it. +A hand-written fixture would only prove the checker reads its own output format. +""" +import json +import subprocess +import sys +import tempfile +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parent / "pin_contract.py" +FAILS = [] + + +def check(name, cond, extra=""): + print(f"{'PASS' if cond else 'FAIL'} {name}" + (f" :: {extra}" if extra and not cond else "")) + if not cond: + FAILS.append(name) + + +def git(repo, *args): + return subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@t", *args], + cwd=repo, capture_output=True, text=True) + + +ARCH_H_BASE = """\ +enum llm_arch { + LLM_ARCH_LLAMA, + LLM_ARCH_UNKNOWN, +}; +""" +MODEL_CPP_BASE = """\ +void build_model(llm_arch arch) { + switch (arch) { + case LLM_ARCH_LLAMA: + build_llama(); + break; + } +} +""" + + +def make_repo(): + """A base tag `b1` plus a pin branch adding one architecture, merged.""" + d = Path(tempfile.mkdtemp(prefix="pc_")) + git(d, "init", "-q", "-b", "main") + (d / "src").mkdir() + (d / "src" / "llama-arch.h").write_text(ARCH_H_BASE) + (d / "src" / "llama-model.cpp").write_text(MODEL_CPP_BASE) + git(d, "add", "-A"); git(d, "commit", "-qm", "base") + git(d, "tag", "b1") + + git(d, "checkout", "-qb", "pin") + (d / "src" / "llama-arch.h").write_text( + ARCH_H_BASE.replace(" LLM_ARCH_UNKNOWN,", + " LLM_ARCH_INKLING,\n LLM_ARCH_UNKNOWN,")) + (d / "src" / "llama-model.cpp").write_text( + MODEL_CPP_BASE.replace(" }\n}", + " case LLM_ARCH_INKLING:\n" + " build_inkling_with_banded_bias();\n" + " break;\n }\n}")) + (d / "src" / "inkling.cpp").write_text( + "void build_inkling_with_banded_bias() { do_the_banded_thing(); }\n") + git(d, "add", "-A"); git(d, "commit", "-qm", "add inkling") + sha = git(d, "rev-parse", "HEAD").stdout.strip() + + git(d, "checkout", "-q", "main") + git(d, "merge", "-q", "--no-ff", "--no-edit", "-m", "merge pin", "pin") + + # Outside the work tree on purpose: a test that commits after this would + # otherwise sweep the pin file into the pin's own diff. + pr_set = Path(tempfile.mkdtemp(prefix="pcset_")) / "pr-set.json" + pr_set.write_text(json.dumps({"prs": [ + f"https://github.com/unslothai/llama.cpp/pull/1/commits/{sha}"]})) + return d, pr_set, sha + + +def run(repo, pr_set, *extra): + rep = repo / "r.json" + p = subprocess.run([sys.executable, str(SCRIPT), "--root", str(repo), + "--pr-set", str(pr_set), "--base", "b1", + "--report", str(rep), *extra], + capture_output=True, text=True) + return p.returncode, (json.loads(rep.read_text()) if rep.exists() else {}), p.stderr + + +# --- 1. an intact merge passes ------------------------------------------- +repo, pr_set, sha = make_repo() +rc, rep, err = run(repo, pr_set) +check("intact merge passes", rc == 0 and rep["ok"], err) +check("intact merge finds the new arch", + "LLM_ARCH_INKLING" in json.dumps(rep["pins"][0]["symbols"]), rep) +check("intact merge reports no notices", rep["notices"] == [], rep) + +# --- 2. the arm is dropped from ONE file: a tree-wide grep would pass ------ +# The real shape: LLM_ARCH_INKLING survives in the enum and the dispatch arm +# that makes it do anything is gone. +repo, pr_set, sha = make_repo() +p = repo / "src" / "llama-model.cpp" +p.write_text(MODEL_CPP_BASE) +rc, rep, err = run(repo, pr_set) +check("a dropped dispatch arm fails", rc == 1 and not rep["ok"], err) +check("the failure names the file, not just the symbol", + any("llama-model.cpp" in x for x in rep["pins"][0]["problems"]), rep) +check("the enum copy of the symbol does not rescue it", + "LLM_ARCH_INKLING" in (repo / "src" / "llama-arch.h").read_text()) + +# --- 3. a whole added file goes missing ---------------------------------- +repo, pr_set, sha = make_repo() +(repo / "src" / "inkling.cpp").unlink() +rc, rep, err = run(repo, pr_set) +check("a missing added file fails", rc == 1, err) +check("the failure names the file", + any("inkling.cpp" in x for x in rep["pins"][0]["problems"]), rep) + +# --- 4. a hunk is eaten without touching a symbol ------------------------- +repo, pr_set, sha = make_repo() +(repo / "src" / "inkling.cpp").write_text( + "void build_inkling_with_banded_bias() { }\n") # body gone, name kept +rc, rep, err = run(repo, pr_set) +check("an eaten body fails on line survival", rc == 1, err) +check("line survival names what went missing", + any("do_the_banded_thing" in x for x in rep["pins"][0]["problems"]), rep) + +# --- 5. redundancy: the base already has everything the pin adds ---------- +# Built the way it happens for real: upstream lands the same work, so the base +# tag has it and the pin is not an ancestor of anything. +d = Path(tempfile.mkdtemp(prefix="pc_")) +git(d, "init", "-q", "-b", "main") +(d / "src").mkdir() +(d / "src" / "f.cpp").write_text("int a() { return 1; }\n") +git(d, "add", "-A"); git(d, "commit", "-qm", "root") +git(d, "checkout", "-qb", "pin") +(d / "src" / "f.cpp").write_text( + "int a() { return 1; }\nint the_new_helper() { return 42; }\n") +git(d, "add", "-A"); git(d, "commit", "-qm", "pin work") +sha5 = git(d, "rev-parse", "HEAD").stdout.strip() +git(d, "checkout", "-q", "main") +(d / "src" / "f.cpp").write_text( # upstream squashed the same work + "int a() { return 1; }\nint the_new_helper() { return 42; }\n") +git(d, "add", "-A"); git(d, "commit", "-qm", "upstream squash of the same change") +git(d, "tag", "b1") +ps5 = Path(tempfile.mkdtemp(prefix="pcset_")) / "pr-set.json" +ps5.write_text(json.dumps({"prs": [ + f"https://github.com/unslothai/llama.cpp/pull/1/commits/{sha5}"]})) +rc, rep, err = run(d, ps5) +check("a pin the base already carries is reported", rep["notices"], rep) +check("redundancy says to delete the entry", + "deleted from pr-set.json" in " ".join(rep["notices"]), rep) +check("redundancy is NOT fatal", rc == 0, err) + +# --- 6. --emit checks nothing --------------------------------------------- +repo, pr_set, sha = make_repo() +(repo / "src" / "inkling.cpp").unlink() +rc, rep, err = run(repo, pr_set, "--emit") +check("--emit does not check", rc == 0 and rep["ok"], err) +check("--emit still derives the contract", + rep["pins"][0]["added_files"] == ["src/inkling.cpp"], rep) + +# --- 7. a comment is not a contract --------------------------------------- +# unslothai#70 has a comment naming GGML_OP_SSM_SCAN to say it does NOT use it. +# Holding comment wording would fail the moment upstream rewords it. +repo, pr_set, sha = make_repo() +git(repo, "checkout", "-q", "pin") +(repo / "src" / "note.cpp").write_text( + "// unlike LLM_ARCH_MISTRAL this one does its own thing\nint g() { return 0; }\n") +git(repo, "add", "-A"); git(repo, "commit", "-qm", "comment") +sha7 = git(repo, "rev-parse", "HEAD").stdout.strip() +git(repo, "checkout", "-q", "main") +git(repo, "merge", "-q", "--no-ff", "--no-edit", "-m", "m2", "pin") +(repo / "src" / "note.cpp").write_text( # comment reworded, code kept + "// this one does its own thing\nint g() { return 0; }\n") +pr_set.write_text(json.dumps({"prs": [ + f"https://github.com/unslothai/llama.cpp/pull/1/commits/{sha7}"]})) +rc, rep, err = run(repo, pr_set) +check("a reworded comment does not fail the pin", rc == 0, err) +check("no symbol was harvested from the comment", + "LLM_ARCH_MISTRAL" not in json.dumps(rep["pins"][0]["symbols"]), rep) + +print() +print(f"{len(FAILS)} failure(s)" + (": " + ", ".join(FAILS) if FAILS else "")) +sys.exit(1 if FAILS else 0)