docs(perf): the 1.84x acc0 gap is stale — re-measured it is 1.12x at t=1/4/8

crates/onnx-runtime-ep-cpu/benches/acc0_gap_matrix.py

@@ -52,13 +52,52 @@
 Separating the two requires re-running both arms interleaved on a quiet host,
 which is what `wait_quiet` and `competing_load` below exist to guarantee. Until
 that has been done this cell has no published ratio.
+
+Two preconditions added 2026-08-23, both of which this script previously
+assumed rather than checked
+---------------------------------------------------------------------------
+* **The two arms must get the same machine.** `ONNX_GENAI_CPU_DECODE_THREADS=w`
+  confines the *whole native process* to `w` CPUs (it prints so). Pinning ORT
+  to all 16 even CPUs, as this script did at every thread count, gave ORT a
+  16-core machine while native had `w` -- more L3 and more memory controllers,
+  on a workload that is bandwidth-bound by construction. See `native_pin`.
+* **The native width must be non-vacuous.** The realized width is read back
+  from the binary and a cell is refused unless it equals the request. Timings
+  cannot detect this: a sweep that silently runs one width in every row looks
+  perfectly stable, because it is -- it is the same configuration each time.
+
+Distribution, orientation and token budget
+------------------------------------------
+* `--launches N` repeats the whole matrix `N` times. Every arm is already a
+  fresh process, so this distributes over **process launches**, which is the
+  larger source of variance and the one `--reps` cannot see: reps live inside
+  one process and inherit whatever state it started in. The per-width summary
+  at the end medians the gap over *paired* cells -- native and ORT run seconds
+  apart inside one launch, so pairing cancels drift that a ratio of two
+  independently-medianed columns would keep -- and prints the cell count and
+  the full range beside it, because a median of two cells and a median of six
+  are not the same claim.
+* `--tokens` accepts either a single count or a per-width map,
+  `1:64,4:192,8:384`. A fixed count is not width-neutral: at `t=1` a token
+  costs ~16x what it costs at `t=16`, so a flat budget spends the most wall
+  time on the narrow widths and still gives them the fewest samples relative
+  to their variance.
+* Two ratio columns are printed, and this is deliberate. `ratio` is
+  **native/ORT** (below 1.0 means we are behind); `gap` is its reciprocal, the
+  "Nx behind ORT" figure the documents quote. Printing both means neither
+  orientation has to be inferred from which side of 1.0 a number happens to
+  sit -- which is exactly the kind of inference that produced a published
+  reciprocal once already.
 """
 import argparse
+import collections
 import json
 import os
 import re
+import statistics
 import subprocess
 import sys
+import threading
 import time
 
 HERE = os.path.dirname(os.path.abspath(__file__))
@@ -71,7 +110,27 @@
 }
 
 # Even CPUs are distinct physical cores on this SMT host (siblings adjacent).
-PIN = ",".join(str(c) for c in range(0, 32, 2))
+EVEN = list(range(0, 32, 2))
+PIN = ",".join(str(c) for c in EVEN)
+
+
+def native_pin(threads):
+    """The CPUs the *native* arm actually gets at this thread count.
+
+    `ONNX_GENAI_CPU_DECODE_THREADS=w` does not merely size a pool: it confines
+    the whole process to `w` CPUs, and says so on stderr --
+
+        CPU decode budget 4 confined the process to 4 CPUs [0, 2, 4, 6]
+
+    So pinning both arms to all 16 even CPUs, as this script used to, hands ORT
+    a 16-core machine while native has `w`. That is not a thread-count
+    comparison, it is a machine-size comparison, and at every `w < 16` it
+    flatters ORT with more L3 and more memory controllers than native can
+    reach. It went unnoticed because the only published gap cell was `t=1`,
+    where ORT's `intra_op_num_threads=1` means one thread regardless, and
+    because at `t=16` the two pins coincide exactly.
+    """
+    return ",".join(str(c) for c in EVEN[:threads])
 
 
 def weight_bytes(model, block):
@@ -107,17 +166,37 @@ def native(binary, model, block, acc, threads, sessions, tokens, reps, extra=Non
     if extra:
         env.update(extra)
     r = sh(f"taskset -c {PIN} {binary}", env)
-    for line in r.stdout.splitlines():
+    steady, width_line = None, None
+    for line in r.stdout.splitlines() + r.stderr.splitlines():
         if line.strip().startswith("steady"):
             f = line.split()
-            return {"ms_token": float(f[1]), "p90": float(f[2]),
-                    "tps": float(f[3]), "spread": float(f[4])}
-    sys.stderr.write(r.stdout + r.stderr)
-    raise RuntimeError("native arm produced no steady row")
-
-
-def ort(model, block, acc, threads, sessions, tokens, reps):
-    cmd = (f"taskset -c {PIN} python3 ort_matmulnbits_baseline.py "
+            steady = {"ms_token": float(f[1]), "p90": float(f[2]),
+                      "tps": float(f[3]), "spread": float(f[4])}
+        if line.strip().startswith("decode_width"):
+            width_line = line.strip()
+    if steady is None:
+        sys.stderr.write(r.stdout + r.stderr)
+        raise RuntimeError("native arm produced no steady row")
+    # Non-vacuity, checked rather than assumed. A width sweep whose rows all
+    # report the same width is the failure that produced the retracted
+    # "t=1 == t=2" claim, and it is invisible in the timings themselves --
+    # they look stable, they are just all the same configuration. The binary
+    # reads the realized width back out of the pool, so ask it.
+    if width_line is None:
+        raise RuntimeError("native arm did not report decode_width")
+    steady["decode_width"] = width_line
+    if not width_line.endswith("as_requested"):
+        # Width 1 legitimately builds no pool at all (`allowed.len() == 1`
+        # declines), so `path=flat` there is correct, not a reduction. Any
+        # other mismatch invalidates the row's label.
+        if not (threads == 1 and "path=flat" in width_line):
+            raise RuntimeError(f"native width vacuous: {width_line}")
+    return steady
+
+
+def ort(model, block, acc, threads, sessions, tokens, reps, pin=None):
+    pin = pin or native_pin(threads)
+    cmd = (f"taskset -c {pin} python3 ort_matmulnbits_baseline.py "
            f"--model {model} --block {block} --accuracy {acc} --threads {threads} "
            f"--tokens {tokens} --reps {reps} --sessions {sessions}")
     r = subprocess.run(cmd, shell=True, capture_output=True, text=True, cwd=BENCH,
@@ -126,7 +205,7 @@ def ort(model, block, acc, threads, sessions, tokens, reps):
     if not m:
         sys.stderr.write(r.stdout + r.stderr)
         raise RuntimeError("ORT arm produced no throughput")
-    return {"tps": float(m.group(1)), "spread": float(m.group(2))}
+    return {"tps": float(m.group(1)), "spread": float(m.group(2)), "pin": pin}
 
 
 def competing_load():
@@ -181,6 +260,61 @@ def wait_quiet(threshold=3.0, limit=900):
     return os.getloadavg()[0], competing_load()
 
 
+class LoadWatch:
+    """Peak runnable count *during* an arm, not merely before it.
+
+    `wait_quiet` is a pre-check, and a pre-check cannot see a competitor that
+    starts after the cell does. That is not hypothetical either: a sibling
+    agent's `cargo test` began mid-matrix and four cells that had passed the
+    pre-check were measured against a saturated box, at spreads of 20-63%.
+
+    Two details matter. The instantaneous **runnable** count (field 4 of
+    `/proc/loadavg`) is used rather than load average, which is a 1-minute
+    exponential average that both lags a job that just started and stays high
+    long after one ends. And the threshold scales with the thread count,
+    because at `t` threads our own arm legitimately contributes ~`t` runnable
+    threads -- a constant like "runnable > 4" would refuse every honest cell
+    at `t >= 8`.
+
+    It is also worth being explicit that this is a *necessary*, not a
+    sufficient, condition. Ten launches at width 16 split into a fast and a
+    slow mode 1.8x apart in wall time while burning identical CPU-seconds
+    (14.4 vs 14.1 CPU-s per wall-s): the affected threads were never
+    descheduled, they just retired fewer instructions per cycle. No load,
+    CPU-efficiency or context-switch guard can see that.
+    """
+
+    def __init__(self, period=1.0):
+        self.period = period
+        self.peak = 0
+        self._stop = threading.Event()
+        self._thread = None
+
+    @staticmethod
+    def runnable():
+        try:
+            with open("/proc/loadavg") as f:
+                return int(f.read().split()[3].split("/")[0])
+        except Exception:
+            return -1
+
+    def _loop(self):
+        while not self._stop.is_set():
+            self.peak = max(self.peak, self.runnable())
+            self._stop.wait(self.period)
+
+    def __enter__(self):
+        self.peak = self.runnable()
+        self._thread = threading.Thread(target=self._loop, daemon=True)
+        self._thread.start()
+        return self
+
+    def __exit__(self, *exc):
+        self._stop.set()
+        self._thread.join(timeout=2 * self.period)
+        return False
+
+
 def main():
     ap = argparse.ArgumentParser()
     ap.add_argument("--binary", required=True)
@@ -189,22 +323,65 @@ def main():
     ap.add_argument("--sessions", default="1,2,4")
     ap.add_argument("--block", type=int, default=32)
     ap.add_argument("--acc", type=int, default=0)
-    ap.add_argument("--tokens", type=int, default=24)
+    ap.add_argument("--tokens", default="24",
+                    help="measured tokens per session. Either a single number, "
+                         "or a per-width map `1:64,4:192,8:384` -- a fixed token "
+                         "count makes a t=1 cell 16x longer in wall time than a "
+                         "t=16 cell, so the narrow widths get the fewest samples "
+                         "exactly where the variance is worst")
     ap.add_argument("--reps", type=int, default=3)
+    ap.add_argument("--launches", type=int, default=1,
+                    help="independent repetitions of the whole matrix. Every "
+                         "arm is a fresh process, so this distributes over "
+                         "process launches rather than over reps inside one "
+                         "process -- the launch-to-launch spread is the larger "
+                         "of the two and is invisible to --reps")
     ap.add_argument("--out", default="acc0_gap.json")
     ap.add_argument("--aa", action="store_true", help="per-cell interleaved A/A")
+    ap.add_argument("--slack", type=int, default=4,
+                    help="peak runnable count above `--threads` that still "
+                         "counts as a quiet host during a cell")
+    ap.add_argument("--ort-pin", choices=["matched", "wide", "both"],
+                    default="matched",
+                    help="CPUs for the ORT arm: the same `t` the native process "
+                         "confines itself to (matched, the only comparison), all "
+                         "16 physical cores (wide, what this script used to do), "
+                         "or both so the asymmetry is quantified")
     args = ap.parse_args()
 
+    # `--tokens 192` or `--tokens 1:64,4:192,8:384`.
+    threads = [int(x) for x in args.threads.split(",")]
+    if ":" in args.tokens:
+        tokens_for = {}
+        for pair in args.tokens.split(","):
+            key, value = pair.split(":")
+            tokens_for[int(key)] = int(value)
+        # A per-width map that is missing a width would otherwise die with a
+        # bare KeyError partway through a matrix that has already spent
+        # minutes in `wait_quiet`. Fail before any measurement starts.
+        missing = sorted(set(threads) - set(tokens_for))
+        if missing:
+            ap.error(
+                f"--tokens map has no entry for width(s) {missing}; it must "
+                f"cover every --threads value. Got {sorted(tokens_for)}, need "
+                f"{sorted(threads)}. Use a scalar (--tokens 192) for one "
+                f"token count at every width.")
+    else:
+        fixed = int(args.tokens)
+        tokens_for = collections.defaultdict(lambda: fixed)
+
     rows = []
-    hdr = (f"{'model':>6} {'t':>3} {'s':>2} {'nat_tps':>9} {'nat_sp%':>8} "
-           f"{'ort_tps':>9} {'ort_sp%':>8} {'ratio':>7} {'nat_GB/s':>9} "
+    hdr = (f"{'model':>6} {'L':>2} {'t':>3} {'s':>2} {'nat_tps':>9} {'nat_sp%':>8} "
+           f"{'ort_tps':>9} {'ort_sp%':>8} {'ratio':>7} {'gap':>7} {'nat_GB/s':>9} "
            f"{'ort_GB/s':>9} {'A/A':>6}")
     print(hdr)
     print("-" * len(hdr))
 
-    for model in args.models.split(","):
+    for launch in range(args.launches):
+      for model in args.models.split(","):
         wb = weight_bytes(model, args.block)
-        for t in [int(x) for x in args.threads.split(",")]:
+        for t in threads:
+            tokens = tokens_for[t]
             for s in [int(x) for x in args.sessions.split(",")]:
                 load, busy = wait_quiet()
                 if busy:
@@ -214,34 +391,94 @@ def main():
                     for pid, pcpu, cmd in busy[:3]:
                         sys.stderr.write(f"    pid={pid} cpu={pcpu:.0f}% {cmd}\n")
                 # Interleaved: native, ORT, native again (the A/A partner).
-                a1 = native(args.binary, model, args.block, args.acc, t, s,
-                            args.tokens, args.reps)
-                o = ort(model, args.block, args.acc, t, s, args.tokens, args.reps)
-                aa = ""
-                if args.aa:
-                    a2 = native(args.binary, model, args.block, args.acc, t, s,
-                                args.tokens, args.reps)
-                    aa = f"{a2['tps'] / a1['tps']:.3f}"
+                # Every arm runs inside a LoadWatch so a competitor that
+                # arrives mid-cell is caught, not just one that was already
+                # there when `wait_quiet` returned.
+                with LoadWatch() as watch:
+                    a1 = native(args.binary, model, args.block, args.acc, t, s,
+                                tokens, args.reps)
+                    o = None
+                    o_wide = None
+                    if args.ort_pin in ("matched", "both"):
+                        o = ort(model, args.block, args.acc, t, s, tokens,
+                                args.reps, pin=native_pin(t))
+                    if args.ort_pin in ("wide", "both"):
+                        o_wide = ort(model, args.block, args.acc, t, s,
+                                     tokens, args.reps, pin=PIN)
+                    if o is None:
+                        o = o_wide
+                    aa = ""
+                    if args.aa:
+                        a2 = native(args.binary, model, args.block, args.acc, t,
+                                    s, tokens, args.reps)
+                        aa = f"{a2['tps'] / a1['tps']:.3f}"
+                if watch.peak > t + args.slack:
+                    sys.stderr.write(
+                        f"WARNING {model} t={t} s={s}: peak runnable "
+                        f"{watch.peak} > {t} + {args.slack} during the cell; "
+                        f"cell is UNTRUSTED\n")
+                    busy = busy or [("-", 0.0, f"peak runnable {watch.peak}")]
                 nat_bw = a1["tps"] * wb / 1e9
                 ort_bw = o["tps"] * wb / 1e9
-                row = {"model": model, "threads": t, "sessions": s,
+                row = {"model": model, "launch": launch, "threads": t,
+                       "sessions": s, "tokens": tokens,
                        "native": a1, "ort": o, "ratio": a1["tps"] / o["tps"],
+                       # `ratio` is native/ORT (below 1 means we are behind);
+                       # `gap` is its reciprocal, the "Nx behind ORT" the docs
+                       # quote. Both are printed so neither orientation has to
+                       # be inferred from which side of 1.0 a number sits.
+                       "gap": o["tps"] / a1["tps"],
+                       "ort_wide": o_wide,
+                       "ratio_wide": (a1["tps"] / o_wide["tps"]) if o_wide else None,
                        "native_gbs": nat_bw, "ort_gbs": ort_bw,
                        "weight_bytes": wb, "loadavg_at_start": load,
+                       "peak_runnable": watch.peak,
                        "trusted": not busy,
                        "competitors": [c[2] for c in busy],
                        "aa": float(aa) if aa else None}
                 rows.append(row)
                 flag = "" if not busy else "  !CONTENDED"
-                print(f"{model:>6} {t:>3} {s:>2} {a1['tps']:>9.1f} "
+                wide = ""
+                if o_wide is not None and row["ratio_wide"] is not None:
+                    wide = (f"  wide_ort={o_wide['tps']:.1f} "
+                            f"ratio_wide={row['ratio_wide']:.3f}")
+                print(f"{model:>6} {launch:>2} {t:>3} {s:>2} {a1['tps']:>9.1f} "
                       f"{a1['spread']:>8.1f} {o['tps']:>9.1f} {o['spread']:>8.1f} "
-                      f"{row['ratio']:>7.3f} {nat_bw:>9.1f} {ort_bw:>9.1f} {aa:>6}"
-                      f"{flag}")
+                      f"{row['ratio']:>7.3f} {row['gap']:>7.3f} "
+                      f"{nat_bw:>9.1f} {ort_bw:>9.1f} {aa:>6}"
+                      f"{wide}{flag}")
                 sys.stdout.flush()
                 with open(os.path.join(HERE, args.out), "w") as f:
                     json.dump(rows, f, indent=1)
 
     print()
+    # Per-width summary over launches. The gap is medianed over *paired* cells
+    # -- native and ORT run seconds apart inside one launch, so pairing cancels
+    # drift that a ratio of two independently-medianed columns would keep. The
+    # cell count and the full range are printed beside it because a median of
+    # two cells and a median of six are not the same claim, and a range that
+    # straddles the A/A null means the cell did not resolve the gap at all.
+    if args.launches > 1 or args.aa:
+        print(f"{'model':>6} {'t':>3} {'s':>2} {'gap_med':>8} {'gap_min':>8} "
+              f"{'gap_max':>8} {'cells':>6} {'aa_min':>7} {'aa_max':>7}")
+        for model in args.models.split(","):
+            for t in threads:
+                for s in [int(x) for x in args.sessions.split(",")]:
+                    kept = [r for r in rows
+                            if r["model"] == model and r["threads"] == t
+                            and r["sessions"] == s and r["trusted"]]
+                    if not kept:
+                        print(f"{model:>6} {t:>3} {s:>2} "
+                              f"{'no trusted cells':>40}")
+                        continue
+                    gaps = sorted(r["gap"] for r in kept)
+                    aas = sorted(r["aa"] for r in kept if r["aa"] is not None)
+                    print(f"{model:>6} {t:>3} {s:>2} "
+                          f"{statistics.median(gaps):>8.3f} {gaps[0]:>8.3f} "
+                          f"{gaps[-1]:>8.3f} {len(gaps):>6} "
+                          f"{(f'{aas[0]:.3f}' if aas else '-'):>7} "
+                          f"{(f'{aas[-1]:.3f}' if aas else '-'):>7}")
+        print()
     print("weight bytes per token per session:",
           {m: f"{weight_bytes(m, args.block) / 1e6:.1f} MB" for m in args.models.split(",")})