From 2228b40cff1db585fec8625a700bf9f6293a474a Mon Sep 17 00:00:00 2001 From: Yingyi Huang Date: Sun, 1 Nov 2026 01:18:58 -0800 Subject: [PATCH 1/3] perf(cake_gdn_train_kcp): one stream context per entry body, cached launch resolution The forward and backward bodies of the non-deterministic KCP package now run in one tvm_ffi.use_torch_stream() scope (cake_backend.launch_scope) instead of entering a context per launch; a launch outside every scope, or after a stream switch inside one, still enters its own. The scope is entered inside a CUDA graph capture when there is one, so it binds the capture stream. The record, grid, test flag and merge columns of each launch key are resolved once and kept with the context's tables (_Tables.launches); every launch still rebuilds its public values and appends its record. The launches, grids and streams are unchanged: host time only. --- .../experimental/cake_gdn_train_kcp/README.md | 5 + .../cake_gdn_train_kcp/cake_backend.py | 55 +++- .../cake_gdn_train_kcp/cake_kcp.py | 94 ++++++- tests/experimental/test_cake_gdn_train_kcp.py | 245 ++++++++++++++++++ .../test_cake_gdn_train_kcp_cp1.py | 81 ++++++ 5 files changed, 471 insertions(+), 9 deletions(-) diff --git a/flashinfer/experimental/cake_gdn_train_kcp/README.md b/flashinfer/experimental/cake_gdn_train_kcp/README.md index 0587abbcb36..485614df117 100644 --- a/flashinfer/experimental/cake_gdn_train_kcp/README.md +++ b/flashinfer/experimental/cake_gdn_train_kcp/README.md @@ -173,6 +173,11 @@ The packed GDN training routes come in four combinations: Gradient accumulation (`.backward()` twice) accumulates. * No cooperative launch anywhere in the package; every program is a plain stream-ordered launch. +* Host launch path: the record, grid and merge columns of a launch are + resolved once per launch key and kept with the context's tables; each + forward or backward body, eager or captured, binds the caller's current + stream for tvm-ffi once (`tvm_ffi.use_torch_stream()`), and a launch enters + its own context only outside such a body or after a stream switch inside it. ### Partition and context diff --git a/flashinfer/experimental/cake_gdn_train_kcp/cake_backend.py b/flashinfer/experimental/cake_gdn_train_kcp/cake_backend.py index f7b8abc3204..f9998d2d73b 100644 --- a/flashinfer/experimental/cake_gdn_train_kcp/cake_backend.py +++ b/flashinfer/experimental/cake_gdn_train_kcp/cake_backend.py @@ -40,16 +40,22 @@ parameters onto the package's logical vocabulary (``cake_jit.LOGICAL_TENSORS`` / ``LOGICAL_SCALARS``), the grid comes from the host plan (``cake_jit.launch_grid``) and the kernel runs on the current torch stream. +A launch enters ``tvm_ffi.use_torch_stream()`` itself unless an enclosing +:func:`launch_scope` (one per package entry: the forward or backward body) +already bound torch's current stream on this thread; a launch whose current +stream is not the bound one takes its own context, as without a scope. Every launch is a plain stream-ordered launch: nothing cooperative, no PDL. Nothing here synchronizes, reads device memory or allocates. """ from __future__ import annotations +import contextlib import functools import hashlib import json -from collections.abc import Mapping +import threading +from collections.abc import Iterator, Mapping from pathlib import Path from typing import Any @@ -64,6 +70,7 @@ "arch_for_capability", "device_arch", "is_prebuilt", + "launch_scope", "library_name", "load", "load_test", @@ -273,6 +280,45 @@ def test_specs_for_arch(arch: str) -> list[Any]: # Launching one record +# The stream a launch scope bound on this thread, ``(device index, raw handle)``; tvm-ffi's environment stream is +# per thread too. +_SCOPE = threading.local() + + +def _current_stream() -> tuple[int, int]: + """``(device index, raw handle)`` of torch's current stream on the current device: what + ``tvm_ffi.use_torch_stream()`` binds.""" + import torch + + index = torch.cuda.current_device() + raw = getattr(torch._C, "_cuda_getCurrentRawStream", None) + if raw is not None: + return index, int(raw(index)) + return index, int(torch.cuda.current_stream(index).cuda_stream) + + +@contextlib.contextmanager +def launch_scope() -> Iterator[None]: + """One ``tvm_ffi.use_torch_stream()`` context for the launches of a package entry. + + It binds torch's current stream as this thread's tvm-ffi environment stream once; :func:`run_module` launches + inside the scope skip their own context while torch's current stream is still the bound one, and enter it as + before otherwise (a stream switch inside the scope, a launch outside every scope). A scope inside a scope of + the same stream adds nothing; one on another stream (a CUDA graph capture inside the scope) binds its own stream + until it exits. The kernels run on the same stream either way.""" + stream = _current_stream() + outer = getattr(_SCOPE, "stream", None) + if outer == stream: + yield + return + with tvm_ffi.use_torch_stream(): + _SCOPE.stream = stream + try: + yield + finally: + _SCOPE.stream = outer + + @functools.cache def _argument_plan(name: str) -> tuple[tuple[str | None, int], ...]: """The record's plan as ``(logical name, -1)`` per kernel parameter and ``(None, axis)`` per grid argument, in @@ -322,5 +368,10 @@ def run_module( ) from None if module is None: module = load(name, device_arch(device_index)) + entry = getattr(module, found["ffi_entry"]) + bound = getattr(_SCOPE, "stream", None) + if bound is not None and bound == _current_stream(): + entry(*args) # the enclosing launch scope bound this stream + return with tvm_ffi.use_torch_stream(): - getattr(module, found["ffi_entry"])(*args) + entry(*args) diff --git a/flashinfer/experimental/cake_gdn_train_kcp/cake_kcp.py b/flashinfer/experimental/cake_gdn_train_kcp/cake_kcp.py index c2de0e20615..902bb953f79 100644 --- a/flashinfer/experimental/cake_gdn_train_kcp/cake_kcp.py +++ b/flashinfer/experimental/cake_gdn_train_kcp/cake_kcp.py @@ -1070,6 +1070,11 @@ def begin_call(self, call: CallKcp) -> None: def end_call(self, call: CallKcp) -> None: """Hook at the end of a call.""" + def launch_scope(self, call: CallKcp) -> Any: + """The context manager the forward / backward body of ``call`` runs in (its launches, eagerly or inside a + capture), or ``None`` (the default): no scope.""" + return None + @staticmethod def transpose_tiles( *, @@ -1195,6 +1200,52 @@ def resolve_launch( return str(record), values, grid, is_test +def _launch_key(name: str, args: dict[str, Any], cols_per_cta: int) -> tuple: + """Every input :func:`resolve_launch` reads besides the launch's tensors: the program, the static knobs, the grid + rule's arguments (in the call site's order), the walk-form inputs, the merge's columns and the caller's grid x.""" + grid_args = args.get("_grid_args") + return ( + name, + args.get("_precision_mode"), + args.get("_transfer_walk"), + tuple(grid_args.items()) if grid_args else None, + args.get("_sm_count"), + args.get("_fwdh_vh"), + args.get("_dhu_vh"), + args.get("ncols"), + args.get("cols_per_cta"), + args.get("_grid"), + int(cols_per_cta), + ) + + +def _resolve_cached( + name: str, args: dict[str, Any], *, cols_per_cta: int = 8 +) -> tuple[str, dict[str, Any], tuple[int, int, int], bool]: + """:func:`resolve_launch` through the launch cache of the call's tables (``_Tables.launches``, one per context + plan): the record, the grid, the test flag and the merge's ``cols_per_cta`` of a launch key are resolved once; + every launch rebuilds its public values and appends ``(record, grid)`` to the call's records as + :func:`resolve_launch` does.""" + call = args.get("_call") + cache = getattr(getattr(call, "tb", None), "launches", None) + if cache is None: + return resolve_launch(name, args, cols_per_cta=cols_per_cta) + key = _launch_key(name, args, cols_per_cta) + hit = cache.get(key) + if hit is None: + record, values, grid, is_test = resolve_launch( + name, args, cols_per_cta=cols_per_cta + ) + cache[key] = (record, grid, is_test, values.get("cols_per_cta")) + return record, values, grid, is_test + record, grid, is_test, cols = hit + values = _public_args(args) + if cols is not None: + values["cols_per_cta"] = cols + call.records.append((record, grid)) + return record, values, grid, is_test + + class KernelBackend(Backend): """The shipped backend: the exported programs through the package's launch glue (:mod:`.cake_backend` ``run_module(record, values, *, grid, device_index, module=None)``, imported at the first launch so this module @@ -1250,7 +1301,7 @@ def run(self, name: str, args: dict[str, Any]) -> None: **args, "_grid_args": {**grid_args, "arch": be.device_arch(device_index)}, } - record, values, grid, is_test = resolve_launch( + record, values, grid, is_test = _resolve_cached( name, args, cols_per_cta=self.cols_per_cta ) module = None @@ -1266,6 +1317,12 @@ def run(self, name: str, args: dict[str, Any]) -> None: record, values, grid=grid, device_index=device_index, module=module ) + def launch_scope(self, call: CallKcp) -> Any: + """The launch glue's stream scope (``cake_backend.launch_scope``: one ``tvm_ffi.use_torch_stream()`` context + for the body's launches instead of one per launch); ``None`` when the glue has no scope.""" + scope = getattr(self.launcher(), "launch_scope", None) + return None if scope is None else scope() + class FakeBackend(Backend): """The torch emulation of the program ABI (``backend="fake"``; the package's :mod:`.cake_fake_backend`, imported at @@ -1290,7 +1347,7 @@ def module(self) -> Any: def run(self, name: str, args: dict[str, Any]) -> None: mod = self.module() call: CallKcp = args["_call"] - record, values, grid, _is_test = resolve_launch(name, args) + record, values, grid, _is_test = _resolve_cached(name, args) run_module = getattr(mod, "run_module", None) if ( run_module is not None @@ -1500,6 +1557,9 @@ def __init__( self, ctx: Any, HV: int, *, fused_walks: int = 1, P: int | None = None ) -> None: self.ctx = ctx + # launch key -> (record, grid, is_test, merge cols_per_cta): the launch cache of _resolve_cached (host data of + # this plan, filled at the first launch of each key) + self.launches: dict[tuple, tuple] = {} self.H = H = int(HV) # every pool, split and chain is per VALUE head self.fused_walks = int( fused_walks @@ -4254,6 +4314,17 @@ def chunk_gdn_fwd_kcp( ) +def _in_launch_scope(call: CallKcp, body: Any, *args: Any, **kw: Any) -> Any: + """``body(*args, **kw)`` -- a forward or backward body, eagerly or inside a capture -- in the backend's launch + scope of ``call`` (:meth:`Backend.launch_scope`), entered where the body's launches run (inside the capture + when there is one), so the scope binds their stream; without a scope the body runs as it is.""" + scope = call.backend.launch_scope(call) + if scope is None: + return body(*args, **kw) + with scope: + return body(*args, **kw) + + def _fwd_entry( q: torch.Tensor, k: torch.Tensor, @@ -4317,8 +4388,13 @@ def _fwd_entry( "fwd", entry, call, - lambda: _fwd_body( - call, save=save, autograd_node=autograd_node, executable=executable + lambda: _in_launch_scope( + call, + _fwd_body, + call, + save=save, + autograd_node=autograd_node, + executable=executable, ), ) if res is not None: @@ -4370,7 +4446,9 @@ def _fwd_entry( call.device, ) call.scratch["recompute_handle"] = handle_mode - out, saved = _fwd_body( + out, saved = _in_launch_scope( + call, + _fwd_body, call, save=save, hand_off=hand_off and call.T_r > 0, @@ -4725,7 +4803,9 @@ def _bwd_entry( if saved is not None: _check_saved(saved, call) if entry is not None and entry.seen > 0 and entry.disabled is None: - res = _graph_capture("bwd", entry, call, lambda: _bwd_body(call, saved)) + res = _graph_capture( + "bwd", entry, call, lambda: _in_launch_scope(call, _bwd_body, call, saved) + ) if res is not None: entry.result = (res, batched) return tuple(_ranked(t.detach(), batched) for t in res) @@ -4745,7 +4825,7 @@ def _bwd_entry( ) if saved is not None: _check_saved(saved, call) - grads = _bwd_body(call, saved) + grads = _in_launch_scope(call, _bwd_body, call, saved) _record("bwd", call) if opts.graph != "off": _graph_eager("bwd", entry) diff --git a/tests/experimental/test_cake_gdn_train_kcp.py b/tests/experimental/test_cake_gdn_train_kcp.py index 6e2a1eb7323..1440f5e0cb6 100644 --- a/tests/experimental/test_cake_gdn_train_kcp.py +++ b/tests/experimental/test_cake_gdn_train_kcp.py @@ -51,6 +51,7 @@ import random import re import threading +import types import pytest import torch @@ -4715,6 +4716,250 @@ def test_launch_recorder_lists_the_programs_of_the_call(monkeypatch) -> None: ) +# --------------------------------------------------------------------------- host launch path: launch cache, stream scope +# Host data only (record names, grids, launch order, context entries): exact checks. + + +def _harvest_launches(kcp, ctx, ops, **knobs) -> list[tuple[str, dict]]: + """``(program, args)`` of every launch of one forward + backward on the fake backend (``args`` without the call).""" + seen: list[tuple[str, dict]] = [] + + class Recording(kcp.FakeBackend): + def run(self, name, args): + seen.append((name, {k: v for k, v in args.items() if k != "_call"})) + super().run(name, args) + + previous = kcp.set_backend("fake", Recording()) + try: + args = [ops[n] for n in LEAVES] + _out, saved = kcp.chunk_gdn_fwd_kcp( + *args, ctx, backend="fake", deterministic=False, **knobs + ) + kcp.chunk_gdn_bwd_kcp( + *args, + ctx, + ops["dout"], + saved=saved, + backend="fake", + deterministic=False, + **knobs, + ) + finally: + kcp.set_backend("fake", previous) + return seen + + +def _same_launch(got: tuple, want: tuple) -> bool: + """Two ``(record, values, grid, is_test)`` resolutions name the same record, grid and flag and the same values + (the same tensor objects, equal scalars).""" + (r1, v1, g1, t1), (r2, v2, g2, t2) = got, want + if (r1, tuple(g1), t1) != (r2, tuple(g2), t2) or v1.keys() != v2.keys(): + return False + return all( + v1[k] is v2[k] if isinstance(v2[k], torch.Tensor) else v1[k] == v2[k] + for k in v2 + ) + + +LAUNCH_CACHE_KNOBS = ( + {}, + {"segments": 2}, + { + "segments": 2, + "test_knobs": {"cols_rule": "by_chains", "fwdh_vh": 64, "dhu_vh": 128}, + }, + {"segments": 2, "test_knobs": {"schedule": "overlap", "fwdh_vh": 128}}, +) + + +def test_launch_cache_resolves_every_launch_as_resolve_launch(monkeypatch) -> None: + """``_resolve_cached`` (the launch cache of the context tables) answers every launch of fake-backend forward + + backward runs on a context with and without an SM count -- default options, two segments, forced walk forms with + the by-chains merge columns, the overlap schedule; each persistent launch also with the kernels backend's ``arch`` + grid argument, each walk's arguments also as a transfer walk of every precision mode (the test-only records) -- + under both merge column defaults with the record, public values, grid and test flag of ``resolve_launch``, on the + first launch of its key (the one call of ``resolve_launch``) and on every later one, and appends + ``(record, grid)`` to the call's records each time.""" + kcp, ctx, ops = _ckpt_setup() + part = gp.partition_kcp(torch.tensor(CKPT_CU, dtype=torch.int64), 1, "contiguous") + ctx_sm = gp.build_context_kcp(part, None, 0, CPU, sm_count=SM_COUNT) + launches: list[tuple[str, dict]] = [] + for c in (ctx, ctx_sm): + for knobs in LAUNCH_CACHE_KNOBS: + launches += _harvest_launches(kcp, c, ops, **knobs) + extra = [] + for name, args in launches: + grid_args = args.get("_grid_args") or {} + if "sm_count" in grid_args: + extra.append( + (name, {**args, "_grid_args": {**grid_args, "arch": "sm_100a"}}) + ) + if name in kcp.FWDH_PROGRAMS + kcp.DHU_PROGRAMS: + extra += [ + ("kcp_transfer_walk", {**args, "_precision_mode": pm}) + for pm in (0, 1, 2) + ] + launches += extra + names = {n for n, _ in launches} + assert {"kcp_affine_merge", "kcp_transfer_walk", "fwd_o", "bwd_dqkg"} <= names + assert names & set(kcp.FWDH_PROGRAMS) and names & set(kcp.DHU_PROGRAMS) + real = kcp.resolve_launch + assert any(real(n, dict(a))[3] for n, a in launches) # a test-only record + misses: list[tuple] = [] + + def counting(name, args, *, cols_per_cta=8): + misses.append(kcp._launch_key(name, args, cols_per_cta)) + return real(name, args, cols_per_cta=cols_per_cta) + + monkeypatch.setattr(kcp, "resolve_launch", counting) + call = types.SimpleNamespace(tb=types.SimpleNamespace(launches={}), records=[]) + for _round in range(2): + for cols in (8, 16): + for name, args in launches: + want = real(name, dict(args), cols_per_cta=cols) + got = kcp._resolve_cached( + name, {**args, "_call": call}, cols_per_cta=cols + ) + assert _same_launch(got, want), (name, cols) + assert call.records[-1] == (want[0], want[2]), (name, cols) + keys = {kcp._launch_key(n, a, c) for n, a in launches for c in (8, 16)} + assert len(misses) == len(set(misses)) == len(keys) == len(call.tb.launches) + assert len(call.records) == 4 * len(launches) + # no tables, or tables without a cache: resolve_launch itself (nothing cached) + for stub in (types.SimpleNamespace(records=[]), None): + name, args = launches[0] + before = len(misses) + got = kcp._resolve_cached(name, {**args, "_call": stub}) + assert _same_launch(got, real(name, dict(args))) and len(misses) == before + 1 + + +def test_launch_cache_keeps_every_calls_launch_records(monkeypatch) -> None: + """One context on the fake backend: a run on cold launch caches, a run on warm ones and a run with the cache + bypassed (``resolve_launch`` per launch) record the same launches -- records and grids in order, forward and + backward, direct and checkpointed -- the tables cache one entry per launch key, and every run's results are within + the budget of the FP64 reference and of each other.""" + kcp, ctx, ops = _ckpt_setup() + ref = _ckpt_reference() + tb = kcp._context_tables(ctx, CKPT_HEADS[1], fused_walks=1, P=2) + assert tb.launches == {} + arms, records = {}, {} + for arm in ("cold", "warm", "bypassed"): + res = {} + for ckpt in (False, True): + with monkeypatch.context() as m: + if arm == "bypassed": + m.setattr( + kcp, + "_resolve_cached", + lambda name, args, *, cols_per_cta=8: kcp.resolve_launch( + name, args, cols_per_cta=cols_per_cta + ), + ) + res[ckpt] = _ckpt_autograd(kcp, ctx, ops, ckpt=ckpt, segments=2) + records[(arm, ckpt)] = { + k: list(kcp.LAST_CALL[k]["records"]) for k in ("fwd", "bwd") + } + arms[arm] = res + if arm == "cold": + n_keys = len(tb.launches) + assert n_keys > 0 + assert len(tb.launches) == n_keys # warm and bypassed runs add no key + for ckpt in (False, True): + assert ( + records[("cold", ckpt)] + == records[("warm", ckpt)] + == records[("bypassed", ckpt)] + ), ckpt + _arms_within_budget( + {arm: arms[arm][ckpt] for arm in arms}, + ref, + ("out",) + CKPT_GRADS, + f"launch cache arms (checkpoint={ckpt})", + ) + launched = { + (r, tuple(g)) for rec in records.values() for k in rec for r, g in rec[k] + } + assert launched <= {(r, g) for r, g, _t, _c in tb.launches.values()} + + +def test_launch_scope_binds_the_stream_once_per_entry_body(monkeypatch) -> None: + """``cake_backend.launch_scope`` with a counting stand-in for ``tvm_ffi`` and a set current stream (CPU): a launch + outside every scope enters its own ``use_torch_stream()`` context; inside a scope it enters none while the current + stream is the bound one and its own after a stream switch; a nested scope on the same stream enters nothing, one + on another stream binds that stream until it exits; another thread sees no scope; an exception restores the + binding. The kernels backend hands out its glue's scope; a glue without one and the fake backend none.""" + be = _package("cake_backend") + kcp = _package("cake_kcp") + state = {"stream": (0, 11), "enters": 0, "bound": [], "launched": []} + + class _Context: + def __enter__(self): + state["enters"] += 1 + state["bound"].append(state["stream"]) + return self + + def __exit__(self, *exc): + state["bound"].pop() + + monkeypatch.setattr( + be, + "tvm_ffi", + types.SimpleNamespace(use_torch_stream=lambda: _Context()), + ) + monkeypatch.setattr(be, "_current_stream", lambda: state["stream"]) + name = "fwd_prep" + values = {n: n for n, _axis in be._argument_plan(name) if n is not None} + module = types.SimpleNamespace( + **{ + be.record(name)["ffi_entry"]: lambda *a: state["launched"].append( + state["bound"][-1] if state["bound"] else None + ) + } + ) + + def launch() -> None: + be.run_module(name, values, grid=(1, 1, 1), device_index=0, module=module) + + def enters_after(fn) -> int: + n = state["enters"] + fn() + return state["enters"] - n + + assert enters_after(launch) == 1 and state["launched"][-1] == (0, 11) + with be.launch_scope(): + assert state["enters"] == 2 and be._SCOPE.stream == (0, 11) + assert enters_after(launch) == 0 and enters_after(launch) == 0 + assert state["launched"][-1] == (0, 11) + with be.launch_scope(): + assert enters_after(launch) == 0 + assert be._SCOPE.stream == (0, 11) and state["bound"] == [(0, 11)] + state["stream"] = (0, 22) # a stream switch inside the scope + assert enters_after(launch) == 1 and state["launched"][-1] == (0, 22) + with be.launch_scope(): # a scope on the other stream binds it + assert be._SCOPE.stream == (0, 22) + assert enters_after(launch) == 0 and state["launched"][-1] == (0, 22) + assert be._SCOPE.stream == (0, 11) + assert enters_after(launch) == 1 + state["stream"] = (0, 11) + worker = threading.Thread(target=launch) + assert enters_after(lambda: (worker.start(), worker.join())) == 1 + assert enters_after(launch) == 0 + assert getattr(be._SCOPE, "stream", None) is None and state["bound"] == [] + with pytest.raises(RuntimeError, match="inside the scope"), be.launch_scope(): + raise RuntimeError("inside the scope") + assert getattr(be._SCOPE, "stream", None) is None and state["bound"] == [] + assert enters_after(launch) == 1 + # the backends' scopes + assert kcp.FakeBackend().launch_scope(None) is None + assert ( + kcp.KernelBackend(launcher=types.SimpleNamespace()).launch_scope(None) is None + ) + with kcp.KernelBackend(launcher=be).launch_scope(None): + assert be._SCOPE.stream == (0, 11) + assert enters_after(launch) == 0 + assert getattr(be._SCOPE, "stream", None) is None + + # --------------------------------------------------------------------------- numerics harness hooks (fake backend) NONDET_REPEATS = 8 # the worst case over this many runs is gated per tensor diff --git a/tests/experimental/test_cake_gdn_train_kcp_cp1.py b/tests/experimental/test_cake_gdn_train_kcp_cp1.py index 737a2950abc..ca8a09174f3 100644 --- a/tests/experimental/test_cake_gdn_train_kcp_cp1.py +++ b/tests/experimental/test_cake_gdn_train_kcp_cp1.py @@ -695,3 +695,84 @@ def test_two_caller_streams_with_cached_workspaces_are_within_budget(device) -> TENSORS, f"stream {key}", ) + + +# --------------------------------------------------------------------------- the launch scope (one GPU) + + +def test_launch_scope_enters_one_stream_context_per_entry_body( + device, monkeypatch +) -> None: + """The kernels backend's launch path on a caller stream: an autograd forward + backward, direct and under a + non-reentrant checkpoint, enters ``tvm_ffi.use_torch_stream()`` once per entry body (the launch scope) instead + of once per launch, launches the same records on the same grids as with the scope off (one context per launch), + and both arms' results are within the budget of the FP64 reference and of each other.""" + from torch.utils.checkpoint import checkpoint + + be = pytest.importorskip("flashinfer.experimental.cake_gdn_train_kcp.cake_backend") + kcp = pytest.importorskip("flashinfer.experimental.cake_gdn_train_kcp.cake_kcp") + real = be.tvm_ffi + entered = [0] + + class _Counting: + def __getattr__(self, name): + return getattr(real, name) + + @staticmethod + def use_torch_stream(*args, **kwargs): + entered[0] += 1 + return real.use_torch_stream(*args, **kwargs) + + monkeypatch.setattr(be, "tvm_ffi", _Counting()) + cu, heads = [0, 700, 1900, 3000], (2, 6) + inputs = make_inputs(cu, *heads) + ref, _ = reference_pack(inputs, device) + ctx = _local_context(cu, device) + stream = torch.cuda.Stream(device=device) + torch.cuda.synchronize(device) + results, launches, contexts = {}, {}, {} + for arm in ("scope", "per-launch"): + if arm == "per-launch": + monkeypatch.setattr( + kcp.KernelBackend, "launch_scope", lambda self, call: None + ) + for ckpt in (False, True): + entered[0] = 0 + with torch.cuda.stream(stream): + leaves = [ + inputs[n].to(device).detach().requires_grad_(True) for n in OPERANDS + ] + + def layer(q, k, v, g, beta): + return gdn_api.chunk_gdn_kcp( + q, k, v, g, beta, ctx, deterministic=False + ) + + out = ( + checkpoint(layer, *leaves, use_reentrant=False) + if ckpt + else layer(*leaves) + ) + grads = torch.autograd.grad(out, leaves, inputs["dout"].to(device)) + stream.synchronize() + results[(arm, ckpt)] = { + "out": out.detach()[0], + **{n: t.detach()[0] for n, t in zip(GRAD_NAMES, grads, strict=True)}, + } + launches[(arm, ckpt)] = { + k: [(r, tuple(g)) for r, g in kcp.LAST_CALL[k]["records"]] + for k in ("fwd", "bwd") + } + contexts[(arm, ckpt)] = entered[0] + for ckpt in (False, True): + bodies = 3 if ckpt else 2 # forward, (the checkpoint's recompute,) backward + n_launches = sum(len(v) for v in launches[("per-launch", ckpt)].values()) + assert launches[("scope", ckpt)] == launches[("per-launch", ckpt)], ckpt + assert contexts[("scope", ckpt)] == bodies, (ckpt, contexts) + assert contexts[("per-launch", ckpt)] >= n_launches > bodies, (ckpt, contexts) + _arms_within_budget( + {arm: results[(arm, ckpt)] for arm in ("scope", "per-launch")}, + ref, + TENSORS, + f"launch scope (checkpoint={ckpt})", + ) From 7e5a4a30dba6dc8fe9feef8146c0496b1789c724 Mon Sep 17 00:00:00 2001 From: Yingyi Huang Date: Sun, 1 Nov 2026 04:28:59 -0800 Subject: [PATCH 2/3] perf(cake_gdn_train_kcp): cache the facade's package modules _pkg() and _det() ran a function-level import on every facade call. Cache the module objects with functools.cache: the first call still imports lazily, later calls reuse the module. Routing, exceptions and the lazy type resolution are unchanged. A CPU test checks that routed calls on either route run no import after first use. --- flashinfer/cake_gdn_train_kcp.py | 4 +++ .../test_cake_gdn_train_kcp_api.py | 27 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/flashinfer/cake_gdn_train_kcp.py b/flashinfer/cake_gdn_train_kcp.py index 87f2f6ff734..89c25e23c62 100644 --- a/flashinfer/cake_gdn_train_kcp.py +++ b/flashinfer/cake_gdn_train_kcp.py @@ -72,6 +72,7 @@ from __future__ import annotations import dataclasses +import functools import warnings from typing import Any, Optional @@ -85,12 +86,15 @@ DEFAULT_SCALE = 128**-0.5 +# The routed modules are imported on first use and cached: a call does not repeat the import. +@functools.cache def _pkg(): from .experimental import cake_gdn_train_kcp # noqa: PLC0415 - the backend package, imported on first use return cake_gdn_train_kcp +@functools.cache def _det(): from . import cake_gdn_det_train_kcp # noqa: PLC0415 - the deterministic route, imported on first use diff --git a/tests/experimental/test_cake_gdn_train_kcp_api.py b/tests/experimental/test_cake_gdn_train_kcp_api.py index f668976ab0b..9f427aac15a 100644 --- a/tests/experimental/test_cake_gdn_train_kcp_api.py +++ b/tests/experimental/test_cake_gdn_train_kcp_api.py @@ -26,6 +26,7 @@ from __future__ import annotations import ast +import builtins import dataclasses import importlib import inspect @@ -963,6 +964,32 @@ def test_facade_routes_by_mode(monkeypatch) -> None: facade.is_gdn_train_kcp_supported("cpu", deterministic=None) +def test_facade_imports_each_route_once(monkeypatch) -> None: + """The facade imports each routed module on first use only: later calls on either route run no import and reach + the same module objects.""" + facade = importlib.import_module(FACADE) + _route_recorders(monkeypatch) + calls = _facade_calls(facade).values() + for call in calls: # first use of both routes + call() + call(deterministic=False) + imports: list = [] + real_import = builtins.__import__ + + def counting_import(name, *args, **kwargs): + imports.append(name) + return real_import(name, *args, **kwargs) + + with monkeypatch.context() as m: + m.setattr(builtins, "__import__", counting_import) + for call in calls: + call() + call(deterministic=False) + assert imports == [], imports + assert facade._pkg() is importlib.import_module(PACKAGE) + assert facade._det() is importlib.import_module(DET_FACADE) + + def test_facade_context_graph_replay_hint(monkeypatch) -> None: """``build_context_kcp(..., graph_replay=...)``: keyword-only, default ``False``, a non-bool is a ``TypeError``; the non-deterministic route receives the hint, the deterministic route (whose plan is the graph plan already) does From 704529f27ca1f9a2938d4b3d8c5eb9e41a4a6a19 Mon Sep 17 00:00:00 2001 From: Yingyi Huang Date: Sun, 1 Nov 2026 07:11:56 -0800 Subject: [PATCH 3/3] docs(cake_gdn_train_kcp): state the launch-cache key invariant resolve_launch derives the record, grid, test flag and merge cols_per_cta from the scalars in _launch_key alone; say so next to the key, so that an input added to resolve_launch is added to the key as well. --- flashinfer/experimental/cake_gdn_train_kcp/cake_kcp.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flashinfer/experimental/cake_gdn_train_kcp/cake_kcp.py b/flashinfer/experimental/cake_gdn_train_kcp/cake_kcp.py index 902bb953f79..924638d1a2e 100644 --- a/flashinfer/experimental/cake_gdn_train_kcp/cake_kcp.py +++ b/flashinfer/experimental/cake_gdn_train_kcp/cake_kcp.py @@ -1202,7 +1202,11 @@ def resolve_launch( def _launch_key(name: str, args: dict[str, Any], cols_per_cta: int) -> tuple: """Every input :func:`resolve_launch` reads besides the launch's tensors: the program, the static knobs, the grid - rule's arguments (in the call site's order), the walk-form inputs, the merge's columns and the caller's grid x.""" + rule's arguments (in the call site's order), the walk-form inputs, the merge's columns and the caller's grid x. + + The launch cache is valid only because :func:`resolve_launch` derives the record, the grid, the test flag and the + merge's ``cols_per_cta`` from these scalars alone and reads nothing from the launch's tensors: an input added to + :func:`resolve_launch` (a new grid-rule argument, for example) must be added to this key as well.""" grid_args = args.get("_grid_args") return ( name,