From 170794cacb6876ffa241ddc5f17b6165d6607335 Mon Sep 17 00:00:00 2001 From: Yingyi Huang Date: Tue, 3 Nov 2026 09:09:24 -0800 Subject: [PATCH] perf(cake_dense_projection_gemm): cache the router entry points' calls by their own tensors With deterministic=False, router_forward, router_dgrad and router_wgrad now keep one more convenience cache, keyed by the entry point's own tensors, the operation, the split count, the PDL flag, the raw stream and the device. Each entry holds the cached launch that the entry point's own path takes for the key (the same _K2_SPLITK_CACHE or _K2_ATOMIC_CACHE entry, planned under the same key), and the router table memo and the ROUTES value it was routed under. A call goes from its key to the bound launch: it builds no transposed operand, takes no route and builds no second key. A router table change or a ROUTES change routes the key again. A plain-dict router table, tensors that are not on one CUDA device, or a launch that is not bound for the entry point's own views take the entry point's own path. The K2 launches now pass the call's views to the FFI calls as torch tensors, enter the arena's stream context (made once per arena) and call torch.sum with a positional dimension. The programs, plans, split counts, numeric results and prepared launches do not change. --- .../dense_projection_gemm/README.md | 9 +- .../dense_projection_gemm/cake_backend.py | 473 +++++++++++++----- .../test_cake_dense_projection_gemm.py | 423 +++++++++++++++- 3 files changed, 786 insertions(+), 119 deletions(-) diff --git a/flashinfer/experimental/dense_projection_gemm/README.md b/flashinfer/experimental/dense_projection_gemm/README.md index 39d418c1d79..e36c146cbb1 100644 --- a/flashinfer/experimental/dense_projection_gemm/README.md +++ b/flashinfer/experimental/dense_projection_gemm/README.md @@ -267,7 +267,8 @@ it through a cached binding of this package: the deterministic package's plan an arena, keyed by the entry point's own tensors. A batched forward that the deterministic package runs on its swapped small-token program makes that package's call (with `enable_pdl` and `weights_static`). A cached K2 launch of the convenience path orders its arguments once per cache key -and converts the arena's BF16x3 stack once per arena growth, so a call converts only the caller's views. A K2 call +and converts the arena's BF16x3 stack once per arena growth, so a call passes only the caller's views (each +converted in its FFI call) and enters the arena's stream context, made once. A K2 call that runs the deterministic program -- the input gradient, the forward without a split count, a call without a split count whose row has no verdict in the router table of the device's SM count (`ROUTER_RULES`, or the part's own table in `ROUTER_RULES_BY_SM`; not planned here) -- launches it through a cached binding of this package: the @@ -276,7 +277,11 @@ partials in this package's arena, bound once per arena growth, and the entry poi views. It issues the same launches as the deterministic package's convenience path in fewer host operations. Whether a row has a verdict is kept per device and view geometry, and whether any router table gives the forward one is kept too, until a router table changes: the router tables count their changes, so a verdict set or removed at run time is seen at the next call (a table rebound to a plain `dict` is read on every -call). +call). The router entry points (`router_forward`, `router_dgrad`, `router_wgrad`) cache each call once more by their +own tensors, the operation, the split count and the PDL flag: a call goes from that key to the cached launch its +route takes, without building the transposed operand, routing the row or building the launch's own key; the key is +routed again when a router table or `ROUTES` changes (with a table rebound to a plain `dict`, every call takes the +full path). ## Benchmark diff --git a/flashinfer/experimental/dense_projection_gemm/cake_backend.py b/flashinfer/experimental/dense_projection_gemm/cake_backend.py index 45bea822cd6..68dcbdf1ab7 100644 --- a/flashinfer/experimental/dense_projection_gemm/cake_backend.py +++ b/flashinfer/experimental/dense_projection_gemm/cake_backend.py @@ -2137,10 +2137,14 @@ def _atomic_k2( # ``_K2_SPLITK_CACHE``): the deterministic package's plan, programs and ``torch.sum`` of the split-K partials, so its # bits, with this package's arena for the stack and the partials. A K2 entry binds its argument lists once # (:class:`_K2Binding`: the plan's values in the program's order, the arena's BF16x3 stack -- and the deterministic -# program's partials -- carved and converted once per arena growth), so a call converts only the caller's views (an -# entry point's own tensors when they are the launch views, :func:`_launch_views`) and fills ``out`` with the bound -# geometry. After a launch that did not complete, :func:`clear_convenience_caches` drops the arenas (fresh tickets -# on the next call). +# program's partials -- carved and converted once per arena growth), so a call passes only the caller's views (an +# entry point's own tensors when they are the launch views, :func:`_launch_views`; torch tensors, each converted in +# its FFI call) and fills ``out`` with the bound geometry; the K2 launches of an arena enter its stream context (made +# once) under its lock. The router entry points' calls (``router_forward`` / ``router_dgrad`` / ``router_wgrad``) +# are cached once more by the entry point's own tensors (``_K2_ENTRY_CACHE``, :func:`_k2_entry`): each holds the +# cached launch of ``_K2_SPLITK_CACHE`` / ``_K2_ATOMIC_CACHE`` that the entry point's path takes for the key and the +# router-table memo and route it was routed under, so a call goes from its key to the bound launch. After a launch +# that did not complete, :func:`clear_convenience_caches` drops the arenas (fresh tickets on the next call). #: workspace kind -> (dtype, zeroed when allocated) _ATOMIC_ARENA_KINDS: dict[str, tuple[torch.dtype, bool]] = { @@ -2177,6 +2181,9 @@ class _AtomicArena: lock: threading.Lock = field( default_factory=threading.Lock, repr=False, compare=False ) + # the tvm-ffi stream context of the (device, stream), made by the first K2 launch and entered by every K2 launch + # under ``lock`` (:func:`_k2_stream_context`) + stream_context: Any = field(default=None, repr=False, compare=False) def require(self, kind: str, numel: int) -> torch.Tensor: """The buffer of ``kind`` with at least ``numel`` elements, grown when the present one is smaller (called @@ -2366,7 +2373,7 @@ class _CachedAtomic: def _release_atomic_entry( - entry: Union[_CachedAtomic, _CachedSplitK2, _CachedDetK1], + entry: Union[_CachedAtomic, _CachedSplitK2, _CachedDetK1, _CachedK2Entry], ) -> None: """``on_evict`` of the convenience caches: the arena of an evicted entry loses a reference and is released when no cached entry uses it any more.""" @@ -2379,8 +2386,10 @@ def _release_atomic_entry( #: An entry of the convenience caches (:data:`_K1_ATOMIC_CACHE`, :data:`_K1_DET_CACHE`, :data:`_K2_ATOMIC_CACHE`, -#: :data:`_K2_SPLITK_CACHE`). -_CacheEntry = TypeVar("_CacheEntry", "_CachedAtomic", "_CachedDetK1", "_CachedSplitK2") +#: :data:`_K2_SPLITK_CACHE`, :data:`_K2_ENTRY_CACHE`). +_CacheEntry = TypeVar( + "_CacheEntry", "_CachedAtomic", "_CachedDetK1", "_CachedSplitK2", "_CachedK2Entry" +) def _cache_insert( @@ -2901,6 +2910,129 @@ def _launch_views( return _k2_views(A, B, plan) +def _plan_atomic_k2( + A: torch.Tensor, + B: torch.Tensor, + out: torch.Tensor, + splits: Optional[int], + index: int, + stream: int, + pdl: int, +) -> _CachedAtomic: + """Plan this package's K2 program for a key of :func:`_launch_cached_atomic_k2` (under ``_ATOMIC_PLAN_LOCK``): + :func:`plan_atomic_router` on the device's architecture and SM count (``plan=None``: the planner keeps the row on + the deterministic program), the program's and fused split's entries, the BF16x3 stack in the arena of (device + ``index``, raw ``stream``) and the argument lists bound with the PDL flag ``pdl`` (0 or 1).""" + det = deterministic_backend() + device = out.device + arch = det.require_arch(device) + plan, _ = plan_atomic_router( + A, + B, + out, + arch=arch, + sm_count=det.device_facts(device).sm_count, + splits=splits, + ) + if plan is None: + return _CachedAtomic(plan=None, arch=arch) + _, split_src = _k2_views(A, B, plan) + split_module = det._fused_split_for(arch, split_src) + arena = _atomic_arena_for(index, stream, device) + arena.require("k2_stack", 3 * split_src.numel()) + if split_module is None: + arena.require("k2_split_scratch", split_src.numel()) + module_name = cake_jit.select_module(arch, plan.template) + fn, arg_plan = _atomic_entry(module_name, plan.grid, arch) + split_fn, split_arg_plan = None, None + if split_module is not None: + record = det.MODULES[split_module] + split_fn = getattr( + det.load_cake_dense_projection_det_gemm_module(split_module, arch), + record["ffi_entry"], + ) + split_arg_plan = det.arg_plan_of(record) + if plan.part_tiles: + _cu_driver() + binding = _k2_binding( + plan, + out, + split_src, + module_name, + arg_plan, + split_module, + split_arg_plan, + pdl, + ) + return _CachedAtomic( + plan=plan, + arch=arch, + module_name=module_name, + entry=fn, + arg_plan=arg_plan, + arena=arena, + split_module=split_module, + split_entry=split_fn, + split_arg_plan=split_arg_plan, + k2=binding, + ) + + +def _k2_stream_context( + det: ModuleType, arena: _AtomicArena, index: int, stream: int +) -> Any: + """The tvm-ffi stream context of ``arena``'s (device ``index``, raw ``stream``), made once and kept in the arena: + a K2 launch enters it under the arena's lock, so one thread at a time does.""" + context = arena.stream_context + if context is None: + context = arena.stream_context = det._ffi_stream_context(index, stream) + return context + + +def _launch_atomic_k2_binding( + det: ModuleType, + entry: _CachedAtomic, + out: torch.Tensor, + a_view: torch.Tensor, + split_src: torch.Tensor, + index: int, + stream: int, + out_ptr: int, +) -> None: + """Launch a planned K2 program of this package (:func:`_plan_atomic_k2`) into ``out`` (data pointer ``out_ptr``) + on the raw ``stream`` of device ``index`` with the call's launch views ``a_view`` / ``split_src``: the split, the + zero fill of ``out`` (part tiles) and the program back to back under the arena's lock, the stack carved again if + the arena grew. ``out`` is converted once for both of its arguments; the other views go to the FFI calls as + torch tensors (each converted there, once).""" + arena, fn, bound = entry.arena, entry.entry, entry.k2 + assert arena is not None and fn is not None and bound is not None + split_fn = entry.split_entry + # the split, the memset and the kernel share the arena's BF16x3 stack + with arena.lock: + if bound.version != arena.version: + bound.carve(arena, split_fn is None) + if split_fn is None: + det.split_fp32_to_bf16x3_det_(split_src, bound.parts, bound.resid) + out_ffi = tvm_ffi.from_dlpack(out, require_contiguous=False) + views = ( + a_view, + out_ffi if bound.out_contiguous else det.flat_alias(out), + out_ffi, + None if split_fn is None else split_src, + ) + arguments, split_arguments = bound.arguments(views) + fill = bound.fill + if fill is not None and _CU_DRIVER is None: + _cu_driver() + with _k2_stream_context(det, arena, index, stream): + if split_fn is not None and split_arguments is not None: + split_fn(*split_arguments) + if fill is not None: + rows, cols, pitch = fill + _fill_rows(out, index, out_ptr, rows, cols, pitch, stream) + fn(*arguments) + + def _launch_cached_atomic_k2( A: torch.Tensor, B: torch.Tensor, @@ -2930,99 +3062,18 @@ def _launch_cached_atomic_k2( with _ATOMIC_PLAN_LOCK: entry = det._cache_get(_K2_ATOMIC_CACHE, key) if entry is None: - device = out.device - arch = det.require_arch(device) - plan, _ = plan_atomic_router( - A, - B, - out, - arch=arch, - sm_count=det.device_facts(device).sm_count, - splits=splits, + entry = _cache_insert( + _K2_ATOMIC_CACHE, + key, + _plan_atomic_k2(A, B, out, splits, index, stream, pdl), ) - if plan is None: - entry = _CachedAtomic(plan=None, arch=arch) - else: - _, split_src = _k2_views(A, B, plan) - split_module = det._fused_split_for(arch, split_src) - arena = _atomic_arena_for(index, stream, device) - arena.require("k2_stack", 3 * split_src.numel()) - if split_module is None: - arena.require("k2_split_scratch", split_src.numel()) - module_name = cake_jit.select_module(arch, plan.template) - fn, arg_plan = _atomic_entry(module_name, plan.grid, arch) - split_fn, split_arg_plan = None, None - if split_module is not None: - record = det.MODULES[split_module] - split_fn = getattr( - det.load_cake_dense_projection_det_gemm_module( - split_module, arch - ), - record["ffi_entry"], - ) - split_arg_plan = det.arg_plan_of(record) - if plan.part_tiles: - _cu_driver() - binding = _k2_binding( - plan, - out, - split_src, - module_name, - arg_plan, - split_module, - split_arg_plan, - pdl, - ) - entry = _CachedAtomic( - plan=plan, - arch=arch, - module_name=module_name, - entry=fn, - arg_plan=arg_plan, - arena=arena, - split_module=split_module, - split_entry=split_fn, - split_arg_plan=split_arg_plan, - k2=binding, - ) - entry = _cache_insert(_K2_ATOMIC_CACHE, key, entry) plan = entry.plan if plan is None: return _launch_cached_splitk_k2(A, B, out, splits, index, geometry, pdl, views) a_view, split_src = _launch_views(A, B, plan, views) - arena, fn, bound = entry.arena, entry.entry, entry.k2 - assert arena is not None and fn is not None and bound is not None - split_fn = entry.split_entry - # the split, the memset and the kernel share the arena's BF16x3 stack - with arena.lock: - if bound.version != arena.version: - bound.carve(arena, split_fn is None) - if split_fn is None: - det.split_fp32_to_bf16x3_det_(split_src, bound.parts, bound.resid) - # the caller's views are converted before the first launch (the launches go back to back) into the argument - # lists bound when the key was planned; ``out`` is converted once for both of its arguments - out_ffi = tvm_ffi.from_dlpack(out, require_contiguous=False) - views = ( - tvm_ffi.from_dlpack(a_view, require_contiguous=False), - out_ffi - if bound.out_contiguous - else tvm_ffi.from_dlpack(det.flat_alias(out), require_contiguous=False), - out_ffi, - None - if split_fn is None - else tvm_ffi.from_dlpack(split_src, require_contiguous=False), - ) - arguments, split_arguments = bound.arguments(views) - fill = bound.fill - if fill is not None and _CU_DRIVER is None: - _cu_driver() - with det._ffi_stream_context(index, stream): - if split_fn is not None and split_arguments is not None: - split_fn(*split_arguments) - if fill is not None: - rows, cols, pitch = fill - _fill_rows(out, index, out_ptr, rows, cols, pitch, stream) - fn(*arguments) + _launch_atomic_k2_binding( + det, entry, out, a_view, split_src, index, stream, out_ptr + ) return out @@ -3128,11 +3179,12 @@ def _launch_cached_splitk_k2( ) -> torch.Tensor: """The deterministic package's K2 GEMM ``out = A @ B`` (its plan, programs and fixed-order ``torch.sum`` of the split-K partials: its bits) on this package's convenience path: planned once per key (:func:`_plan_splitk_k2`), then - the bound launch -- the caller's views converted once per call, the stack and the partials once per arena growth - (:class:`_K2Binding`). The split, the program and the reduction share the arena and hold its lock, as the - deterministic package's convenience path does. ``index`` / ``geometry`` are the call's (:func:`_k2_index`, - :func:`_k2_geometry`); ``pdl`` the launch's PDL flag (0 or 1: the split and the program launch as programmatic - dependent launches); ``views`` an entry point's own launch views (:func:`_launch_views`).""" + the bound launch (:func:`_launch_splitk_k2_binding`) -- the caller's views passed per call, the stack and the + partials converted once per arena growth (:class:`_K2Binding`). The split, the program and the reduction share + the arena and hold its lock, as the deterministic package's convenience path does. ``index`` / ``geometry`` are + the call's (:func:`_k2_index`, :func:`_k2_geometry`); ``pdl`` the launch's PDL flag (0 or 1: the split and the + program launch as programmatic dependent launches); ``views`` an entry point's own launch views + (:func:`_launch_views`).""" det = deterministic_backend() stream = det._current_raw_stream(index) # the PDL flag is bound into the argument lists: a key per flag @@ -3147,38 +3199,50 @@ def _launch_cached_splitk_k2( key, _plan_splitk_k2(A, B, out, splits, index, stream, pdl), ) - bound, fn, arena = entry.k2, entry.entry, entry.arena - if bound is None or fn is None or arena is None: + if entry.k2 is None or entry.entry is None or entry.arena is None: return getattr(det, DETERMINISTIC_NAMES["router_fp32_gemm"])( A, B, out, splits=splits, enable_pdl=bool(pdl) ) a_view, split_src = _launch_views(A, B, entry.plan, views) + _launch_splitk_k2_binding(det, entry, out, a_view, split_src, index, stream) + return out + + +def _launch_splitk_k2_binding( + det: ModuleType, + entry: _CachedSplitK2, + out: torch.Tensor, + a_view: torch.Tensor, + split_src: torch.Tensor, + index: int, + stream: int, +) -> None: + """Launch a bound deterministic K2 program (:func:`_plan_splitk_k2`) into ``out`` on the raw ``stream`` of device + ``index`` with the call's launch views ``a_view`` / ``split_src``: the split, the program and the deterministic + package's fixed-order ``torch.sum`` of the split-K partials under the arena's lock, the stack and the partials + carved again if the arena grew. The call's views go to the FFI calls as torch tensors (each converted there, + once).""" + bound, fn, arena = entry.k2, entry.entry, entry.arena + assert bound is not None and fn is not None and arena is not None split_fn = entry.split_entry with arena.lock: if bound.version != arena.version: bound.carve(arena, split_fn is None) if split_fn is None: det.split_fp32_to_bf16x3_det_(split_src, bound.parts, bound.resid) - # the caller's views, converted before the first launch (the launches go back to back) - a = tvm_ffi.from_dlpack(a_view, require_contiguous=False) out32: Any = None if bound.partials_shape is None: # one split: the program stores into ``out`` - out32 = tvm_ffi.from_dlpack( - out if bound.out_contiguous else det.flat_alias(out), - require_contiguous=False, - ) - src: Any = None - if split_fn is not None: - src = tvm_ffi.from_dlpack(split_src, require_contiguous=False) - arguments, split_arguments = bound.arguments((a, out32, None, src)) - with det._ffi_stream_context(index, stream): + out32 = out if bound.out_contiguous else det.flat_alias(out) + arguments, split_arguments = bound.arguments( + (a_view, out32, None, None if split_fn is None else split_src) + ) + with _k2_stream_context(det, arena, index, stream): if split_fn is not None and split_arguments is not None: split_fn(*split_arguments) fn(*arguments) if bound.workspace is not None: - # the deterministic package's fixed-order reduction of the partials - torch.sum(bound.workspace, dim=0, out=out) - return out + # the deterministic package's fixed-order reduction of the partials (its overload and kernel) + torch.sum(bound.workspace, 0, out=out) def _splitk_k2( @@ -3198,6 +3262,176 @@ def _splitk_k2( ) +#: The cached calls of the router entry points with ``deterministic=False`` (``router_forward``, ``router_dgrad``, +#: ``router_wgrad``), keyed by the entry point's own tensors (:func:`_k1_key`), the operation, its split count and the +#: PDL flag (:class:`_CachedK2Entry`). +_K2_ENTRY_CACHE: "OrderedDict[tuple, Any]" = OrderedDict() + + +@dataclass +class _CachedK2Entry: + """The launch a router entry point's call of a key of :data:`_K2_ENTRY_CACHE` takes (:func:`_plan_k2_entry`): an + entry of :data:`_K2_SPLITK_CACHE` (the deterministic program) or, with ``atomic``, of :data:`_K2_ATOMIC_CACHE` (this + package's program), each planned and bound there as the entry point's own path plans it, with the entry point's + tensors as its launch views; ``None``: the call takes the entry point's own path. ``memo`` is the router-table + memo (:func:`_router_memo`) and ``route`` the :data:`ROUTES` value of the layout class ``layout`` (``None``: a + route that does not read it) the call was routed under; ``arena`` the launch's (the cache's reference to it).""" + + memo: Any + layout: Optional[str] = None + route: Optional[str] = None + launch: Optional[Union[_CachedSplitK2, _CachedAtomic]] = field( + default=None, repr=False + ) + atomic: bool = False + arena: Optional[_AtomicArena] = field(default=None, repr=False) + + +def _k2_entry_operands( + op: str, P: torch.Tensor, Q: torch.Tensor +) -> tuple[torch.Tensor, torch.Tensor, tuple[bool, bool, bool]]: + """``(A, B, flags)`` of a router entry point's tensors ``P``, ``Q`` (``"fwd"``: ``X``, ``W``; ``"dgrad"``: ``G``, + ``W``; ``"wgrad"``: ``G``, ``X``): the launched GEMM ``A @ B`` and the plan flags under which the entry point's own + tensors are the launch views (:data:`_FORWARD_VIEWS` / :data:`_DGRAD_VIEWS` / :data:`_WGRAD_VIEWS`).""" + if op == "fwd": + return P, Q.t(), _FORWARD_VIEWS + if op == "dgrad": + return P, Q, _DGRAD_VIEWS + return P.t(), Q, _WGRAD_VIEWS + + +def _plan_k2_entry( + op: str, + P: torch.Tensor, + Q: torch.Tensor, + out: torch.Tensor, + splits: Optional[int], + index: int, + stream: int, + pdl: int, + memo: _RouterMemo, +) -> _CachedK2Entry: + """The :class:`_CachedK2Entry` of a router entry point's call (under ``_ATOMIC_PLAN_LOCK``): the route the entry + point's own path takes under the router tables of ``memo`` -- the input gradient, the forward without a split count + while no router table gives the forward a verdict, a layout routed ``"det"``, a call without a split count whose + row has no verdict (:func:`_k2_runs_without_verdict`) and a row the planner keeps on the deterministic program run + the deterministic program, the others this package's program -- and that path's cached launch of the route (looked + up in, or planned into, :data:`_K2_SPLITK_CACHE` / :data:`_K2_ATOMIC_CACHE` under the same key). A launch that is + not bound, or whose plan does not take the entry point's tensors as its views, leaves the call to the entry + point's own path.""" + det = deterministic_backend() + A, B, flags = _k2_entry_operands(op, P, Q) + geometry = _k2_geometry(A, B, out) + key = _k2_key(index, stream, (splits, pdl), A, B, out, geometry, out.data_ptr()) + launch: Optional[Union[_CachedSplitK2, _CachedAtomic]] = None + atomic = False + # the input gradient, and the forward without a split count while no table gives it a verdict, do not read ROUTES + layout = route = None + if op != "dgrad" and (splits is not None or op == "wgrad" or memo.forward_verdicts): + layout = _k2_layout(A, B) + route = _route(layout) + if ( + route is not None + and route != "det" + and not (splits is None and _k2_runs_without_verdict(geometry, index)) + ): + cached = det._cache_get(_K2_ATOMIC_CACHE, key) + if cached is None: + cached = _cache_insert( + _K2_ATOMIC_CACHE, + key, + _plan_atomic_k2(A, B, out, splits, index, stream, pdl), + ) + if cached.plan is not None: + launch, atomic = cached, True + if launch is None: + launch = det._cache_get(_K2_SPLITK_CACHE, key) + if launch is None: + launch = _cache_insert( + _K2_SPLITK_CACHE, + key, + _plan_splitk_k2(A, B, out, splits, index, stream, pdl), + ) + plan = launch.plan + if ( + launch.k2 is None + or launch.entry is None + or launch.arena is None + or (plan.swapped, plan.a_mn, plan.b_mn) != flags + ): + return _CachedK2Entry(memo=memo, layout=layout, route=route) + return _CachedK2Entry( + memo=memo, + layout=layout, + route=route, + launch=launch, + atomic=atomic, + arena=launch.arena, + ) + + +def _k2_entry_stale(entry: Optional[_CachedK2Entry], memo: _RouterMemo) -> bool: + """Whether a call plans its key of :data:`_K2_ENTRY_CACHE` (again): no entry, or one routed under other router + tables (``memo``) or another :data:`ROUTES` value of its layout class.""" + return ( + entry is None + or entry.memo is not memo + or (entry.layout is not None and _route(entry.layout) != entry.route) + ) + + +def _k2_entry( + op: str, + P: torch.Tensor, + Q: torch.Tensor, + out: torch.Tensor, + splits: Optional[int], + enable_pdl: Optional[bool], +) -> Optional[torch.Tensor]: + """A router entry point's call of ``deterministic=False`` (``op`` and its tensors ``P``, ``Q`` as for + :func:`_k2_entry_operands`, into ``out``) through :data:`_K2_ENTRY_CACHE`: planned once per key of the entry + point's own tensors, the operation, ``splits`` and the PDL flag (``enable_pdl`` resolved for the device), and again + after a router table or the route of its layout class changed (:func:`_plan_k2_entry`), then the bound launch with + the entry point's tensors as the launch views (``X[None]`` / ``G[None]`` and the split source), so a call builds + no transposed operand, takes no route and reads no second key. Returns ``out``, or ``None`` when the call takes + the entry point's own path: tensors not on one CUDA device, router tables the memo does not track, a launch not + bound for the key.""" + index = _k1_index(P, Q, out) + if index < 0: + return None + det = deterministic_backend() + stream = det._current_raw_stream(index) + pdl = int(det.pdl_enabled(enable_pdl, index)) + key = _k1_key(index, stream, (op, splits, pdl), P, Q, out) + memo = _router_memo() + entry = det._cache_get(_K2_ENTRY_CACHE, key) + if _k2_entry_stale(entry, memo): + if not memo.tracked: + return None + with _ATOMIC_PLAN_LOCK: + entry = det._cache_get(_K2_ENTRY_CACHE, key) + if _k2_entry_stale(entry, memo): + entry = _cache_insert( + _K2_ENTRY_CACHE, + key, + _plan_k2_entry(op, P, Q, out, splits, index, stream, pdl, memo), + ) + launch = entry.launch + if launch is None: + return None + if op == "wgrad": + a_view, split_src = Q.unsqueeze(0), P + else: + a_view, split_src = P.unsqueeze(0), Q + if entry.atomic: + _launch_atomic_k2_binding( + det, launch, out, a_view, split_src, index, stream, out.data_ptr() + ) + else: + _launch_splitk_k2_binding(det, launch, out, a_view, split_src, index, stream) + return out + + def _drop_convenience_entries() -> None: """Drop this package's cached convenience-path launches, what the convenience path keeps from the router tables (:func:`_router_memo`) and this package's workspace arenas.""" @@ -3207,6 +3441,7 @@ def _drop_convenience_entries() -> None: _K1_DET_CACHE.clear() _K2_ATOMIC_CACHE.clear() _K2_SPLITK_CACHE.clear() + _K2_ENTRY_CACHE.clear() _ATOMIC_ARENAS.clear() _ROUTER_MEMO = _RouterMemo(-1, None, None, False, True) @@ -3666,6 +3901,8 @@ def router_forward( ) if out is None: out = torch.empty(X.shape[0], W.shape[0], device=X.device, dtype=torch.float32) + if _k2_entry("fwd", X, W, out, splits, enable_pdl) is not None: + return out views = (X.unsqueeze(0), W, _FORWARD_VIEWS) if splits is None and not _router_memo().forward_verdicts: # no router table gives the forward a verdict: the deterministic program (the plan the planner would keep), @@ -3690,6 +3927,8 @@ def router_dgrad( return _det_entry("router_dgrad")(G, W, out, **_pdl_kw(enable_pdl)) if out is None: out = torch.empty(G.shape[0], W.shape[1], device=G.device, dtype=torch.float32) + if _k2_entry("dgrad", G, W, out, 1, enable_pdl) is not None: + return out return _splitk_k2(G, W, out, 1, (G.unsqueeze(0), W, _DGRAD_VIEWS), enable_pdl) @@ -3713,6 +3952,8 @@ def router_wgrad( ) if out is None: out = torch.empty(G.shape[1], X.shape[1], device=G.device, dtype=torch.float32) + if _k2_entry("wgrad", G, X, out, splits, enable_pdl) is not None: + return out # a row without a verdict (kept per device and geometry, :func:`_router_memo`) runs the deterministic program views = (X.unsqueeze(0), G, _WGRAD_VIEWS) return _router_gemm(G.t(), X, out, splits, False, enable_pdl, views) diff --git a/tests/experimental/test_cake_dense_projection_gemm.py b/tests/experimental/test_cake_dense_projection_gemm.py index 408900de909..a80d5961483 100644 --- a/tests/experimental/test_cake_dense_projection_gemm.py +++ b/tests/experimental/test_cake_dense_projection_gemm.py @@ -33,7 +33,7 @@ import threading import warnings from pathlib import Path -from types import ModuleType +from types import ModuleType, SimpleNamespace import pytest import torch @@ -3425,6 +3425,277 @@ def test_k2_bound_arguments_are_the_ordered_arguments(monkeypatch, case): assert bound.fill is None +def _cpu_k2_launches(monkeypatch, arch="sm_103a"): + """CPU binding tests of the router entry points' K2 launches (:func:`_cpu_k1_launches`, nothing is loaded or + launched): the entry points' tensors taken as on device 0, the zero fill of ``out`` (part tiles) recorded, fresh + K2 caches and router-table memo; returns the stand-in module's entry and the fill recorder.""" + entry_fn = _cpu_k1_launches(monkeypatch, arch) + monkeypatch.setattr(cake_backend, "_k1_index", lambda *views: 0) + monkeypatch.setattr(cake_backend, "_k2_index", lambda A, B, out: 0) + monkeypatch.setattr(cake_backend, "_CU_DRIVER", False) + fill = Recorder() + monkeypatch.setattr(cake_backend, "_fill_rows", fill) + for cache in ("_K2_ATOMIC_CACHE", "_K2_SPLITK_CACHE", "_K2_ENTRY_CACHE"): + monkeypatch.setattr(cake_backend, cache, type(getattr(cake_backend, cache))()) + memo = cake_backend._RouterMemo(-1, None, None, False, True) + monkeypatch.setattr(cake_backend, "_ROUTER_MEMO", memo) + return entry_fn, fill + + +# A router entry point's call on a 152-SM sm_103a part: (op, T, hidden, splits, padded out, padded G, whether it runs +# this package's program -- else the deterministic program through this package's binding) +# fmt: off +ROUTER_ENTRY_CASES = { + "fwd": ("fwd", 64, 5120, None, False, False, False), + "fwd_two_splits": ("fwd", 64, 5120, 2, False, False, True), + "fwd_padded_out": ("fwd", 64, 5120, None, True, False, False), + "dgrad": ("dgrad", 64, 5120, None, False, False, False), + "dgrad_padded_out": ("dgrad", 64, 5120, None, True, False, False), + "wgrad_waves": ("wgrad", 64, 5120, None, False, False, True), + "wgrad_head_tail": ("wgrad", 16172, 5120, None, False, False, True), + "wgrad_no_verdict": ("wgrad", 1000, 6144, None, False, False, False), + "wgrad_padded_out": ("wgrad", 64, 5120, 2, True, False, True), + "wgrad_torch_chain": ("wgrad", 64, 5120, None, False, True, True), +} +# fmt: on + + +def _router_entry_call(case): + """``(call, out)``: a router entry point's ``deterministic=False`` call of ``case`` (:data:`ROUTER_ENTRY_CASES`) + on fresh CPU tensors, and its output.""" + op, T, H, splits, pad_out, pad_g, _ = ROUTER_ENTRY_CASES[case] + X, W = torch.empty(T, H), torch.empty(256, H) + G = torch.empty(T, 256 + 16)[:, :256] if pad_g else torch.empty(T, 256) + rows, cols = {"fwd": (T, 256), "dgrad": (T, H), "wgrad": (256, H)}[op] + out = torch.empty(rows, cols + 64)[:, :cols] if pad_out else torch.empty(rows, cols) + kw = ( + dict(deterministic=False) + if op == "dgrad" + else dict(deterministic=False, splits=splits) + ) + call = { + "fwd": lambda: cake_backend.router_forward(X, W, out, **kw), + "dgrad": lambda: cake_backend.router_dgrad(G, W, out, **kw), + "wgrad": lambda: cake_backend.router_wgrad(G, X, out, **kw), + }[op] + return call, out + + +@pytest.mark.parametrize("case", sorted(ROUTER_ENTRY_CASES)) +def test_router_entry_points_launch_their_paths_launches(monkeypatch, case): + """With ``deterministic=False`` a router entry point's call goes through ``_K2_ENTRY_CACHE``: planned once per key + of its own tensors into the cached launch its own path takes (the same ``_K2_SPLITK_CACHE`` / ``_K2_ATOMIC_CACHE`` + entry, under the same key), it issues that path's launches -- the same programs with the same arguments, the + same split and zero fill -- and the next calls go from the key to the bound launch (no geometry, layout, key of + the launch or planning).""" + entry_fn, fill = _cpu_k2_launches(monkeypatch) + split = Recorder() # the torch chain's split + monkeypatch.setattr(det_backend, "split_fp32_to_bf16x3_det_", split) + op, *_, atomic = ROUTER_ENTRY_CASES[case] + call, out = _router_entry_call(case) + with monkeypatch.context() as m: + m.setattr(cake_backend, "_k2_entry", lambda *args: None) + assert call() is out + recorders = (entry_fn, fill, split) + want = [list(r.calls) for r in recorders] + assert len(want[0]) == (1 if case.endswith("torch_chain") else 2) + assert len(want[2]) == (1 if case.endswith("torch_chain") else 0) + inner = cake_backend._K2_ATOMIC_CACHE if atomic else cake_backend._K2_SPLITK_CACHE + ((inner_key, launch),) = [(k, e) for k, e in inner.items() if e.plan is not None] + for again in range(3): + for r in recorders: + r.calls.clear() + assert call() is out + for r, expected in zip(recorders, want, strict=True): + assert len(r.calls) == len(expected) + for (args, kwargs), (w_args, w_kwargs) in zip( + r.calls, expected, strict=True + ): + _same_arguments(list(args), list(w_args)) + assert kwargs == w_kwargs + ((key, entry),) = cake_backend._K2_ENTRY_CACHE.items() + assert key[2][0] == op and key[2][2] == 1 # the PDL flag: the device's default + assert entry.launch is launch and entry.atomic == atomic + assert entry.arena is launch.arena + assert inner[inner_key] is launch + if again == 0: + for name in ("_k2_geometry", "_k2_layout", "_k2_key", "_plan_k2_entry"): + monkeypatch.setattr( + cake_backend, + name, + lambda *args, _n=name: pytest.fail(f"{_n} on a cached call"), + ) + assert len(fill.calls) == int(atomic and launch.k2.fill is not None) + + +def test_router_entry_cache_keys_and_routes_again(monkeypatch): + """``_K2_ENTRY_CACHE`` keys a call by the device, the raw stream, the operation, the split count, the PDL flag and + the entry point's tensors, and routes a key again after a router table changes (a forward verdict set at run time + runs this package's program, its removal the deterministic one) or a ``ROUTES`` value does; a call whose tensors + are not on one CUDA device, or made while a router table is a plain ``dict``, takes the entry point's own path. + ``clear_convenience_caches`` drops the entries.""" + arch, sm_count = "sm_103a", 152 + X, W, out = torch.empty(64, 5120), torch.empty(256, 5120), torch.empty(64, 256) + assert cake_backend._k2_entry("fwd", X, W, out, None, None) is None # CPU tensors + entry_fn, _ = _cpu_k2_launches(monkeypatch, arch) + cache = cake_backend._K2_ENTRY_CACHE + + def fwd(**kw): + n = len(entry_fn.calls) + assert cake_backend.router_forward(X, W, out, deterministic=False, **kw) is out + assert len(entry_fn.calls) > n + return cache[next(reversed(cache))] + + forward_verdicts = cake_backend._router_memo().forward_verdicts + fwd() + fwd(enable_pdl=False) + fwd(splits=2) + monkeypatch.setattr(det_backend, "_current_raw_stream", lambda index: 7) + fwd() + monkeypatch.setattr(det_backend, "_current_raw_stream", lambda index: 0) + keys = list(cache) + assert [k[:3] for k in keys] == [ + (0, 0, ("fwd", None, 1)), + (0, 0, ("fwd", None, 0)), + (0, 0, ("fwd", 2, 1)), + (0, 7, ("fwd", None, 1)), + ] + assert [k[3:] for k in keys] == [keys[0][3:]] * 4 + assert cache[keys[2]].atomic and (cache[keys[2]].layout, cache[keys[2]].route) == ( + "k2_kk", + "atomic", + ) + assert [cache[k].arena.key for k in keys] == [(0, 0), (0, 0), (0, 0), (0, 7)] + if not forward_verdicts: # the forward without a split count reads no route + assert all(cache[k].layout is None and not cache[k].atomic for k in keys[:2]) + # a forward verdict set at run time, and removed: this package's program, then the deterministic one + table, _ = cake_backend.router_rules(arch, sm_count) + if table is cake_backend.ROUTER_RULES: + monkeypatch.setitem(cake_backend.ROUTER_SM_COUNT, arch, sm_count) + X, W, out = torch.empty(1000, 6144), torch.empty(256, 6144), torch.empty(1000, 256) + key = (arch, "kk", 6144, 256) + assert table.get(key) is None # no verdict on any architecture + before = fwd() + assert not before.atomic and before.launch is not None + monkeypatch.setitem(table, key, "forward") + after = fwd() + assert after.atomic and after.memo is not before.memo and after.route == "atomic" + assert after.launch.plan is not None and after.launch.k2 is not None + table[key] = None + assert not fwd().atomic + # a ROUTES value changed at run time + X, G, out = torch.empty(64, 5120), torch.empty(64, 256), torch.empty(256, 5120) + + def wgrad(): + n = len(entry_fn.calls) + assert cake_backend.router_wgrad(G, X, out, deterministic=False) is out + assert len(entry_fn.calls) > n + return cache[next(reversed(cache))] + + assert wgrad().atomic + monkeypatch.setitem(cake_backend.ROUTES, "k2_nn_t", "det") + routed = wgrad() + assert not routed.atomic and routed.launch is not None + assert (routed.layout, routed.route) == ("k2_nn_t", "det") + monkeypatch.setitem(cake_backend.ROUTES, "k2_nn_t", "atomic") + assert wgrad().atomic + # a router table rebound to a plain dict: every call takes the entry point's own path + size = len(cache) + with monkeypatch.context() as m: + m.setattr(cake_backend, "ROUTER_RULES", dict(cake_backend.ROUTER_RULES)) + assert cake_backend._k2_entry("wgrad", G, X, out, None, None) is None + n = len(entry_fn.calls) + assert cake_backend.router_wgrad(G, X, out, deterministic=False) is out + assert len(entry_fn.calls) > n and len(cache) == size + cake_backend.clear_convenience_caches() + assert not cache and not cake_backend._ATOMIC_ARENAS + + +def test_router_entry_without_a_bound_launch_takes_the_entry_points_path(monkeypatch): + """A key whose launch is not bound, or whose plan does not take the entry point's tensors as its launch views, + is kept without a launch: its calls take the entry point's own path (without planning the key again), and a plan + with the entry point's flags is bound.""" + _cpu_k2_launches(monkeypatch) + G, W, out = torch.empty(64, 256), torch.empty(256, 5120), torch.empty(64, 5120) + arena = cake_backend._AtomicArena(torch.device("cpu"), (0, 0)) + memo = cake_backend._router_memo() + + def plan(flags, bound): + swapped, a_mn, b_mn = flags + return cake_backend._CachedSplitK2( + plan=SimpleNamespace(swapped=swapped, a_mn=a_mn, b_mn=b_mn), + entry=Recorder(), + arena=arena, + k2=bound, + ) + + dgrad_flags = cake_backend._DGRAD_VIEWS + for launch, kept in ( + (plan(dgrad_flags, None), False), + (plan(cake_backend._FORWARD_VIEWS, object()), False), + (plan(dgrad_flags, object()), True), + ): + cake_backend._K2_SPLITK_CACHE.clear() + monkeypatch.setattr( + cake_backend, "_plan_splitk_k2", lambda *args, launch=launch: launch + ) + entry = cake_backend._plan_k2_entry("dgrad", G, W, out, 1, 0, 0, 1, memo) + assert (entry.launch is launch) == kept and (entry.arena is arena) == kept + assert entry.memo is memo and entry.layout is None and not entry.atomic + monkeypatch.setattr( + cake_backend, "_plan_splitk_k2", lambda *args: plan(dgrad_flags, None) + ) + cake_backend._K2_SPLITK_CACHE.clear() + path = Recorder() + monkeypatch.setattr(cake_backend, "_splitk_k2", path) + assert cake_backend.router_dgrad(G, W, out, deterministic=False) is path.result + ((key, entry),) = cake_backend._K2_ENTRY_CACHE.items() + assert entry.launch is None and key[2] == ("dgrad", 1, 1) + monkeypatch.setattr( + cake_backend, "_plan_k2_entry", lambda *args: pytest.fail("planned again") + ) + assert cake_backend.router_dgrad(G, W, out, deterministic=False) is path.result + assert len(path.calls) == 2 + + +def test_router_entry_cache_holds_its_arenas(monkeypatch): + """An entry of ``_K2_ENTRY_CACHE`` holds a reference to its launch's arena, as the launch's own entry does: an + arena's reference count is the number of cached entries of every cache that use it, eviction (the cache bound) + drops them, and an arena no entry uses any more is released.""" + _cpu_k2_launches(monkeypatch) + monkeypatch.setattr(det_backend, "CONVENIENCE_CACHE_SIZE", 3) + W = torch.empty(256, 6144) + caches = ( + cake_backend._K2_ENTRY_CACHE, + cake_backend._K2_SPLITK_CACHE, + cake_backend._K2_ATOMIC_CACHE, + ) + + def dgrad(T): + G, out = torch.empty(T, 256), torch.empty(T, 6144) + assert cake_backend.router_dgrad(G, W, out, deterministic=False) is out + arenas = list(cake_backend._ATOMIC_ARENAS.values()) + for arena in arenas: + users = sum(e.arena is arena for c in caches for e in c.values()) + assert arena.refs == users > 0 + assert all( + any(e.arena is a for a in arenas) for c in caches for e in c.values() + ) + return {a.key: a.refs for a in arenas} + + for T in (64, 128, 192, 256, 320): + refs = dgrad(T) + assert len(caches[0]) == len(caches[1]) == 3 and refs == {(0, 0): 6} + monkeypatch.setattr(det_backend, "_current_raw_stream", lambda index: 7) + assert dgrad(64) == {(0, 0): 4, (0, 7): 2} # an arena of the stream's own + monkeypatch.setattr(det_backend, "_current_raw_stream", lambda index: 0) + for T in (64, 128, 192): + refs = dgrad(T) + assert refs == {(0, 0): 6} # evicted from both caches: released + cake_backend.clear_convenience_caches() + assert not cake_backend._ATOMIC_ARENAS and not any(caches) + + def test_deterministic_backend_follows_the_binding_and_sys_modules(monkeypatch): """``deterministic_backend`` returns the module an import of the bound package's ``cake_backend`` returns: a rebinding and a replaced ``sys.modules`` entry are seen on the next call.""" @@ -5220,6 +5491,156 @@ def test_gpu_router_misaligned_out_takes_the_deterministic_path(): cake_backend.router_fp32_gemm(X, W.t(), out, splits=1, deterministic=False) +def _router_entry_of(op, P, Q, out, splits, stream): + """The ``_K2_ENTRY_CACHE`` entry of a router entry point's call (``op`` and its tensors ``P``, ``Q``, ``out``) + on ``stream`` with the device's default PDL flag, or ``None``.""" + index = out.get_device() + pdl = int(det_backend.pdl_enabled(None, index)) + key = cake_backend._k1_key(index, stream.cuda_stream, (op, splits, pdl), P, Q, out) + return cake_backend._K2_ENTRY_CACHE.get(key) + + +def _router_entry_bits(entry) -> bool: + """Whether a cached router entry's launch repeats bit for bit: the deterministic program, or a reproducible plan + of this package's.""" + return not entry.atomic or entry.launch.plan.reproducible + + +@pytest.mark.parametrize("hidden, T", [(5120, 64), (4096, 256), (6144, 1000)]) +def test_gpu_router_entry_points_give_their_paths_results(monkeypatch, hidden, T): + """``deterministic=False`` through ``_K2_ENTRY_CACHE``: every router entry point's call -- the forward with and + without a split count, the input gradient, the weight gradient with and without one; dense and padded outputs; + another stream -- binds the launch its own path takes and gives that path's results on repeated calls: the + deterministic program and a reproducible plan bit for bit (the deterministic program the deterministic package's + bits), every launch within the gate; the padding keeps its values.""" + device, arch = _atomic_device_or_skip() + cake_backend.clear_convenience_caches() + X, W, G = _router_inputs(T, device, SEED + 100 + T, K=hidden) + tensors = {"fwd": (X, W), "dgrad": (G, W), "wgrad": (G, X)} + calls = { + "fwd": cake_backend.router_forward, + "dgrad": cake_backend.router_dgrad, + "wgrad": cake_backend.router_wgrad, + } + shapes = {"fwd": (T, 256), "dgrad": (T, hidden), "wgrad": (256, hidden)} + side = torch.cuda.Stream(device) + ran = {"bits": 0, "gate": 0} + # fmt: off + cases = ( + ("fwd", None, False, False), ("fwd", 2, False, False), ("fwd", None, True, True), + ("dgrad", None, False, False), ("dgrad", None, True, True), + ("wgrad", None, False, False), ("wgrad", 2, True, False), ("wgrad", None, False, True), + ) + # fmt: on + for op, splits, pad, on_side in cases: + P, Q = tensors[op] + kw = dict(deterministic=False) + if op != "dgrad": + kw["splits"] = splits + rows, cols = shapes[op] + storage = torch.full((rows, cols + (64 if pad else 0)), 7.0, device=device) + out = storage[:, :cols] + ref = (P.double().t() if op == "wgrad" else P.double()) @ ( + Q.double().t() if op == "fwd" else Q.double() + ) + stream = side if on_side else torch.cuda.current_stream(device) + stream.wait_stream(torch.cuda.current_stream(device)) + with torch.cuda.stream(stream): + with monkeypatch.context() as m: + m.setattr(cake_backend, "_k2_entry", lambda *args: None) + want = calls[op](P, Q, out, **kw).clone() + for _ in range(3): + out.fill_(float("nan")) + assert calls[op](P, Q, out, **kw) is out + stream.synchronize() + entry = _router_entry_of( + op, P, Q, out, splits if op != "dgrad" else 1, stream + ) + assert entry is not None and entry.launch is not None, (op, splits, pad) + assert _rel_fro(out, ref) <= ROUTER_REL_FRO_MAX, (op, splits, pad) + assert bool((storage[:, cols:] == 7.0).all()), (op, splits, pad) + if _router_entry_bits(entry): + assert torch.equal(out, want), (op, splits, pad) + if not entry.atomic: + det_kw = {} if op == "dgrad" or splits is None else {"splits": splits} + det_out = calls[op](P, Q, torch.empty_like(out), **det_kw) + stream.synchronize() + assert torch.equal(out, det_out), (op, splits, pad) + ran["bits" if _router_entry_bits(entry) else "gate"] += 1 + print(f"ROUTER_ENTRY {arch} H{hidden} T{T}: {ran}") + + +def test_gpu_router_entry_points_capture_in_a_cuda_graph(monkeypatch): + """The router entry points' ``deterministic=False`` calls through ``_K2_ENTRY_CACHE`` (the forward with and + without a split count, the input gradient, the weight gradient of a row with a verdict and of one without), their + keys cached on the capturing stream, capture in a CUDA graph without planning: every replay gives the eager calls' + results -- bit for bit for the deterministic program and a reproducible plan -- within the gate.""" + device, arch = _atomic_device_or_skip() + cake_backend.clear_convenience_caches() + stream = torch.cuda.Stream(device) + cases = [] + for hidden, T in ((5120, 64), (6144, 1000)): + X, W, G = _router_inputs(T, device, SEED + 110 + T, K=hidden) + # name -> (op, P, Q, splits, out) + # fmt: off + cases += [ + ("fwd", X, W, None, torch.empty(T, 256, device=device)), + ("fwd", X, W, 2, torch.empty(T, 256, device=device)), + ("dgrad", G, W, 1, torch.empty(T, hidden, device=device)), + ("wgrad", G, X, None, torch.empty(256, hidden, device=device)), + ] + # fmt: on + + def run_all(): + for op, P, Q, splits, out in cases: + if op == "fwd": + cake_backend.router_forward( + P, Q, out, splits=splits, deterministic=False + ) + elif op == "dgrad": + cake_backend.router_dgrad(P, Q, out, deterministic=False) + else: + cake_backend.router_wgrad(P, Q, out, splits=splits, deterministic=False) + + stream.wait_stream(torch.cuda.current_stream(device)) + with torch.cuda.stream(stream): + for _ in range(2): + run_all() + stream.synchronize() + eager = [out.clone() for *_, out in cases] + entries = [_router_entry_of(op, P, Q, out, s, stream) for op, P, Q, s, out in cases] + assert all(e is not None and e.launch is not None for e in entries) + keys = set(cake_backend._K2_ENTRY_CACHE) + graph = torch.cuda.CUDAGraph() + with monkeypatch.context() as m: + m.setattr( + cake_backend, + "_plan_k2_entry", + lambda *args: pytest.fail("planned in capture"), + ) + with torch.cuda.graph(graph, stream=stream): + run_all() + assert set(cake_backend._K2_ENTRY_CACHE) == keys + for _ in range(3): + for *_, out in cases: + out.fill_(float("nan")) + graph.replay() + torch.cuda.synchronize() + for (op, P, Q, splits, out), want, entry in zip( + cases, eager, entries, strict=True + ): + ref = (P.double().t() if op == "wgrad" else P.double()) @ ( + Q.double().t() if op == "fwd" else Q.double() + ) + assert _rel_fro(out, ref) <= ROUTER_REL_FRO_MAX, (op, splits) + if _router_entry_bits(entry): + assert torch.equal(out, want), (op, splits) + print( + f"ROUTER_ENTRY_GRAPH {arch}: {len(cases)} launches, " + f"{sum(map(_router_entry_bits, entries))} bit for bit" + ) + + # --------------------------------------------------------------------------- # GPU: streams, graphs, allocation-free prepared launches # ---------------------------------------------------------------------------