diff --git a/flashinfer/cake_gdn_train_kcp.py b/flashinfer/cake_gdn_train_kcp.py index 87f2f6ff734..89c25e23c62 100644 --- a/flashinfer/cake_gdn_train_kcp.py +++ b/flashinfer/cake_gdn_train_kcp.py @@ -72,6 +72,7 @@ from __future__ import annotations import dataclasses +import functools import warnings from typing import Any, Optional @@ -85,12 +86,15 @@ DEFAULT_SCALE = 128**-0.5 +# The routed modules are imported on first use and cached: a call does not repeat the import. +@functools.cache def _pkg(): from .experimental import cake_gdn_train_kcp # noqa: PLC0415 - the backend package, imported on first use return cake_gdn_train_kcp +@functools.cache def _det(): from . import cake_gdn_det_train_kcp # noqa: PLC0415 - the deterministic route, imported on first use diff --git a/flashinfer/experimental/cake_gdn_train_kcp/README.md b/flashinfer/experimental/cake_gdn_train_kcp/README.md index 0587abbcb36..485614df117 100644 --- a/flashinfer/experimental/cake_gdn_train_kcp/README.md +++ b/flashinfer/experimental/cake_gdn_train_kcp/README.md @@ -173,6 +173,11 @@ The packed GDN training routes come in four combinations: Gradient accumulation (`.backward()` twice) accumulates. * No cooperative launch anywhere in the package; every program is a plain stream-ordered launch. +* Host launch path: the record, grid and merge columns of a launch are + resolved once per launch key and kept with the context's tables; each + forward or backward body, eager or captured, binds the caller's current + stream for tvm-ffi once (`tvm_ffi.use_torch_stream()`), and a launch enters + its own context only outside such a body or after a stream switch inside it. ### Partition and context diff --git a/flashinfer/experimental/cake_gdn_train_kcp/cake_backend.py b/flashinfer/experimental/cake_gdn_train_kcp/cake_backend.py index f7b8abc3204..f9998d2d73b 100644 --- a/flashinfer/experimental/cake_gdn_train_kcp/cake_backend.py +++ b/flashinfer/experimental/cake_gdn_train_kcp/cake_backend.py @@ -40,16 +40,22 @@ parameters onto the package's logical vocabulary (``cake_jit.LOGICAL_TENSORS`` / ``LOGICAL_SCALARS``), the grid comes from the host plan (``cake_jit.launch_grid``) and the kernel runs on the current torch stream. +A launch enters ``tvm_ffi.use_torch_stream()`` itself unless an enclosing +:func:`launch_scope` (one per package entry: the forward or backward body) +already bound torch's current stream on this thread; a launch whose current +stream is not the bound one takes its own context, as without a scope. Every launch is a plain stream-ordered launch: nothing cooperative, no PDL. Nothing here synchronizes, reads device memory or allocates. """ from __future__ import annotations +import contextlib import functools import hashlib import json -from collections.abc import Mapping +import threading +from collections.abc import Iterator, Mapping from pathlib import Path from typing import Any @@ -64,6 +70,7 @@ "arch_for_capability", "device_arch", "is_prebuilt", + "launch_scope", "library_name", "load", "load_test", @@ -273,6 +280,45 @@ def test_specs_for_arch(arch: str) -> list[Any]: # Launching one record +# The stream a launch scope bound on this thread, ``(device index, raw handle)``; tvm-ffi's environment stream is +# per thread too. +_SCOPE = threading.local() + + +def _current_stream() -> tuple[int, int]: + """``(device index, raw handle)`` of torch's current stream on the current device: what + ``tvm_ffi.use_torch_stream()`` binds.""" + import torch + + index = torch.cuda.current_device() + raw = getattr(torch._C, "_cuda_getCurrentRawStream", None) + if raw is not None: + return index, int(raw(index)) + return index, int(torch.cuda.current_stream(index).cuda_stream) + + +@contextlib.contextmanager +def launch_scope() -> Iterator[None]: + """One ``tvm_ffi.use_torch_stream()`` context for the launches of a package entry. + + It binds torch's current stream as this thread's tvm-ffi environment stream once; :func:`run_module` launches + inside the scope skip their own context while torch's current stream is still the bound one, and enter it as + before otherwise (a stream switch inside the scope, a launch outside every scope). A scope inside a scope of + the same stream adds nothing; one on another stream (a CUDA graph capture inside the scope) binds its own stream + until it exits. The kernels run on the same stream either way.""" + stream = _current_stream() + outer = getattr(_SCOPE, "stream", None) + if outer == stream: + yield + return + with tvm_ffi.use_torch_stream(): + _SCOPE.stream = stream + try: + yield + finally: + _SCOPE.stream = outer + + @functools.cache def _argument_plan(name: str) -> tuple[tuple[str | None, int], ...]: """The record's plan as ``(logical name, -1)`` per kernel parameter and ``(None, axis)`` per grid argument, in @@ -322,5 +368,10 @@ def run_module( ) from None if module is None: module = load(name, device_arch(device_index)) + entry = getattr(module, found["ffi_entry"]) + bound = getattr(_SCOPE, "stream", None) + if bound is not None and bound == _current_stream(): + entry(*args) # the enclosing launch scope bound this stream + return with tvm_ffi.use_torch_stream(): - getattr(module, found["ffi_entry"])(*args) + entry(*args) diff --git a/flashinfer/experimental/cake_gdn_train_kcp/cake_kcp.py b/flashinfer/experimental/cake_gdn_train_kcp/cake_kcp.py index c2de0e20615..924638d1a2e 100644 --- a/flashinfer/experimental/cake_gdn_train_kcp/cake_kcp.py +++ b/flashinfer/experimental/cake_gdn_train_kcp/cake_kcp.py @@ -1070,6 +1070,11 @@ def begin_call(self, call: CallKcp) -> None: def end_call(self, call: CallKcp) -> None: """Hook at the end of a call.""" + def launch_scope(self, call: CallKcp) -> Any: + """The context manager the forward / backward body of ``call`` runs in (its launches, eagerly or inside a + capture), or ``None`` (the default): no scope.""" + return None + @staticmethod def transpose_tiles( *, @@ -1195,6 +1200,56 @@ def resolve_launch( return str(record), values, grid, is_test +def _launch_key(name: str, args: dict[str, Any], cols_per_cta: int) -> tuple: + """Every input :func:`resolve_launch` reads besides the launch's tensors: the program, the static knobs, the grid + rule's arguments (in the call site's order), the walk-form inputs, the merge's columns and the caller's grid x. + + The launch cache is valid only because :func:`resolve_launch` derives the record, the grid, the test flag and the + merge's ``cols_per_cta`` from these scalars alone and reads nothing from the launch's tensors: an input added to + :func:`resolve_launch` (a new grid-rule argument, for example) must be added to this key as well.""" + grid_args = args.get("_grid_args") + return ( + name, + args.get("_precision_mode"), + args.get("_transfer_walk"), + tuple(grid_args.items()) if grid_args else None, + args.get("_sm_count"), + args.get("_fwdh_vh"), + args.get("_dhu_vh"), + args.get("ncols"), + args.get("cols_per_cta"), + args.get("_grid"), + int(cols_per_cta), + ) + + +def _resolve_cached( + name: str, args: dict[str, Any], *, cols_per_cta: int = 8 +) -> tuple[str, dict[str, Any], tuple[int, int, int], bool]: + """:func:`resolve_launch` through the launch cache of the call's tables (``_Tables.launches``, one per context + plan): the record, the grid, the test flag and the merge's ``cols_per_cta`` of a launch key are resolved once; + every launch rebuilds its public values and appends ``(record, grid)`` to the call's records as + :func:`resolve_launch` does.""" + call = args.get("_call") + cache = getattr(getattr(call, "tb", None), "launches", None) + if cache is None: + return resolve_launch(name, args, cols_per_cta=cols_per_cta) + key = _launch_key(name, args, cols_per_cta) + hit = cache.get(key) + if hit is None: + record, values, grid, is_test = resolve_launch( + name, args, cols_per_cta=cols_per_cta + ) + cache[key] = (record, grid, is_test, values.get("cols_per_cta")) + return record, values, grid, is_test + record, grid, is_test, cols = hit + values = _public_args(args) + if cols is not None: + values["cols_per_cta"] = cols + call.records.append((record, grid)) + return record, values, grid, is_test + + class KernelBackend(Backend): """The shipped backend: the exported programs through the package's launch glue (:mod:`.cake_backend` ``run_module(record, values, *, grid, device_index, module=None)``, imported at the first launch so this module @@ -1250,7 +1305,7 @@ def run(self, name: str, args: dict[str, Any]) -> None: **args, "_grid_args": {**grid_args, "arch": be.device_arch(device_index)}, } - record, values, grid, is_test = resolve_launch( + record, values, grid, is_test = _resolve_cached( name, args, cols_per_cta=self.cols_per_cta ) module = None @@ -1266,6 +1321,12 @@ def run(self, name: str, args: dict[str, Any]) -> None: record, values, grid=grid, device_index=device_index, module=module ) + def launch_scope(self, call: CallKcp) -> Any: + """The launch glue's stream scope (``cake_backend.launch_scope``: one ``tvm_ffi.use_torch_stream()`` context + for the body's launches instead of one per launch); ``None`` when the glue has no scope.""" + scope = getattr(self.launcher(), "launch_scope", None) + return None if scope is None else scope() + class FakeBackend(Backend): """The torch emulation of the program ABI (``backend="fake"``; the package's :mod:`.cake_fake_backend`, imported at @@ -1290,7 +1351,7 @@ def module(self) -> Any: def run(self, name: str, args: dict[str, Any]) -> None: mod = self.module() call: CallKcp = args["_call"] - record, values, grid, _is_test = resolve_launch(name, args) + record, values, grid, _is_test = _resolve_cached(name, args) run_module = getattr(mod, "run_module", None) if ( run_module is not None @@ -1500,6 +1561,9 @@ def __init__( self, ctx: Any, HV: int, *, fused_walks: int = 1, P: int | None = None ) -> None: self.ctx = ctx + # launch key -> (record, grid, is_test, merge cols_per_cta): the launch cache of _resolve_cached (host data of + # this plan, filled at the first launch of each key) + self.launches: dict[tuple, tuple] = {} self.H = H = int(HV) # every pool, split and chain is per VALUE head self.fused_walks = int( fused_walks @@ -4254,6 +4318,17 @@ def chunk_gdn_fwd_kcp( ) +def _in_launch_scope(call: CallKcp, body: Any, *args: Any, **kw: Any) -> Any: + """``body(*args, **kw)`` -- a forward or backward body, eagerly or inside a capture -- in the backend's launch + scope of ``call`` (:meth:`Backend.launch_scope`), entered where the body's launches run (inside the capture + when there is one), so the scope binds their stream; without a scope the body runs as it is.""" + scope = call.backend.launch_scope(call) + if scope is None: + return body(*args, **kw) + with scope: + return body(*args, **kw) + + def _fwd_entry( q: torch.Tensor, k: torch.Tensor, @@ -4317,8 +4392,13 @@ def _fwd_entry( "fwd", entry, call, - lambda: _fwd_body( - call, save=save, autograd_node=autograd_node, executable=executable + lambda: _in_launch_scope( + call, + _fwd_body, + call, + save=save, + autograd_node=autograd_node, + executable=executable, ), ) if res is not None: @@ -4370,7 +4450,9 @@ def _fwd_entry( call.device, ) call.scratch["recompute_handle"] = handle_mode - out, saved = _fwd_body( + out, saved = _in_launch_scope( + call, + _fwd_body, call, save=save, hand_off=hand_off and call.T_r > 0, @@ -4725,7 +4807,9 @@ def _bwd_entry( if saved is not None: _check_saved(saved, call) if entry is not None and entry.seen > 0 and entry.disabled is None: - res = _graph_capture("bwd", entry, call, lambda: _bwd_body(call, saved)) + res = _graph_capture( + "bwd", entry, call, lambda: _in_launch_scope(call, _bwd_body, call, saved) + ) if res is not None: entry.result = (res, batched) return tuple(_ranked(t.detach(), batched) for t in res) @@ -4745,7 +4829,7 @@ def _bwd_entry( ) if saved is not None: _check_saved(saved, call) - grads = _bwd_body(call, saved) + grads = _in_launch_scope(call, _bwd_body, call, saved) _record("bwd", call) if opts.graph != "off": _graph_eager("bwd", entry) diff --git a/tests/experimental/test_cake_gdn_train_kcp.py b/tests/experimental/test_cake_gdn_train_kcp.py index 6e2a1eb7323..1440f5e0cb6 100644 --- a/tests/experimental/test_cake_gdn_train_kcp.py +++ b/tests/experimental/test_cake_gdn_train_kcp.py @@ -51,6 +51,7 @@ import random import re import threading +import types import pytest import torch @@ -4715,6 +4716,250 @@ def test_launch_recorder_lists_the_programs_of_the_call(monkeypatch) -> None: ) +# --------------------------------------------------------------------------- host launch path: launch cache, stream scope +# Host data only (record names, grids, launch order, context entries): exact checks. + + +def _harvest_launches(kcp, ctx, ops, **knobs) -> list[tuple[str, dict]]: + """``(program, args)`` of every launch of one forward + backward on the fake backend (``args`` without the call).""" + seen: list[tuple[str, dict]] = [] + + class Recording(kcp.FakeBackend): + def run(self, name, args): + seen.append((name, {k: v for k, v in args.items() if k != "_call"})) + super().run(name, args) + + previous = kcp.set_backend("fake", Recording()) + try: + args = [ops[n] for n in LEAVES] + _out, saved = kcp.chunk_gdn_fwd_kcp( + *args, ctx, backend="fake", deterministic=False, **knobs + ) + kcp.chunk_gdn_bwd_kcp( + *args, + ctx, + ops["dout"], + saved=saved, + backend="fake", + deterministic=False, + **knobs, + ) + finally: + kcp.set_backend("fake", previous) + return seen + + +def _same_launch(got: tuple, want: tuple) -> bool: + """Two ``(record, values, grid, is_test)`` resolutions name the same record, grid and flag and the same values + (the same tensor objects, equal scalars).""" + (r1, v1, g1, t1), (r2, v2, g2, t2) = got, want + if (r1, tuple(g1), t1) != (r2, tuple(g2), t2) or v1.keys() != v2.keys(): + return False + return all( + v1[k] is v2[k] if isinstance(v2[k], torch.Tensor) else v1[k] == v2[k] + for k in v2 + ) + + +LAUNCH_CACHE_KNOBS = ( + {}, + {"segments": 2}, + { + "segments": 2, + "test_knobs": {"cols_rule": "by_chains", "fwdh_vh": 64, "dhu_vh": 128}, + }, + {"segments": 2, "test_knobs": {"schedule": "overlap", "fwdh_vh": 128}}, +) + + +def test_launch_cache_resolves_every_launch_as_resolve_launch(monkeypatch) -> None: + """``_resolve_cached`` (the launch cache of the context tables) answers every launch of fake-backend forward + + backward runs on a context with and without an SM count -- default options, two segments, forced walk forms with + the by-chains merge columns, the overlap schedule; each persistent launch also with the kernels backend's ``arch`` + grid argument, each walk's arguments also as a transfer walk of every precision mode (the test-only records) -- + under both merge column defaults with the record, public values, grid and test flag of ``resolve_launch``, on the + first launch of its key (the one call of ``resolve_launch``) and on every later one, and appends + ``(record, grid)`` to the call's records each time.""" + kcp, ctx, ops = _ckpt_setup() + part = gp.partition_kcp(torch.tensor(CKPT_CU, dtype=torch.int64), 1, "contiguous") + ctx_sm = gp.build_context_kcp(part, None, 0, CPU, sm_count=SM_COUNT) + launches: list[tuple[str, dict]] = [] + for c in (ctx, ctx_sm): + for knobs in LAUNCH_CACHE_KNOBS: + launches += _harvest_launches(kcp, c, ops, **knobs) + extra = [] + for name, args in launches: + grid_args = args.get("_grid_args") or {} + if "sm_count" in grid_args: + extra.append( + (name, {**args, "_grid_args": {**grid_args, "arch": "sm_100a"}}) + ) + if name in kcp.FWDH_PROGRAMS + kcp.DHU_PROGRAMS: + extra += [ + ("kcp_transfer_walk", {**args, "_precision_mode": pm}) + for pm in (0, 1, 2) + ] + launches += extra + names = {n for n, _ in launches} + assert {"kcp_affine_merge", "kcp_transfer_walk", "fwd_o", "bwd_dqkg"} <= names + assert names & set(kcp.FWDH_PROGRAMS) and names & set(kcp.DHU_PROGRAMS) + real = kcp.resolve_launch + assert any(real(n, dict(a))[3] for n, a in launches) # a test-only record + misses: list[tuple] = [] + + def counting(name, args, *, cols_per_cta=8): + misses.append(kcp._launch_key(name, args, cols_per_cta)) + return real(name, args, cols_per_cta=cols_per_cta) + + monkeypatch.setattr(kcp, "resolve_launch", counting) + call = types.SimpleNamespace(tb=types.SimpleNamespace(launches={}), records=[]) + for _round in range(2): + for cols in (8, 16): + for name, args in launches: + want = real(name, dict(args), cols_per_cta=cols) + got = kcp._resolve_cached( + name, {**args, "_call": call}, cols_per_cta=cols + ) + assert _same_launch(got, want), (name, cols) + assert call.records[-1] == (want[0], want[2]), (name, cols) + keys = {kcp._launch_key(n, a, c) for n, a in launches for c in (8, 16)} + assert len(misses) == len(set(misses)) == len(keys) == len(call.tb.launches) + assert len(call.records) == 4 * len(launches) + # no tables, or tables without a cache: resolve_launch itself (nothing cached) + for stub in (types.SimpleNamespace(records=[]), None): + name, args = launches[0] + before = len(misses) + got = kcp._resolve_cached(name, {**args, "_call": stub}) + assert _same_launch(got, real(name, dict(args))) and len(misses) == before + 1 + + +def test_launch_cache_keeps_every_calls_launch_records(monkeypatch) -> None: + """One context on the fake backend: a run on cold launch caches, a run on warm ones and a run with the cache + bypassed (``resolve_launch`` per launch) record the same launches -- records and grids in order, forward and + backward, direct and checkpointed -- the tables cache one entry per launch key, and every run's results are within + the budget of the FP64 reference and of each other.""" + kcp, ctx, ops = _ckpt_setup() + ref = _ckpt_reference() + tb = kcp._context_tables(ctx, CKPT_HEADS[1], fused_walks=1, P=2) + assert tb.launches == {} + arms, records = {}, {} + for arm in ("cold", "warm", "bypassed"): + res = {} + for ckpt in (False, True): + with monkeypatch.context() as m: + if arm == "bypassed": + m.setattr( + kcp, + "_resolve_cached", + lambda name, args, *, cols_per_cta=8: kcp.resolve_launch( + name, args, cols_per_cta=cols_per_cta + ), + ) + res[ckpt] = _ckpt_autograd(kcp, ctx, ops, ckpt=ckpt, segments=2) + records[(arm, ckpt)] = { + k: list(kcp.LAST_CALL[k]["records"]) for k in ("fwd", "bwd") + } + arms[arm] = res + if arm == "cold": + n_keys = len(tb.launches) + assert n_keys > 0 + assert len(tb.launches) == n_keys # warm and bypassed runs add no key + for ckpt in (False, True): + assert ( + records[("cold", ckpt)] + == records[("warm", ckpt)] + == records[("bypassed", ckpt)] + ), ckpt + _arms_within_budget( + {arm: arms[arm][ckpt] for arm in arms}, + ref, + ("out",) + CKPT_GRADS, + f"launch cache arms (checkpoint={ckpt})", + ) + launched = { + (r, tuple(g)) for rec in records.values() for k in rec for r, g in rec[k] + } + assert launched <= {(r, g) for r, g, _t, _c in tb.launches.values()} + + +def test_launch_scope_binds_the_stream_once_per_entry_body(monkeypatch) -> None: + """``cake_backend.launch_scope`` with a counting stand-in for ``tvm_ffi`` and a set current stream (CPU): a launch + outside every scope enters its own ``use_torch_stream()`` context; inside a scope it enters none while the current + stream is the bound one and its own after a stream switch; a nested scope on the same stream enters nothing, one + on another stream binds that stream until it exits; another thread sees no scope; an exception restores the + binding. The kernels backend hands out its glue's scope; a glue without one and the fake backend none.""" + be = _package("cake_backend") + kcp = _package("cake_kcp") + state = {"stream": (0, 11), "enters": 0, "bound": [], "launched": []} + + class _Context: + def __enter__(self): + state["enters"] += 1 + state["bound"].append(state["stream"]) + return self + + def __exit__(self, *exc): + state["bound"].pop() + + monkeypatch.setattr( + be, + "tvm_ffi", + types.SimpleNamespace(use_torch_stream=lambda: _Context()), + ) + monkeypatch.setattr(be, "_current_stream", lambda: state["stream"]) + name = "fwd_prep" + values = {n: n for n, _axis in be._argument_plan(name) if n is not None} + module = types.SimpleNamespace( + **{ + be.record(name)["ffi_entry"]: lambda *a: state["launched"].append( + state["bound"][-1] if state["bound"] else None + ) + } + ) + + def launch() -> None: + be.run_module(name, values, grid=(1, 1, 1), device_index=0, module=module) + + def enters_after(fn) -> int: + n = state["enters"] + fn() + return state["enters"] - n + + assert enters_after(launch) == 1 and state["launched"][-1] == (0, 11) + with be.launch_scope(): + assert state["enters"] == 2 and be._SCOPE.stream == (0, 11) + assert enters_after(launch) == 0 and enters_after(launch) == 0 + assert state["launched"][-1] == (0, 11) + with be.launch_scope(): + assert enters_after(launch) == 0 + assert be._SCOPE.stream == (0, 11) and state["bound"] == [(0, 11)] + state["stream"] = (0, 22) # a stream switch inside the scope + assert enters_after(launch) == 1 and state["launched"][-1] == (0, 22) + with be.launch_scope(): # a scope on the other stream binds it + assert be._SCOPE.stream == (0, 22) + assert enters_after(launch) == 0 and state["launched"][-1] == (0, 22) + assert be._SCOPE.stream == (0, 11) + assert enters_after(launch) == 1 + state["stream"] = (0, 11) + worker = threading.Thread(target=launch) + assert enters_after(lambda: (worker.start(), worker.join())) == 1 + assert enters_after(launch) == 0 + assert getattr(be._SCOPE, "stream", None) is None and state["bound"] == [] + with pytest.raises(RuntimeError, match="inside the scope"), be.launch_scope(): + raise RuntimeError("inside the scope") + assert getattr(be._SCOPE, "stream", None) is None and state["bound"] == [] + assert enters_after(launch) == 1 + # the backends' scopes + assert kcp.FakeBackend().launch_scope(None) is None + assert ( + kcp.KernelBackend(launcher=types.SimpleNamespace()).launch_scope(None) is None + ) + with kcp.KernelBackend(launcher=be).launch_scope(None): + assert be._SCOPE.stream == (0, 11) + assert enters_after(launch) == 0 + assert getattr(be._SCOPE, "stream", None) is None + + # --------------------------------------------------------------------------- numerics harness hooks (fake backend) NONDET_REPEATS = 8 # the worst case over this many runs is gated per tensor diff --git a/tests/experimental/test_cake_gdn_train_kcp_api.py b/tests/experimental/test_cake_gdn_train_kcp_api.py index f668976ab0b..9f427aac15a 100644 --- a/tests/experimental/test_cake_gdn_train_kcp_api.py +++ b/tests/experimental/test_cake_gdn_train_kcp_api.py @@ -26,6 +26,7 @@ from __future__ import annotations import ast +import builtins import dataclasses import importlib import inspect @@ -963,6 +964,32 @@ def test_facade_routes_by_mode(monkeypatch) -> None: facade.is_gdn_train_kcp_supported("cpu", deterministic=None) +def test_facade_imports_each_route_once(monkeypatch) -> None: + """The facade imports each routed module on first use only: later calls on either route run no import and reach + the same module objects.""" + facade = importlib.import_module(FACADE) + _route_recorders(monkeypatch) + calls = _facade_calls(facade).values() + for call in calls: # first use of both routes + call() + call(deterministic=False) + imports: list = [] + real_import = builtins.__import__ + + def counting_import(name, *args, **kwargs): + imports.append(name) + return real_import(name, *args, **kwargs) + + with monkeypatch.context() as m: + m.setattr(builtins, "__import__", counting_import) + for call in calls: + call() + call(deterministic=False) + assert imports == [], imports + assert facade._pkg() is importlib.import_module(PACKAGE) + assert facade._det() is importlib.import_module(DET_FACADE) + + def test_facade_context_graph_replay_hint(monkeypatch) -> None: """``build_context_kcp(..., graph_replay=...)``: keyword-only, default ``False``, a non-bool is a ``TypeError``; the non-deterministic route receives the hint, the deterministic route (whose plan is the graph plan already) does diff --git a/tests/experimental/test_cake_gdn_train_kcp_cp1.py b/tests/experimental/test_cake_gdn_train_kcp_cp1.py index 737a2950abc..ca8a09174f3 100644 --- a/tests/experimental/test_cake_gdn_train_kcp_cp1.py +++ b/tests/experimental/test_cake_gdn_train_kcp_cp1.py @@ -695,3 +695,84 @@ def test_two_caller_streams_with_cached_workspaces_are_within_budget(device) -> TENSORS, f"stream {key}", ) + + +# --------------------------------------------------------------------------- the launch scope (one GPU) + + +def test_launch_scope_enters_one_stream_context_per_entry_body( + device, monkeypatch +) -> None: + """The kernels backend's launch path on a caller stream: an autograd forward + backward, direct and under a + non-reentrant checkpoint, enters ``tvm_ffi.use_torch_stream()`` once per entry body (the launch scope) instead + of once per launch, launches the same records on the same grids as with the scope off (one context per launch), + and both arms' results are within the budget of the FP64 reference and of each other.""" + from torch.utils.checkpoint import checkpoint + + be = pytest.importorskip("flashinfer.experimental.cake_gdn_train_kcp.cake_backend") + kcp = pytest.importorskip("flashinfer.experimental.cake_gdn_train_kcp.cake_kcp") + real = be.tvm_ffi + entered = [0] + + class _Counting: + def __getattr__(self, name): + return getattr(real, name) + + @staticmethod + def use_torch_stream(*args, **kwargs): + entered[0] += 1 + return real.use_torch_stream(*args, **kwargs) + + monkeypatch.setattr(be, "tvm_ffi", _Counting()) + cu, heads = [0, 700, 1900, 3000], (2, 6) + inputs = make_inputs(cu, *heads) + ref, _ = reference_pack(inputs, device) + ctx = _local_context(cu, device) + stream = torch.cuda.Stream(device=device) + torch.cuda.synchronize(device) + results, launches, contexts = {}, {}, {} + for arm in ("scope", "per-launch"): + if arm == "per-launch": + monkeypatch.setattr( + kcp.KernelBackend, "launch_scope", lambda self, call: None + ) + for ckpt in (False, True): + entered[0] = 0 + with torch.cuda.stream(stream): + leaves = [ + inputs[n].to(device).detach().requires_grad_(True) for n in OPERANDS + ] + + def layer(q, k, v, g, beta): + return gdn_api.chunk_gdn_kcp( + q, k, v, g, beta, ctx, deterministic=False + ) + + out = ( + checkpoint(layer, *leaves, use_reentrant=False) + if ckpt + else layer(*leaves) + ) + grads = torch.autograd.grad(out, leaves, inputs["dout"].to(device)) + stream.synchronize() + results[(arm, ckpt)] = { + "out": out.detach()[0], + **{n: t.detach()[0] for n, t in zip(GRAD_NAMES, grads, strict=True)}, + } + launches[(arm, ckpt)] = { + k: [(r, tuple(g)) for r, g in kcp.LAST_CALL[k]["records"]] + for k in ("fwd", "bwd") + } + contexts[(arm, ckpt)] = entered[0] + for ckpt in (False, True): + bodies = 3 if ckpt else 2 # forward, (the checkpoint's recompute,) backward + n_launches = sum(len(v) for v in launches[("per-launch", ckpt)].values()) + assert launches[("scope", ckpt)] == launches[("per-launch", ckpt)], ckpt + assert contexts[("scope", ckpt)] == bodies, (ckpt, contexts) + assert contexts[("per-launch", ckpt)] >= n_launches > bodies, (ckpt, contexts) + _arms_within_budget( + {arm: results[(arm, ckpt)] for arm in ("scope", "per-launch")}, + ref, + TENSORS, + f"launch scope (checkpoint={ckpt})", + )