Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions flashinfer/cake_gdn_train_kcp.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@
from __future__ import annotations

import dataclasses
import functools
import warnings
from typing import Any, Optional

Expand All @@ -85,12 +86,15 @@
DEFAULT_SCALE = 128**-0.5


# The routed modules are imported on first use and cached: a call does not repeat the import.
@functools.cache
def _pkg():
from .experimental import cake_gdn_train_kcp # noqa: PLC0415 - the backend package, imported on first use

return cake_gdn_train_kcp


@functools.cache
def _det():
from . import cake_gdn_det_train_kcp # noqa: PLC0415 - the deterministic route, imported on first use

Expand Down
5 changes: 5 additions & 0 deletions flashinfer/experimental/cake_gdn_train_kcp/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,11 @@ The packed GDN training routes come in four combinations:
Gradient accumulation (`.backward()` twice) accumulates.
* No cooperative launch anywhere in the package; every program is a plain
stream-ordered launch.
* Host launch path: the record, grid and merge columns of a launch are
resolved once per launch key and kept with the context's tables; each
forward or backward body, eager or captured, binds the caller's current
stream for tvm-ffi once (`tvm_ffi.use_torch_stream()`), and a launch enters
its own context only outside such a body or after a stream switch inside it.

### Partition and context

Expand Down
55 changes: 53 additions & 2 deletions flashinfer/experimental/cake_gdn_train_kcp/cake_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,16 +40,22 @@
parameters onto the package's logical vocabulary (``cake_jit.LOGICAL_TENSORS``
/ ``LOGICAL_SCALARS``), the grid comes from the host plan
(``cake_jit.launch_grid``) and the kernel runs on the current torch stream.
A launch enters ``tvm_ffi.use_torch_stream()`` itself unless an enclosing
:func:`launch_scope` (one per package entry: the forward or backward body)
already bound torch's current stream on this thread; a launch whose current
stream is not the bound one takes its own context, as without a scope.
Every launch is a plain stream-ordered launch: nothing cooperative, no PDL.
Nothing here synchronizes, reads device memory or allocates.
"""

from __future__ import annotations

import contextlib
import functools
import hashlib
import json
from collections.abc import Mapping
import threading
from collections.abc import Iterator, Mapping
from pathlib import Path
from typing import Any

Expand All @@ -64,6 +70,7 @@
"arch_for_capability",
"device_arch",
"is_prebuilt",
"launch_scope",
"library_name",
"load",
"load_test",
Expand Down Expand Up @@ -273,6 +280,45 @@ def test_specs_for_arch(arch: str) -> list[Any]:
# Launching one record


# The stream a launch scope bound on this thread, ``(device index, raw handle)``; tvm-ffi's environment stream is
# per thread too.
_SCOPE = threading.local()


def _current_stream() -> tuple[int, int]:
"""``(device index, raw handle)`` of torch's current stream on the current device: what
``tvm_ffi.use_torch_stream()`` binds."""
import torch

index = torch.cuda.current_device()
raw = getattr(torch._C, "_cuda_getCurrentRawStream", None)
if raw is not None:
return index, int(raw(index))
return index, int(torch.cuda.current_stream(index).cuda_stream)


@contextlib.contextmanager
def launch_scope() -> Iterator[None]:
"""One ``tvm_ffi.use_torch_stream()`` context for the launches of a package entry.

It binds torch's current stream as this thread's tvm-ffi environment stream once; :func:`run_module` launches
inside the scope skip their own context while torch's current stream is still the bound one, and enter it as
before otherwise (a stream switch inside the scope, a launch outside every scope). A scope inside a scope of
the same stream adds nothing; one on another stream (a CUDA graph capture inside the scope) binds its own stream
until it exits. The kernels run on the same stream either way."""
stream = _current_stream()
outer = getattr(_SCOPE, "stream", None)
if outer == stream:
yield
return
with tvm_ffi.use_torch_stream():
_SCOPE.stream = stream
try:
yield
finally:
_SCOPE.stream = outer


@functools.cache
def _argument_plan(name: str) -> tuple[tuple[str | None, int], ...]:
"""The record's plan as ``(logical name, -1)`` per kernel parameter and ``(None, axis)`` per grid argument, in
Expand Down Expand Up @@ -322,5 +368,10 @@ def run_module(
) from None
if module is None:
module = load(name, device_arch(device_index))
entry = getattr(module, found["ffi_entry"])
bound = getattr(_SCOPE, "stream", None)
if bound is not None and bound == _current_stream():
entry(*args) # the enclosing launch scope bound this stream
return
with tvm_ffi.use_torch_stream():
getattr(module, found["ffi_entry"])(*args)
entry(*args)
94 changes: 87 additions & 7 deletions flashinfer/experimental/cake_gdn_train_kcp/cake_kcp.py
Original file line number Diff line number Diff line change
Expand Up @@ -1070,6 +1070,11 @@ def begin_call(self, call: CallKcp) -> None:
def end_call(self, call: CallKcp) -> None:
"""Hook at the end of a call."""

def launch_scope(self, call: CallKcp) -> Any:
"""The context manager the forward / backward body of ``call`` runs in (its launches, eagerly or inside a
capture), or ``None`` (the default): no scope."""
return None

@staticmethod
def transpose_tiles(
*,
Expand Down Expand Up @@ -1195,6 +1200,52 @@ def resolve_launch(
return str(record), values, grid, is_test


def _launch_key(name: str, args: dict[str, Any], cols_per_cta: int) -> tuple:
"""Every input :func:`resolve_launch` reads besides the launch's tensors: the program, the static knobs, the grid
rule's arguments (in the call site's order), the walk-form inputs, the merge's columns and the caller's grid x."""
grid_args = args.get("_grid_args")
return (
name,
args.get("_precision_mode"),
args.get("_transfer_walk"),
tuple(grid_args.items()) if grid_args else None,
args.get("_sm_count"),
args.get("_fwdh_vh"),
args.get("_dhu_vh"),
args.get("ncols"),
args.get("cols_per_cta"),
args.get("_grid"),
int(cols_per_cta),
)


def _resolve_cached(
name: str, args: dict[str, Any], *, cols_per_cta: int = 8
) -> tuple[str, dict[str, Any], tuple[int, int, int], bool]:
""":func:`resolve_launch` through the launch cache of the call's tables (``_Tables.launches``, one per context
plan): the record, the grid, the test flag and the merge's ``cols_per_cta`` of a launch key are resolved once;
every launch rebuilds its public values and appends ``(record, grid)`` to the call's records as
:func:`resolve_launch` does."""
call = args.get("_call")
cache = getattr(getattr(call, "tb", None), "launches", None)
if cache is None:
return resolve_launch(name, args, cols_per_cta=cols_per_cta)
key = _launch_key(name, args, cols_per_cta)
hit = cache.get(key)
if hit is None:
record, values, grid, is_test = resolve_launch(
name, args, cols_per_cta=cols_per_cta
)
cache[key] = (record, grid, is_test, values.get("cols_per_cta"))
return record, values, grid, is_test
record, grid, is_test, cols = hit
values = _public_args(args)
if cols is not None:
values["cols_per_cta"] = cols
call.records.append((record, grid))
return record, values, grid, is_test


class KernelBackend(Backend):
"""The shipped backend: the exported programs through the package's launch glue (:mod:`.cake_backend`
``run_module(record, values, *, grid, device_index, module=None)``, imported at the first launch so this module
Expand Down Expand Up @@ -1250,7 +1301,7 @@ def run(self, name: str, args: dict[str, Any]) -> None:
**args,
"_grid_args": {**grid_args, "arch": be.device_arch(device_index)},
}
record, values, grid, is_test = resolve_launch(
record, values, grid, is_test = _resolve_cached(
name, args, cols_per_cta=self.cols_per_cta
)
module = None
Expand All @@ -1266,6 +1317,12 @@ def run(self, name: str, args: dict[str, Any]) -> None:
record, values, grid=grid, device_index=device_index, module=module
)

def launch_scope(self, call: CallKcp) -> Any:
"""The launch glue's stream scope (``cake_backend.launch_scope``: one ``tvm_ffi.use_torch_stream()`` context
for the body's launches instead of one per launch); ``None`` when the glue has no scope."""
scope = getattr(self.launcher(), "launch_scope", None)
return None if scope is None else scope()


class FakeBackend(Backend):
"""The torch emulation of the program ABI (``backend="fake"``; the package's :mod:`.cake_fake_backend`, imported at
Expand All @@ -1290,7 +1347,7 @@ def module(self) -> Any:
def run(self, name: str, args: dict[str, Any]) -> None:
mod = self.module()
call: CallKcp = args["_call"]
record, values, grid, _is_test = resolve_launch(name, args)
record, values, grid, _is_test = _resolve_cached(name, args)
run_module = getattr(mod, "run_module", None)
if (
run_module is not None
Expand Down Expand Up @@ -1500,6 +1557,9 @@ def __init__(
self, ctx: Any, HV: int, *, fused_walks: int = 1, P: int | None = None
) -> None:
self.ctx = ctx
# launch key -> (record, grid, is_test, merge cols_per_cta): the launch cache of _resolve_cached (host data of
# this plan, filled at the first launch of each key)
self.launches: dict[tuple, tuple] = {}
self.H = H = int(HV) # every pool, split and chain is per VALUE head
self.fused_walks = int(
fused_walks
Expand Down Expand Up @@ -4254,6 +4314,17 @@ def chunk_gdn_fwd_kcp(
)


def _in_launch_scope(call: CallKcp, body: Any, *args: Any, **kw: Any) -> Any:
"""``body(*args, **kw)`` -- a forward or backward body, eagerly or inside a capture -- in the backend's launch
scope of ``call`` (:meth:`Backend.launch_scope`), entered where the body's launches run (inside the capture
when there is one), so the scope binds their stream; without a scope the body runs as it is."""
scope = call.backend.launch_scope(call)
if scope is None:
return body(*args, **kw)
with scope:
return body(*args, **kw)


def _fwd_entry(
q: torch.Tensor,
k: torch.Tensor,
Expand Down Expand Up @@ -4317,8 +4388,13 @@ def _fwd_entry(
"fwd",
entry,
call,
lambda: _fwd_body(
call, save=save, autograd_node=autograd_node, executable=executable
lambda: _in_launch_scope(
call,
_fwd_body,
call,
save=save,
autograd_node=autograd_node,
executable=executable,
),
)
if res is not None:
Expand Down Expand Up @@ -4370,7 +4446,9 @@ def _fwd_entry(
call.device,
)
call.scratch["recompute_handle"] = handle_mode
out, saved = _fwd_body(
out, saved = _in_launch_scope(
call,
_fwd_body,
call,
save=save,
hand_off=hand_off and call.T_r > 0,
Expand Down Expand Up @@ -4725,7 +4803,9 @@ def _bwd_entry(
if saved is not None:
_check_saved(saved, call)
if entry is not None and entry.seen > 0 and entry.disabled is None:
res = _graph_capture("bwd", entry, call, lambda: _bwd_body(call, saved))
res = _graph_capture(
"bwd", entry, call, lambda: _in_launch_scope(call, _bwd_body, call, saved)
)
if res is not None:
entry.result = (res, batched)
return tuple(_ranked(t.detach(), batched) for t in res)
Expand All @@ -4745,7 +4825,7 @@ def _bwd_entry(
)
if saved is not None:
_check_saved(saved, call)
grads = _bwd_body(call, saved)
grads = _in_launch_scope(call, _bwd_body, call, saved)
_record("bwd", call)
if opts.graph != "off":
_graph_eager("bwd", entry)
Expand Down
Loading
Loading