diff --git a/CLAUDE.md b/CLAUDE.md index b3b97db6c98..609bc023764 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -34,6 +34,9 @@ FlashInfer is a GPU kernel library for LLM serving that uses **JIT (Just-In-Time | Enable GDN strided QKV path | `export FLASHINFER_GDN_WY_STRIDED_QKV=1` | | Enable GDN native A/B tensors | `export FLASHINFER_GDN_WY_NATIVE_AB=1` | | Override CuTe-DSL prefill scheduling | `export FLASHINFER_CUTE_PREFILL_PERSISTENT=0` (non-persistent) or `1` (persistent) | +| Skip MoE EP CuTe-DSL import/version guard | `export FLASHINFER_MOE_EP_SKIP_DSL_CHECK=1` | +| Override MoE EP knob-cache path | `export FLASHINFER_MOE_EP_KNOB_CACHE=/path/to/knobs.json` | +| Disable MoE EP fused staging kernel | `export FLASHINFER_MEGA_FUSED_STAGE=0` | ## Quick Start for Development diff --git a/docs/api/comm.rst b/docs/api/comm.rst index 0988c4f0891..103c0e82453 100644 --- a/docs/api/comm.rst +++ b/docs/api/comm.rst @@ -252,13 +252,16 @@ Topology Probing and Backend Selection .. autosummary:: :toctree: ../generated - resolve_ulysses_backend - decide_ulysses_backend - probe_ulysses_rank_topology UlyssesBackendDecision UlyssesRankTopology UlyssesBackendError +.. autofunction:: resolve_ulysses_backend + +.. autofunction:: decide_ulysses_backend + +.. autofunction:: probe_ulysses_rank_topology + Raw Kernel Entry Points (advanced) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/flashinfer/comm/mixed_comm.py b/flashinfer/comm/mixed_comm.py index 16a21ae0e5e..ea7a0a95b89 100644 --- a/flashinfer/comm/mixed_comm.py +++ b/flashinfer/comm/mixed_comm.py @@ -935,7 +935,17 @@ def checkpoint_prepare(self): @flashinfer_api def checkpoint_restore(self, comm_backend: CommBackend): - """Restore physical VMM backing at retained graph-visible addresses.""" + """Restore physical VMM backing at retained graph-visible addresses. + + Parameters + ---------- + comm_backend : CommBackend + Intra-node communication backend whose ``Get_rank()`` and + ``Get_size()`` match this handler's ``local_rank`` and + ``local_size``. Its ``bcast`` and ``barrier`` operations + coordinate the Unix-socket handle exchange and synchronize VMM + mapping restoration. + """ if self.para_info.use_inter: raise NotImplementedError( "MixedComm checkpointing does not yet support multi-node handlers" diff --git a/flashinfer/comm/ulysses.py b/flashinfer/comm/ulysses.py index e25474e59d5..7819d63cca7 100644 --- a/flashinfer/comm/ulysses.py +++ b/flashinfer/comm/ulysses.py @@ -149,6 +149,29 @@ def __init__( backend: str = "auto", device: Optional[Union[torch.device, str, int]] = None, ): + r"""Construct a Ulysses communicator. + + Parameters + ---------- + group : Optional[ProcessGroup], optional + Process group spanning the participating ranks. ``None`` uses + ``torch.distributed.group.WORLD``. + max_elems : int + Per-rank upper bound on the number of elements communicated by a + single collective call. Used to size the backend workspace. + dtype : torch.dtype + Element dtype for collective operands. Must be one of + ``torch.float16``, ``torch.bfloat16``, or ``torch.float32``. + backend : str, default = "auto" + Backend selection policy. ``"auto"`` probes topology and prefers + NVLink when supported, otherwise falls back to NCCL. ``"nvlink"`` + forces the NVLink backend and raises if unavailable. ``"nccl"`` + forces the NCCL path. + device : Optional[Union[torch.device, str, int]], optional + CUDA device bound to this rank. ``None`` uses the current CUDA + device. Strings and integers are normalized to an explicit CUDA + ordinal. + """ self._state = _CLOSED # flipped to OPEN only when construction succeeds self._nvlink_armed = False # joint property: set on all ranks or none # opaque NVLink-backend handle (a C++ UlyssesA2A* as an int) from @@ -650,6 +673,17 @@ def scatter_heads(self, x: torch.Tensor) -> torch.Tensor: ``[rank * H_local, (rank+1) * H_local)`` of every token. Runs on the current CUDA stream. Returns the input unchanged when ``world_size == 1``. + + Parameters + ---------- + x : torch.Tensor + Contiguous 4-D CUDA tensor with shape ``[B, S_local, H, D]``. + + Returns + ------- + torch.Tensor + Tensor with shape ``[B, S_global, H_local, D]`` on the same device + and dtype as ``x``. """ self._validate(x, "scatter_heads") B, S_local, H, D = x.shape @@ -681,6 +715,17 @@ def gather_heads(self, x: torch.Tensor) -> torch.Tensor: Inverse of :meth:`scatter_heads`: gather all head slices for this rank's local sequence shard. Runs on the current CUDA stream. Returns the input unchanged when ``world_size == 1``. + + Parameters + ---------- + x : torch.Tensor + Contiguous 4-D CUDA tensor with shape ``[B, S_global, H_local, D]``. + + Returns + ------- + torch.Tensor + Tensor with shape ``[B, S_local, H, D]`` on the same device and + dtype as ``x``. """ self._validate(x, "gather_heads") B, S_global, H_local, D = x.shape @@ -958,6 +1003,26 @@ def ulysses_a2a( tensors of the same dtype (float32/float16/bfloat16). All ranks must call with consistent geometry in the same order; a mismatch is a collective failure (hang or corruption), as with any collective. + + Parameters + ---------- + fa : int + Opaque backend handle returned by :func:`init_ulysses_a2a`. + inp : torch.Tensor + Contiguous 4-D CUDA input tensor. + out : torch.Tensor + Contiguous 4-D CUDA output tensor written in place. + B : int + Batch size. + S_local : int + Local sequence length per rank. + H : int + Global head count. + D : int + Head dimension. + mode : int + ``0`` for scatter-heads input all-to-all, ``1`` for gather-heads + output all-to-all. """ if type(fa) is not int or fa == 0: raise ValueError( diff --git a/flashinfer/comm/ulysses_topology.py b/flashinfer/comm/ulysses_topology.py index 4ca9d8db26f..82049d86157 100644 --- a/flashinfer/comm/ulysses_topology.py +++ b/flashinfer/comm/ulysses_topology.py @@ -79,6 +79,19 @@ def probe_ulysses_rank_topology( Never raises: the whole probe (including hostname and device resolution) runs inside an exception envelope, so any failure lands in ``probe_error`` and the (conservative) decision layer falls back to NCCL. + + Parameters + ---------- + device : torch.device or str or int, optional + CUDA device for this rank. ``None`` uses the current CUDA device. + rank : int + Rank id to record in the returned topology object. + + Returns + ------- + UlyssesRankTopology + Per-rank topology information, including any probe error instead of + raising locally. """ topo = UlyssesRankTopology(rank=rank) try: @@ -160,6 +173,20 @@ def decide_ulysses_backend( inconsistent, or unverifiable selects NCCL. Raises :class:`UlyssesBackendError` only when ``requested == "nvlink"`` and the NVLink path cannot be used. + + Parameters + ---------- + requested : str + Requested backend policy: ``"auto"``, ``"nvlink"``, or ``"nccl"``. + topologies : list[UlyssesRankTopology] + Per-rank topology probe results gathered from the process group. + supported_world_sizes : Sequence[int], optional + World sizes for which the fused NVLink kernel is instantiated. + + Returns + ------- + UlyssesBackendDecision + Backend choice and selection or fallback reason. """ if requested not in ULYSSES_BACKENDS: raise ValueError( @@ -267,6 +294,21 @@ def resolve_ulysses_backend( catches the result into an outcome, gathers, and cross-checks. Any disagreement conservatively selects NCCL — or raises for ``backend="nvlink"``. + + Parameters + ---------- + backend : str, default = "auto" + Requested backend policy: ``"auto"``, ``"nvlink"``, or ``"nccl"``. + group : torch.distributed.ProcessGroup, optional + Process group whose ranks must make a consistent backend decision. + Defaults to ``torch.distributed.group.WORLD``. + device : torch.device or str or int, optional + CUDA device to use for probe and metadata collective guards. + + Returns + ------- + UlyssesBackendDecision + Group-consistent backend choice and selection or fallback reason. """ if group is None: group = dist.group.WORLD