From 5392f20dabfb53b8a13541a7158d0a5ca9319041 Mon Sep 17 00:00:00 2001 From: kangbintNV Date: Wed, 29 Jul 2026 13:43:37 +0000 Subject: [PATCH 1/5] docs: fix 10 new documentation issues --- CLAUDE.md | 3 +++ docs/api/comm.rst | 9 +++++--- flashinfer/comm/ulysses.py | 45 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index b3b97db6c98..609bc023764 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -34,6 +34,9 @@ FlashInfer is a GPU kernel library for LLM serving that uses **JIT (Just-In-Time | Enable GDN strided QKV path | `export FLASHINFER_GDN_WY_STRIDED_QKV=1` | | Enable GDN native A/B tensors | `export FLASHINFER_GDN_WY_NATIVE_AB=1` | | Override CuTe-DSL prefill scheduling | `export FLASHINFER_CUTE_PREFILL_PERSISTENT=0` (non-persistent) or `1` (persistent) | +| Skip MoE EP CuTe-DSL import/version guard | `export FLASHINFER_MOE_EP_SKIP_DSL_CHECK=1` | +| Override MoE EP knob-cache path | `export FLASHINFER_MOE_EP_KNOB_CACHE=/path/to/knobs.json` | +| Disable MoE EP fused staging kernel | `export FLASHINFER_MEGA_FUSED_STAGE=0` | ## Quick Start for Development diff --git a/docs/api/comm.rst b/docs/api/comm.rst index 0988c4f0891..eb56b147310 100644 --- a/docs/api/comm.rst +++ b/docs/api/comm.rst @@ -252,9 +252,6 @@ Topology Probing and Backend Selection .. autosummary:: :toctree: ../generated - resolve_ulysses_backend - decide_ulysses_backend - probe_ulysses_rank_topology UlyssesBackendDecision UlyssesRankTopology UlyssesBackendError @@ -272,6 +269,12 @@ verified all-pairs NVLink P2P and owns the IPC workspace lifecycle. dispose_ulysses_a2a ulysses_a2a +.. autofunction:: resolve_ulysses_backend + +.. autofunction:: decide_ulysses_backend + +.. autofunction:: probe_ulysses_rank_topology + MNNVL (Multi-Node NVLink) ------------------------- diff --git a/flashinfer/comm/ulysses.py b/flashinfer/comm/ulysses.py index e25474e59d5..205e19764db 100644 --- a/flashinfer/comm/ulysses.py +++ b/flashinfer/comm/ulysses.py @@ -149,6 +149,29 @@ def __init__( backend: str = "auto", device: Optional[Union[torch.device, str, int]] = None, ): + r"""Construct a Ulysses communicator. + + Parameters + ---------- + group : Optional[ProcessGroup], optional + Process group spanning the participating ranks. ``None`` uses + ``torch.distributed.group.WORLD``. + max_elems : int + Per-rank upper bound on the number of elements communicated by a + single collective call. Used to size the backend workspace. + dtype : torch.dtype + Element dtype for collective operands. Must be one of + ``torch.float16``, ``torch.bfloat16``, or ``torch.float32``. + backend : str, default = "auto" + Backend selection policy. ``"auto"`` probes topology and prefers + NVLink when supported, otherwise falls back to NCCL. ``"nvlink"`` + forces the NVLink backend and raises if unavailable. ``"nccl"`` + forces the NCCL path. + device : Optional[Union[torch.device, str, int]], optional + CUDA device bound to this rank. ``None`` uses the current CUDA + device. Strings and integers are normalized to an explicit CUDA + ordinal. + """ self._state = _CLOSED # flipped to OPEN only when construction succeeds self._nvlink_armed = False # joint property: set on all ranks or none # opaque NVLink-backend handle (a C++ UlyssesA2A* as an int) from @@ -650,6 +673,17 @@ def scatter_heads(self, x: torch.Tensor) -> torch.Tensor: ``[rank * H_local, (rank+1) * H_local)`` of every token. Runs on the current CUDA stream. Returns the input unchanged when ``world_size == 1``. + + Parameters + ---------- + x : torch.Tensor + Contiguous 4-D CUDA tensor with shape ``[B, S_local, H, D]``. + + Returns + ------- + torch.Tensor + Tensor with shape ``[B, S_global, H_local, D]`` on the same device + and dtype as ``x``. """ self._validate(x, "scatter_heads") B, S_local, H, D = x.shape @@ -681,6 +715,17 @@ def gather_heads(self, x: torch.Tensor) -> torch.Tensor: Inverse of :meth:`scatter_heads`: gather all head slices for this rank's local sequence shard. Runs on the current CUDA stream. Returns the input unchanged when ``world_size == 1``. + + Parameters + ---------- + x : torch.Tensor + Contiguous 4-D CUDA tensor with shape ``[B, S_global, H_local, D]``. + + Returns + ------- + torch.Tensor + Tensor with shape ``[B, S_local, H, D]`` on the same device and + dtype as ``x``. """ self._validate(x, "gather_heads") B, S_global, H_local, D = x.shape From 630f973dc55f8359077a7e21bec0e499c745c77d Mon Sep 17 00:00:00 2001 From: kangbint Date: Fri, 31 Jul 2026 13:52:43 +0800 Subject: [PATCH 2/5] docs: address comm doc check coverage --- docs/api/comm.rst | 12 ++++---- flashinfer/comm/mixed_comm.py | 9 +++++- flashinfer/comm/ulysses.py | 20 ++++++++++++ flashinfer/comm/ulysses_topology.py | 47 +++++++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 7 deletions(-) diff --git a/docs/api/comm.rst b/docs/api/comm.rst index eb56b147310..103c0e82453 100644 --- a/docs/api/comm.rst +++ b/docs/api/comm.rst @@ -256,6 +256,12 @@ Topology Probing and Backend Selection UlyssesRankTopology UlyssesBackendError +.. autofunction:: resolve_ulysses_backend + +.. autofunction:: decide_ulysses_backend + +.. autofunction:: probe_ulysses_rank_topology + Raw Kernel Entry Points (advanced) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -269,12 +275,6 @@ verified all-pairs NVLink P2P and owns the IPC workspace lifecycle. dispose_ulysses_a2a ulysses_a2a -.. autofunction:: resolve_ulysses_backend - -.. autofunction:: decide_ulysses_backend - -.. autofunction:: probe_ulysses_rank_topology - MNNVL (Multi-Node NVLink) ------------------------- diff --git a/flashinfer/comm/mixed_comm.py b/flashinfer/comm/mixed_comm.py index 16a21ae0e5e..4ac6f481a76 100644 --- a/flashinfer/comm/mixed_comm.py +++ b/flashinfer/comm/mixed_comm.py @@ -935,7 +935,14 @@ def checkpoint_prepare(self): @flashinfer_api def checkpoint_restore(self, comm_backend: CommBackend): - """Restore physical VMM backing at retained graph-visible addresses.""" + """Restore physical VMM backing at retained graph-visible addresses. + + Parameters + ---------- + comm_backend : CommBackend + Intra-node communication backend used to exchange and synchronize + the restored VMM mappings. + """ if self.para_info.use_inter: raise NotImplementedError( "MixedComm checkpointing does not yet support multi-node handlers" diff --git a/flashinfer/comm/ulysses.py b/flashinfer/comm/ulysses.py index 205e19764db..7819d63cca7 100644 --- a/flashinfer/comm/ulysses.py +++ b/flashinfer/comm/ulysses.py @@ -1003,6 +1003,26 @@ def ulysses_a2a( tensors of the same dtype (float32/float16/bfloat16). All ranks must call with consistent geometry in the same order; a mismatch is a collective failure (hang or corruption), as with any collective. + + Parameters + ---------- + fa : int + Opaque backend handle returned by :func:`init_ulysses_a2a`. + inp : torch.Tensor + Contiguous 4-D CUDA input tensor. + out : torch.Tensor + Contiguous 4-D CUDA output tensor written in place. + B : int + Batch size. + S_local : int + Local sequence length per rank. + H : int + Global head count. + D : int + Head dimension. + mode : int + ``0`` for scatter-heads input all-to-all, ``1`` for gather-heads + output all-to-all. """ if type(fa) is not int or fa == 0: raise ValueError( diff --git a/flashinfer/comm/ulysses_topology.py b/flashinfer/comm/ulysses_topology.py index 4ca9d8db26f..1ac3594bed7 100644 --- a/flashinfer/comm/ulysses_topology.py +++ b/flashinfer/comm/ulysses_topology.py @@ -21,6 +21,8 @@ import torch import torch.distributed as dist +from ..api_logging import flashinfer_api + # world sizes for which the fused-transpose NVLink kernel is instantiated; # lives here (the policy layer) so the dependency direction stays # ulysses.py -> ulysses_topology.py with no cycle @@ -70,6 +72,7 @@ class UlyssesBackendDecision: reason: str +@flashinfer_api def probe_ulysses_rank_topology( device: Optional[Union[torch.device, str, int]], rank: int ) -> UlyssesRankTopology: @@ -79,6 +82,19 @@ def probe_ulysses_rank_topology( Never raises: the whole probe (including hostname and device resolution) runs inside an exception envelope, so any failure lands in ``probe_error`` and the (conservative) decision layer falls back to NCCL. + + Parameters + ---------- + device : torch.device or str or int, optional + CUDA device for this rank. ``None`` uses the current CUDA device. + rank : int + Rank id to record in the returned topology object. + + Returns + ------- + UlyssesRankTopology + Per-rank topology information, including any probe error instead of + raising locally. """ topo = UlyssesRankTopology(rank=rank) try: @@ -148,6 +164,7 @@ def _handle(idx: int): return topo +@flashinfer_api def decide_ulysses_backend( requested: str, topologies: List[UlyssesRankTopology], @@ -160,6 +177,20 @@ def decide_ulysses_backend( inconsistent, or unverifiable selects NCCL. Raises :class:`UlyssesBackendError` only when ``requested == "nvlink"`` and the NVLink path cannot be used. + + Parameters + ---------- + requested : str + Requested backend policy: ``"auto"``, ``"nvlink"``, or ``"nccl"``. + topologies : list[UlyssesRankTopology] + Per-rank topology probe results gathered from the process group. + supported_world_sizes : Sequence[int], optional + World sizes for which the fused NVLink kernel is instantiated. + + Returns + ------- + UlyssesBackendDecision + Backend choice and selection or fallback reason. """ if requested not in ULYSSES_BACKENDS: raise ValueError( @@ -236,6 +267,7 @@ def fallback(reason: str) -> UlyssesBackendDecision: ) +@flashinfer_api def resolve_ulysses_backend( backend: str = "auto", group: Optional[dist.ProcessGroup] = None, @@ -267,6 +299,21 @@ def resolve_ulysses_backend( catches the result into an outcome, gathers, and cross-checks. Any disagreement conservatively selects NCCL — or raises for ``backend="nvlink"``. + + Parameters + ---------- + backend : str, default = "auto" + Requested backend policy: ``"auto"``, ``"nvlink"``, or ``"nccl"``. + group : torch.distributed.ProcessGroup, optional + Process group whose ranks must make a consistent backend decision. + Defaults to ``torch.distributed.group.WORLD``. + device : torch.device or str or int, optional + CUDA device to use for probe and metadata collective guards. + + Returns + ------- + UlyssesBackendDecision + Group-consistent backend choice and selection or fallback reason. """ if group is None: group = dist.group.WORLD From 4c309eb39a1e54b163423b69d73ed642bf2b2371 Mon Sep 17 00:00:00 2001 From: kangbint Date: Fri, 31 Jul 2026 14:01:05 +0800 Subject: [PATCH 3/5] docs: keep Ulysses topology helpers internal --- docs/api/comm.rst | 6 ------ flashinfer/comm/ulysses_topology.py | 5 ----- 2 files changed, 11 deletions(-) diff --git a/docs/api/comm.rst b/docs/api/comm.rst index 103c0e82453..6e2a78c9961 100644 --- a/docs/api/comm.rst +++ b/docs/api/comm.rst @@ -256,12 +256,6 @@ Topology Probing and Backend Selection UlyssesRankTopology UlyssesBackendError -.. autofunction:: resolve_ulysses_backend - -.. autofunction:: decide_ulysses_backend - -.. autofunction:: probe_ulysses_rank_topology - Raw Kernel Entry Points (advanced) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/flashinfer/comm/ulysses_topology.py b/flashinfer/comm/ulysses_topology.py index 1ac3594bed7..82049d86157 100644 --- a/flashinfer/comm/ulysses_topology.py +++ b/flashinfer/comm/ulysses_topology.py @@ -21,8 +21,6 @@ import torch import torch.distributed as dist -from ..api_logging import flashinfer_api - # world sizes for which the fused-transpose NVLink kernel is instantiated; # lives here (the policy layer) so the dependency direction stays # ulysses.py -> ulysses_topology.py with no cycle @@ -72,7 +70,6 @@ class UlyssesBackendDecision: reason: str -@flashinfer_api def probe_ulysses_rank_topology( device: Optional[Union[torch.device, str, int]], rank: int ) -> UlyssesRankTopology: @@ -164,7 +161,6 @@ def _handle(idx: int): return topo -@flashinfer_api def decide_ulysses_backend( requested: str, topologies: List[UlyssesRankTopology], @@ -267,7 +263,6 @@ def fallback(reason: str) -> UlyssesBackendDecision: ) -@flashinfer_api def resolve_ulysses_backend( backend: str = "auto", group: Optional[dist.ProcessGroup] = None, From b148c2304539ebfeab3b5496916b192f3e150a12 Mon Sep 17 00:00:00 2001 From: kangbint Date: Fri, 31 Jul 2026 14:19:00 +0800 Subject: [PATCH 4/5] docs: keep Ulysses topology helpers documented --- docs/api/comm.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/api/comm.rst b/docs/api/comm.rst index 6e2a78c9961..103c0e82453 100644 --- a/docs/api/comm.rst +++ b/docs/api/comm.rst @@ -256,6 +256,12 @@ Topology Probing and Backend Selection UlyssesRankTopology UlyssesBackendError +.. autofunction:: resolve_ulysses_backend + +.. autofunction:: decide_ulysses_backend + +.. autofunction:: probe_ulysses_rank_topology + Raw Kernel Entry Points (advanced) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 1e5ba0e2e94cd26b33ffe8f7cbe79934a10b2543 Mon Sep 17 00:00:00 2001 From: kangbint Date: Fri, 31 Jul 2026 14:21:44 +0800 Subject: [PATCH 5/5] docs: clarify mixed comm checkpoint backend --- flashinfer/comm/mixed_comm.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/flashinfer/comm/mixed_comm.py b/flashinfer/comm/mixed_comm.py index 4ac6f481a76..ea7a0a95b89 100644 --- a/flashinfer/comm/mixed_comm.py +++ b/flashinfer/comm/mixed_comm.py @@ -940,8 +940,11 @@ def checkpoint_restore(self, comm_backend: CommBackend): Parameters ---------- comm_backend : CommBackend - Intra-node communication backend used to exchange and synchronize - the restored VMM mappings. + Intra-node communication backend whose ``Get_rank()`` and + ``Get_size()`` match this handler's ``local_rank`` and + ``local_size``. Its ``bcast`` and ``barrier`` operations + coordinate the Unix-socket handle exchange and synchronize VMM + mapping restoration. """ if self.para_info.use_inter: raise NotImplementedError(