From 994d5cfd6b7db8157124aedc4b1f360c7ddd4c4b Mon Sep 17 00:00:00 2001 From: Caio Rocha Date: Tue, 21 Jul 2026 21:33:00 +0000 Subject: [PATCH 01/11] wip --- .../srt/distributed/device_communicators/pymscclpp.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/sglang/srt/distributed/device_communicators/pymscclpp.py b/python/sglang/srt/distributed/device_communicators/pymscclpp.py index 261e8d6cdfda..853bbe20a5b9 100644 --- a/python/sglang/srt/distributed/device_communicators/pymscclpp.py +++ b/python/sglang/srt/distributed/device_communicators/pymscclpp.py @@ -20,7 +20,7 @@ class PyMscclppCommunicator: - _SUPPORTED_WORLD_SIZES = [8, 16, 32] + _SUPPORTED_WORLD_SIZES = [8, 16, 32, 64] _SUPPORTED_DTYPE = [torch.float, torch.float16, torch.bfloat16] def _is_symm_mem_enabled(self) -> bool: @@ -47,7 +47,7 @@ def _get_tuned_config(self, size): def _create_dsl_algorithms(self): dsl_algos_config = [] n_nodes = self.world_size // self.nranks_per_node - if n_nodes == 2 or n_nodes == 4: + if n_nodes == 2 or n_nodes == 4 or n_nodes == 8: for tbg in [1, 2, 4, 8]: for num_threads_per_block in [256, 512, 768, 1024]: spec = self.mscclpp.language.AlgoSpec( @@ -121,7 +121,7 @@ def _create_algorithms(self): if self.world_size == 8: self.algos_config = self._create_native_algorithms() self._tune(5, 10, 20, self.algos_config) - elif self.world_size == 16 or self.world_size == 32: + elif self.world_size == 16 or self.world_size == 32 or self.world_size == 64: self.dsl_algos_config = self._create_dsl_algorithms() self._tune(5, 10, 20, self.dsl_algos_config) From be0e6191060dc61a215d5c6d160a1cdeec93f925 Mon Sep 17 00:00:00 2001 From: Caio Rocha Date: Tue, 18 Aug 2026 17:36:30 +0000 Subject: [PATCH 02/11] wip --- .../kernels/all_gather/benchmark_mscclpp.py | 282 ++++++ .../device_communicators/pymscclpp.py | 920 +++++++++++++++--- .../sglang/srt/distributed/parallel_state.py | 8 + python/sglang/srt/server_args.py | 2 +- 4 files changed, 1079 insertions(+), 133 deletions(-) create mode 100644 benchmark/kernels/all_gather/benchmark_mscclpp.py diff --git a/benchmark/kernels/all_gather/benchmark_mscclpp.py b/benchmark/kernels/all_gather/benchmark_mscclpp.py new file mode 100644 index 000000000000..7f19ec690227 --- /dev/null +++ b/benchmark/kernels/all_gather/benchmark_mscclpp.py @@ -0,0 +1,282 @@ +"""Benchmark multi-node MSCCL++ AllGather against NCCL.""" + +import argparse +import os + +import torch +import torch.distributed as dist + +from sglang.srt.distributed import init_distributed_environment +from sglang.srt.distributed.parallel_state import ( + cleanup_dist_env_and_memory, + get_tensor_model_parallel_group, + initialize_model_parallel, + set_mscclpp_all_reduce, +) + + +def _max_across_ranks(value: float, group) -> float: + tensor = torch.tensor([value], dtype=torch.float64) + dist.all_reduce(tensor, op=dist.ReduceOp.MAX, group=group) + return tensor.item() + + +def _time_eager(func, group, warmup: int = 2, repeat: int = 10) -> float: + for _ in range(warmup): + func() + torch.cuda.synchronize() + dist.barrier(group=group) + + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + start.record() + for _ in range(repeat): + func() + end.record() + end.synchronize() + return _max_across_ranks(start.elapsed_time(end) * 1000 / repeat, group) + + +def _time_cuda_graph( + func, + group, + warmup: int = 2, + graph_loop: int = 10, + repeat: int = 10, +) -> float: + current_stream = torch.cuda.current_stream() + capture_stream = torch.cuda.Stream() + capture_stream.wait_stream(current_stream) + with torch.cuda.stream(capture_stream): + func(capture_stream) + capture_stream.synchronize() + + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=capture_stream): + for _ in range(graph_loop): + func(capture_stream) + + with torch.cuda.stream(capture_stream): + for _ in range(warmup): + graph.replay() + capture_stream.synchronize() + + dist.barrier(group=group) + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + start.record(capture_stream) + with torch.cuda.stream(capture_stream): + for _ in range(repeat): + graph.replay() + end.record(capture_stream) + end.synchronize() + torch.cuda.current_stream().wait_stream(capture_stream) + return _max_across_ranks( + start.elapsed_time(end) * 1000 / (repeat * graph_loop), + group, + ) + + +def _validate_cuda_graph_updates( + func, + reference_func, + input_tensor: torch.Tensor, + actual: torch.Tensor, + expected: torch.Tensor, + group, +) -> None: + current_stream = torch.cuda.current_stream() + capture_stream = torch.cuda.Stream() + capture_stream.wait_stream(current_stream) + with torch.cuda.stream(capture_stream): + func(capture_stream) + capture_stream.synchronize() + + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=capture_stream): + func(capture_stream) + + input_tensor.neg_() + reference_func() + current_stream.synchronize() + capture_stream.wait_stream(current_stream) + with torch.cuda.stream(capture_stream): + graph.replay() + capture_stream.synchronize() + + if not torch.equal(actual, expected): + raise AssertionError("MSCCL++ CUDA graph replay used stale AllGather data") + dist.barrier(group=group) + + +def _message_sizes(min_bytes: int, max_bytes: int): + size = 1 << (min_bytes - 1).bit_length() + while size <= max_bytes: + yield size + size <<= 1 + + +def human_readable_size(size, decimal_places=1): + for unit in ["B", "KiB", "MiB", "GiB", "TiB", "PiB"]: + if size < 1024.0 or unit == "PiB": + break + size /= 1024.0 + return f"{size:.{decimal_places}f} {unit}" + + +try: + from tabulate import tabulate +except ImportError: + print("tabulate not installed, skipping table printing") + tabulate = None + + +def print_markdown_table(data): + if tabulate is not None: + print(tabulate(data, headers="keys", tablefmt="github")) + return + headers = data[0].keys() + header_row = "| " + " | ".join(headers) + " |" + separator = "| " + " | ".join(["---"] * len(headers)) + " |" + rows = [] + for item in data: + row = "| " + " | ".join(str(item[key]) for key in headers) + " |" + rows.append(row) + markdown_table = "\n".join([header_row, separator] + rows) + print(markdown_table) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--min-bytes", type=int, default=1 << 10) + parser.add_argument("--max-bytes", type=int, default=8 << 20) + args = parser.parse_args() + if not (1 << 10) <= args.min_bytes <= args.max_bytes <= (8 << 20): + parser.error("Require 1 KiB <= min-bytes <= max-bytes <= 8 MiB") + + if not dist.is_initialized(): + dist.init_process_group(backend="nccl") + world_size = dist.get_world_size() + rank = dist.get_rank() + local_rank = int(os.environ.get("LOCAL_RANK", rank % torch.cuda.device_count())) + torch.cuda.set_device(local_rank) + device = torch.cuda.current_device() + set_mscclpp_all_reduce(True) + init_distributed_environment( + world_size=world_size, + rank=rank, + local_rank=local_rank, + ) + initialize_model_parallel(tensor_model_parallel_size=world_size) + tp_group = get_tensor_model_parallel_group() + gpu_group = tp_group.device_group + cpu_group = tp_group.cpu_group + pynccl_comm = tp_group.pynccl_comm + mscclpp_comm = tp_group.pymscclpp_comm + assert pynccl_comm is not None + assert mscclpp_comm is not None + if args.min_bytes % world_size != 0 or args.max_bytes % world_size != 0: + parser.error("Total message sizes must be divisible by WORLD_SIZE") + + if not mscclpp_comm.allgather_best_configs: + raise RuntimeError("No tuned MSCCL++ AllGather configuration is available") + + result = [] + + for total_bytes in _message_sizes(args.min_bytes, args.max_bytes): + input_bytes = total_bytes // world_size + if input_bytes % torch.bfloat16.itemsize != 0: + continue + elements = input_bytes // torch.bfloat16.itemsize + input_tensor = ( + torch.arange( + elements, + dtype=torch.long, + device=device, + ) + % 193 + + rank + ).to(torch.bfloat16) + nccl_output = torch.empty( + elements * world_size, + dtype=input_tensor.dtype, + device=device, + ) + mscclpp_output = torch.empty_like(nccl_output) + + def torch_nccl(): + dist.all_gather_into_tensor( + nccl_output, + input_tensor, + group=gpu_group, + ) + + def py_nccl(): + pynccl_comm.all_gather(nccl_output, input_tensor) + + def mscclpp(): + mscclpp_comm.all_gather(mscclpp_output, input_tensor) + + def mscclpp_on_stream(stream): + mscclpp_comm.all_gather( + mscclpp_output, + input_tensor, + stream=stream, + ) + + def py_nccl_on_stream(stream): + pynccl_comm.cp_all_gather_into_tensor( + nccl_output, + input_tensor, + stream=stream, + ) + + torch_nccl() + mscclpp() + py_nccl() + torch.cuda.synchronize() + if not torch.equal(mscclpp_output, nccl_output): + raise AssertionError( + f"MSCCL++ AllGather mismatch at {input_bytes} input bytes" + ) + _validate_cuda_graph_updates( + mscclpp_on_stream, + torch_nccl, + input_tensor, + mscclpp_output, + nccl_output, + cpu_group, + ) + + nccl_eager_us = _time_eager(torch_nccl, cpu_group) + mscclpp_eager_us = _time_eager(mscclpp, cpu_group) + mscclpp_graph_us = _time_cuda_graph(mscclpp_on_stream, cpu_group) + pynccl_graph_us = _time_cuda_graph(py_nccl_on_stream, cpu_group) + + result.append( + { + "msg_size": human_readable_size(total_bytes), + "torch eager time": nccl_eager_us, + "msccl eager time": mscclpp_eager_us, + "msccl graph time": mscclpp_graph_us, + "pynccl graph time": pynccl_graph_us, + } + ) + if rank == 0: + print( + f"sz={total_bytes // input_tensor.element_size()}, " + f"dtype={input_tensor.dtype}: correctness check PASS!" + ) + + torch.cuda.empty_cache() + + if rank == 0: + print_markdown_table(result) + + mscclpp_comm.destroy() + dist.barrier() + cleanup_dist_env_and_memory() + + +if __name__ == "__main__": + main() diff --git a/python/sglang/srt/distributed/device_communicators/pymscclpp.py b/python/sglang/srt/distributed/device_communicators/pymscclpp.py index 853bbe20a5b9..eaca6ae12155 100644 --- a/python/sglang/srt/distributed/device_communicators/pymscclpp.py +++ b/python/sglang/srt/distributed/device_communicators/pymscclpp.py @@ -1,7 +1,9 @@ import importlib import logging +import os from contextlib import contextmanager -from typing import Optional, Union +from dataclasses import dataclass +from typing import Any, Optional, Union import torch import torch.distributed as dist @@ -19,9 +21,98 @@ logger = logging.getLogger(__name__) +@dataclass(frozen=True) +class _TuningCandidate: + algorithm: Any + nblocks: tuple[int, ...] + nthreads: tuple[int, ...] + min_message_size: int + max_message_size: int + + +@dataclass(frozen=True) +class _TwoKernelAllReduce: + name: str + reduce_scatter: Any + allgather: Any + rank: int + world_size: int + + @property + def message_size_range(self) -> tuple[int, int]: + reduce_scatter_range = self.reduce_scatter.message_size_range + allgather_range = self.allgather.message_size_range + return ( + max(reduce_scatter_range[0], allgather_range[0]), + min(reduce_scatter_range[1], allgather_range[1]), + ) + + def execute( + self, + *, + comm, + executor, + input_buffer, + output_buffer, + input_size, + output_size, + dtype, + op, + stream, + nblocks, + nthreads_per_block, + symmetric_memory, + ): + if input_buffer != output_buffer: + raise ValueError("RSAG AllReduce requires in-place buffers") + if input_size % (16 * self.world_size) != 0: + raise ValueError( + f"RSAG AllReduce input size {input_size} must be divisible " + f"by {16 * self.world_size}" + ) + shard_size = input_size // self.world_size + + result = self.reduce_scatter.execute( + comm=comm, + executor=executor, + input_buffer=input_buffer, + output_buffer=output_buffer, + input_size=input_size, + output_size=shard_size, + dtype=dtype, + op=op, + stream=stream, + nblocks=nblocks, + nthreads_per_block=nthreads_per_block, + symmetric_memory=symmetric_memory, + ) + if result != 0: + return result + return self.allgather.execute( + comm=comm, + executor=executor, + input_buffer=output_buffer + self.rank * shard_size, + output_buffer=output_buffer, + input_size=shard_size, + output_size=output_size, + dtype=dtype, + op=op, + stream=stream, + nblocks=nblocks, + nthreads_per_block=nthreads_per_block, + symmetric_memory=symmetric_memory, + ) + + def reset(self): + self.reduce_scatter.reset() + self.allgather.reset() + + class PyMscclppCommunicator: - _SUPPORTED_WORLD_SIZES = [8, 16, 32, 64] + _SUPPORTED_WORLD_SIZES = [4, 8, 16, 32, 64] _SUPPORTED_DTYPE = [torch.float, torch.float16, torch.bfloat16] + _ALLGATHER_MIN_TOTAL_BYTES = 1 << 10 + _ALLGATHER_MAX_TOTAL_BYTES = 8 << 20 def _is_symm_mem_enabled(self) -> bool: try: @@ -35,50 +126,184 @@ def _is_weak_contiguous(self, inp: torch.Tensor): == inp.numel() * inp.element_size() ) - def _get_tuned_config(self, size): + def _get_allreduce_tuned_config(self, size): if size <= 512: target_size = 512 elif size > 256 * 1024 * 1024: target_size = 256 * 1024 * 1024 else: target_size = 1 << (size - 1).bit_length() - return self.best_configs.get(target_size) - - def _create_dsl_algorithms(self): - dsl_algos_config = [] - n_nodes = self.world_size // self.nranks_per_node - if n_nodes == 2 or n_nodes == 4 or n_nodes == 8: - for tbg in [1, 2, 4, 8]: - for num_threads_per_block in [256, 512, 768, 1024]: - spec = self.mscclpp.language.AlgoSpec( - name=f"allreduce_{n_nodes}node_{tbg}TBG_{num_threads_per_block}TPB", + return self.allreduce_best_configs.get(target_size) + + def _get_allgather_tuned_config(self, size): + total_size = size * self.world_size + if not ( + self._ALLGATHER_MIN_TOTAL_BYTES + <= total_size + <= self._ALLGATHER_MAX_TOTAL_BYTES + ): + return None + target_size = 1 << (size - 1).bit_length() + return self.allgather_best_configs.get(target_size) + + def _compile_dsl_candidate( + self, + *, + name, + collective, + algorithm_builder, + num_threads_per_block, + plan_min_message_size, + plan_max_message_size, + tuning_min_message_size=None, + tuning_max_message_size=None, + instr_fusion=True, + compile_kwargs=None, + ): + spec = self.mscclpp.language.AlgoSpec( + name=name, + collective=collective, + nranks_per_node=self.nranks_per_ipc_domain, + world_size=self.world_size, + in_place=collective.inplace, + instances=1, + protocol="LL", + auto_sync=False, + instr_fusion=instr_fusion, + num_threads_per_block=num_threads_per_block, + reuse_resources=True, + use_double_scratch_buffer=True, + min_message_size=plan_min_message_size, + max_message_size=plan_max_message_size, + tags={"default": 1}, + ) + algorithm = self.mscclpp.compile( + algorithm_builder, + spec, + self.rank, + **(compile_kwargs or {}), + ) + message_range = algorithm.message_size_range + return _TuningCandidate( + algorithm=algorithm, + nblocks=(0,), + nthreads=(0,), + min_message_size=( + message_range[0] + if tuning_min_message_size is None + else tuning_min_message_size + ), + max_message_size=( + message_range[1] + if tuning_max_message_size is None + else tuning_max_message_size + ), + ) + + def _create_dsl_allreduce_algorithms(self): + algorithms = [] + n_ipc_domains = self.world_size // self.nranks_per_ipc_domain + if n_ipc_domains not in (2, 4, 8): + return algorithms + + for tbg in (1, 2, 4, 8): + for num_threads_per_block in (256, 512, 768, 1024): + algorithms.append( + self._compile_dsl_candidate( + name=( + f"allreduce_{n_ipc_domains}node_" + f"{tbg}TBG_{num_threads_per_block}TPB" + ), collective=self.mscclpp.language.collectives.AllReduce( - self.world_size, 1, True + self.world_size, + 1, + True, ), - nranks_per_node=self.nranks_per_node, - world_size=self.world_size, - in_place=True, - instances=1, - protocol="LL", - auto_sync=False, + algorithm_builder=self.def_algo.allreduce_multi_nodes, num_threads_per_block=num_threads_per_block, - reuse_resources=True, - use_double_scratch_buffer=True, - min_message_size=tbg * (1 << 10), - max_message_size=8 << 20, - tags={"default": 1}, + plan_min_message_size=tbg * (1 << 10), + plan_max_message_size=8 << 20, + compile_kwargs={"thread_block_group_size": tbg}, ) - algo = self.mscclpp.compile( - self.def_algo.allreduce_multi_nodes, - spec, - self.rank, - thread_block_group_size=tbg, + ) + return algorithms + + def _create_dsl_allgather_algorithms(self): + if not hasattr(self.def_algo, "allgather_multi_nodes"): + raise RuntimeError( + "The installed MSCCL++ package does not provide " + "default_algos.allgather_multi_nodes" + ) + + algorithms = [] + n_ipc_domains = self.world_size // self.nranks_per_ipc_domain + if n_ipc_domains not in (2, 4, 8): + return algorithms + + input_min = self._ALLGATHER_MIN_TOTAL_BYTES // self.world_size + input_max = self._ALLGATHER_MAX_TOTAL_BYTES // self.world_size + for num_threads_per_block in self.allgather_tuning_threads: + algorithms.append( + self._compile_dsl_candidate( + name=( + f"allgather_{n_ipc_domains}node_1TBG_" + f"{num_threads_per_block}TPB" + ), + collective=self.mscclpp.language.collectives.AllGather( + self.world_size, + 1, + False, + ), + algorithm_builder=self.def_algo.allgather_multi_nodes, + num_threads_per_block=num_threads_per_block, + plan_min_message_size=self._ALLGATHER_MIN_TOTAL_BYTES, + plan_max_message_size=self._ALLGATHER_MAX_TOTAL_BYTES, + tuning_min_message_size=input_min, + tuning_max_message_size=input_max, + ) + ) + return algorithms + + def _create_dsl_reducescatter_algorithms(self): + if not hasattr(self.def_algo, "reducescatter_multi_nodes"): + return [] + + algorithms = [] + n_ipc_domains = self.world_size // self.nranks_per_ipc_domain + tbg_min_message_size = { + 1: 1 << 10, + 2: 1 << 20, + 4: 2 << 20, + 8: 8 << 20, + } + for tbg in (1, 2, 4, 8): + for num_threads_per_block in (256, 512, 768, 1024): + algorithms.append( + self._compile_dsl_candidate( + name=( + f"reducescatter_{n_ipc_domains}node_" + "directowner_v4_inplace_unfused_" + f"{tbg}TBG_{num_threads_per_block}TPB" + ), + collective=self.mscclpp.language.collectives.ReduceScatter( + self.world_size, + 1, + True, + ), + algorithm_builder=self.def_algo.reducescatter_multi_nodes, + num_threads_per_block=num_threads_per_block, + plan_min_message_size=1 << 10, + plan_max_message_size=8 << 20, + tuning_min_message_size=tbg_min_message_size[tbg], + instr_fusion=False, + compile_kwargs={"thread_block_group_size": tbg}, ) - dsl_algos_config.append((algo, [0], [0])) - return dsl_algos_config + ) + return algorithms - def _create_native_algorithms(self): - navitve_algorithms_config = [] + def _create_native_allreduce_algorithms(self): + native_algorithms_config = [] + force_disable_nvls = os.getenv("MSCCLPP_FORCE_DISABLE_NVLS") == "1" dlpack = self.mscclpp.RawGpuBuffer(1 << 27).to_dlpack( data_type=str(torch.float16) ) @@ -91,180 +316,503 @@ def _create_native_algorithms(self): ) for algo in algos: + if force_disable_nvls and "nvls" in algo.name: + continue if algo.name == "default_allreduce_nvls_packet": algo.set_message_size_range(0, 512 << 10) - navitve_algorithms_config.append( - (algo, [4, 8, 12, 16], [256, 512, 768, 1024]) + native_algorithms_config.append( + _TuningCandidate( + algorithm=algo, + nblocks=(4, 8, 12, 16), + nthreads=(256, 512, 768, 1024), + min_message_size=0, + max_message_size=512 << 10, + ) ) if algo.name == "default_allreduce_packet": algo.set_message_size_range(0, 2 << 20) - navitve_algorithms_config.append( - (algo, [14, 21, 28, 42, 56], [256, 512, 768, 1024]) + native_algorithms_config.append( + _TuningCandidate( + algorithm=algo, + nblocks=(14, 21, 28, 42, 56), + nthreads=(256, 512, 768, 1024), + min_message_size=0, + max_message_size=2 << 20, + ) ) if algo.name == "default_allreduce_rsag_zero_copy": algo.set_message_size_range(512 << 10, 4 << 30) - navitve_algorithms_config.append( - (algo, [32, 48, 64, 128], [256, 512, 768, 1024]) + native_algorithms_config.append( + _TuningCandidate( + algorithm=algo, + nblocks=(32, 48, 64, 128), + nthreads=(256, 512, 768, 1024), + min_message_size=512 << 10, + max_message_size=4 << 30, + ) ) if ( self.symm_mem_enabled and algo.name == "default_allreduce_nvls_zero_copy" ): algo.set_message_size_range(512 << 10, 4 << 30) - navitve_algorithms_config.append( - (algo, [4, 8, 12, 16, 32], [256, 512, 768, 1024]) + native_algorithms_config.append( + _TuningCandidate( + algorithm=algo, + nblocks=(4, 8, 12, 16, 32), + nthreads=(256, 512, 768, 1024), + min_message_size=512 << 10, + max_message_size=4 << 30, + ) ) - return navitve_algorithms_config + return native_algorithms_config + + def _tune_collective(self, collective, algorithms): + if not algorithms: + raise RuntimeError(f"No MSCCL++ {collective} algorithms were compiled") + self._tune( + collective, + algorithms, + n_warmup=5, + n_graph_launches=20, + n_ops_per_graph=20, + ) + best_configs = { + "allreduce": self.allreduce_best_configs, + "allgather": self.allgather_best_configs, + "reducescatter": self.reducescatter_best_configs, + }[collective] + if not best_configs: + raise RuntimeError( + f"MSCCL++ {collective} produced no usable tuned configurations" + ) def _create_algorithms(self): - if self.world_size == 8: - self.algos_config = self._create_native_algorithms() - self._tune(5, 10, 20, self.algos_config) - elif self.world_size == 16 or self.world_size == 32 or self.world_size == 64: - self.dsl_algos_config = self._create_dsl_algorithms() - self._tune(5, 10, 20, self.dsl_algos_config) + n_ipc_domains = self.world_size // self.nranks_per_ipc_domain + if n_ipc_domains == 1: + if "allreduce" in self.collectives: + self._tune_collective( + "allreduce", + self._create_native_allreduce_algorithms(), + ) + return + + if n_ipc_domains not in (2, 4, 8): + return + + need_rsag = "allreduce" in self.collectives and hasattr( + self.def_algo, + "reducescatter_multi_nodes", + ) + if "allgather" in self.collectives or need_rsag: + self._tune_collective( + "allgather", + self._create_dsl_allgather_algorithms(), + ) + + if "allreduce" in self.collectives: + self._tune_collective( + "allreduce", + self._create_dsl_allreduce_algorithms(), + ) + if need_rsag: + self._tune_collective( + "reducescatter", + self._create_dsl_reducescatter_algorithms(), + ) + self._merge_rsag_configs( + n_warmup=5, + n_graph_launches=20, + n_ops_per_graph=20, + ) + + def _merge_rsag_configs( + self, + *, + n_warmup, + n_graph_launches, + n_ops_per_graph, + ): + n_ipc_domains = self.world_size // self.nranks_per_ipc_domain + element_size = torch.empty((), dtype=torch.bfloat16).element_size() + size = 1 << 10 + while size <= 8 << 20: + reduce_scatter_config = self.reducescatter_best_configs.get(size) + allgather_config = self.allgather_best_configs.get(size // self.world_size) + if reduce_scatter_config is None or allgather_config is None: + size <<= 1 + continue + + reduce_scatter = reduce_scatter_config[0] + allgather = allgather_config[0] + if size not in self.allreduce_best_configs: + size <<= 1 + continue + reduce_scatter_prefix = ( + f"reducescatter_{n_ipc_domains}node_" "directowner_v4_inplace_unfused_" + ) + allgather_prefix = f"allgather_{n_ipc_domains}node_1TBG_" + algorithm = _TwoKernelAllReduce( + name=( + f"allreduce_rsag_{n_ipc_domains}node_" + f"RS{reduce_scatter.name.removeprefix(reduce_scatter_prefix)}_" + f"AG{allgather.name.removeprefix(allgather_prefix)}" + ), + reduce_scatter=reduce_scatter, + allgather=allgather, + rank=self.rank, + world_size=self.world_size, + ) + tensor = torch.empty( + size // element_size, + dtype=torch.bfloat16, + device=self.device, + ) + elapsed = self._get_time( + "allreduce", + algorithm, + tensor, + tensor, + size, + 0, + 0, + n_warmup, + n_graph_launches, + n_ops_per_graph, + symmetric_memory=False, + ) + legacy_time = self.allreduce_best_config_times.get( + size, + float("inf"), + ) + if elapsed < legacy_time: + self.allreduce_best_configs[size] = (algorithm, 0, 0) + self.allreduce_best_config_times[size] = elapsed + if self.rank == 0: + logger.info( + "MSCCL++ allreduce RSAG merge: input_bytes=%d " + "reduce_scatter=%s allgather=%s time_ms=%.4f " + "selected=%s", + size, + reduce_scatter.name, + allgather.name, + elapsed, + elapsed < legacy_time, + ) + del tensor + size <<= 1 def _get_time( self, + collective, algo, - tune_tensor, - size, - nb, - nt, + input_tensor, + output_tensor, + input_size, + nblocks, + nthreads, n_warmup, n_graph_launches, n_ops_per_graph, + symmetric_memory=False, ): - # Check if the algorithm can run with the given configuration - if self._run_algo(algo, tune_tensor, size, nb, nt, True) != 0: - return float("inf") + def run(stream=None): + return self._run_algo( + collective, + algo, + input_tensor, + nblocks, + nthreads, + output_tensor=output_tensor, + input_size=input_size, + stream=stream, + symmetric_memory=symmetric_memory, + ) + + result = run() + if result != 0: + raise RuntimeError( + f"MSCCL++ {collective} tuning failed with error code {result}" + ) - # Warmup iterations to stabilize performance for _ in range(n_warmup): - self._run_algo(algo, tune_tensor, size, nb, nt, True) + run() - # Warmup on capture stream capture_stream = torch.cuda.Stream() capture_stream.wait_stream(torch.cuda.current_stream()) with torch.cuda.stream(capture_stream): - self._run_algo(algo, tune_tensor, size, nb, nt, True) + run(capture_stream) capture_stream.synchronize() - # Capture the algorithm execution in a CUDA graph - g = torch.cuda.CUDAGraph() - with torch.cuda.graph(g, stream=capture_stream): + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=capture_stream): for _ in range(n_ops_per_graph): - self._run_algo(algo, tune_tensor, size, nb, nt, True) + run(capture_stream) - # Measure the execution time of the captured graph + capture_stream.synchronize() + dist.barrier(group=self.group) start_event = torch.cuda.Event(enable_timing=True) end_event = torch.cuda.Event(enable_timing=True) start_event.record(capture_stream) with torch.cuda.stream(capture_stream): for _ in range(n_graph_launches): - g.replay() + graph.replay() end_event.record(capture_stream) end_event.synchronize() - elapsed = start_event.elapsed_time(end_event) - - # Synchronize timing results across all ranks to ensure consistent algorithm selection - # replicate n times such due to algo limitations - time_tensor = torch.full( - (self.world_size,), elapsed, dtype=torch.float64, device="cuda" - ).to(dtype=torch.float32) torch.cuda.current_stream().wait_stream(capture_stream) - if self.rank == 0: - avg_time = time_tensor[self.rank].item() / self.world_size - tensor = torch.tensor([avg_time]) - else: - tensor = torch.empty(1) - dist.broadcast(tensor, src=0, group=self.group) - avg_time = tensor.item() - - return avg_time - def _tune(self, n_warmup, n_graph_launches, n_ops_per_graph, algos_config): - sizes = [1 << i for i in range(9, 24)] - dlpack = self.mscclpp.RawGpuBuffer(1 << 27).to_dlpack( - data_type=str(torch.float16) + elapsed = start_event.elapsed_time(end_event) / ( + n_graph_launches * n_ops_per_graph ) - tune_tensor = torch.utils.dlpack.from_dlpack(dlpack) + max_elapsed = torch.tensor([elapsed], dtype=torch.float64) + dist.all_reduce(max_elapsed, op=ReduceOp.MAX, group=self.group) + return max_elapsed.item() + + def _tune( + self, + collective: str, + algos_config: list[_TuningCandidate], + *, + n_warmup: int = 5, + n_graph_launches: int = 20, + n_ops_per_graph: int = 1, + ): + if collective not in {"allreduce", "allgather", "reducescatter"}: + raise ValueError(f"Unsupported tuning collective: {collective}") + + if collective == "allreduce": + size = 1 << 9 + max_size = 1 << 23 + dlpack = self.mscclpp.RawGpuBuffer(1 << 27).to_dlpack( + data_type=str(torch.float16) + ) + allreduce_tensor = torch.utils.dlpack.from_dlpack(dlpack) + elif collective == "allgather": + size = self._ALLGATHER_MIN_TOTAL_BYTES // self.world_size + max_size = self._ALLGATHER_MAX_TOTAL_BYTES // self.world_size + allreduce_tensor = None + else: + size = self._ALLGATHER_MIN_TOTAL_BYTES + max_size = self._ALLGATHER_MAX_TOTAL_BYTES + allreduce_tensor = None + + while size <= max_size: + warmup = n_warmup + graph_launches = n_graph_launches + if collective == "allgather": + elements = ( + size + // torch.empty( + (), + dtype=torch.bfloat16, + ).element_size() + ) + input_tensor = torch.empty( + elements, + dtype=torch.bfloat16, + device=self.device, + ) + output_tensor = torch.empty( + elements * self.world_size, + dtype=input_tensor.dtype, + device=self.device, + ) + elif collective == "reducescatter": + elements = ( + size + // torch.empty( + (), + dtype=torch.bfloat16, + ).element_size() + ) + input_tensor = torch.empty( + elements, + dtype=torch.bfloat16, + device=self.device, + ) + output_tensor = input_tensor[: elements // self.world_size] + else: + input_tensor = allreduce_tensor + output_tensor = allreduce_tensor - for size in sizes: best_time = float("inf") best_config = None - for i in range(len(algos_config)): - algo, candidates_nblocks, candidates_nthreads = algos_config[i] - if ( - size >= algo.message_size_range[0] - and size <= algo.message_size_range[1] + for candidate in algos_config: + if not ( + candidate.min_message_size <= size <= candidate.max_message_size ): - for nb in candidates_nblocks: - for nt in candidates_nthreads: - avg_time = self._get_time( - algo, - tune_tensor, + continue + for nblocks in candidate.nblocks: + for nthreads in candidate.nthreads: + if self.rank == 0: + logger.info( + "MSCCL++ %s tuning candidate: input_bytes=%d " + "algorithm=%s nblocks=%d nthreads=%d", + collective, size, - nb, - nt, - n_warmup, - n_graph_launches, - n_ops_per_graph, + candidate.algorithm.name, + nblocks, + nthreads, ) - if avg_time < best_time: - best_time = avg_time - best_config = (algo, nb, nt) - if best_config: - self.best_configs[size] = best_config + avg_time = self._get_time( + collective, + candidate.algorithm, + input_tensor, + output_tensor, + size, + nblocks, + nthreads, + warmup, + graph_launches, + n_ops_per_graph, + symmetric_memory=collective == "allreduce", + ) + config = ( + candidate.algorithm, + nblocks, + nthreads, + ) + if avg_time < best_time: + best_time = avg_time + best_config = config + + if best_config is not None: + if collective == "allreduce": + self.allreduce_best_configs[size] = best_config + self.allreduce_best_config_times[size] = best_time + elif collective == "allgather": + self.allgather_best_configs[size] = best_config + self.allgather_best_config_times[size] = best_time + else: + self.reducescatter_best_configs[size] = best_config + self.reducescatter_best_config_times[size] = best_time + if self.rank == 0: + logger.info( + "MSCCL++ %s tuning: input_bytes=%d " + "algorithm=%s nblocks=%d nthreads=%d time_ms=%.4f", + collective, + size, + best_config[0].name, + best_config[1], + best_config[2], + best_time, + ) + + if collective in {"allgather", "reducescatter"}: + del input_tensor, output_tensor + size <<= 1 torch.cuda.synchronize() - for algo, _, _ in algos_config: - algo.reset() + if collective in {"allreduce", "reducescatter"}: + for candidate in algos_config: + candidate.algorithm.reset() + + def _run_algo( + self, + collective, + algo, + input_tensor, + nblocks, + nthreads, + *, + output_tensor=None, + input_size=None, + stream=None, + symmetric_memory=False, + ): + if collective not in {"allreduce", "allgather", "reducescatter"}: + raise ValueError(f"Unsupported MSCCL++ collective: {collective}") + if stream is None: + stream = torch.cuda.current_stream() + if input_size is None: + input_size = input_tensor.nbytes + if output_tensor is None: + output_tensor = input_tensor + + if collective == "allreduce": + output_size = input_size + reduce_op = self.mscclpp.ReduceOp.SUM + elif collective == "allgather": + output_size = output_tensor.nbytes + reduce_op = self.mscclpp.ReduceOp.NOP + else: + output_size = output_tensor.nbytes + reduce_op = self.mscclpp.ReduceOp.SUM - def _run_algo(self, algo, tensor, size, nblocks, nthreads, sym_mem_enabled=False): return algo.execute( comm=self.comm.communicator, executor=self.executor, - input_buffer=tensor.data_ptr(), - output_buffer=tensor.data_ptr(), - input_size=size, - output_size=size, - dtype=self.dtype_to_mscclpp_dtype(tensor.dtype), - op=self.mscclpp.ReduceOp.SUM, - stream=torch.cuda.current_stream().cuda_stream, + input_buffer=input_tensor.data_ptr(), + output_buffer=output_tensor.data_ptr(), + input_size=input_size, + output_size=output_size, + dtype=self.dtype_to_mscclpp_dtype(input_tensor.dtype), + op=reduce_op, + stream=stream.cuda_stream, nblocks=nblocks, nthreads_per_block=nthreads, - symmetric_memory=sym_mem_enabled, + symmetric_memory=symmetric_memory, ) def __init__( self, group: ProcessGroup, device: Union[int, str, torch.device], + group_name: str = "anonymous", + collectives: Optional[set[str]] = None, + allgather_tuning_threads: tuple[int, ...] = (256, 512, 768, 1024), ) -> None: """Args: group: the process group to work on. If None, it will use the default process group. device: the device to bind the CustomAllreduce to. If None, it will be bind to f"cuda:{local_rank}". + group_name: a human-readable process-group name for logging. It is the caller's responsibility to make sure each communicator is bind to a unique device, and all communicators in this group are in the same node. """ self._IS_CAPTURING = False self.disabled = True + self.available = False + self.allreduce_best_configs = {} + self.allreduce_best_config_times = {} + self.allgather_best_configs = {} + self.allgather_best_config_times = {} + self.reducescatter_best_configs = {} + self.reducescatter_best_config_times = {} + self._logged_allgather_sizes = set() + self.scratch_buffer = None + self.flag_buffer = None + self.collectives = frozenset( + ("allreduce", "allgather") if collectives is None else collectives + ) + unsupported_collectives = self.collectives - {"allreduce", "allgather"} + if unsupported_collectives: + raise ValueError( + f"Unsupported MSCCL++ collectives: {unsupported_collectives}" + ) + if not allgather_tuning_threads or any( + threads <= 0 or threads > 1024 for threads in allgather_tuning_threads + ): + raise ValueError( + "allgather_tuning_threads must contain values in [1, 1024]" + ) + self.allgather_tuning_threads = tuple(allgather_tuning_threads) try: self.mscclpp = importlib.import_module("mscclpp") self.mscclpp_ext = importlib.import_module("mscclpp.ext") self.def_algo = importlib.import_module("mscclpp.default_algos") - except ImportError: - self.available = False + except ImportError as exc: + logger.warning( + "PyMscclpp is unavailable because its dependencies failed to import: %s", + exc, + ) self.mscclpp = None return - self.available = True self.group = group assert ( @@ -273,6 +821,8 @@ def __init__( rank = dist.get_rank(group=self.group) world_size = dist.get_world_size(group=self.group) + self.rank = rank + self.world_size = world_size if world_size == 1: # No need to initialize mscclpp for single GPU case. return @@ -288,7 +838,6 @@ def __init__( return self.ranks = torch.distributed.get_process_group_ranks(group) - self.nranks_per_node = torch.cuda.device_count() # for now mscclpp with stride in the communicator is not tested if not (abs(self.ranks[-1] - self.ranks[0]) == world_size - 1): logger.warning( @@ -307,19 +856,46 @@ def __init__( assert isinstance(device, torch.device) self.device = device - self.rank = rank - self.world_size = world_size self.comm = self.mscclpp.CommGroup( torch_group=self.group, rank=rank, size=world_size ) + nranks_per_ipc_domain = getattr(self.comm, "nranks_per_ipc_domain", None) + self.nranks_per_ipc_domain = ( + self.comm.nranks_per_node + if nranks_per_ipc_domain is None + else nranks_per_ipc_domain + ) self.executor = self.mscclpp.Executor(self.comm.communicator) self.symm_mem_enabled = self._is_symm_mem_enabled() - self.best_configs = {} - self._create_algorithms() + try: + self._create_algorithms() + self.executor.reset() + except Exception: + logger.exception("Failed to initialize MSCCL++ collectives") + raise + self.available = bool( + self.allreduce_best_configs or self.allgather_best_configs + ) + if not self.available: + logger.warning("PyMscclpp did not produce any usable tuned configurations.") + else: + logger.info( + "Created MSCCL++ communicator: group=%s world_size=%d device=%s " + "allreduce_configs=%d allgather_configs=%d", + group_name, + world_size, + self.device, + len(self.allreduce_best_configs), + len(self.allgather_best_configs), + ) def destroy(self): - self.algos_config = None - self.best_configs = None + self.allreduce_best_configs = None + self.allreduce_best_config_times = None + self.allgather_best_configs = None + self.allgather_best_config_times = None + self.reducescatter_best_configs = None + self.reducescatter_best_config_times = None self.executor = None self.scratch_buffer = None self.flag_buffer = None @@ -339,7 +915,7 @@ def should_mscclpp_allreduce( return False if op is not ReduceOp.SUM: return False - if self._get_tuned_config(inp.numel() * inp.element_size()) is None: + if self._get_allreduce_tuned_config(inp.numel() * inp.element_size()) is None: return False # mscclpp must not be used during any piecewise CUDA graph phase # (compile, capture, or replay) as it changes the allreduce dispatch @@ -352,6 +928,40 @@ def should_mscclpp_allreduce( return False return True + def should_mscclpp_allgather( + self, + output_tensor: torch.Tensor, + input_tensor: torch.Tensor, + ) -> bool: + if ( + self.disabled + or self.world_size not in PyMscclppCommunicator._SUPPORTED_WORLD_SIZES + ): + return False + if input_tensor.dtype not in PyMscclppCommunicator._SUPPORTED_DTYPE: + return False + if output_tensor.dtype != input_tensor.dtype: + return False + if not input_tensor.is_contiguous() or not output_tensor.is_contiguous(): + return False + if output_tensor.numel() != input_tensor.numel() * self.world_size: + return False + if output_tensor.device != input_tensor.device: + return False + nbytes = input_tensor.numel() * input_tensor.element_size() + if nbytes % 16 != 0: + return False + config = self._get_allgather_tuned_config(nbytes) + if config is None: + return False + if ( + is_in_tc_piecewise_cuda_graph() + or is_in_torch_compile_warmup() + or get_pcg_capture_stream() is not None + ): + return False + return True + def dtype_to_mscclpp_dtype(self, dtype: torch.dtype): if dtype == torch.float16: return self.mscclpp.DataType.float16 @@ -372,10 +982,56 @@ def all_reduce( ): assert op == torch.distributed.ReduceOp.SUM nbytes = tensor.numel() * tensor.element_size() - algo, nblocks, nthreads = self._get_tuned_config(nbytes) - self._run_algo(algo, tensor, nbytes, nblocks, nthreads, self.symm_mem_enabled) + algo, nblocks, nthreads = self._get_allreduce_tuned_config(nbytes) + result = self._run_algo( + "allreduce", + algo, + tensor, + nblocks, + nthreads, + input_size=nbytes, + symmetric_memory=self.symm_mem_enabled, + stream=stream, + ) + if result != 0: + raise RuntimeError(f"MSCCL++ AllReduce failed with error code {result}") return tensor + def all_gather( + self, + output_tensor: torch.Tensor, + input_tensor: torch.Tensor, + stream: torch.cuda.Stream = None, + ): + nbytes = input_tensor.numel() * input_tensor.element_size() + config = self._get_allgather_tuned_config(nbytes) + if config is None: + raise RuntimeError( + f"No tuned MSCCL++ AllGather configuration for {nbytes} bytes" + ) + algo, nblocks, nthreads = config + if self.rank == 0 and nbytes not in self._logged_allgather_sizes: + self._logged_allgather_sizes.add(nbytes) + logger.info( + "Dispatching MSCCL++ AllGather: input_bytes=%d " + "total_bytes=%d algorithm=%s", + nbytes, + nbytes * self.world_size, + algo.name, + ) + result = self._run_algo( + "allgather", + algo, + input_tensor, + nblocks, + nthreads, + output_tensor=output_tensor, + stream=stream, + ) + if result != 0: + raise RuntimeError(f"MSCCL++ AllGather failed with error code {result}") + return output_tensor + @contextmanager def change_state( self, diff --git a/python/sglang/srt/distributed/parallel_state.py b/python/sglang/srt/distributed/parallel_state.py index b1d26d32f723..4000bdc2c321 100644 --- a/python/sglang/srt/distributed/parallel_state.py +++ b/python/sglang/srt/distributed/parallel_state.py @@ -431,6 +431,7 @@ def __init__( self.pymscclpp_comm = PyMscclppCommunicator( group=self.cpu_group, device=self.device, + group_name=group_name, ) self.ca_comm: Optional[Any] = None @@ -1018,6 +1019,13 @@ def _all_gather_into_tensor(self, output: torch.Tensor, input: torch.Tensor): ca_comm.all_gather_unreg(input, out=output, dim=0) return + pymscclpp_comm = self.pymscclpp_comm + if pymscclpp_comm is not None and pymscclpp_comm.should_mscclpp_allgather( + output, input + ): + pymscclpp_comm.all_gather(output, input) + return + pynccl_comm = self.pynccl_comm if pynccl_comm is not None and ( not pynccl_comm.disabled or self.is_symmetric_memory_enabled() diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index be7028baf28f..fdfb5fd86596 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -1629,7 +1629,7 @@ class ServerArgs: ] = False enable_mscclpp: A[ bool, - "Enable using mscclpp for small messages for all-reduce kernel and fall back to NCCL.", + "Enable MSCCL++ for tuned AllReduce and AllGather messages, with NCCL fallback.", ] = False enable_torch_symm_mem: A[ bool, From 5d9b0c750ab63504428bce42286727c18b2702d7 Mon Sep 17 00:00:00 2001 From: Caio Rocha Date: Tue, 1 Sep 2026 14:45:48 +0000 Subject: [PATCH 03/11] wip --- .../device_communicators/pymscclpp.py | 1354 +++++++++-------- 1 file changed, 751 insertions(+), 603 deletions(-) diff --git a/python/sglang/srt/distributed/device_communicators/pymscclpp.py b/python/sglang/srt/distributed/device_communicators/pymscclpp.py index eaca6ae12155..ee29019c4aa5 100644 --- a/python/sglang/srt/distributed/device_communicators/pymscclpp.py +++ b/python/sglang/srt/distributed/device_communicators/pymscclpp.py @@ -1,9 +1,9 @@ import importlib import logging -import os +from abc import ABC, abstractmethod from contextlib import contextmanager -from dataclasses import dataclass -from typing import Any, Optional, Union +from dataclasses import dataclass, replace +from typing import Any, ClassVar, Optional, Union import torch import torch.distributed as dist @@ -20,14 +20,399 @@ logger = logging.getLogger(__name__) +_SUPPORTED_COLLECTIVES = {"allreduce", "allgather", "reducescatter"} +_DEFAULT_SUPPORTED_DTYPES = (torch.float, torch.float16, torch.bfloat16) +_DEFAULT_GPU_BUFFER_SIZE = 1 << 27 +_TUNING_MESSAGE_SIZES = tuple(1 << exponent for exponent in range(9, 24)) + @dataclass(frozen=True) -class _TuningCandidate: - algorithm: Any +class _MessageSizeRange: + minimum: int + maximum: int + + def __post_init__(self): + if self.minimum < 0 or self.maximum < self.minimum: + raise ValueError(f"Invalid message size range: {self}") + + def contains(self, size: int) -> bool: + return self.minimum <= size <= self.maximum + + +@dataclass(frozen=True, kw_only=True) +class _AlgorithmConfig(ABC): + implementation: ClassVar[str] + name: str + collective: str + world_sizes: tuple[int, ...] + ipc_domain_counts: tuple[int, ...] + threads_per_block: tuple[int, ...] + message_size_range: _MessageSizeRange + reduce_op: str + supported_dtypes: tuple[torch.dtype, ...] = _DEFAULT_SUPPORTED_DTYPES + in_place: bool = True + requires_nvls: bool = False + algorithm: Any = None + + def __post_init__(self): + if self.collective not in _SUPPORTED_COLLECTIVES: + raise ValueError(f"Unsupported collective: {self.collective}") + if self.reduce_op not in {"SUM", "NOP"}: + raise ValueError(f"Unsupported reduction operation: {self.reduce_op}") + if not self.world_sizes or not self.ipc_domain_counts: + raise ValueError("Algorithm topology constraints cannot be empty") + if not self.threads_per_block: + raise ValueError("Algorithm topology constraints cannot be empty") + if not self.supported_dtypes: + raise ValueError("Algorithm supported dtypes cannot be empty") + + def supports_topology(self, world_size: int, ipc_domain_count: int) -> bool: + return ( + world_size in self.world_sizes + and ipc_domain_count in self.ipc_domain_counts + ) + + def requirements_satisfied( + self, + world_size: int, + ipc_domain_count: int, + nvls_supported: bool, + symmetric_memory: bool, + ) -> bool: + return self.supports_topology( + world_size, ipc_domain_count + ) and self.support_nvls(nvls_supported, symmetric_memory) + + def support_nvls(self, nvls_supported: bool, symmetric_memory: bool) -> bool: + return not self.requires_nvls or (nvls_supported and symmetric_memory) + + def supports_message_size(self, message_size: int) -> bool: + return self.message_size_range.contains(message_size) + + def supports_dtype(self, dtype: torch.dtype) -> bool: + return dtype in self.supported_dtypes + + def resolve_reduce_op(self, reduce_ops): + return getattr(reduce_ops, self.reduce_op) + + def input_buffer_size(self, message_size: int, world_size: int) -> int: + if self.collective == "allgather": + if message_size % world_size != 0: + raise ValueError( + f"All-gather output size {message_size} must be divisible " + f"by world size {world_size}" + ) + return message_size // world_size + return message_size + + def output_buffer_size(self, message_size: int, world_size: int) -> int: + if self.collective == "reducescatter": + if message_size % world_size != 0: + raise ValueError( + f"Reduce-scatter input size {message_size} must be divisible " + f"by world size {world_size}" + ) + return message_size // world_size + return message_size + + @abstractmethod + def tuning_launches(self) -> tuple[tuple[int, int], ...]: + raise NotImplementedError + + def bind(self, algorithm) -> "_AlgorithmConfig": + return replace(self, algorithm=algorithm) + + def select(self, nblocks: int, threads_per_block: int) -> "_AlgorithmConfig": + if self.algorithm is None: + raise RuntimeError(f"Algorithm {self.name} has not been bound") + if (nblocks, threads_per_block) not in self.tuning_launches(): + raise ValueError(f"Invalid launch selection for algorithm {self.name}") + return self + + def selected_launch(self) -> tuple[int, int]: + if self.algorithm is None: + raise RuntimeError(f"Algorithm {self.name} has not been bound") + return self.tuning_launches()[0] + + def reset(self): + if self.algorithm is None: + raise RuntimeError(f"Algorithm {self.name} has not been bound") + self.algorithm.reset() + + @staticmethod + def compose_rsag( + reduce_scatter: "_AlgorithmConfig", + allgather: "_AlgorithmConfig", + *, + rank: int, + world_size: int, + ipc_domain_count: int, + reduce_ops: Any, + ) -> "_CompositeAlgorithmConfig": + if ( + reduce_scatter.collective != "reducescatter" + or allgather.collective != "allgather" + ): + raise ValueError("RSAG requires reduce-scatter and all-gather algorithms") + if reduce_scatter.algorithm is None or allgather.algorithm is None: + raise RuntimeError("RSAG composition requires bound algorithms") + message_size_range = _MessageSizeRange( + minimum=max( + reduce_scatter.message_size_range.minimum, + allgather.message_size_range.minimum, + ), + maximum=min( + reduce_scatter.message_size_range.maximum, + allgather.message_size_range.maximum, + ), + ) + algorithm = _TwoKernelAllReduce( + name=( + f"allreduce_rsag_{reduce_scatter.algorithm.name}_" + f"{allgather.algorithm.name}" + ), + reduce_scatter=reduce_scatter.algorithm, + allgather=allgather.algorithm, + allgather_op=allgather.resolve_reduce_op(reduce_ops), + rank=rank, + world_size=world_size, + ) + return _CompositeAlgorithmConfig( + name=algorithm.name, + collective="allreduce", + world_sizes=(world_size,), + ipc_domain_counts=(ipc_domain_count,), + threads_per_block=(0,), + message_size_range=message_size_range, + reduce_op="SUM", + supported_dtypes=tuple( + dtype + for dtype in reduce_scatter.supported_dtypes + if dtype in allgather.supported_dtypes + ), + algorithm=algorithm, + ) + + +@dataclass(frozen=True, kw_only=True) +class _DslAlgorithmConfig(_AlgorithmConfig): + implementation: ClassVar[str] = "dsl" + algo_spec: Any + algorithm_kwargs: tuple[dict[str, Any], ...] = () + name_variant: str = "" + + def __post_init__(self): + super().__post_init__() + if not self.threads_per_block: + raise ValueError("DSL compile thread candidates cannot be empty") + if any(threads <= 0 for threads in self.threads_per_block): + raise ValueError("DSL compile thread candidates must be positive") + if self.algorithm is None: + if self.algo_spec.world_size != 0 or self.algo_spec.nranks_per_node != 0: + raise ValueError("DSL AlgoSpec templates require zero topology values") + if self.algo_spec.name != self.name: + raise ValueError("DSL AlgoSpec and algorithm names must match") + elif self.algo_spec.world_size <= 0 or self.algo_spec.nranks_per_node <= 0: + raise ValueError("Bound DSL algorithms require a materialized AlgoSpec") + if self.algo_spec.collective.name != self.collective: + raise ValueError("DSL AlgoSpec and algorithm collectives must match") + if self.algo_spec.in_place != self.in_place: + raise ValueError("DSL AlgoSpec and algorithm buffer modes must match") + + def tuning_launches(self) -> tuple[tuple[int, int], ...]: + return ((0, 0),) + + def bind(self, algorithm, algo_spec) -> "_AlgorithmConfig": + return replace(self, algorithm=algorithm, algo_spec=algo_spec) + + def dsl_name( + self, + ipc_domain_count: int, + threads_per_block: int, + algorithm_kwargs: dict[str, Any], + ) -> str: + variant_parts = [self.name_variant] if self.name_variant else [] + variant_parts.extend( + f"{key}_{value}" for key, value in sorted(algorithm_kwargs.items()) + ) + variant = f"{'_'.join(variant_parts)}_" if variant_parts else "" + return f"{self.name}_{ipc_domain_count}node_{variant}" f"{threads_per_block}TPB" + + +@dataclass(frozen=True, kw_only=True) +class _NativeAlgorithmConfig(_AlgorithmConfig): + implementation: ClassVar[str] = "native" nblocks: tuple[int, ...] - nthreads: tuple[int, ...] - min_message_size: int - max_message_size: int + selected_launch_parameters: Optional[tuple[int, int]] = None + + def __post_init__(self): + super().__post_init__() + if not self.nblocks: + raise ValueError("Native launch candidates cannot be empty") + if self.selected_launch_parameters is not None: + if self.algorithm is None: + raise ValueError("Only bound native algorithms can select a launch") + if self.selected_launch_parameters not in self.tuning_launches(): + raise ValueError("Invalid native launch selection") + + def tuning_launches(self) -> tuple[tuple[int, int], ...]: + return tuple( + (nblocks, threads_per_block) + for nblocks in self.nblocks + for threads_per_block in self.threads_per_block + ) + + def select(self, nblocks: int, threads_per_block: int) -> "_AlgorithmConfig": + if self.algorithm is None: + raise RuntimeError(f"Algorithm {self.name} has not been bound") + launch = (nblocks, threads_per_block) + if launch not in self.tuning_launches(): + raise ValueError(f"Invalid launch selection for algorithm {self.name}") + return replace(self, selected_launch_parameters=launch) + + def selected_launch(self) -> tuple[int, int]: + if self.selected_launch_parameters is None: + raise RuntimeError(f"Algorithm {self.name} has not been tuned") + return self.selected_launch_parameters + + +@dataclass(frozen=True, kw_only=True) +class _CompositeAlgorithmConfig(_AlgorithmConfig): + implementation: ClassVar[str] = "composite" + + def __post_init__(self): + super().__post_init__() + if self.algorithm is None: + raise ValueError("Composite algorithms must be bound when constructed") + + def tuning_launches(self) -> tuple[tuple[int, int], ...]: + return ((0, 0),) + + +_DEFAULT_THREADS_PER_BLOCK = (256, 512, 768, 1024) +_DEFAULT_THREAD_BLOCK_GROUP_SIZES = (1, 2, 4, 8) +_SUPPORTED_WORLD_SIZES = (4, 8, 16, 32, 64) +_NATIVE_IPC_DOMAIN_COUNTS = (1,) +_MULTI_NODE_IPC_DOMAIN_COUNTS = (2, 4, 8) + + +_NATIVE_ALGORITHM_CONFIGS = ( + _NativeAlgorithmConfig( + name="default_allreduce_nvls_packet", + collective="allreduce", + world_sizes=_SUPPORTED_WORLD_SIZES, + ipc_domain_counts=_NATIVE_IPC_DOMAIN_COUNTS, + message_size_range=_MessageSizeRange(0, 512 << 10), + nblocks=(4, 8, 12, 16), + threads_per_block=_DEFAULT_THREADS_PER_BLOCK, + reduce_op="SUM", + requires_nvls=True, + ), + _NativeAlgorithmConfig( + name="default_allreduce_packet", + collective="allreduce", + world_sizes=_SUPPORTED_WORLD_SIZES, + ipc_domain_counts=_NATIVE_IPC_DOMAIN_COUNTS, + message_size_range=_MessageSizeRange(0, 2 << 20), + nblocks=(14, 21, 28, 42, 56), + threads_per_block=_DEFAULT_THREADS_PER_BLOCK, + reduce_op="SUM", + ), + _NativeAlgorithmConfig( + name="default_allreduce_rsag_zero_copy", + collective="allreduce", + world_sizes=_SUPPORTED_WORLD_SIZES, + ipc_domain_counts=_NATIVE_IPC_DOMAIN_COUNTS, + message_size_range=_MessageSizeRange(512 << 10, 4 << 30), + nblocks=(32, 48, 64, 128), + threads_per_block=_DEFAULT_THREADS_PER_BLOCK, + reduce_op="SUM", + ), + _NativeAlgorithmConfig( + name="default_allreduce_nvls_zero_copy", + collective="allreduce", + world_sizes=_SUPPORTED_WORLD_SIZES, + ipc_domain_counts=_NATIVE_IPC_DOMAIN_COUNTS, + message_size_range=_MessageSizeRange(512 << 10, 4 << 30), + nblocks=(4, 8, 12, 16, 32), + threads_per_block=_DEFAULT_THREADS_PER_BLOCK, + reduce_op="SUM", + requires_nvls=True, + ), +) + + +def _create_algorithm_configs(language) -> tuple[_AlgorithmConfig, ...]: + default_spec = language.AlgoSpec( + name="allreduce_multi_nodes", + collective=language.collectives.AllReduce(0, 1, True), + nranks_per_node=0, + world_size=0, + in_place=True, + instances=1, + protocol="LL", + instr_fusion=True, + auto_sync=False, + replication_policy=language.ReplicationPolicy.interleaved, + reuse_resources=True, + use_double_scratch_buffer=True, + buffer_alignment=16, + ) + allgather_spec = replace( + default_spec, + name="allgather_multi_nodes", + collective=language.collectives.AllGather(0, 1, False), + in_place=False, + ) + reduce_scatter_spec = replace( + default_spec, + name="reducescatter_multi_nodes", + collective=language.collectives.ReduceScatter(0, 1, True), + instr_fusion=False, + ) + dsl_configs = ( + _DslAlgorithmConfig( + name="allreduce_multi_nodes", + collective="allreduce", + world_sizes=_SUPPORTED_WORLD_SIZES, + ipc_domain_counts=_MULTI_NODE_IPC_DOMAIN_COUNTS, + message_size_range=_MessageSizeRange(1 << 10, 8 << 20), + reduce_op="SUM", + algo_spec=default_spec, + threads_per_block=_DEFAULT_THREADS_PER_BLOCK, + algorithm_kwargs=tuple( + {"thread_block_group_size": thread_block_group_size} + for thread_block_group_size in _DEFAULT_THREAD_BLOCK_GROUP_SIZES + ), + ), + _DslAlgorithmConfig( + name="allgather_multi_nodes", + collective="allgather", + world_sizes=_SUPPORTED_WORLD_SIZES, + ipc_domain_counts=_MULTI_NODE_IPC_DOMAIN_COUNTS, + message_size_range=_MessageSizeRange(1 << 10, 8 << 20), + reduce_op="NOP", + in_place=False, + algo_spec=allgather_spec, + threads_per_block=_DEFAULT_THREADS_PER_BLOCK, + ), + _DslAlgorithmConfig( + name="reducescatter_multi_nodes", + collective="reducescatter", + world_sizes=_SUPPORTED_WORLD_SIZES, + ipc_domain_counts=_MULTI_NODE_IPC_DOMAIN_COUNTS, + message_size_range=_MessageSizeRange(1 << 10, 8 << 20), + reduce_op="SUM", + algo_spec=reduce_scatter_spec, + threads_per_block=_DEFAULT_THREADS_PER_BLOCK, + algorithm_kwargs=tuple( + {"thread_block_group_size": thread_block_group_size} + for thread_block_group_size in _DEFAULT_THREAD_BLOCK_GROUP_SIZES + ), + name_variant="directowner_v4_inplace_unfused", + ), + ) + return (*dsl_configs, *_NATIVE_ALGORITHM_CONFIGS) @dataclass(frozen=True) @@ -35,18 +420,10 @@ class _TwoKernelAllReduce: name: str reduce_scatter: Any allgather: Any + allgather_op: Any rank: int world_size: int - @property - def message_size_range(self) -> tuple[int, int]: - reduce_scatter_range = self.reduce_scatter.message_size_range - allgather_range = self.allgather.message_size_range - return ( - max(reduce_scatter_range[0], allgather_range[0]), - min(reduce_scatter_range[1], allgather_range[1]), - ) - def execute( self, *, @@ -96,7 +473,7 @@ def execute( input_size=shard_size, output_size=output_size, dtype=dtype, - op=op, + op=self.allgather_op, stream=stream, nblocks=nblocks, nthreads_per_block=nthreads_per_block, @@ -109,11 +486,6 @@ def reset(self): class PyMscclppCommunicator: - _SUPPORTED_WORLD_SIZES = [4, 8, 16, 32, 64] - _SUPPORTED_DTYPE = [torch.float, torch.float16, torch.bfloat16] - _ALLGATHER_MIN_TOTAL_BYTES = 1 << 10 - _ALLGATHER_MAX_TOTAL_BYTES = 8 << 20 - def _is_symm_mem_enabled(self) -> bool: try: return get_server_args().enable_symm_mem @@ -126,414 +498,248 @@ def _is_weak_contiguous(self, inp: torch.Tensor): == inp.numel() * inp.element_size() ) - def _get_allreduce_tuned_config(self, size): - if size <= 512: - target_size = 512 - elif size > 256 * 1024 * 1024: - target_size = 256 * 1024 * 1024 - else: - target_size = 1 << (size - 1).bit_length() - return self.allreduce_best_configs.get(target_size) - - def _get_allgather_tuned_config(self, size): - total_size = size * self.world_size - if not ( - self._ALLGATHER_MIN_TOTAL_BYTES - <= total_size - <= self._ALLGATHER_MAX_TOTAL_BYTES - ): + def _get_tuned_config(self, collective: str, message_size: int): + if message_size <= 0: return None - target_size = 1 << (size - 1).bit_length() - return self.allgather_best_configs.get(target_size) + configs = self._best_configs[collective] + target_size = 1 << (message_size - 1).bit_length() + config = configs.get(target_size) + if config is not None and config.supports_message_size(message_size): + return config - def _compile_dsl_candidate( - self, - *, - name, - collective, - algorithm_builder, - num_threads_per_block, - plan_min_message_size, - plan_max_message_size, - tuning_min_message_size=None, - tuning_max_message_size=None, - instr_fusion=True, - compile_kwargs=None, - ): - spec = self.mscclpp.language.AlgoSpec( - name=name, - collective=collective, - nranks_per_node=self.nranks_per_ipc_domain, - world_size=self.world_size, - in_place=collective.inplace, - instances=1, - protocol="LL", - auto_sync=False, - instr_fusion=instr_fusion, - num_threads_per_block=num_threads_per_block, - reuse_resources=True, - use_double_scratch_buffer=True, - min_message_size=plan_min_message_size, - max_message_size=plan_max_message_size, - tags={"default": 1}, - ) - algorithm = self.mscclpp.compile( - algorithm_builder, - spec, - self.rank, - **(compile_kwargs or {}), - ) - message_range = algorithm.message_size_range - return _TuningCandidate( - algorithm=algorithm, - nblocks=(0,), - nthreads=(0,), - min_message_size=( - message_range[0] - if tuning_min_message_size is None - else tuning_min_message_size - ), - max_message_size=( - message_range[1] - if tuning_max_message_size is None - else tuning_max_message_size - ), - ) - - def _create_dsl_allreduce_algorithms(self): - algorithms = [] - n_ipc_domains = self.world_size // self.nranks_per_ipc_domain - if n_ipc_domains not in (2, 4, 8): - return algorithms - - for tbg in (1, 2, 4, 8): - for num_threads_per_block in (256, 512, 768, 1024): - algorithms.append( - self._compile_dsl_candidate( - name=( - f"allreduce_{n_ipc_domains}node_" - f"{tbg}TBG_{num_threads_per_block}TPB" - ), - collective=self.mscclpp.language.collectives.AllReduce( - self.world_size, - 1, - True, - ), - algorithm_builder=self.def_algo.allreduce_multi_nodes, - num_threads_per_block=num_threads_per_block, - plan_min_message_size=tbg * (1 << 10), - plan_max_message_size=8 << 20, - compile_kwargs={"thread_block_group_size": tbg}, - ) - ) - return algorithms + if not configs: + return None + minimum_tuned_size = min(configs) + minimum_config = configs[minimum_tuned_size] + if target_size < minimum_tuned_size and minimum_config.supports_message_size( + message_size + ): + return minimum_config + return None + + def _get_allreduce_tuned_config(self, message_size): + return self._get_tuned_config("allreduce", message_size) + + def _get_allgather_tuned_config(self, message_size): + return self._get_tuned_config("allgather", message_size) + + def _algorithm_configs(self, ipc_domain_count: int): + return [ + config + for config in self._registered_algorithm_configs + if config.requirements_satisfied( + self.world_size, + ipc_domain_count, + nvls_supported=self.mscclpp.is_nvls_supported(), + symmetric_memory=self.symm_mem_enabled, + ) + ] - def _create_dsl_allgather_algorithms(self): - if not hasattr(self.def_algo, "allgather_multi_nodes"): + def _compile_dsl_algorithm( + self, + config: _DslAlgorithmConfig, + ipc_domain_count: int, + ) -> list[_DslAlgorithmConfig]: + algorithm_builder = getattr(self.def_algo, config.name, None) + if algorithm_builder is None: raise RuntimeError( "The installed MSCCL++ package does not provide " - "default_algos.allgather_multi_nodes" + f"default_algos.{config.name}" ) - + template_collective = config.algo_spec.collective + collective = type(template_collective)( + self.world_size, + template_collective.chunk_factor, + config.in_place, + ) algorithms = [] - n_ipc_domains = self.world_size // self.nranks_per_ipc_domain - if n_ipc_domains not in (2, 4, 8): - return algorithms - - input_min = self._ALLGATHER_MIN_TOTAL_BYTES // self.world_size - input_max = self._ALLGATHER_MAX_TOTAL_BYTES // self.world_size - for num_threads_per_block in self.allgather_tuning_threads: - algorithms.append( - self._compile_dsl_candidate( - name=( - f"allgather_{n_ipc_domains}node_1TBG_" - f"{num_threads_per_block}TPB" + algorithm_kwargs_variants = config.algorithm_kwargs or ({},) + for algorithm_kwargs in algorithm_kwargs_variants: + for threads_per_block in config.threads_per_block: + spec = replace( + config.algo_spec, + name=config.dsl_name( + ipc_domain_count, + threads_per_block, + algorithm_kwargs, ), - collective=self.mscclpp.language.collectives.AllGather( - self.world_size, - 1, - False, - ), - algorithm_builder=self.def_algo.allgather_multi_nodes, - num_threads_per_block=num_threads_per_block, - plan_min_message_size=self._ALLGATHER_MIN_TOTAL_BYTES, - plan_max_message_size=self._ALLGATHER_MAX_TOTAL_BYTES, - tuning_min_message_size=input_min, - tuning_max_message_size=input_max, + collective=collective, + nranks_per_node=self.nranks_per_ipc_domain, + world_size=self.world_size, + num_threads_per_block=threads_per_block, ) - ) + algorithm = self.mscclpp.compile( + algorithm_builder, + spec, + self.rank, + **algorithm_kwargs, + ) + algorithms.append(config.bind(algorithm, spec)) return algorithms - def _create_dsl_reducescatter_algorithms(self): - if not hasattr(self.def_algo, "reducescatter_multi_nodes"): - return [] - + def _create_dsl_algorithms( + self, + configs: list[_DslAlgorithmConfig], + ipc_domain_count: int, + ) -> list[_DslAlgorithmConfig]: algorithms = [] - n_ipc_domains = self.world_size // self.nranks_per_ipc_domain - tbg_min_message_size = { - 1: 1 << 10, - 2: 1 << 20, - 4: 2 << 20, - 8: 8 << 20, - } - for tbg in (1, 2, 4, 8): - for num_threads_per_block in (256, 512, 768, 1024): - algorithms.append( - self._compile_dsl_candidate( - name=( - f"reducescatter_{n_ipc_domains}node_" - "directowner_v4_inplace_unfused_" - f"{tbg}TBG_{num_threads_per_block}TPB" - ), - collective=self.mscclpp.language.collectives.ReduceScatter( - self.world_size, - 1, - True, - ), - algorithm_builder=self.def_algo.reducescatter_multi_nodes, - num_threads_per_block=num_threads_per_block, - plan_min_message_size=1 << 10, - plan_max_message_size=8 << 20, - tuning_min_message_size=tbg_min_message_size[tbg], - instr_fusion=False, - compile_kwargs={"thread_block_group_size": tbg}, - ) - ) + for config in configs: + algorithms.extend(self._compile_dsl_algorithm(config, ipc_domain_count)) return algorithms - def _create_native_allreduce_algorithms(self): - native_algorithms_config = [] - force_disable_nvls = os.getenv("MSCCLPP_FORCE_DISABLE_NVLS") == "1" - dlpack = self.mscclpp.RawGpuBuffer(1 << 27).to_dlpack( + def _create_native_algorithms( + self, configs: list[_NativeAlgorithmConfig] + ) -> list[_NativeAlgorithmConfig]: + dlpack = self.mscclpp.RawGpuBuffer(_DEFAULT_GPU_BUFFER_SIZE).to_dlpack( data_type=str(torch.float16) ) self.scratch_buffer = torch.utils.dlpack.from_dlpack(dlpack) - self.flag_buffer = torch.ones(128, dtype=torch.uint32, device="cuda") algos = self.mscclpp_ext.AlgorithmCollectionBuilder().build_default_algorithms( scratch_buffer=self.scratch_buffer.data_ptr(), scratch_buffer_size=self.scratch_buffer.nbytes, rank=self.rank, ) + configs_by_name = {config.name: config for config in configs} + algorithms = [] for algo in algos: - if force_disable_nvls and "nvls" in algo.name: + config = configs_by_name.get(algo.name) + if config is None: continue - if algo.name == "default_allreduce_nvls_packet": - algo.set_message_size_range(0, 512 << 10) - native_algorithms_config.append( - _TuningCandidate( - algorithm=algo, - nblocks=(4, 8, 12, 16), - nthreads=(256, 512, 768, 1024), - min_message_size=0, - max_message_size=512 << 10, - ) - ) - if algo.name == "default_allreduce_packet": - algo.set_message_size_range(0, 2 << 20) - native_algorithms_config.append( - _TuningCandidate( - algorithm=algo, - nblocks=(14, 21, 28, 42, 56), - nthreads=(256, 512, 768, 1024), - min_message_size=0, - max_message_size=2 << 20, - ) - ) - if algo.name == "default_allreduce_rsag_zero_copy": - algo.set_message_size_range(512 << 10, 4 << 30) - native_algorithms_config.append( - _TuningCandidate( - algorithm=algo, - nblocks=(32, 48, 64, 128), - nthreads=(256, 512, 768, 1024), - min_message_size=512 << 10, - max_message_size=4 << 30, - ) - ) - if ( - self.symm_mem_enabled - and algo.name == "default_allreduce_nvls_zero_copy" - ): - algo.set_message_size_range(512 << 10, 4 << 30) - native_algorithms_config.append( - _TuningCandidate( - algorithm=algo, - nblocks=(4, 8, 12, 16, 32), - nthreads=(256, 512, 768, 1024), - min_message_size=512 << 10, - max_message_size=4 << 30, - ) - ) + message_range = config.message_size_range + algo.set_message_size_range(message_range.minimum, message_range.maximum) + algorithms.append(config.bind(algo)) - return native_algorithms_config - - def _tune_collective(self, collective, algorithms): - if not algorithms: - raise RuntimeError(f"No MSCCL++ {collective} algorithms were compiled") - self._tune( - collective, - algorithms, - n_warmup=5, - n_graph_launches=20, - n_ops_per_graph=20, - ) - best_configs = { - "allreduce": self.allreduce_best_configs, - "allgather": self.allgather_best_configs, - "reducescatter": self.reducescatter_best_configs, - }[collective] - if not best_configs: - raise RuntimeError( - f"MSCCL++ {collective} produced no usable tuned configurations" + return algorithms + + def _create_algorithm_candidates( + self, + configs: list[_AlgorithmConfig], + ipc_domain_count: int, + ) -> list[_AlgorithmConfig]: + native_configs = [ + config for config in configs if isinstance(config, _NativeAlgorithmConfig) + ] + dsl_configs = [ + config for config in configs if isinstance(config, _DslAlgorithmConfig) + ] + algorithms = [] + if native_configs: + algorithms.extend(self._create_native_algorithms(native_configs)) + if dsl_configs: + algorithms.extend( + self._create_dsl_algorithms(dsl_configs, ipc_domain_count) ) + return algorithms def _create_algorithms(self): - n_ipc_domains = self.world_size // self.nranks_per_ipc_domain - if n_ipc_domains == 1: - if "allreduce" in self.collectives: - self._tune_collective( - "allreduce", - self._create_native_allreduce_algorithms(), - ) - return - - if n_ipc_domains not in (2, 4, 8): - return - - need_rsag = "allreduce" in self.collectives and hasattr( - self.def_algo, - "reducescatter_multi_nodes", + ipc_domain_count = self.world_size // self.nranks_per_ipc_domain + algorithms = self._create_algorithm_candidates( + self._algorithm_configs(ipc_domain_count), + ipc_domain_count, + ) + algorithms_by_collective = { + collective: [ + algorithm + for algorithm in algorithms + if algorithm.collective == collective + ] + for collective in ("allreduce", "allgather", "reducescatter") + } + allreduce_algorithms = algorithms_by_collective["allreduce"] + allgather_algorithms = algorithms_by_collective["allgather"] + reduce_scatter_algorithms = algorithms_by_collective["reducescatter"] + need_rsag = ( + "allreduce" in self.collectives + and bool(allgather_algorithms) + and bool(reduce_scatter_algorithms) ) if "allgather" in self.collectives or need_rsag: - self._tune_collective( - "allgather", - self._create_dsl_allgather_algorithms(), - ) + if allgather_algorithms: + self._tune(allgather_algorithms) - if "allreduce" in self.collectives: - self._tune_collective( - "allreduce", - self._create_dsl_allreduce_algorithms(), - ) + if "allreduce" in self.collectives and allreduce_algorithms: + self._tune(allreduce_algorithms) if need_rsag: - self._tune_collective( - "reducescatter", - self._create_dsl_reducescatter_algorithms(), - ) - self._merge_rsag_configs( - n_warmup=5, - n_graph_launches=20, - n_ops_per_graph=20, - ) - - def _merge_rsag_configs( - self, - *, - n_warmup, - n_graph_launches, - n_ops_per_graph, - ): - n_ipc_domains = self.world_size // self.nranks_per_ipc_domain - element_size = torch.empty((), dtype=torch.bfloat16).element_size() - size = 1 << 10 - while size <= 8 << 20: - reduce_scatter_config = self.reducescatter_best_configs.get(size) - allgather_config = self.allgather_best_configs.get(size // self.world_size) - if reduce_scatter_config is None or allgather_config is None: - size <<= 1 + self._tune(reduce_scatter_algorithms) + self._merge_rsag_configs() + + def _merge_rsag_configs(self): + allreduce_configs = self._best_configs["allreduce"] + allgather_configs = self._best_configs["allgather"] + reduce_scatter_configs = self._best_configs["reducescatter"] + allreduce_times = self._best_config_times["allreduce"] + allgather_times = self._best_config_times["allgather"] + reduce_scatter_times = self._best_config_times["reducescatter"] + for message_size, reduce_scatter_config in sorted( + reduce_scatter_configs.items() + ): + allreduce_config = allreduce_configs.get(message_size) + allgather_config = allgather_configs.get(message_size) + if allreduce_config is None or allgather_config is None: continue - reduce_scatter = reduce_scatter_config[0] - allgather = allgather_config[0] - if size not in self.allreduce_best_configs: - size <<= 1 - continue - reduce_scatter_prefix = ( - f"reducescatter_{n_ipc_domains}node_" "directowner_v4_inplace_unfused_" - ) - allgather_prefix = f"allgather_{n_ipc_domains}node_1TBG_" - algorithm = _TwoKernelAllReduce( - name=( - f"allreduce_rsag_{n_ipc_domains}node_" - f"RS{reduce_scatter.name.removeprefix(reduce_scatter_prefix)}_" - f"AG{allgather.name.removeprefix(allgather_prefix)}" - ), - reduce_scatter=reduce_scatter, - allgather=allgather, + composite = _AlgorithmConfig.compose_rsag( + reduce_scatter_config, + allgather_config, rank=self.rank, world_size=self.world_size, + ipc_domain_count=(self.world_size // self.nranks_per_ipc_domain), + reduce_ops=self.mscclpp.ReduceOp, ) - tensor = torch.empty( - size // element_size, - dtype=torch.bfloat16, - device=self.device, - ) - elapsed = self._get_time( - "allreduce", - algorithm, - tensor, - tensor, - size, - 0, - 0, - n_warmup, - n_graph_launches, - n_ops_per_graph, - symmetric_memory=False, - ) - legacy_time = self.allreduce_best_config_times.get( - size, - float("inf"), - ) - if elapsed < legacy_time: - self.allreduce_best_configs[size] = (algorithm, 0, 0) - self.allreduce_best_config_times[size] = elapsed + try: + composite_time = ( + reduce_scatter_times[message_size] + allgather_times[message_size] + ) + allreduce_time = allreduce_times[message_size] + except KeyError as exc: + raise RuntimeError( + "RSAG merge requires timings for all tuned algorithms" + ) from exc + selected = composite_time < allreduce_time + if selected: + allreduce_configs[message_size] = composite + allreduce_times[message_size] = composite_time if self.rank == 0: logger.info( "MSCCL++ allreduce RSAG merge: input_bytes=%d " - "reduce_scatter=%s allgather=%s time_ms=%.4f " + "reduce_scatter=%s allgather=%s estimated_time_ms=%.4f " "selected=%s", - size, - reduce_scatter.name, - allgather.name, - elapsed, - elapsed < legacy_time, + message_size, + reduce_scatter_config.algorithm.name, + allgather_config.algorithm.name, + composite_time, + selected, ) - del tensor - size <<= 1 def _get_time( self, - collective, - algo, + algorithm_config, input_tensor, output_tensor, - input_size, + message_size, nblocks, nthreads, n_warmup, n_graph_launches, n_ops_per_graph, - symmetric_memory=False, ): def run(stream=None): - return self._run_algo( - collective, - algo, + return self._run_algorithm( + algorithm_config, input_tensor, + output_tensor, + message_size, nblocks, nthreads, - output_tensor=output_tensor, - input_size=input_size, stream=stream, - symmetric_memory=symmetric_memory, ) result = run() if result != 0: raise RuntimeError( - f"MSCCL++ {collective} tuning failed with error code {result}" + f"MSCCL++ {algorithm_config.collective} tuning failed " + f"with error code {result}" ) for _ in range(n_warmup): @@ -569,178 +775,140 @@ def run(stream=None): dist.all_reduce(max_elapsed, op=ReduceOp.MAX, group=self.group) return max_elapsed.item() + def _allocate_tuning_tensor(self, nbytes: int, dtype: torch.dtype) -> torch.Tensor: + element_size = torch.empty((), dtype=dtype).element_size() + if nbytes <= 0 or nbytes % element_size != 0: + raise ValueError( + f"Tuning buffer size {nbytes} is invalid for dtype {dtype}" + ) + with torch.cuda.device(self.device): + dlpack = self.mscclpp.RawGpuBuffer(nbytes).to_dlpack(data_type=str(dtype)) + return torch.utils.dlpack.from_dlpack(dlpack) + def _tune( self, - collective: str, - algos_config: list[_TuningCandidate], - *, + algorithms: list[_AlgorithmConfig], n_warmup: int = 5, n_graph_launches: int = 20, - n_ops_per_graph: int = 1, + n_ops_per_graph: int = 20, ): - if collective not in {"allreduce", "allgather", "reducescatter"}: - raise ValueError(f"Unsupported tuning collective: {collective}") - - if collective == "allreduce": - size = 1 << 9 - max_size = 1 << 23 - dlpack = self.mscclpp.RawGpuBuffer(1 << 27).to_dlpack( - data_type=str(torch.float16) + dtype = torch.bfloat16 + element_size = torch.empty((), dtype=dtype).element_size() + tuning_cases = [] + for message_size in _TUNING_MESSAGE_SIZES: + candidates = [ + candidate + for candidate in algorithms + if candidate.supports_message_size(message_size) + ] + if not candidates: + continue + input_size = max( + candidate.input_buffer_size(message_size, self.world_size) + for candidate in candidates ) - allreduce_tensor = torch.utils.dlpack.from_dlpack(dlpack) - elif collective == "allgather": - size = self._ALLGATHER_MIN_TOTAL_BYTES // self.world_size - max_size = self._ALLGATHER_MAX_TOTAL_BYTES // self.world_size - allreduce_tensor = None - else: - size = self._ALLGATHER_MIN_TOTAL_BYTES - max_size = self._ALLGATHER_MAX_TOTAL_BYTES - allreduce_tensor = None - - while size <= max_size: - warmup = n_warmup - graph_launches = n_graph_launches - if collective == "allgather": - elements = ( - size - // torch.empty( - (), - dtype=torch.bfloat16, - ).element_size() - ) - input_tensor = torch.empty( - elements, - dtype=torch.bfloat16, - device=self.device, - ) - output_tensor = torch.empty( - elements * self.world_size, - dtype=input_tensor.dtype, - device=self.device, - ) - elif collective == "reducescatter": - elements = ( - size - // torch.empty( - (), - dtype=torch.bfloat16, - ).element_size() - ) - input_tensor = torch.empty( - elements, - dtype=torch.bfloat16, - device=self.device, - ) - output_tensor = input_tensor[: elements // self.world_size] - else: - input_tensor = allreduce_tensor - output_tensor = allreduce_tensor + output_size = max( + candidate.output_buffer_size(message_size, self.world_size) + for candidate in candidates + ) + tuning_cases.append((message_size, candidates, input_size, output_size)) + + if tuning_cases: + input_buffer = self._allocate_tuning_tensor( + max(case[2] for case in tuning_cases), dtype + ) + output_buffer = self._allocate_tuning_tensor( + max(case[3] for case in tuning_cases), dtype + ) + + for message_size, candidates, input_size, output_size in tuning_cases: + input_tensor = input_buffer[: input_size // element_size] + output_tensor = output_buffer[: output_size // element_size] best_time = float("inf") best_config = None - for candidate in algos_config: - if not ( - candidate.min_message_size <= size <= candidate.max_message_size - ): - continue - for nblocks in candidate.nblocks: - for nthreads in candidate.nthreads: - if self.rank == 0: - logger.info( - "MSCCL++ %s tuning candidate: input_bytes=%d " - "algorithm=%s nblocks=%d nthreads=%d", - collective, - size, - candidate.algorithm.name, - nblocks, - nthreads, - ) - avg_time = self._get_time( - collective, - candidate.algorithm, - input_tensor, - output_tensor, - size, - nblocks, - nthreads, - warmup, - graph_launches, - n_ops_per_graph, - symmetric_memory=collective == "allreduce", + for candidate in candidates: + candidate_output = input_tensor if candidate.in_place else output_tensor + for nblocks, nthreads in candidate.tuning_launches(): + if self.rank == 0: + threads_per_block = ( + candidate.algo_spec.num_threads_per_block + if isinstance(candidate, _DslAlgorithmConfig) + else nthreads ) - config = ( - candidate.algorithm, + logger.debug( + "MSCCL++ %s tuning candidate: message_bytes=%d " + "algorithm=%s nblocks=%d nthreads=%d", + candidate.collective, + message_size, + candidate.algorithm.name, nblocks, - nthreads, + threads_per_block, ) - if avg_time < best_time: - best_time = avg_time - best_config = config + avg_time = self._get_time( + candidate, + input_tensor, + candidate_output, + message_size, + nblocks, + nthreads, + n_warmup, + n_graph_launches, + n_ops_per_graph, + ) + config = (candidate, nblocks, nthreads) + if avg_time < best_time: + best_time = avg_time + best_config = config if best_config is not None: - if collective == "allreduce": - self.allreduce_best_configs[size] = best_config - self.allreduce_best_config_times[size] = best_time - elif collective == "allgather": - self.allgather_best_configs[size] = best_config - self.allgather_best_config_times[size] = best_time - else: - self.reducescatter_best_configs[size] = best_config - self.reducescatter_best_config_times[size] = best_time + self._best_configs[candidate.collective][message_size] = best_config[ + 0 + ].select( + best_config[1], + best_config[2], + ) + self._best_config_times[candidate.collective][message_size] = best_time if self.rank == 0: + threads_per_block = ( + best_config[0].algo_spec.num_threads_per_block + if isinstance(best_config[0], _DslAlgorithmConfig) + else best_config[2] + ) logger.info( - "MSCCL++ %s tuning: input_bytes=%d " + "MSCCL++ %s tuning: message_bytes=%d " "algorithm=%s nblocks=%d nthreads=%d time_ms=%.4f", - collective, - size, - best_config[0].name, + candidate.collective, + message_size, + best_config[0].algorithm.name, best_config[1], - best_config[2], + threads_per_block, best_time, ) - - if collective in {"allgather", "reducescatter"}: - del input_tensor, output_tensor - size <<= 1 - + del input_tensor, output_tensor torch.cuda.synchronize() - if collective in {"allreduce", "reducescatter"}: - for candidate in algos_config: - candidate.algorithm.reset() + for candidate in algorithms: + candidate.reset() - def _run_algo( + def _run_algorithm( self, - collective, - algo, + config: _AlgorithmConfig, input_tensor, + output_tensor, + message_size, nblocks, nthreads, *, - output_tensor=None, - input_size=None, stream=None, - symmetric_memory=False, ): - if collective not in {"allreduce", "allgather", "reducescatter"}: - raise ValueError(f"Unsupported MSCCL++ collective: {collective}") if stream is None: stream = torch.cuda.current_stream() - if input_size is None: - input_size = input_tensor.nbytes - if output_tensor is None: - output_tensor = input_tensor - - if collective == "allreduce": - output_size = input_size - reduce_op = self.mscclpp.ReduceOp.SUM - elif collective == "allgather": - output_size = output_tensor.nbytes - reduce_op = self.mscclpp.ReduceOp.NOP - else: - output_size = output_tensor.nbytes - reduce_op = self.mscclpp.ReduceOp.SUM - return algo.execute( + if config.algorithm is None: + raise RuntimeError(f"Algorithm {config.name} has not been bound") + input_size = config.input_buffer_size(message_size, self.world_size) + output_size = config.output_buffer_size(message_size, self.world_size) + return config.algorithm.execute( comm=self.comm.communicator, executor=self.executor, input_buffer=input_tensor.data_ptr(), @@ -748,11 +916,11 @@ def _run_algo( input_size=input_size, output_size=output_size, dtype=self.dtype_to_mscclpp_dtype(input_tensor.dtype), - op=reduce_op, + op=config.resolve_reduce_op(self.mscclpp.ReduceOp), stream=stream.cuda_stream, nblocks=nblocks, nthreads_per_block=nthreads, - symmetric_memory=symmetric_memory, + symmetric_memory=self.symm_mem_enabled, ) def __init__( @@ -761,30 +929,27 @@ def __init__( device: Union[int, str, torch.device], group_name: str = "anonymous", collectives: Optional[set[str]] = None, - allgather_tuning_threads: tuple[int, ...] = (256, 512, 768, 1024), ) -> None: """Args: - group: the process group to work on. If None, it will use the - default process group. - device: the device to bind the CustomAllreduce to. If None, - it will be bind to f"cuda:{local_rank}". + group: A non-NCCL process group used to bootstrap MSCCL++ and + synchronize tuning results. + device: The CUDA device used by this rank. group_name: a human-readable process-group name for logging. - It is the caller's responsibility to make sure each communicator - is bind to a unique device, and all communicators in this group - are in the same node. + collectives: Public collectives to compile and tune. Reduce-scatter + may be tuned internally to form an all-reduce candidate. + + Ranks must be consecutive, and each rank must be bound to its own + device. Supported multi-node layouts contain 2, 4, or 8 IPC domains. """ - self._IS_CAPTURING = False self.disabled = True self.available = False - self.allreduce_best_configs = {} - self.allreduce_best_config_times = {} - self.allgather_best_configs = {} - self.allgather_best_config_times = {} - self.reducescatter_best_configs = {} - self.reducescatter_best_config_times = {} - self._logged_allgather_sizes = set() + self._best_configs = { + collective: {} for collective in ("allreduce", "allgather", "reducescatter") + } + self._best_config_times = { + collective: {} for collective in ("allreduce", "allgather", "reducescatter") + } self.scratch_buffer = None - self.flag_buffer = None self.collectives = frozenset( ("allreduce", "allgather") if collectives is None else collectives ) @@ -793,13 +958,6 @@ def __init__( raise ValueError( f"Unsupported MSCCL++ collectives: {unsupported_collectives}" ) - if not allgather_tuning_threads or any( - threads <= 0 or threads > 1024 for threads in allgather_tuning_threads - ): - raise ValueError( - "allgather_tuning_threads must contain values in [1, 1024]" - ) - self.allgather_tuning_threads = tuple(allgather_tuning_threads) try: self.mscclpp = importlib.import_module("mscclpp") @@ -813,33 +971,35 @@ def __init__( self.mscclpp = None return + self._registered_algorithm_configs = _create_algorithm_configs( + self.mscclpp.language + ) + self.group = group assert ( dist.get_backend(group) != dist.Backend.NCCL ), "CustomAllreduce should be attached to a non-NCCL group." - rank = dist.get_rank(group=self.group) - world_size = dist.get_world_size(group=self.group) - self.rank = rank - self.world_size = world_size - if world_size == 1: + self.rank = dist.get_rank(group=self.group) + self.world_size = dist.get_world_size(group=self.group) + if self.world_size == 1: # No need to initialize mscclpp for single GPU case. return - if world_size not in PyMscclppCommunicator._SUPPORTED_WORLD_SIZES: + if self.world_size not in _SUPPORTED_WORLD_SIZES: logger.warning( "PyMscclpp is disabled due to an unsupported world" " size: %d. Supported world sizes: %s. To silence this " "warning, specify disable_mscclpp=True explicitly.", - world_size, - str(PyMscclppCommunicator._SUPPORTED_WORLD_SIZES), + self.world_size, + str(_SUPPORTED_WORLD_SIZES), ) return self.ranks = torch.distributed.get_process_group_ranks(group) # for now mscclpp with stride in the communicator is not tested - if not (abs(self.ranks[-1] - self.ranks[0]) == world_size - 1): + if not (abs(self.ranks[-1] - self.ranks[0]) == self.world_size - 1): logger.warning( "PyMscclpp is disabled due to an unsupported group %s." "Please ensure all ranks in the group are consecutive." @@ -857,7 +1017,7 @@ def __init__( self.device = device self.comm = self.mscclpp.CommGroup( - torch_group=self.group, rank=rank, size=world_size + torch_group=self.group, rank=self.rank, size=self.world_size ) nranks_per_ipc_domain = getattr(self.comm, "nranks_per_ipc_domain", None) self.nranks_per_ipc_domain = ( @@ -874,7 +1034,7 @@ def __init__( logger.exception("Failed to initialize MSCCL++ collectives") raise self.available = bool( - self.allreduce_best_configs or self.allgather_best_configs + self._best_configs["allreduce"] or self._best_configs["allgather"] ) if not self.available: logger.warning("PyMscclpp did not produce any usable tuned configurations.") @@ -883,39 +1043,30 @@ def __init__( "Created MSCCL++ communicator: group=%s world_size=%d device=%s " "allreduce_configs=%d allgather_configs=%d", group_name, - world_size, + self.world_size, self.device, - len(self.allreduce_best_configs), - len(self.allgather_best_configs), + len(self._best_configs["allreduce"]), + len(self._best_configs["allgather"]), ) def destroy(self): - self.allreduce_best_configs = None - self.allreduce_best_config_times = None - self.allgather_best_configs = None - self.allgather_best_config_times = None - self.reducescatter_best_configs = None - self.reducescatter_best_config_times = None + self._best_configs = None + self._best_config_times = None self.executor = None self.scratch_buffer = None - self.flag_buffer = None self.comm = None def should_mscclpp_allreduce( self, inp: torch.Tensor, op: ReduceOp = ReduceOp.SUM ) -> bool: - if ( - self.disabled - or self.world_size not in PyMscclppCommunicator._SUPPORTED_WORLD_SIZES - ): - return False - if inp.dtype not in PyMscclppCommunicator._SUPPORTED_DTYPE: + if self.disabled or self.world_size not in _SUPPORTED_WORLD_SIZES: return False if not self._is_weak_contiguous(inp): return False if op is not ReduceOp.SUM: return False - if self._get_allreduce_tuned_config(inp.numel() * inp.element_size()) is None: + config = self._get_allreduce_tuned_config(inp.numel() * inp.element_size()) + if config is None or not config.supports_dtype(inp.dtype): return False # mscclpp must not be used during any piecewise CUDA graph phase # (compile, capture, or replay) as it changes the allreduce dispatch @@ -933,12 +1084,7 @@ def should_mscclpp_allgather( output_tensor: torch.Tensor, input_tensor: torch.Tensor, ) -> bool: - if ( - self.disabled - or self.world_size not in PyMscclppCommunicator._SUPPORTED_WORLD_SIZES - ): - return False - if input_tensor.dtype not in PyMscclppCommunicator._SUPPORTED_DTYPE: + if self.disabled or self.world_size not in _SUPPORTED_WORLD_SIZES: return False if output_tensor.dtype != input_tensor.dtype: return False @@ -948,11 +1094,12 @@ def should_mscclpp_allgather( return False if output_tensor.device != input_tensor.device: return False - nbytes = input_tensor.numel() * input_tensor.element_size() - if nbytes % 16 != 0: + input_nbytes = input_tensor.numel() * input_tensor.element_size() + if input_nbytes % 16 != 0: return False - config = self._get_allgather_tuned_config(nbytes) - if config is None: + output_nbytes = output_tensor.numel() * output_tensor.element_size() + config = self._get_allgather_tuned_config(output_nbytes) + if config is None or not config.supports_dtype(input_tensor.dtype): return False if ( is_in_tc_piecewise_cuda_graph() @@ -980,17 +1127,22 @@ def all_reduce( op: ReduceOp = ReduceOp.SUM, stream: torch.cuda.Stream = None, ): - assert op == torch.distributed.ReduceOp.SUM + if op != torch.distributed.ReduceOp.SUM: + raise ValueError("MSCCL++ AllReduce only supports SUM") nbytes = tensor.numel() * tensor.element_size() - algo, nblocks, nthreads = self._get_allreduce_tuned_config(nbytes) - result = self._run_algo( - "allreduce", - algo, + config = self._get_allreduce_tuned_config(nbytes) + if config is None: + raise RuntimeError( + f"No tuned MSCCL++ AllReduce configuration for {nbytes} bytes" + ) + nblocks, threads_per_block = config.selected_launch() + result = self._run_algorithm( + config, + tensor, tensor, + nbytes, nblocks, - nthreads, - input_size=nbytes, - symmetric_memory=self.symm_mem_enabled, + threads_per_block, stream=stream, ) if result != 0: @@ -1003,29 +1155,27 @@ def all_gather( input_tensor: torch.Tensor, stream: torch.cuda.Stream = None, ): - nbytes = input_tensor.numel() * input_tensor.element_size() - config = self._get_allgather_tuned_config(nbytes) + input_nbytes = input_tensor.numel() * input_tensor.element_size() + output_nbytes = output_tensor.numel() * output_tensor.element_size() + if output_nbytes != input_nbytes * self.world_size: + raise ValueError( + f"MSCCL++ AllGather output has {output_nbytes} bytes; expected " + f"{input_nbytes * self.world_size} bytes" + ) + config = self._get_allgather_tuned_config(output_nbytes) if config is None: raise RuntimeError( - f"No tuned MSCCL++ AllGather configuration for {nbytes} bytes" + "No tuned MSCCL++ AllGather configuration for " + f"{output_nbytes} output bytes" ) - algo, nblocks, nthreads = config - if self.rank == 0 and nbytes not in self._logged_allgather_sizes: - self._logged_allgather_sizes.add(nbytes) - logger.info( - "Dispatching MSCCL++ AllGather: input_bytes=%d " - "total_bytes=%d algorithm=%s", - nbytes, - nbytes * self.world_size, - algo.name, - ) - result = self._run_algo( - "allgather", - algo, + nblocks, threads_per_block = config.selected_launch() + result = self._run_algorithm( + config, input_tensor, + output_tensor, + output_nbytes, nblocks, - nthreads, - output_tensor=output_tensor, + threads_per_block, stream=stream, ) if result != 0: @@ -1038,13 +1188,11 @@ def change_state( enable: Optional[bool] = None, ): if enable is None or self.available is False: - # guess a default value when not specified - # DO: Decided if raise an exception here or not enable = self.available old_disable = self.disabled self.disabled = not enable - - yield - - self.disabled = old_disable + try: + yield + finally: + self.disabled = old_disable From da91b557d2ab2df828c57a9b1ce71c2bc02dba4c Mon Sep 17 00:00:00 2001 From: Caio Rocha Date: Tue, 1 Sep 2026 15:37:32 +0000 Subject: [PATCH 04/11] wip --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index f318a292c825..945bdf9d5254 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -21,7 +21,7 @@ ARG GITHUB_ARTIFACTORY=github.com ARG INSTALL_FLASHINFER_JIT_CACHE=0 ARG FLASHINFER_VERSION=0.6.14 ARG MOONCAKE_VERSION=0.3.11.post1 -ARG MSCCLPP_VERSION=sglang-v0.9.1 +ARG MSCCLPP_VERSION=sglang-v0.10.1 #if need other arg please add in MOONCAKE_COMPILE_ARG ARG MOONCAKE_COMPILE_ARG="-DUSE_HTTP=ON -DUSE_MNNVL=ON -DUSE_CUDA=ON -DWITH_EP=ON" From bc42096b73811846070264acb60030a775c2b8b5 Mon Sep 17 00:00:00 2001 From: Caio Rocha Date: Tue, 8 Sep 2026 14:09:26 +0000 Subject: [PATCH 05/11] wip --- .../device_communicators/pymscclpp.py | 239 ++++++++++++------ 1 file changed, 158 insertions(+), 81 deletions(-) diff --git a/python/sglang/srt/distributed/device_communicators/pymscclpp.py b/python/sglang/srt/distributed/device_communicators/pymscclpp.py index ee29019c4aa5..bf01eb08a7ca 100644 --- a/python/sglang/srt/distributed/device_communicators/pymscclpp.py +++ b/python/sglang/srt/distributed/device_communicators/pymscclpp.py @@ -2,8 +2,9 @@ import logging from abc import ABC, abstractmethod from contextlib import contextmanager -from dataclasses import dataclass, replace -from typing import Any, ClassVar, Optional, Union +from dataclasses import dataclass, field, replace +from typing import Any, Callable, ClassVar, Optional, Union +from itertools import product import torch import torch.distributed as dist @@ -24,6 +25,7 @@ _DEFAULT_SUPPORTED_DTYPES = (torch.float, torch.float16, torch.bfloat16) _DEFAULT_GPU_BUFFER_SIZE = 1 << 27 _TUNING_MESSAGE_SIZES = tuple(1 << exponent for exponent in range(9, 24)) +_ParameterAdapter = Callable[..., None] @dataclass(frozen=True) @@ -39,7 +41,7 @@ def contains(self, size: int) -> bool: return self.minimum <= size <= self.maximum -@dataclass(frozen=True, kw_only=True) +@dataclass(kw_only=True) class _AlgorithmConfig(ABC): implementation: ClassVar[str] name: str @@ -49,6 +51,7 @@ class _AlgorithmConfig(ABC): threads_per_block: tuple[int, ...] message_size_range: _MessageSizeRange reduce_op: str + parameter_adapter: Optional[_ParameterAdapter] = None supported_dtypes: tuple[torch.dtype, ...] = _DEFAULT_SUPPORTED_DTYPES in_place: bool = True requires_nvls: bool = False @@ -95,6 +98,17 @@ def supports_dtype(self, dtype: torch.dtype) -> bool: def resolve_reduce_op(self, reduce_ops): return getattr(reduce_ops, self.reduce_op) + def adapt_to_topology( + self, world_size: int, nranks_per_ipc_domain: int + ): + if self.parameter_adapter is None: + return + self.parameter_adapter( + self, + world_size=world_size, + nranks_per_ipc_domain=nranks_per_ipc_domain, + ) + def input_buffer_size(self, message_size: int, world_size: int) -> int: if self.collective == "allgather": if message_size % world_size != 0: @@ -194,12 +208,11 @@ def compose_rsag( ) -@dataclass(frozen=True, kw_only=True) +@dataclass(kw_only=True) class _DslAlgorithmConfig(_AlgorithmConfig): implementation: ClassVar[str] = "dsl" algo_spec: Any - algorithm_kwargs: tuple[dict[str, Any], ...] = () - name_variant: str = "" + algorithm_kwargs: dict[str, tuple[Any, ...]] = field(default_factory=dict) def __post_init__(self): super().__post_init__() @@ -207,6 +220,8 @@ def __post_init__(self): raise ValueError("DSL compile thread candidates cannot be empty") if any(threads <= 0 for threads in self.threads_per_block): raise ValueError("DSL compile thread candidates must be positive") + if any(not values for values in self.algorithm_kwargs.values()): + raise ValueError("DSL algorithm kwarg candidates cannot be empty") if self.algorithm is None: if self.algo_spec.world_size != 0 or self.algo_spec.nranks_per_node != 0: raise ValueError("DSL AlgoSpec templates require zero topology values") @@ -222,6 +237,22 @@ def __post_init__(self): def tuning_launches(self) -> tuple[tuple[int, int], ...]: return ((0, 0),) + def adapt_to_topology( + self, + world_size: int, + nranks_per_ipc_domain: int, + algorithm_kwargs: Optional[dict[str, Any]] = None, + ): + if self.parameter_adapter is None: + return + self.parameter_adapter( + self, + world_size=world_size, + nranks_per_ipc_domain=nranks_per_ipc_domain, + instances=self.algo_spec.instances, + algorithm_kwargs=algorithm_kwargs or {}, + ) + def bind(self, algorithm, algo_spec) -> "_AlgorithmConfig": return replace(self, algorithm=algorithm, algo_spec=algo_spec) @@ -231,15 +262,11 @@ def dsl_name( threads_per_block: int, algorithm_kwargs: dict[str, Any], ) -> str: - variant_parts = [self.name_variant] if self.name_variant else [] - variant_parts.extend( - f"{key}_{value}" for key, value in sorted(algorithm_kwargs.items()) - ) + variant_parts = [f"{key}_{value}" for key, value in sorted(algorithm_kwargs.items())] variant = f"{'_'.join(variant_parts)}_" if variant_parts else "" return f"{self.name}_{ipc_domain_count}node_{variant}" f"{threads_per_block}TPB" - -@dataclass(frozen=True, kw_only=True) +@dataclass(kw_only=True) class _NativeAlgorithmConfig(_AlgorithmConfig): implementation: ClassVar[str] = "native" nblocks: tuple[int, ...] @@ -276,7 +303,7 @@ def selected_launch(self) -> tuple[int, int]: return self.selected_launch_parameters -@dataclass(frozen=True, kw_only=True) +@dataclass(kw_only=True) class _CompositeAlgorithmConfig(_AlgorithmConfig): implementation: ClassVar[str] = "composite" @@ -296,50 +323,89 @@ def tuning_launches(self) -> tuple[tuple[int, int], ...]: _MULTI_NODE_IPC_DOMAIN_COUNTS = (2, 4, 8) -_NATIVE_ALGORITHM_CONFIGS = ( - _NativeAlgorithmConfig( - name="default_allreduce_nvls_packet", - collective="allreduce", - world_sizes=_SUPPORTED_WORLD_SIZES, - ipc_domain_counts=_NATIVE_IPC_DOMAIN_COUNTS, - message_size_range=_MessageSizeRange(0, 512 << 10), - nblocks=(4, 8, 12, 16), - threads_per_block=_DEFAULT_THREADS_PER_BLOCK, - reduce_op="SUM", - requires_nvls=True, - ), - _NativeAlgorithmConfig( - name="default_allreduce_packet", - collective="allreduce", - world_sizes=_SUPPORTED_WORLD_SIZES, - ipc_domain_counts=_NATIVE_IPC_DOMAIN_COUNTS, - message_size_range=_MessageSizeRange(0, 2 << 20), - nblocks=(14, 21, 28, 42, 56), - threads_per_block=_DEFAULT_THREADS_PER_BLOCK, - reduce_op="SUM", - ), - _NativeAlgorithmConfig( - name="default_allreduce_rsag_zero_copy", - collective="allreduce", - world_sizes=_SUPPORTED_WORLD_SIZES, - ipc_domain_counts=_NATIVE_IPC_DOMAIN_COUNTS, - message_size_range=_MessageSizeRange(512 << 10, 4 << 30), - nblocks=(32, 48, 64, 128), - threads_per_block=_DEFAULT_THREADS_PER_BLOCK, - reduce_op="SUM", - ), - _NativeAlgorithmConfig( - name="default_allreduce_nvls_zero_copy", - collective="allreduce", - world_sizes=_SUPPORTED_WORLD_SIZES, - ipc_domain_counts=_NATIVE_IPC_DOMAIN_COUNTS, - message_size_range=_MessageSizeRange(512 << 10, 4 << 30), - nblocks=(4, 8, 12, 16, 32), - threads_per_block=_DEFAULT_THREADS_PER_BLOCK, - reduce_op="SUM", - requires_nvls=True, - ), -) +def _adapt_allreduce_packet( + config: _AlgorithmConfig, + *, + world_size: int, + nranks_per_ipc_domain: int, +) -> bool: + if not isinstance(config, _NativeAlgorithmConfig): + raise TypeError("AllReduce packet adaptation requires a native config") + if world_size != nranks_per_ipc_domain: + return False + min_blocks = nranks_per_ipc_domain - 1 + nblocks = tuple(nblocks for nblocks in config.nblocks if nblocks >= min_blocks) + if not nblocks: + return False + config.nblocks = nblocks + return True + + +def _adapt_dsl_message_size_range( + config: _AlgorithmConfig, + *, + world_size: int, + nranks_per_ipc_domain: int, + algorithm_kwargs: dict[str, Any], + **_: Any, +) -> bool: + if not isinstance(config, _DslAlgorithmConfig): + raise TypeError("DSL message size adaptation requires a DSL config") + ipc_domain_count = world_size // nranks_per_ipc_domain + thread_block_group_size = algorithm_kwargs.get("thread_block_group_size", 1) + config.message_size_range = _MessageSizeRange( + minimum=config.message_size_range.minimum * thread_block_group_size, + maximum=config.message_size_range.maximum * ipc_domain_count, + ) + return True + + +def _create_native_algorithm_configs() -> tuple[_NativeAlgorithmConfig, ...]: + return ( + _NativeAlgorithmConfig( + name="default_allreduce_nvls_packet", + collective="allreduce", + world_sizes=_SUPPORTED_WORLD_SIZES, + ipc_domain_counts=_NATIVE_IPC_DOMAIN_COUNTS, + message_size_range=_MessageSizeRange(0, 512 << 10), + nblocks=(4, 8, 12, 16), + threads_per_block=_DEFAULT_THREADS_PER_BLOCK, + reduce_op="SUM", + requires_nvls=True, + ), + _NativeAlgorithmConfig( + name="default_allreduce_packet", + collective="allreduce", + world_sizes=_SUPPORTED_WORLD_SIZES, + ipc_domain_counts=_NATIVE_IPC_DOMAIN_COUNTS, + message_size_range=_MessageSizeRange(0, 2 << 20), + nblocks=(14, 21, 28, 42, 56), + threads_per_block=_DEFAULT_THREADS_PER_BLOCK, + reduce_op="SUM", + parameter_adapter=_adapt_allreduce_packet, + ), + _NativeAlgorithmConfig( + name="default_allreduce_rsag_zero_copy", + collective="allreduce", + world_sizes=(4, 8, 16, 32), + ipc_domain_counts=_NATIVE_IPC_DOMAIN_COUNTS, + message_size_range=_MessageSizeRange(512 << 10, 4 << 30), + nblocks=(32, 48, 64, 128), + threads_per_block=_DEFAULT_THREADS_PER_BLOCK, + reduce_op="SUM", + ), + _NativeAlgorithmConfig( + name="default_allreduce_nvls_zero_copy", + collective="allreduce", + world_sizes=_SUPPORTED_WORLD_SIZES, + ipc_domain_counts=_NATIVE_IPC_DOMAIN_COUNTS, + message_size_range=_MessageSizeRange(512 << 10, 4 << 30), + nblocks=(4, 8, 12, 16, 32), + threads_per_block=_DEFAULT_THREADS_PER_BLOCK, + reduce_op="SUM", + requires_nvls=True, + ), + ) def _create_algorithm_configs(language) -> tuple[_AlgorithmConfig, ...]: @@ -372,47 +438,50 @@ def _create_algorithm_configs(language) -> tuple[_AlgorithmConfig, ...]: ) dsl_configs = ( _DslAlgorithmConfig( - name="allreduce_multi_nodes", - collective="allreduce", - world_sizes=_SUPPORTED_WORLD_SIZES, - ipc_domain_counts=_MULTI_NODE_IPC_DOMAIN_COUNTS, - message_size_range=_MessageSizeRange(1 << 10, 8 << 20), - reduce_op="SUM", - algo_spec=default_spec, - threads_per_block=_DEFAULT_THREADS_PER_BLOCK, - algorithm_kwargs=tuple( - {"thread_block_group_size": thread_block_group_size} - for thread_block_group_size in _DEFAULT_THREAD_BLOCK_GROUP_SIZES - ), + name="allreduce_multi_nodes", + collective="allreduce", + world_sizes=_SUPPORTED_WORLD_SIZES, + ipc_domain_counts=_MULTI_NODE_IPC_DOMAIN_COUNTS, + message_size_range=_MessageSizeRange(1 << 10, 1 << 20), + reduce_op="SUM", + algo_spec=default_spec, + threads_per_block=_DEFAULT_THREADS_PER_BLOCK, + algorithm_kwargs={ + "thread_block_group_size": _DEFAULT_THREAD_BLOCK_GROUP_SIZES + }, + parameter_adapter=_adapt_dsl_message_size_range, ), _DslAlgorithmConfig( name="allgather_multi_nodes", collective="allgather", world_sizes=_SUPPORTED_WORLD_SIZES, - ipc_domain_counts=_MULTI_NODE_IPC_DOMAIN_COUNTS, - message_size_range=_MessageSizeRange(1 << 10, 8 << 20), + ipc_domain_counts=( + *_NATIVE_IPC_DOMAIN_COUNTS, + *_MULTI_NODE_IPC_DOMAIN_COUNTS, + ), + message_size_range=_MessageSizeRange(1 << 10, 1 << 20), reduce_op="NOP", in_place=False, algo_spec=allgather_spec, threads_per_block=_DEFAULT_THREADS_PER_BLOCK, + parameter_adapter=_adapt_dsl_message_size_range, ), _DslAlgorithmConfig( name="reducescatter_multi_nodes", collective="reducescatter", world_sizes=_SUPPORTED_WORLD_SIZES, ipc_domain_counts=_MULTI_NODE_IPC_DOMAIN_COUNTS, - message_size_range=_MessageSizeRange(1 << 10, 8 << 20), + message_size_range=_MessageSizeRange(1 << 10, 1 << 20), reduce_op="SUM", algo_spec=reduce_scatter_spec, threads_per_block=_DEFAULT_THREADS_PER_BLOCK, - algorithm_kwargs=tuple( - {"thread_block_group_size": thread_block_group_size} - for thread_block_group_size in _DEFAULT_THREAD_BLOCK_GROUP_SIZES - ), - name_variant="directowner_v4_inplace_unfused", + algorithm_kwargs={ + "thread_block_group_size": _DEFAULT_THREAD_BLOCK_GROUP_SIZES + }, + parameter_adapter=_adapt_dsl_message_size_range, ), ) - return (*dsl_configs, *_NATIVE_ALGORITHM_CONFIGS) + return (*dsl_configs, *_create_native_algorithm_configs()) @dataclass(frozen=True) @@ -553,8 +622,9 @@ def _compile_dsl_algorithm( config.in_place, ) algorithms = [] - algorithm_kwargs_variants = config.algorithm_kwargs or ({},) - for algorithm_kwargs in algorithm_kwargs_variants: + algorithm_kwarg_names = tuple(config.algorithm_kwargs) + for values in product(*config.algorithm_kwargs.values()): + algorithm_kwargs = dict(zip(algorithm_kwarg_names, values)) for threads_per_block in config.threads_per_block: spec = replace( config.algo_spec, @@ -568,13 +638,19 @@ def _compile_dsl_algorithm( world_size=self.world_size, num_threads_per_block=threads_per_block, ) + candidate = replace(config) + candidate.adapt_to_topology( + self.world_size, + self.nranks_per_ipc_domain, + algorithm_kwargs, + ) algorithm = self.mscclpp.compile( algorithm_builder, spec, self.rank, **algorithm_kwargs, ) - algorithms.append(config.bind(algorithm, spec)) + algorithms.append(candidate.bind(algorithm, spec)) return algorithms def _create_dsl_algorithms( @@ -608,6 +684,7 @@ def _create_native_algorithms( continue message_range = config.message_size_range algo.set_message_size_range(message_range.minimum, message_range.maximum) + config.adapt_to_topology(self.world_size, self.nranks_per_ipc_domain) algorithms.append(config.bind(algo)) return algorithms From b933802fdd8fd999a39d755d3dd16a154d01375d Mon Sep 17 00:00:00 2001 From: Caio Rocha Date: Thu, 10 Sep 2026 18:53:42 +0000 Subject: [PATCH 06/11] wip --- .../kernels/all_gather/benchmark_mscclpp.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/benchmark/kernels/all_gather/benchmark_mscclpp.py b/benchmark/kernels/all_gather/benchmark_mscclpp.py index 7f19ec690227..1da2f24fa69b 100644 --- a/benchmark/kernels/all_gather/benchmark_mscclpp.py +++ b/benchmark/kernels/all_gather/benchmark_mscclpp.py @@ -40,9 +40,9 @@ def _time_eager(func, group, warmup: int = 2, repeat: int = 10) -> float: def _time_cuda_graph( func, group, - warmup: int = 2, + warmup: int = 100, graph_loop: int = 10, - repeat: int = 10, + repeat: int = 100, ) -> float: current_stream = torch.cuda.current_stream() capture_stream = torch.cuda.Stream() @@ -154,12 +154,15 @@ def main() -> None: if not (1 << 10) <= args.min_bytes <= args.max_bytes <= (8 << 20): parser.error("Require 1 KiB <= min-bytes <= max-bytes <= 8 MiB") + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + torch.cuda.set_device(local_rank) if not dist.is_initialized(): - dist.init_process_group(backend="nccl") + dist.init_process_group( + backend="nccl", + device_id=torch.device("cuda", local_rank), + ) world_size = dist.get_world_size() rank = dist.get_rank() - local_rank = int(os.environ.get("LOCAL_RANK", rank % torch.cuda.device_count())) - torch.cuda.set_device(local_rank) device = torch.cuda.current_device() set_mscclpp_all_reduce(True) init_distributed_environment( @@ -178,7 +181,7 @@ def main() -> None: if args.min_bytes % world_size != 0 or args.max_bytes % world_size != 0: parser.error("Total message sizes must be divisible by WORLD_SIZE") - if not mscclpp_comm.allgather_best_configs: + if not mscclpp_comm._best_configs["allgather"]: raise RuntimeError("No tuned MSCCL++ AllGather configuration is available") result = [] @@ -224,12 +227,9 @@ def mscclpp_on_stream(stream): stream=stream, ) - def py_nccl_on_stream(stream): - pynccl_comm.cp_all_gather_into_tensor( - nccl_output, - input_tensor, - stream=stream, - ) + def py_nccl_on_stream(_stream): + with pynccl_comm.change_state(enable=True): + pynccl_comm.all_gather(nccl_output, input_tensor) torch_nccl() mscclpp() From b3c171025509001c1998f9d589f40fb9d2ad172d Mon Sep 17 00:00:00 2001 From: Caio Rocha Date: Fri, 11 Sep 2026 16:10:39 +0000 Subject: [PATCH 07/11] wip --- docker/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index ac9684462bd9..638d1cf49c4a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -828,12 +828,13 @@ COPY --from=framework_final /root/.cache/huggingface /root/.cache/huggingface COPY --from=framework_final /root/.cache/sglang /root/.cache/sglang # Copy GDRCopy runtime libraries (but not the build artifacts) -COPY --from=framework_final /usr/lib/libgdrapi.so* /usr/lib/ +COPY --from=framework_final /usr/lib/*-linux-gnu/libgdrapi.so* /usr/lib/ COPY --from=framework_final /usr/bin/gdrcopy_* /usr/bin/ COPY --from=framework_final /usr/src/gdrdrv-2.5.1 /usr/src/gdrdrv-2.5.1 # Fix DeepEP IBGDA symlink in runtime -RUN ln -sf /usr/lib/$(uname -m)-linux-gnu/libmlx5.so.1 /usr/lib/$(uname -m)-linux-gnu/libmlx5.so +RUN ln -sf /usr/lib/$(uname -m)-linux-gnu/libmlx5.so.1 /usr/lib/$(uname -m)-linux-gnu/libmlx5.so \ + && ldconfig WORKDIR /sgl-workspace/sglang From 48de585f02e945dd93a19f2340194f9fb0ec322e Mon Sep 17 00:00:00 2001 From: Caio Rocha Date: Fri, 11 Sep 2026 21:00:33 +0000 Subject: [PATCH 08/11] wip --- .../kernels/all_reduce/benchmark_mscclpp.py | 86 +++++++++++++------ .../device_communicators/pymscclpp.py | 7 +- 2 files changed, 63 insertions(+), 30 deletions(-) diff --git a/benchmark/kernels/all_reduce/benchmark_mscclpp.py b/benchmark/kernels/all_reduce/benchmark_mscclpp.py index 5b72d2f460d1..7fb4a6fe5c39 100644 --- a/benchmark/kernels/all_reduce/benchmark_mscclpp.py +++ b/benchmark/kernels/all_reduce/benchmark_mscclpp.py @@ -12,17 +12,22 @@ --master_port $MASTER_PORT benchmark/kernels/all_reduce/benchmark_mscclpp.py """ +import argparse import os from contextlib import nullcontext -from typing import List import torch import torch.distributed as dist from torch.distributed import ProcessGroup +from sglang.srt.runtime_context import publish +from sglang.srt.server_args import ServerArgs from sglang.srt.distributed import init_distributed_environment from sglang.srt.distributed.device_communicators.pymscclpp import PyMscclppCommunicator from sglang.srt.distributed.device_communicators.pynccl import PyNcclCommunicator +from sglang.srt.distributed.device_communicators.pynccl_allocator import ( + use_symmetric_memory, +) from sglang.srt.distributed.parallel_state import ( cleanup_dist_env_and_memory, get_tensor_model_parallel_group, @@ -50,14 +55,21 @@ def pynccl_allreduce( return msccl_input -def _bench_graph_time(func, inp_randn, warmup_loop=2, graph_loop=10, test_loop=10): - graph_input = inp_randn.clone() - graph_input_snapshot = inp_randn.clone() +def _bench_graph_time( + func, + inp_randn, + tp_group, + warmup_loop=2, + graph_loop=100, + test_loop=10, +): + with use_symmetric_memory(tp_group): + graph_input = inp_randn.clone() with graph_capture() as graph_capture_context: graph = torch.cuda.CUDAGraph() with torch.cuda.graph(graph, stream=graph_capture_context.stream): for _ in range(graph_loop): - graph_input.copy_(graph_input_snapshot) + graph_input.copy_(inp_randn) graph_out = func(graph_input) graph.replay() @@ -70,7 +82,7 @@ def _bench_graph_time(func, inp_randn, warmup_loop=2, graph_loop=10, test_loop=1 start_event = torch.cuda.Event(enable_timing=True) end_event = torch.cuda.Event(enable_timing=True) - latencies: List[float] = [] + latencies = [] for _ in range(test_loop): torch.cuda.synchronize() dist.barrier() @@ -84,8 +96,9 @@ def _bench_graph_time(func, inp_randn, warmup_loop=2, graph_loop=10, test_loop=1 return func_output, func_cost_us -def _bench_eager_time(func, inp_randn, warmup_loop=2, test_loop=10): - eager_input = inp_randn.clone() +def _bench_eager_time(func, inp_randn, tp_group, warmup_loop=2, test_loop=10): + with use_symmetric_memory(tp_group): + eager_input = inp_randn.clone() eager_output = func(eager_input) func_output = eager_output.clone() @@ -155,56 +168,79 @@ def print_markdown_table(data): if __name__ == "__main__": import logging + parser = argparse.ArgumentParser() + parser.add_argument( + "--enable-symm-mem", + action="store_true", + help="Allocate collective inputs from NCCL symmetric memory.", + ) + args = parser.parse_args() + logging.basicConfig( level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S", force=True, ) + local_rank = int(os.environ["LOCAL_RANK"]) + torch.cuda.set_device(local_rank) + device = torch.cuda.current_device() if not dist.is_initialized(): - dist.init_process_group(backend="nccl") + dist.init_process_group( + backend="nccl", device_id=torch.device("cuda", local_rank) + ) world, world_size = dist.group.WORLD, dist.get_world_size() rank = dist.get_rank() - torch.cuda.set_device(rank % 8) - device = torch.cuda.current_device() set_mscclpp_all_reduce(True) init_distributed_environment( world_size=world_size, rank=rank, - local_rank=rank % 8, + local_rank=local_rank, + ) + publish( + ServerArgs(model_path="dummy", enable_symm_mem=args.enable_symm_mem), + role="scheduler", + ) + initialize_model_parallel( + tensor_model_parallel_size=world_size, + enable_symm_mem=args.enable_symm_mem, ) - initialize_model_parallel(tensor_model_parallel_size=world_size) - group = get_tensor_model_parallel_group().device_group - cpu_group = get_tensor_model_parallel_group().cpu_group - pynccl_comm = get_tensor_model_parallel_group().pynccl_comm - pymscclpp_comm = get_tensor_model_parallel_group().pymscclpp_comm + tp_group = get_tensor_model_parallel_group() + group = tp_group.device_group + pynccl_comm = tp_group.pynccl_comm + pymscclpp_comm = tp_group.pymscclpp_comm dist.barrier() profile = False dtype = torch.bfloat16 ctx = get_torch_prof_ctx(profile) result = [] + max_message_bytes = 1 << 20 with ctx: for i in range(10, 20): sz = 2**i - if sz * dtype.itemsize > 2**20: + if sz * dtype.itemsize > max_message_bytes: break - inp_randn = torch.randint(1, 16, (sz,), dtype=dtype, device=device) + inp_randn = torch.randint(0, 2, (sz,), dtype=dtype, device=device) - memory = torch.empty_like(inp_randn) - memory_out = torch.empty_like(memory) torch_eager_output, torch_eager_time = _bench_eager_time( - lambda inp: torch_allreduce(inp, group), inp_randn + lambda inp: torch_allreduce(inp, group), inp_randn, tp_group ) msccl_eager_output, msccl_eager_time = _bench_eager_time( - lambda inp: msccl_allreduce(inp, pymscclpp_comm), inp_randn + lambda inp: msccl_allreduce(inp, pymscclpp_comm), + inp_randn, + tp_group, ) msccl_graph_output, msccl_graph_time = _bench_graph_time( - lambda inp: msccl_allreduce(inp, pymscclpp_comm), inp_randn + lambda inp: msccl_allreduce(inp, pymscclpp_comm), + inp_randn, + tp_group, ) # since pynccl is inplace op, this return result is not correct if graph loop > 1 _, pynccl_graph_time = _bench_graph_time( - lambda inp: pynccl_allreduce(inp, pynccl_comm), inp_randn + lambda inp: pynccl_allreduce(inp, pynccl_comm), + inp_randn, + tp_group, ) torch.testing.assert_close(torch_eager_output, msccl_graph_output) torch.testing.assert_close(torch_eager_output, msccl_eager_output) diff --git a/python/sglang/srt/distributed/device_communicators/pymscclpp.py b/python/sglang/srt/distributed/device_communicators/pymscclpp.py index 5c7f736ecca2..b81224bd9975 100644 --- a/python/sglang/srt/distributed/device_communicators/pymscclpp.py +++ b/python/sglang/srt/distributed/device_communicators/pymscclpp.py @@ -399,7 +399,7 @@ def _create_native_algorithm_configs() -> tuple[_NativeAlgorithmConfig, ...]: collective="allreduce", world_sizes=_SUPPORTED_WORLD_SIZES, ipc_domain_counts=_NATIVE_IPC_DOMAIN_COUNTS, - message_size_range=_MessageSizeRange(512 << 10, 4 << 30), + message_size_range=_MessageSizeRange(1 << 10, 4 << 30), nblocks=(4, 8, 12, 16, 32), threads_per_block=_DEFAULT_THREADS_PER_BLOCK, reduce_op="SUM", @@ -455,10 +455,7 @@ def _create_algorithm_configs(language) -> tuple[_AlgorithmConfig, ...]: name="allgather_multi_nodes", collective="allgather", world_sizes=_SUPPORTED_WORLD_SIZES, - ipc_domain_counts=( - *_NATIVE_IPC_DOMAIN_COUNTS, - *_MULTI_NODE_IPC_DOMAIN_COUNTS, - ), + ipc_domain_counts=_MULTI_NODE_IPC_DOMAIN_COUNTS, message_size_range=_MessageSizeRange(1 << 10, 1 << 20), reduce_op="NOP", in_place=False, From b0d2e1bc02440b624bebb0dcde38d3e9a7bed6ec Mon Sep 17 00:00:00 2001 From: Caio Rocha Date: Mon, 14 Sep 2026 22:59:41 +0000 Subject: [PATCH 09/11] wip --- .../kernels/all_reduce/benchmark_mscclpp.py | 6 ++-- .../device_communicators/pymscclpp.py | 35 ++++++++++--------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/benchmark/kernels/all_reduce/benchmark_mscclpp.py b/benchmark/kernels/all_reduce/benchmark_mscclpp.py index 7fb4a6fe5c39..0081bade04df 100644 --- a/benchmark/kernels/all_reduce/benchmark_mscclpp.py +++ b/benchmark/kernels/all_reduce/benchmark_mscclpp.py @@ -20,8 +20,6 @@ import torch.distributed as dist from torch.distributed import ProcessGroup -from sglang.srt.runtime_context import publish -from sglang.srt.server_args import ServerArgs from sglang.srt.distributed import init_distributed_environment from sglang.srt.distributed.device_communicators.pymscclpp import PyMscclppCommunicator from sglang.srt.distributed.device_communicators.pynccl import PyNcclCommunicator @@ -35,6 +33,8 @@ initialize_model_parallel, set_mscclpp_all_reduce, ) +from sglang.srt.runtime_context import publish +from sglang.srt.server_args import ServerArgs def torch_allreduce(torch_input: torch.Tensor, group: ProcessGroup) -> torch.Tensor: @@ -60,7 +60,7 @@ def _bench_graph_time( inp_randn, tp_group, warmup_loop=2, - graph_loop=100, + graph_loop=10, test_loop=10, ): with use_symmetric_memory(tp_group): diff --git a/python/sglang/srt/distributed/device_communicators/pymscclpp.py b/python/sglang/srt/distributed/device_communicators/pymscclpp.py index b81224bd9975..5c469ecbdc01 100644 --- a/python/sglang/srt/distributed/device_communicators/pymscclpp.py +++ b/python/sglang/srt/distributed/device_communicators/pymscclpp.py @@ -3,8 +3,8 @@ from abc import ABC, abstractmethod from contextlib import contextmanager from dataclasses import dataclass, field, replace -from typing import Any, Callable, ClassVar, Optional, Union from itertools import product +from typing import Any, Callable, ClassVar, Optional, Union import torch import torch.distributed as dist @@ -98,9 +98,7 @@ def supports_dtype(self, dtype: torch.dtype) -> bool: def resolve_reduce_op(self, reduce_ops): return getattr(reduce_ops, self.reduce_op) - def adapt_to_topology( - self, world_size: int, nranks_per_ipc_domain: int - ): + def adapt_to_topology(self, world_size: int, nranks_per_ipc_domain: int): if self.parameter_adapter is None: return self.parameter_adapter( @@ -262,10 +260,13 @@ def dsl_name( threads_per_block: int, algorithm_kwargs: dict[str, Any], ) -> str: - variant_parts = [f"{key}_{value}" for key, value in sorted(algorithm_kwargs.items())] + variant_parts = [ + f"{key}_{value}" for key, value in sorted(algorithm_kwargs.items()) + ] variant = f"{'_'.join(variant_parts)}_" if variant_parts else "" return f"{self.name}_{ipc_domain_count}node_{variant}" f"{threads_per_block}TPB" + @dataclass(kw_only=True) class _NativeAlgorithmConfig(_AlgorithmConfig): implementation: ClassVar[str] = "native" @@ -438,18 +439,18 @@ def _create_algorithm_configs(language) -> tuple[_AlgorithmConfig, ...]: ) dsl_configs = ( _DslAlgorithmConfig( - name="allreduce_multi_nodes", - collective="allreduce", - world_sizes=_SUPPORTED_WORLD_SIZES, - ipc_domain_counts=_MULTI_NODE_IPC_DOMAIN_COUNTS, - message_size_range=_MessageSizeRange(1 << 10, 1 << 20), - reduce_op="SUM", - algo_spec=default_spec, - threads_per_block=_DEFAULT_THREADS_PER_BLOCK, - algorithm_kwargs={ - "thread_block_group_size": _DEFAULT_THREAD_BLOCK_GROUP_SIZES - }, - parameter_adapter=_adapt_dsl_message_size_range, + name="allreduce_multi_nodes", + collective="allreduce", + world_sizes=_SUPPORTED_WORLD_SIZES, + ipc_domain_counts=_MULTI_NODE_IPC_DOMAIN_COUNTS, + message_size_range=_MessageSizeRange(1 << 10, 1 << 20), + reduce_op="SUM", + algo_spec=default_spec, + threads_per_block=_DEFAULT_THREADS_PER_BLOCK, + algorithm_kwargs={ + "thread_block_group_size": _DEFAULT_THREAD_BLOCK_GROUP_SIZES + }, + parameter_adapter=_adapt_dsl_message_size_range, ), _DslAlgorithmConfig( name="allgather_multi_nodes", From 42d576412fb53cb92198a2f8f24338d19cf07b20 Mon Sep 17 00:00:00 2001 From: Caio Rocha Date: Fri, 18 Sep 2026 17:02:26 +0000 Subject: [PATCH 10/11] addressing comments --- .../device_communicators/pymscclpp.py | 120 +++++++++--------- 1 file changed, 59 insertions(+), 61 deletions(-) diff --git a/python/sglang/srt/distributed/device_communicators/pymscclpp.py b/python/sglang/srt/distributed/device_communicators/pymscclpp.py index 0fe1a0a43cb7..f0d69754419b 100644 --- a/python/sglang/srt/distributed/device_communicators/pymscclpp.py +++ b/python/sglang/srt/distributed/device_communicators/pymscclpp.py @@ -1,11 +1,11 @@ import importlib import logging -from abc import ABC, abstractmethod from contextlib import contextmanager -from dataclasses import dataclass, field, replace +from dataclasses import replace as dataclass_replace from itertools import product from typing import Any, Callable, ClassVar, Optional, Union +import msgspec import torch import torch.distributed as dist from torch.distributed import ProcessGroup, ReduceOp @@ -25,11 +25,10 @@ _DEFAULT_SUPPORTED_DTYPES = (torch.float, torch.float16, torch.bfloat16) _DEFAULT_GPU_BUFFER_SIZE = 1 << 27 _TUNING_MESSAGE_SIZES = tuple(1 << exponent for exponent in range(9, 24)) -_ParameterAdapter = Callable[..., None] +_ParameterAdapter = Callable[..., "_AlgorithmConfig"] -@dataclass(frozen=True) -class _MessageSizeRange: +class _MessageSizeRange(msgspec.Struct, frozen=True): minimum: int maximum: int @@ -41,8 +40,7 @@ def contains(self, size: int) -> bool: return self.minimum <= size <= self.maximum -@dataclass(kw_only=True) -class _AlgorithmConfig(ABC): +class _AlgorithmConfig(msgspec.Struct, frozen=True, kw_only=True): implementation: ClassVar[str] name: str collective: str @@ -98,10 +96,12 @@ def supports_dtype(self, dtype: torch.dtype) -> bool: def resolve_reduce_op(self, reduce_ops): return getattr(reduce_ops, self.reduce_op) - def adapt_to_topology(self, world_size: int, nranks_per_ipc_domain: int): + def adapt_to_topology( + self, world_size: int, nranks_per_ipc_domain: int + ) -> "_AlgorithmConfig": if self.parameter_adapter is None: - return - self.parameter_adapter( + return self + return self.parameter_adapter( self, world_size=world_size, nranks_per_ipc_domain=nranks_per_ipc_domain, @@ -127,12 +127,11 @@ def output_buffer_size(self, message_size: int, world_size: int) -> int: return message_size // world_size return message_size - @abstractmethod def tuning_launches(self) -> tuple[tuple[int, int], ...]: raise NotImplementedError def bind(self, algorithm) -> "_AlgorithmConfig": - return replace(self, algorithm=algorithm) + return msgspec.structs.replace(self, algorithm=algorithm) def select(self, nblocks: int, threads_per_block: int) -> "_AlgorithmConfig": if self.algorithm is None: @@ -206,11 +205,10 @@ def compose_rsag( ) -@dataclass(kw_only=True) class _DslAlgorithmConfig(_AlgorithmConfig): implementation: ClassVar[str] = "dsl" algo_spec: Any - algorithm_kwargs: dict[str, tuple[Any, ...]] = field(default_factory=dict) + algorithm_kwargs: dict[str, tuple[Any, ...]] = msgspec.field(default_factory=dict) def __post_init__(self): super().__post_init__() @@ -240,10 +238,10 @@ def adapt_to_topology( world_size: int, nranks_per_ipc_domain: int, algorithm_kwargs: Optional[dict[str, Any]] = None, - ): + ) -> "_AlgorithmConfig": if self.parameter_adapter is None: - return - self.parameter_adapter( + return self + return self.parameter_adapter( self, world_size=world_size, nranks_per_ipc_domain=nranks_per_ipc_domain, @@ -252,7 +250,7 @@ def adapt_to_topology( ) def bind(self, algorithm, algo_spec) -> "_AlgorithmConfig": - return replace(self, algorithm=algorithm, algo_spec=algo_spec) + return msgspec.structs.replace(self, algorithm=algorithm, algo_spec=algo_spec) def dsl_name( self, @@ -267,7 +265,6 @@ def dsl_name( return f"{self.name}_{ipc_domain_count}node_{variant}{threads_per_block}TPB" -@dataclass(kw_only=True) class _NativeAlgorithmConfig(_AlgorithmConfig): implementation: ClassVar[str] = "native" nblocks: tuple[int, ...] @@ -296,7 +293,7 @@ def select(self, nblocks: int, threads_per_block: int) -> "_AlgorithmConfig": launch = (nblocks, threads_per_block) if launch not in self.tuning_launches(): raise ValueError(f"Invalid launch selection for algorithm {self.name}") - return replace(self, selected_launch_parameters=launch) + return msgspec.structs.replace(self, selected_launch_parameters=launch) def selected_launch(self) -> tuple[int, int]: if self.selected_launch_parameters is None: @@ -304,7 +301,6 @@ def selected_launch(self) -> tuple[int, int]: return self.selected_launch_parameters -@dataclass(kw_only=True) class _CompositeAlgorithmConfig(_AlgorithmConfig): implementation: ClassVar[str] = "composite" @@ -320,6 +316,7 @@ def tuning_launches(self) -> tuple[tuple[int, int], ...]: _DEFAULT_THREADS_PER_BLOCK = (256, 512, 768, 1024) _DEFAULT_THREAD_BLOCK_GROUP_SIZES = (1, 2, 4, 8) _SUPPORTED_WORLD_SIZES = (4, 8, 16, 32, 64) +_NATIVE_SUPPORTED_WORLD_SIZES = (4, 8, 16, 32) _NATIVE_IPC_DOMAIN_COUNTS = (1,) _MULTI_NODE_IPC_DOMAIN_COUNTS = (2, 4, 8) @@ -329,17 +326,16 @@ def _adapt_allreduce_packet( *, world_size: int, nranks_per_ipc_domain: int, -) -> bool: +) -> _AlgorithmConfig: if not isinstance(config, _NativeAlgorithmConfig): raise TypeError("AllReduce packet adaptation requires a native config") if world_size != nranks_per_ipc_domain: - return False + return config min_blocks = nranks_per_ipc_domain - 1 nblocks = tuple(nblocks for nblocks in config.nblocks if nblocks >= min_blocks) if not nblocks: - return False - config.nblocks = nblocks - return True + return config + return msgspec.structs.replace(config, nblocks=nblocks) def _adapt_dsl_message_size_range( @@ -349,16 +345,18 @@ def _adapt_dsl_message_size_range( nranks_per_ipc_domain: int, algorithm_kwargs: dict[str, Any], **_: Any, -) -> bool: +) -> _AlgorithmConfig: if not isinstance(config, _DslAlgorithmConfig): raise TypeError("DSL message size adaptation requires a DSL config") ipc_domain_count = world_size // nranks_per_ipc_domain thread_block_group_size = algorithm_kwargs.get("thread_block_group_size", 1) - config.message_size_range = _MessageSizeRange( - minimum=config.message_size_range.minimum * thread_block_group_size, - maximum=config.message_size_range.maximum * ipc_domain_count, + return msgspec.structs.replace( + config, + message_size_range=_MessageSizeRange( + minimum=config.message_size_range.minimum * thread_block_group_size, + maximum=config.message_size_range.maximum * ipc_domain_count, + ), ) - return True def _create_native_algorithm_configs() -> tuple[_NativeAlgorithmConfig, ...]: @@ -366,7 +364,7 @@ def _create_native_algorithm_configs() -> tuple[_NativeAlgorithmConfig, ...]: _NativeAlgorithmConfig( name="default_allreduce_nvls_packet", collective="allreduce", - world_sizes=_SUPPORTED_WORLD_SIZES, + world_sizes=_NATIVE_SUPPORTED_WORLD_SIZES, ipc_domain_counts=_NATIVE_IPC_DOMAIN_COUNTS, message_size_range=_MessageSizeRange(0, 512 << 10), nblocks=(4, 8, 12, 16), @@ -377,7 +375,7 @@ def _create_native_algorithm_configs() -> tuple[_NativeAlgorithmConfig, ...]: _NativeAlgorithmConfig( name="default_allreduce_packet", collective="allreduce", - world_sizes=_SUPPORTED_WORLD_SIZES, + world_sizes=_NATIVE_SUPPORTED_WORLD_SIZES, ipc_domain_counts=_NATIVE_IPC_DOMAIN_COUNTS, message_size_range=_MessageSizeRange(0, 2 << 20), nblocks=(14, 21, 28, 42, 56), @@ -388,7 +386,7 @@ def _create_native_algorithm_configs() -> tuple[_NativeAlgorithmConfig, ...]: _NativeAlgorithmConfig( name="default_allreduce_rsag_zero_copy", collective="allreduce", - world_sizes=(4, 8, 16, 32), + world_sizes=_NATIVE_SUPPORTED_WORLD_SIZES, ipc_domain_counts=_NATIVE_IPC_DOMAIN_COUNTS, message_size_range=_MessageSizeRange(512 << 10, 4 << 30), nblocks=(32, 48, 64, 128), @@ -398,7 +396,7 @@ def _create_native_algorithm_configs() -> tuple[_NativeAlgorithmConfig, ...]: _NativeAlgorithmConfig( name="default_allreduce_nvls_zero_copy", collective="allreduce", - world_sizes=_SUPPORTED_WORLD_SIZES, + world_sizes=_NATIVE_SUPPORTED_WORLD_SIZES, ipc_domain_counts=_NATIVE_IPC_DOMAIN_COUNTS, message_size_range=_MessageSizeRange(1 << 10, 4 << 30), nblocks=(4, 8, 12, 16, 32), @@ -425,13 +423,13 @@ def _create_algorithm_configs(language) -> tuple[_AlgorithmConfig, ...]: use_double_scratch_buffer=True, buffer_alignment=16, ) - allgather_spec = replace( + allgather_spec = dataclass_replace( default_spec, name="allgather_multi_nodes", collective=language.collectives.AllGather(0, 1, False), in_place=False, ) - reduce_scatter_spec = replace( + reduce_scatter_spec = dataclass_replace( default_spec, name="reducescatter_multi_nodes", collective=language.collectives.ReduceScatter(0, 1, True), @@ -482,8 +480,7 @@ def _create_algorithm_configs(language) -> tuple[_AlgorithmConfig, ...]: return (*dsl_configs, *_create_native_algorithm_configs()) -@dataclass(frozen=True) -class _TwoKernelAllReduce: +class _TwoKernelAllReduce(msgspec.Struct, frozen=True): name: str reduce_scatter: Any allgather: Any @@ -507,13 +504,6 @@ def execute( nthreads_per_block, symmetric_memory, ): - if input_buffer != output_buffer: - raise ValueError("RSAG AllReduce requires in-place buffers") - if input_size % (16 * self.world_size) != 0: - raise ValueError( - f"RSAG AllReduce input size {input_size} must be divisible " - f"by {16 * self.world_size}" - ) shard_size = input_size // self.world_size result = self.reduce_scatter.execute( @@ -624,7 +614,7 @@ def _compile_dsl_algorithm( for values in product(*config.algorithm_kwargs.values()): algorithm_kwargs = dict(zip(algorithm_kwarg_names, values)) for threads_per_block in config.threads_per_block: - spec = replace( + spec = dataclass_replace( config.algo_spec, name=config.dsl_name( ipc_domain_count, @@ -636,8 +626,7 @@ def _compile_dsl_algorithm( world_size=self.world_size, num_threads_per_block=threads_per_block, ) - candidate = replace(config) - candidate.adapt_to_topology( + candidate = config.adapt_to_topology( self.world_size, self.nranks_per_ipc_domain, algorithm_kwargs, @@ -682,7 +671,9 @@ def _create_native_algorithms( continue message_range = config.message_size_range algo.set_message_size_range(message_range.minimum, message_range.maximum) - config.adapt_to_topology(self.world_size, self.nranks_per_ipc_domain) + config = config.adapt_to_topology( + self.world_size, self.nranks_per_ipc_domain + ) algorithms.append(config.bind(algo)) return algorithms @@ -811,11 +802,23 @@ def run(stream=None): ) result = run() - if result != 0: - raise RuntimeError( - f"MSCCL++ {algorithm_config.collective} tuning failed " - f"with error code {result}" - ) + failed = torch.tensor([int(result != 0)], dtype=torch.int32) + dist.all_reduce(failed, op=ReduceOp.MAX, group=self.group) + if failed.item(): + if result != 0: + logger.warning( + "MSCCL++ %s tuning candidate failed on rank %d with error " + "code %s; skipping algorithm=%s message_bytes=%d " + "nblocks=%d nthreads=%d", + algorithm_config.collective, + self.rank, + result, + algorithm_config.name, + message_size, + nblocks, + nthreads, + ) + return float("inf") for _ in range(n_warmup): run() @@ -1094,12 +1097,7 @@ def __init__( self.comm = self.mscclpp.CommGroup( torch_group=self.group, rank=self.rank, size=self.world_size ) - nranks_per_ipc_domain = getattr(self.comm, "nranks_per_ipc_domain", None) - self.nranks_per_ipc_domain = ( - self.comm.nranks_per_node - if nranks_per_ipc_domain is None - else nranks_per_ipc_domain - ) + self.nranks_per_ipc_domain = self.comm.nranks_per_ipc_domain self.executor = self.mscclpp.Executor(self.comm.communicator) self.symm_mem_enabled = self._is_symm_mem_enabled() try: From b7c4c2887926d593ca031283e63273254bb37a59 Mon Sep 17 00:00:00 2001 From: Caio Rocha Date: Thu, 24 Sep 2026 22:10:37 +0000 Subject: [PATCH 11/11] addressing comments --- .../kernels/all_gather/benchmark_mscclpp.py | 8 +++--- .../kernels/all_reduce/benchmark_mscclpp.py | 4 +-- python/sglang/srt/distributed/bootstrap.py | 4 +-- .../device_communicators/pymscclpp.py | 17 ++++++++---- .../sglang/srt/distributed/parallel_state.py | 26 +++++++++---------- .../unit/distributed/test_parallel_state.py | 2 +- 6 files changed, 35 insertions(+), 26 deletions(-) diff --git a/benchmark/kernels/all_gather/benchmark_mscclpp.py b/benchmark/kernels/all_gather/benchmark_mscclpp.py index 1da2f24fa69b..1e902ca0d9c8 100644 --- a/benchmark/kernels/all_gather/benchmark_mscclpp.py +++ b/benchmark/kernels/all_gather/benchmark_mscclpp.py @@ -11,8 +11,9 @@ cleanup_dist_env_and_memory, get_tensor_model_parallel_group, initialize_model_parallel, - set_mscclpp_all_reduce, + set_mscclpp, ) +from sglang.test.test_utils import publish_build_topology def _max_across_ranks(value: float, group) -> float: @@ -164,13 +165,14 @@ def main() -> None: world_size = dist.get_world_size() rank = dist.get_rank() device = torch.cuda.current_device() - set_mscclpp_all_reduce(True) + set_mscclpp(True) init_distributed_environment( world_size=world_size, rank=rank, local_rank=local_rank, ) - initialize_model_parallel(tensor_model_parallel_size=world_size) + publish_build_topology(world_rank=rank, tp_size=world_size) + initialize_model_parallel() tp_group = get_tensor_model_parallel_group() gpu_group = tp_group.device_group cpu_group = tp_group.cpu_group diff --git a/benchmark/kernels/all_reduce/benchmark_mscclpp.py b/benchmark/kernels/all_reduce/benchmark_mscclpp.py index 55b9b3eea6d8..5c131619a221 100644 --- a/benchmark/kernels/all_reduce/benchmark_mscclpp.py +++ b/benchmark/kernels/all_reduce/benchmark_mscclpp.py @@ -30,7 +30,7 @@ cleanup_dist_env_and_memory, graph_capture, initialize_model_parallel, - set_mscclpp_all_reduce, + set_mscclpp, ) from sglang.srt.runtime_context import get_parallel from sglang.test.test_utils import publish_build_topology @@ -190,7 +190,7 @@ def print_markdown_table(data): ) world, world_size = dist.group.WORLD, dist.get_world_size() rank = dist.get_rank() - set_mscclpp_all_reduce(True) + set_mscclpp(True) init_distributed_environment( world_size=world_size, rank=rank, diff --git a/python/sglang/srt/distributed/bootstrap.py b/python/sglang/srt/distributed/bootstrap.py index 6f82a92f3eb2..6af6992c6d93 100644 --- a/python/sglang/srt/distributed/bootstrap.py +++ b/python/sglang/srt/distributed/bootstrap.py @@ -20,7 +20,7 @@ initialize_model_parallel, set_custom_all_reduce, set_flashinfer_allreduce_only, - set_mscclpp_all_reduce, + set_mscclpp, set_torch_symm_mem_all_reduce, ) from sglang.srt.environ import envs @@ -240,7 +240,7 @@ def _resolve_dist_init_method(*, dist_port: int) -> str: def _set_all_reduce_flags() -> None: set_custom_all_reduce(not get_exec().comm.disable_custom_all_reduce) - set_mscclpp_all_reduce(get_exec().comm.enable_mscclpp) + set_mscclpp(get_exec().comm.enable_mscclpp) set_torch_symm_mem_all_reduce(get_exec().comm.enable_torch_symm_mem) set_flashinfer_allreduce_only( get_exec().comm.flashinfer_allreduce_fusion_backend is not None diff --git a/python/sglang/srt/distributed/device_communicators/pymscclpp.py b/python/sglang/srt/distributed/device_communicators/pymscclpp.py index f0d69754419b..2442c9fc5b15 100644 --- a/python/sglang/srt/distributed/device_communicators/pymscclpp.py +++ b/python/sglang/srt/distributed/device_communicators/pymscclpp.py @@ -53,6 +53,7 @@ class _AlgorithmConfig(msgspec.Struct, frozen=True, kw_only=True): supported_dtypes: tuple[torch.dtype, ...] = _DEFAULT_SUPPORTED_DTYPES in_place: bool = True requires_nvls: bool = False + requires_symmetric_memory: bool = False algorithm: Any = None def __post_init__(self): @@ -80,12 +81,17 @@ def requirements_satisfied( nvls_supported: bool, symmetric_memory: bool, ) -> bool: - return self.supports_topology( - world_size, ipc_domain_count - ) and self.support_nvls(nvls_supported, symmetric_memory) + return ( + self.supports_topology(world_size, ipc_domain_count) + and self.support_nvls(nvls_supported) + and self.support_symmetric_memory(symmetric_memory) + ) + + def support_nvls(self, nvls_supported: bool) -> bool: + return not self.requires_nvls or nvls_supported - def support_nvls(self, nvls_supported: bool, symmetric_memory: bool) -> bool: - return not self.requires_nvls or (nvls_supported and symmetric_memory) + def support_symmetric_memory(self, symmetric_memory: bool) -> bool: + return not self.requires_symmetric_memory or symmetric_memory def supports_message_size(self, message_size: int) -> bool: return self.message_size_range.contains(message_size) @@ -403,6 +409,7 @@ def _create_native_algorithm_configs() -> tuple[_NativeAlgorithmConfig, ...]: threads_per_block=_DEFAULT_THREADS_PER_BLOCK, reduce_op="SUM", requires_nvls=True, + requires_symmetric_memory=True, ), ) diff --git a/python/sglang/srt/distributed/parallel_state.py b/python/sglang/srt/distributed/parallel_state.py index 3c65fd02635a..fcf4379c8750 100644 --- a/python/sglang/srt/distributed/parallel_state.py +++ b/python/sglang/srt/distributed/parallel_state.py @@ -267,7 +267,7 @@ class GroupCoordinator: cpu_group: ProcessGroup # group for CPU communication device_group: ProcessGroup # group for device communication use_pynccl: bool # a hint of whether to use PyNccl - use_pymscclpp: bool # a hint of whether to use PyMsccl + use_mscclpp: bool # a hint of whether to use MSCCL++ use_custom_allreduce: bool # a hint of whether to use CustomAllreduce use_torch_symm_mem_all_reduce: ( bool # a hint of whether to use TorchSymmMemAllReduce @@ -287,7 +287,7 @@ def __init__( local_rank: int, torch_distributed_backend: Union[str, Backend], use_pynccl: bool, - use_pymscclpp: bool, + use_mscclpp: bool, use_custom_allreduce: bool, use_torch_symm_mem_all_reduce: bool, use_hpu_communicator: bool, @@ -424,7 +424,7 @@ def __init__( # Import communicators self.use_pynccl = use_pynccl - self.use_pymscclpp = use_pymscclpp + self.use_mscclpp = use_mscclpp self.use_custom_allreduce = use_custom_allreduce self.use_torch_symm_mem_all_reduce = use_torch_symm_mem_all_reduce self.use_hpu_communicator = use_hpu_communicator @@ -471,7 +471,7 @@ def __init__( ) self.pymscclpp_comm: Optional[PyMscclppCommunicator] = None - if use_pymscclpp and self.world_size > 1: + if use_mscclpp and self.world_size > 1: self.pymscclpp_comm = PyMscclppCommunicator( group=self.cpu_group, device=self.device, @@ -2095,7 +2095,7 @@ def init_world_group( local_rank=local_rank, torch_distributed_backend=backend, use_pynccl=False, - use_pymscclpp=False, + use_mscclpp=False, use_custom_allreduce=False, use_torch_symm_mem_all_reduce=False, use_hpu_communicator=False, @@ -2115,7 +2115,7 @@ def init_model_parallel_group( use_custom_allreduce: Optional[bool] = None, use_message_queue_broadcaster: bool = False, group_name: Optional[str] = None, - use_mscclpp_allreduce: Optional[bool] = None, + use_mscclpp: Optional[bool] = None, use_torch_symm_mem_allreduce: Optional[bool] = None, recovered_rank: bool = False, rank_offset: int = 0, @@ -2123,8 +2123,8 @@ def init_model_parallel_group( ) -> GroupCoordinator: if use_custom_allreduce is None: use_custom_allreduce = _ENABLE_CUSTOM_ALL_REDUCE - if use_mscclpp_allreduce is None: - use_mscclpp_allreduce = _ENABLE_MSCCLPP_ALL_REDUCE + if use_mscclpp is None: + use_mscclpp = _ENABLE_MSCCLPP if use_torch_symm_mem_allreduce is None: use_torch_symm_mem_allreduce = _ENABLE_TORCH_SYMM_MEM_ALL_REDUCE return GroupCoordinator( @@ -2136,7 +2136,7 @@ def init_model_parallel_group( if use_pynccl is None else use_pynccl ), - use_pymscclpp=use_mscclpp_allreduce, + use_mscclpp=use_mscclpp, use_custom_allreduce=use_custom_allreduce, use_torch_symm_mem_all_reduce=use_torch_symm_mem_allreduce, use_hpu_communicator=True, @@ -2278,7 +2278,7 @@ def graph_capture(stream=None): logger = logging.getLogger(__name__) _ENABLE_CUSTOM_ALL_REDUCE = True -_ENABLE_MSCCLPP_ALL_REDUCE = False +_ENABLE_MSCCLPP = False _ENABLE_TORCH_SYMM_MEM_ALL_REDUCE = False _ENABLE_FLASHINFER_ALLREDUCE_ONLY = False @@ -2288,9 +2288,9 @@ def set_custom_all_reduce(enable: bool): _ENABLE_CUSTOM_ALL_REDUCE = enable -def set_mscclpp_all_reduce(enable: bool): - global _ENABLE_MSCCLPP_ALL_REDUCE - _ENABLE_MSCCLPP_ALL_REDUCE = enable +def set_mscclpp(enable: bool): + global _ENABLE_MSCCLPP + _ENABLE_MSCCLPP = enable def set_torch_symm_mem_all_reduce(enable: bool): diff --git a/test/registered/unit/distributed/test_parallel_state.py b/test/registered/unit/distributed/test_parallel_state.py index edb86d654710..652f909f8f21 100644 --- a/test/registered/unit/distributed/test_parallel_state.py +++ b/test/registered/unit/distributed/test_parallel_state.py @@ -410,7 +410,7 @@ def _read_group_descs(group_name): local_rank=0, torch_distributed_backend="gloo", use_pynccl=False, - use_pymscclpp=False, + use_mscclpp=False, use_custom_allreduce=False, use_torch_symm_mem_all_reduce=False, use_hpu_communicator=False,