[Collective][PR 4/6] NCCL Communicator caching and preliminary stream management (ray-project#13030)

Co-authored-by: Dacheng Li <[email protected]>

Author: 2 people

python/ray/util/collective/__init__.py

@@ -1,11 +1,15 @@
-from ray.util.collective.collective import nccl_available, mpi_available, \
+from ray.util.collective.collective import nccl_available, gloo_available, \
     is_group_initialized, init_collective_group, destroy_collective_group, \
-    get_rank, get_world_size, allreduce, barrier, reduce, broadcast, \
-    allgather, reducescatter, send, recv
+    declare_collective_group, get_rank, get_world_size, allreduce, \
+    allreduce_multigpu, barrier, reduce, reduce_multigpu, broadcast, \
+    broadcast_multigpu, allgather, allgather_multigpu, reducescatter, \
+    reducescatter_multigpu, send, send_multigpu, recv, recv_multigpu
 
 __all__ = [
-    "nccl_available", "mpi_available", "is_group_initialized",
-    "init_collective_group", "destroy_collective_group", "get_rank",
-    "get_world_size", "allreduce", "barrier", "reduce", "broadcast",
-    "allgather", "reducescatter", "send", "recv"
+    "nccl_available", "gloo_available", "is_group_initialized",
+    "init_collective_group", "destroy_collective_group",
+    "declare_collective_group", "get_rank", "get_world_size", "allreduce",
+    "allreduce_multigpu", "barrier", "reduce", "reduce_multigpu", "broadcast",
+    "broadcast_multigpu", "allgather", "allgather_multigpu", "reducescatter",
+    "reducescatter_multigpu", "send", "send_multigpu", "recv", "recv_multigpu"
 ]