diff --git a/.gitlab/test_python.sh b/.gitlab/test_python.sh index 1d9e844ce4..4054794345 100755 --- a/.gitlab/test_python.sh +++ b/.gitlab/test_python.sh @@ -46,6 +46,8 @@ pip3 install --break-system-packages zmq echo "==== Running python tests ====" python3 examples/python/nixl_api_example.py pytest test/python +python3 test/python/prep_xfer_perf.py list +python3 test/python/prep_xfer_perf.py array echo "==== Running python example ====" cd examples/python diff --git a/examples/python/nixl_api_example.py b/examples/python/nixl_api_example.py index d68fa43c59..316e84378d 100755 --- a/examples/python/nixl_api_example.py +++ b/examples/python/nixl_api_example.py @@ -17,6 +17,7 @@ import os +import numpy as np import torch import nixl._utils as nixl_utils @@ -54,6 +55,19 @@ agent1_reg_descs = nixl_agent1.get_reg_descs(agent1_strings, "DRAM", is_sorted=True) agent1_xfer_descs = nixl_agent1.get_xfer_descs(agent1_addrs, "DRAM", is_sorted=True) + # Prefer numpy arrays for performance + agent1_addrs_np = np.array(agent1_addrs) + agent1_xfer_descs_np = nixl_agent1.get_xfer_descs( + agent1_addrs_np, "DRAM", is_sorted=True + ) + agent1_reg_descs_np = nixl_agent1.get_reg_descs( + agent1_addrs_np, "DRAM", is_sorted=True + ) + + assert agent1_xfer_descs == agent1_xfer_descs_np + assert agent1_reg_descs == agent1_reg_descs_np + print(agent1_reg_descs, agent1_reg_descs_np) + # Just for tensor test tensors = [torch.zeros(10, dtype=torch.float32) for _ in range(2)] agent1_tensor_reg_descs = nixl_agent1.get_reg_descs(tensors) diff --git a/src/api/python/_api.py b/src/api/python/_api.py index 54c73e93f1..9a68d32bb3 100644 --- a/src/api/python/_api.py +++ b/src/api/python/_api.py @@ -14,8 +14,9 @@ # limitations under the License. import pickle -from typing import Optional +from typing import Optional, Union +import numpy as np import torch import nixl._bindings as nixlBind @@ -313,7 +314,8 @@ def make_connection(self, remote_agent: str, backends: list[str] = []): this is for a loopback (local) transfer to be used for remote_side handle @param agent_name Name of the agent. It can be "NIXL_INIT_AGENT", local agent name, or remote agent name - @param xfer_list List of transfer descriptors, can be list of memory region tuples, tensors, or nixlXferDList. + @param xfer_list List of transfer descriptors, can be list of memory region tuples, tensors, + Nx3 numpy array, or nixlXferDList. See get_xfer_descs for more details on the structure. @param mem_type Optional memory type necessary for list of memory regions. @param is_sorted Optional bool for whether memory region list or tensor list are sorted. For long lists of transfer descriptors, sorting can speed up transfer preparation. @@ -364,10 +366,10 @@ def estimate_xfer_cost(self, req_handle: nixl_xfer_handle) -> tuple[int, int, in @param operation Type of operation ("WRITE" or "READ"). @param local_xfer_side Handle to the local transfer descriptor list, received from prep_xfer_dlist. - @param local_indices List of indices for selecting local descriptors. + @param local_indices List or numpy array (dtype=int32) of indices for selecting local descriptors. @param remote_xfer_side Handle to the remote (or loopback) transfer descriptor list, received from prep_xfer_dlist. - @param remote_indices List of indices for selecting remote descriptors. + @param remote_indices List or numpy array (dtype=int32) of indices for selecting remote descriptors. @param notif_msg Optional notification message to send after transfer is done. notif_msg should be bytes, as that is what will be returned to the target, but will work with str too. @param backends Optional list of backend names to limit which backends NIXL can use. @@ -379,9 +381,9 @@ def make_prepped_xfer( self, operation: str, local_xfer_side: nixl_prepped_dlist_handle, - local_indices: list[int], + local_indices: Union[list[int], np.ndarray], remote_xfer_side: nixl_prepped_dlist_handle, - remote_indices: list[int], + remote_indices: Union[list[int], np.ndarray], notif_msg: bytes = b"", backends: list[str] = [], skip_desc_merge: bool = False, @@ -747,7 +749,9 @@ def check_remote_metadata( a) list of 3 element tuples (address, len, device ID) alongside a mandatory memory type b) a tensor c) a list of tensors - d) passes along if an xfer_dlist is given. + d) a Nx3 2D numpy array, each row defines a single descriptor (address, len, device ID), + alongside a mandatory memory type + e) passes along if an xfer_dlist is given. @param descs List of any of the above types @param mem_type Optional memory type necessary for (a). @@ -780,6 +784,19 @@ def get_xfer_descs( else: print("3-tuple list needed for transfer") new_descs = None + elif isinstance(descs, np.ndarray): + if mem_type is not None and descs.ndim == 2 and descs.shape[1] == 3: + new_descs = nixlBind.nixlXferDList( + self.nixl_mems[mem_type], descs, is_sorted + ) + elif mem_type is None: + print("Please specify a mem type if not using Tensors") + new_descs = None + else: + print( + "Nx3 shape required for transfer descriptor list from numpy array" + ) + new_descs = None elif isinstance(descs, torch.Tensor): mem_type = "cuda" if str(descs.device).startswith("cuda") else "cpu" base_addr = descs.data_ptr() @@ -794,7 +811,7 @@ def get_xfer_descs( ) elif isinstance(descs[0], torch.Tensor): # List[torch.Tensor]: tensor_type = descs[0].device - dlist = [(0, 0, 0)] * len(descs) + dlist = np.zeros((len(descs), 3), dtype=np.uint64) for i in range(len(descs)): if descs[i].device != tensor_type: @@ -804,7 +821,7 @@ def get_xfer_descs( gpu_id = descs[i].get_device() if gpu_id == -1: # DRAM gpu_id = 0 - dlist[i] = (base_addr, region_len, gpu_id) + dlist[i, :] = (base_addr, region_len, gpu_id) mem_type = "cuda" if str(tensor_type).startswith("cuda") else "cpu" new_descs = nixlBind.nixlXferDList( self.nixl_mems[mem_type], dlist, is_sorted @@ -819,7 +836,9 @@ def get_xfer_descs( a) list of 4 element tuples (address, len, device ID, meta info) alongside a mandatory memory type b) a tensor c) a list of tensors - d) passes along if a reg_dlist is given. + d) a Nx3 2D numpy array, each row defines a single descriptor (address, len, device ID), + alongside a mandatory memory type. Empty meta info will be considered for each descriptor. + e) passes along if a reg_dlist is given. @param descs List of any of the above types @param mem_type Optional memory type necessary for (a). @@ -852,6 +871,19 @@ def get_reg_descs( else: print("4-tuple list needed for registration") new_descs = None + elif isinstance(descs, np.ndarray): + if mem_type is not None and descs.ndim == 2 and descs.shape[1] == 3: + new_descs = nixlBind.nixlRegDList( + self.nixl_mems[mem_type], descs, is_sorted + ) + elif mem_type is None: + print("Please specify a mem type if not using Tensors") + new_descs = None + else: + print( + "Nx3 shape required for transfer descriptor list from numpy array" + ) + new_descs = None elif isinstance(descs, torch.Tensor): mem_type = "cuda" if str(descs.device).startswith("cuda") else "cpu" base_addr = descs.data_ptr() @@ -866,7 +898,7 @@ def get_reg_descs( ) elif isinstance(descs[0], torch.Tensor): # List[torch.Tensor]: tensor_type = descs[0].device - dlist = [(0, 0, 0, "")] * len(descs) + dlist = np.zeros((len(descs), 3), dtype=np.uint64) for i in range(len(descs)): if descs[i].device != tensor_type: @@ -876,7 +908,7 @@ def get_reg_descs( gpu_id = descs[i].get_device() if gpu_id == -1: # DRAM gpu_id = 0 - dlist[i] = (base_addr, region_len, gpu_id, "") + dlist[i, :] = (base_addr, region_len, gpu_id) mem_type = "cuda" if str(tensor_type).startswith("cuda") else "cpu" new_descs = nixlBind.nixlRegDList( self.nixl_mems[mem_type], dlist, is_sorted diff --git a/src/bindings/python/nixl_bindings.cpp b/src/bindings/python/nixl_bindings.cpp index 65a39a04e7..dba1eda5ba 100644 --- a/src/bindings/python/nixl_bindings.cpp +++ b/src/bindings/python/nixl_bindings.cpp @@ -17,9 +17,11 @@ #include #include #include +#include #include #include +#include #include "nixl.h" #include "serdes/serdes.h" @@ -172,27 +174,47 @@ PYBIND11_MODULE(_bindings, m) { py::class_(m, "nixlXferDList") .def(py::init(), py::arg("type"), py::arg("sorted")=false, py::arg("init_size")=0) - .def(py::init([](nixl_mem_t mem, std::vector descs, bool sorted) { + .def(py::init([](nixl_mem_t mem, py::array descs, bool sorted) { + static_assert(sizeof(nixlBasicDesc) == 3 * sizeof(uint64_t), "nixlBasicDesc size mismatch"); + // Check array shape and dtype + if (descs.ndim() != 2 || descs.shape(1) != 3) + throw std::invalid_argument("descs must be a Nx3 numpy array"); + if (!py::dtype::of().equal(descs.dtype()) && !py::dtype::of().equal(descs.dtype())) + throw std::invalid_argument("descs must be a Nx3 numpy array of uint64 or int64"); + if (!(descs.flags() & py::array::c_style)) { + throw std::invalid_argument("descs must be a C-contiguous numpy array"); + } + size_t n = descs.shape(0); + nixl_xfer_dlist_t new_list(mem, sorted, n); + // We assume that the Nx3 array matches the nixlBasicDesc layout so we can simply memcpy + std::memcpy(&new_list[0], descs.data(), descs.size() * sizeof(uint64_t)); + assert(!sorted || new_list.verifySorted()); + return new_list; + }), py::arg("type"), py::arg("descs").noconvert(), py::arg("sorted")=false) + .def(py::init([](nixl_mem_t mem, py::list descs, bool sorted) { nixl_xfer_dlist_t new_list(mem, sorted, descs.size()); - for(long unsigned int i = 0; i(), descs[i][1].cast(), descs[i][2].cast()); - if (sorted) new_list.verifySorted(); + for(size_t i = 0; i < descs.size(); i++) { + if (!py::isinstance(descs[i])) { + throw py::type_error("Each descriptor must be a tuple when provided as a list"); + } + auto desc = py::reinterpret_borrow(descs[i]); + if (desc.size() != 3) { + throw py::value_error("Each descriptor tuple must have exactly 3 elements"); + } + new_list[i] = nixlBasicDesc(desc[0].cast(), desc[1].cast(), desc[2].cast()); + } + assert(!sorted || new_list.verifySorted()); return new_list; - }), py::arg("type"), py::arg("descs"), py::arg("sorted")=false) + }), py::arg("type"), py::arg("descs").noconvert(), py::arg("sorted")=false) .def("getType", &nixl_xfer_dlist_t::getType) .def("descCount", &nixl_xfer_dlist_t::descCount) .def("isEmpty", &nixl_xfer_dlist_t::isEmpty) .def("isSorted", &nixl_xfer_dlist_t::isSorted) .def(py::self == py::self) - .def("__getitem__", [](nixl_xfer_dlist_t &list, unsigned int i) -> - std::tuple { - std::tuple ret; - nixlBasicDesc desc = list[i]; - std::get<0>(ret) = desc.addr; - std::get<1>(ret) = desc.len; - std::get<2>(ret) = desc.devId; - return ret; - }) + .def("__getitem__", [](nixl_xfer_dlist_t &list, unsigned int i) -> py::tuple { + nixlBasicDesc &desc = list[i]; + return py::make_tuple(desc.addr, desc.len, desc.devId); + }) .def("__setitem__", [](nixl_xfer_dlist_t &list, unsigned int i, const py::tuple &desc) { list[i] = nixlBasicDesc(desc[0].cast(), desc[1].cast(), desc[2].cast()); }) @@ -229,11 +251,42 @@ PYBIND11_MODULE(_bindings, m) { py::class_(m, "nixlRegDList") .def(py::init(), py::arg("type"), py::arg("sorted")=false, py::arg("init_size")=0) - .def(py::init([](nixl_mem_t mem, std::vector descs, bool sorted) { + .def(py::init([](nixl_mem_t mem, py::array descs, bool sorted) { + if (descs.ndim() != 2 || descs.shape(1) != 3) + throw std::invalid_argument("descs must be a Nx3 numpy array"); + if (!py::dtype::of().equal(descs.dtype()) && !py::dtype::of().equal(descs.dtype())) + throw std::invalid_argument("descs must be a Nx3 numpy array of uint64 or int64"); + if (!(descs.flags() & py::array::c_style)) { + throw std::invalid_argument("descs must be a C-contiguous numpy array"); + } + size_t n = descs.shape(0); + nixl_reg_dlist_t new_list(mem, sorted, n); + if (py::dtype::of().equal(descs.dtype())) { + auto buffer = descs.unchecked(); + for(size_t i = 0; i < n; i++) { + new_list[i] = nixlBlobDesc(buffer(i, 0), buffer(i, 1), buffer(i, 2), ""); + } + } else { + auto buffer = descs.unchecked(); + for(size_t i = 0; i < n; i++) { + new_list[i] = nixlBlobDesc(buffer(i, 0), buffer(i, 1), buffer(i, 2), ""); + } + } + return new_list; + })) + .def(py::init([](nixl_mem_t mem, py::list descs, bool sorted) { nixl_reg_dlist_t new_list(mem, sorted, descs.size()); - for(long unsigned int i = 0; i(), descs[i][1].cast(), descs[i][2].cast(), descs[i][3].cast()); - if (sorted) new_list.verifySorted(); + for(size_t i = 0; i < descs.size(); i++) { + if (!py::isinstance(descs[i])) { + throw py::type_error("Each descriptor must be a tuple when provided as a list"); + } + auto desc = descs[i].cast(); + if (desc.size() != 4) { + throw py::value_error("Each descriptor tuple must have exactly 4 elements"); + } + new_list[i] = nixlBlobDesc(desc[0].cast(), desc[1].cast(), desc[2].cast(), desc[3].cast()); + } + assert(!sorted || new_list.verifySorted()); return new_list; }), py::arg("type"), py::arg("descs"), py::arg("sorted")=false) .def("getType", &nixl_reg_dlist_t::getType) @@ -241,15 +294,9 @@ PYBIND11_MODULE(_bindings, m) { .def("isEmpty", &nixl_reg_dlist_t::isEmpty) .def("isSorted", &nixl_reg_dlist_t::isSorted) .def(py::self == py::self) - .def("__getitem__", [](nixl_reg_dlist_t &list, unsigned int i) -> - std::tuple { - std::tuple ret; + .def("__getitem__", [](nixl_reg_dlist_t &list, unsigned int i) -> py::tuple { nixlBlobDesc desc = list[i]; - std::get<0>(ret) = desc.addr; - std::get<1>(ret) = desc.len; - std::get<2>(ret) = desc.devId; - std::get<3>(ret) = desc.metaInfo; - return ret; + return py::make_tuple(desc.addr, desc.len, desc.devId, py::bytes(desc.metaInfo)); }) .def("__setitem__", [](nixl_reg_dlist_t &list, unsigned int i, const py::tuple &desc) { list[i] = nixlBlobDesc(desc[0].cast(), desc[1].cast(), desc[2].cast(), desc[3].cast()); @@ -377,9 +424,9 @@ PYBIND11_MODULE(_bindings, m) { .def("makeXferReq", [](nixlAgent &agent, const nixl_xfer_op_t &operation, uintptr_t local_side, - const std::vector &local_indices, + py::object local_indices, uintptr_t remote_side, - const std::vector &remote_indices, + py::object remote_indices, const std::string ¬if_msg, std::vector backends, bool skip_desc_merge) -> uintptr_t { @@ -394,9 +441,33 @@ PYBIND11_MODULE(_bindings, m) { extra_params.hasNotif = true; } extra_params.skipDescMerge = skip_desc_merge; + std::vector local_indices_vec; + std::vector remote_indices_vec; + + auto init_indices_lambda = [](py::object &indices) -> std::vector { + if (py::isinstance(indices)) { + auto indices_array = indices.cast>(); + if (indices_array.ndim() != 1) + throw std::invalid_argument("indices numpy array must be 1D"); + if (!py::dtype::of().equal(indices_array.dtype()) && !py::dtype::of().equal(indices_array.dtype())) + throw std::invalid_argument("indices numpy array must be 1D of uint32 or int32"); + if (!(indices_array.flags() & py::array::c_style)) + throw std::invalid_argument("indices numpy array must be C-contiguous"); + // We assume that the indices array matches the nixlBasicDesc layout so we can simply memcpy + std::vector ret(indices_array.size()); + std::memcpy(ret.data(), indices_array.data(), indices_array.size() * sizeof(uint32_t)); + return ret; + } else { + return indices.cast>(); + } + }; + + local_indices_vec = init_indices_lambda(local_indices); + remote_indices_vec = init_indices_lambda(remote_indices); + throw_nixl_exception(agent.makeXferReq(operation, - (nixlDlistH*) local_side, local_indices, - (nixlDlistH*) remote_side, remote_indices, + (nixlDlistH*)local_side, local_indices_vec, + (nixlDlistH*)remote_side, remote_indices_vec, handle, &extra_params)); return (uintptr_t) handle; diff --git a/test/python/prep_xfer_perf.py b/test/python/prep_xfer_perf.py new file mode 100755 index 0000000000..9f0185c5ed --- /dev/null +++ b/test/python/prep_xfer_perf.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +import numpy as np + +import nixl._utils as nixl_utils +from nixl._api import nixl_agent, nixl_agent_config + + +def init_agent(): + agent_config = nixl_agent_config(backends=["UCX"]) + agent = nixl_agent("agent", agent_config) + return agent + + +def prep_handles(agent: nixl_agent, xfer_dlist, reg_dlist, indices): + start = time.perf_counter() + xfer_dlist_trim = reg_dlist.trim() + elapsed = time.perf_counter() - start + assert xfer_dlist_trim.descCount() == xfer_dlist.descCount() + print(f"Trim nixlRegDList:\t{elapsed:.4f} sec") + + start = time.perf_counter() + assert agent.register_memory(reg_dlist) is not None + elapsed = time.perf_counter() - start + print(f"register_memory:\t{elapsed:.4f} sec") + + start = time.perf_counter() + local_prep_handle = agent.prep_xfer_dlist( + "NIXL_INIT_AGENT", xfer_dlist, "DRAM", False + ) + elapsed = time.perf_counter() - start + assert local_prep_handle + print(f"prep_xfer_dlist INIT:\t{elapsed:.4f} sec") + + start = time.perf_counter() + remote_prep_handle = agent.prep_xfer_dlist("agent", xfer_dlist, "DRAM", False) + elapsed = time.perf_counter() - start + assert remote_prep_handle + print(f"prep_xfer_dlist SELF:\t{elapsed:.4f} sec") + + start = time.perf_counter() + xfer_handle = agent.make_prepped_xfer( + "WRITE", local_prep_handle, indices, remote_prep_handle, indices, b"UUID2" + ) + elapsed = time.perf_counter() - start + assert xfer_handle + print(f"make_prepped_xfer:\t{elapsed:.4f} sec") + + return local_prep_handle, remote_prep_handle, xfer_handle + + +def perf_test_list(num_descs: int, addr_base: int, length: int): + print("-" * 40) + print("Starting list test...") + print("-" * 40) + agent = init_agent() + descs_list = [(addr_base + i * length, length, 0) for i in range(num_descs)] + indices = list(range(num_descs)) + + start = time.perf_counter() + xfer_dlist = agent.get_xfer_descs(descs_list, "DRAM", False) + elapsed = time.perf_counter() - start + assert xfer_dlist.descCount() == num_descs + print(f"get_xfer_descs:\t\t{elapsed:.4f} sec") + + blob_descs_list = [ + (addr_base + i * length, length, 0, b"") for i in range(num_descs) + ] + start = time.perf_counter() + reg_dlist = agent.get_reg_descs(blob_descs_list, "DRAM", False) + elapsed = time.perf_counter() - start + assert reg_dlist.descCount() == num_descs + print(f"get_reg_descs:\t\t{elapsed:.4f} sec") + + local_prep_handle, remote_prep_handle, xfer_handle = prep_handles( + agent, xfer_dlist, reg_dlist, indices + ) + + del agent + + +def perf_test_array(num_descs: int, addr_base: int, length: int): + print("-" * 40) + print("Starting array test...") + print("-" * 40) + agent = init_agent() + descs_np = np.zeros((num_descs, 3), dtype=np.uint64) + indices = np.arange(num_descs) + descs_np[:, 0] = addr_base + indices * length + descs_np[:, 1] = length + descs_np[:, 2] = 0 + + start = time.perf_counter() + xfer_dlist = agent.get_xfer_descs(descs_np, "DRAM", False) + elapsed = time.perf_counter() - start + assert xfer_dlist.descCount() == num_descs + print(f"get_xfer_descs:\t\t{elapsed:.4f} sec") + + start = time.perf_counter() + reg_dlist = agent.get_reg_descs(descs_np, "DRAM", False) + elapsed = time.perf_counter() - start + assert reg_dlist.descCount() == num_descs + print(f"get_reg_descs:\t\t{elapsed:.4f} sec") + + local_prep_handle, remote_prep_handle, xfer_handle = prep_handles( + agent, xfer_dlist, reg_dlist, indices + ) + + del agent + + +if __name__ == "__main__": + import argparse + import os + + parser = argparse.ArgumentParser() + parser.add_argument( + "mode", + choices=["list", "array"], + help="Test mode: list or array based descriptors", + ) + args = parser.parse_args() + + print("Using NIXL Plugins from:") + print(os.environ["NIXL_PLUGIN_DIR"]) + + # Example using nixl_agent_config + agent = init_agent() + + num_descs = 2**20 + length = 64 + addr_base = nixl_utils.malloc_passthru(num_descs * length) + print(f"Performance test: Creating nixlXferDList with {num_descs} descriptors") + + if args.mode == "list": + perf_test_list(num_descs, addr_base, length) + elif args.mode == "array": + perf_test_array(num_descs, addr_base, length)