Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitlab/test_python.sh
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ pip3 install --break-system-packages zmq
echo "==== Running python tests ===="
python3 examples/python/nixl_api_example.py
pytest test/python
python3 test/python/prep_xfer_perf.py list
python3 test/python/prep_xfer_perf.py array

echo "==== Running python example ===="
cd examples/python
Expand Down
14 changes: 14 additions & 0 deletions examples/python/nixl_api_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

import os

import numpy as np
import torch

import nixl._utils as nixl_utils
Expand Down Expand Up @@ -54,6 +55,19 @@
agent1_reg_descs = nixl_agent1.get_reg_descs(agent1_strings, "DRAM", is_sorted=True)
agent1_xfer_descs = nixl_agent1.get_xfer_descs(agent1_addrs, "DRAM", is_sorted=True)

# Prefer numpy arrays for performance
agent1_addrs_np = np.array(agent1_addrs)
agent1_xfer_descs_np = nixl_agent1.get_xfer_descs(
agent1_addrs_np, "DRAM", is_sorted=True
)
agent1_reg_descs_np = nixl_agent1.get_reg_descs(
agent1_addrs_np, "DRAM", is_sorted=True
)

assert agent1_xfer_descs == agent1_xfer_descs_np
assert agent1_reg_descs == agent1_reg_descs_np
print(agent1_reg_descs, agent1_reg_descs_np)
Comment thread
aranadive marked this conversation as resolved.
Resolved

# Just for tensor test
tensors = [torch.zeros(10, dtype=torch.float32) for _ in range(2)]
agent1_tensor_reg_descs = nixl_agent1.get_reg_descs(tensors)
Expand Down
56 changes: 44 additions & 12 deletions src/api/python/_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,9 @@
# limitations under the License.

import pickle
from typing import Optional
from typing import Optional, Union

import numpy as np
import torch

import nixl._bindings as nixlBind
Expand Down Expand Up @@ -313,7 +314,8 @@ def make_connection(self, remote_agent: str, backends: list[str] = []):
this is for a loopback (local) transfer to be used for remote_side handle

@param agent_name Name of the agent. It can be "NIXL_INIT_AGENT", local agent name, or remote agent name
@param xfer_list List of transfer descriptors, can be list of memory region tuples, tensors, or nixlXferDList.
@param xfer_list List of transfer descriptors, can be list of memory region tuples, tensors,
Nx3 numpy array, or nixlXferDList. See get_xfer_descs for more details on the structure.
@param mem_type Optional memory type necessary for list of memory regions.
@param is_sorted Optional bool for whether memory region list or tensor list are sorted.
For long lists of transfer descriptors, sorting can speed up transfer preparation.
Expand Down Expand Up @@ -364,10 +366,10 @@ def estimate_xfer_cost(self, req_handle: nixl_xfer_handle) -> tuple[int, int, in
@param operation Type of operation ("WRITE" or "READ").
@param local_xfer_side Handle to the local transfer descriptor list,
received from prep_xfer_dlist.
@param local_indices List of indices for selecting local descriptors.
@param local_indices List or numpy array (dtype=int32) of indices for selecting local descriptors.
@param remote_xfer_side Handle to the remote (or loopback) transfer descriptor list,
received from prep_xfer_dlist.
@param remote_indices List of indices for selecting remote descriptors.
@param remote_indices List or numpy array (dtype=int32) of indices for selecting remote descriptors.
@param notif_msg Optional notification message to send after transfer is done.
notif_msg should be bytes, as that is what will be returned to the target, but will work with str too.
@param backends Optional list of backend names to limit which backends NIXL can use.
Expand All @@ -379,9 +381,9 @@ def make_prepped_xfer(
self,
operation: str,
local_xfer_side: nixl_prepped_dlist_handle,
local_indices: list[int],
local_indices: Union[list[int], np.ndarray],
remote_xfer_side: nixl_prepped_dlist_handle,
remote_indices: list[int],
remote_indices: Union[list[int], np.ndarray],
notif_msg: bytes = b"",
backends: list[str] = [],
skip_desc_merge: bool = False,
Expand Down Expand Up @@ -747,7 +749,9 @@ def check_remote_metadata(
a) list of 3 element tuples (address, len, device ID) alongside a mandatory memory type
b) a tensor
c) a list of tensors
d) passes along if an xfer_dlist is given.
d) a Nx3 2D numpy array, each row defines a single descriptor (address, len, device ID),
alongside a mandatory memory type
e) passes along if an xfer_dlist is given.

@param descs List of any of the above types
@param mem_type Optional memory type necessary for (a).
Expand Down Expand Up @@ -780,6 +784,19 @@ def get_xfer_descs(
else:
print("3-tuple list needed for transfer")
new_descs = None
elif isinstance(descs, np.ndarray):
if mem_type is not None and descs.ndim == 2 and descs.shape[1] == 3:
new_descs = nixlBind.nixlXferDList(
self.nixl_mems[mem_type], descs, is_sorted
)
elif mem_type is None:
print("Please specify a mem type if not using Tensors")
Comment thread
aranadive marked this conversation as resolved.
Resolved
new_descs = None
else:
print(
"Nx3 shape required for transfer descriptor list from numpy array"
)
new_descs = None
elif isinstance(descs, torch.Tensor):
mem_type = "cuda" if str(descs.device).startswith("cuda") else "cpu"
base_addr = descs.data_ptr()
Expand All @@ -794,7 +811,7 @@ def get_xfer_descs(
)
elif isinstance(descs[0], torch.Tensor): # List[torch.Tensor]:
tensor_type = descs[0].device
dlist = [(0, 0, 0)] * len(descs)
dlist = np.zeros((len(descs), 3), dtype=np.uint64)

for i in range(len(descs)):
if descs[i].device != tensor_type:
Expand All @@ -804,7 +821,7 @@ def get_xfer_descs(
gpu_id = descs[i].get_device()
if gpu_id == -1: # DRAM
gpu_id = 0
dlist[i] = (base_addr, region_len, gpu_id)
dlist[i, :] = (base_addr, region_len, gpu_id)
mem_type = "cuda" if str(tensor_type).startswith("cuda") else "cpu"
new_descs = nixlBind.nixlXferDList(
self.nixl_mems[mem_type], dlist, is_sorted
Expand All @@ -819,7 +836,9 @@ def get_xfer_descs(
a) list of 4 element tuples (address, len, device ID, meta info) alongside a mandatory memory type
b) a tensor
c) a list of tensors
d) passes along if a reg_dlist is given.
d) a Nx3 2D numpy array, each row defines a single descriptor (address, len, device ID),
alongside a mandatory memory type. Empty meta info will be considered for each descriptor.
e) passes along if a reg_dlist is given.

@param descs List of any of the above types
@param mem_type Optional memory type necessary for (a).
Expand Down Expand Up @@ -852,6 +871,19 @@ def get_reg_descs(
else:
print("4-tuple list needed for registration")
new_descs = None
elif isinstance(descs, np.ndarray):
if mem_type is not None and descs.ndim == 2 and descs.shape[1] == 3:
new_descs = nixlBind.nixlRegDList(
self.nixl_mems[mem_type], descs, is_sorted
)
elif mem_type is None:
print("Please specify a mem type if not using Tensors")
new_descs = None
else:
print(
"Nx3 shape required for transfer descriptor list from numpy array"
)
new_descs = None
elif isinstance(descs, torch.Tensor):
mem_type = "cuda" if str(descs.device).startswith("cuda") else "cpu"
base_addr = descs.data_ptr()
Expand All @@ -866,7 +898,7 @@ def get_reg_descs(
)
elif isinstance(descs[0], torch.Tensor): # List[torch.Tensor]:
tensor_type = descs[0].device
dlist = [(0, 0, 0, "")] * len(descs)
dlist = np.zeros((len(descs), 3), dtype=np.uint64)
Comment thread
aranadive marked this conversation as resolved.
Resolved

for i in range(len(descs)):
if descs[i].device != tensor_type:
Expand All @@ -876,7 +908,7 @@ def get_reg_descs(
gpu_id = descs[i].get_device()
if gpu_id == -1: # DRAM
gpu_id = 0
dlist[i] = (base_addr, region_len, gpu_id, "")
dlist[i, :] = (base_addr, region_len, gpu_id)
mem_type = "cuda" if str(tensor_type).startswith("cuda") else "cpu"
new_descs = nixlBind.nixlRegDList(
self.nixl_mems[mem_type], dlist, is_sorted
Expand Down
131 changes: 101 additions & 30 deletions src/bindings/python/nixl_bindings.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,11 @@
#include <pybind11/pybind11.h>
#include <pybind11/stl.h>
#include <pybind11/operators.h>
#include <pybind11/numpy.h>

#include <tuple>
#include <iostream>
#include <cassert>

#include "nixl.h"
#include "serdes/serdes.h"
Expand Down Expand Up @@ -172,27 +174,47 @@ PYBIND11_MODULE(_bindings, m) {

py::class_<nixl_xfer_dlist_t>(m, "nixlXferDList")
.def(py::init<nixl_mem_t, bool, int>(), py::arg("type"), py::arg("sorted")=false, py::arg("init_size")=0)
.def(py::init([](nixl_mem_t mem, std::vector<py::tuple> descs, bool sorted) {
.def(py::init([](nixl_mem_t mem, py::array descs, bool sorted) {
static_assert(sizeof(nixlBasicDesc) == 3 * sizeof(uint64_t), "nixlBasicDesc size mismatch");
// Check array shape and dtype
if (descs.ndim() != 2 || descs.shape(1) != 3)
throw std::invalid_argument("descs must be a Nx3 numpy array");
if (!py::dtype::of<uint64_t>().equal(descs.dtype()) && !py::dtype::of<int64_t>().equal(descs.dtype()))
throw std::invalid_argument("descs must be a Nx3 numpy array of uint64 or int64");
if (!(descs.flags() & py::array::c_style)) {
throw std::invalid_argument("descs must be a C-contiguous numpy array");
}
size_t n = descs.shape(0);
nixl_xfer_dlist_t new_list(mem, sorted, n);
// We assume that the Nx3 array matches the nixlBasicDesc layout so we can simply memcpy
std::memcpy(&new_list[0], descs.data(), descs.size() * sizeof(uint64_t));
assert(!sorted || new_list.verifySorted());
return new_list;
}), py::arg("type"), py::arg("descs").noconvert(), py::arg("sorted")=false)
.def(py::init([](nixl_mem_t mem, py::list descs, bool sorted) {
nixl_xfer_dlist_t new_list(mem, sorted, descs.size());
for(long unsigned int i = 0; i<descs.size(); i++)
new_list[i] = nixlBasicDesc(descs[i][0].cast<uintptr_t>(), descs[i][1].cast<size_t>(), descs[i][2].cast<uint64_t>());
if (sorted) new_list.verifySorted();
for(size_t i = 0; i < descs.size(); i++) {
if (!py::isinstance<py::tuple>(descs[i])) {
throw py::type_error("Each descriptor must be a tuple when provided as a list");
}
auto desc = py::reinterpret_borrow<py::tuple>(descs[i]);
if (desc.size() != 3) {
throw py::value_error("Each descriptor tuple must have exactly 3 elements");
}
new_list[i] = nixlBasicDesc(desc[0].cast<uintptr_t>(), desc[1].cast<size_t>(), desc[2].cast<uint64_t>());
}
assert(!sorted || new_list.verifySorted());
return new_list;
}), py::arg("type"), py::arg("descs"), py::arg("sorted")=false)
}), py::arg("type"), py::arg("descs").noconvert(), py::arg("sorted")=false)
.def("getType", &nixl_xfer_dlist_t::getType)
.def("descCount", &nixl_xfer_dlist_t::descCount)
.def("isEmpty", &nixl_xfer_dlist_t::isEmpty)
.def("isSorted", &nixl_xfer_dlist_t::isSorted)
.def(py::self == py::self)
.def("__getitem__", [](nixl_xfer_dlist_t &list, unsigned int i) ->
std::tuple<uintptr_t, size_t, uint64_t> {
std::tuple<uintptr_t, size_t, uint64_t> ret;
nixlBasicDesc desc = list[i];
std::get<0>(ret) = desc.addr;
std::get<1>(ret) = desc.len;
std::get<2>(ret) = desc.devId;
return ret;
})
.def("__getitem__", [](nixl_xfer_dlist_t &list, unsigned int i) -> py::tuple {
nixlBasicDesc &desc = list[i];
return py::make_tuple(desc.addr, desc.len, desc.devId);
})
.def("__setitem__", [](nixl_xfer_dlist_t &list, unsigned int i, const py::tuple &desc) {
list[i] = nixlBasicDesc(desc[0].cast<uintptr_t>(), desc[1].cast<size_t>(), desc[2].cast<uint64_t>());
})
Expand Down Expand Up @@ -229,27 +251,52 @@ PYBIND11_MODULE(_bindings, m) {

py::class_<nixl_reg_dlist_t>(m, "nixlRegDList")
.def(py::init<nixl_mem_t, bool, int>(), py::arg("type"), py::arg("sorted")=false, py::arg("init_size")=0)
.def(py::init([](nixl_mem_t mem, std::vector<py::tuple> descs, bool sorted) {
.def(py::init([](nixl_mem_t mem, py::array descs, bool sorted) {
if (descs.ndim() != 2 || descs.shape(1) != 3)
throw std::invalid_argument("descs must be a Nx3 numpy array");
if (!py::dtype::of<uint64_t>().equal(descs.dtype()) && !py::dtype::of<int64_t>().equal(descs.dtype()))
throw std::invalid_argument("descs must be a Nx3 numpy array of uint64 or int64");
if (!(descs.flags() & py::array::c_style)) {
throw std::invalid_argument("descs must be a C-contiguous numpy array");
}
size_t n = descs.shape(0);
nixl_reg_dlist_t new_list(mem, sorted, n);
if (py::dtype::of<uint64_t>().equal(descs.dtype())) {
auto buffer = descs.unchecked<uint64_t, 2>();
for(size_t i = 0; i < n; i++) {
new_list[i] = nixlBlobDesc(buffer(i, 0), buffer(i, 1), buffer(i, 2), "");
}
} else {
auto buffer = descs.unchecked<int64_t, 2>();
for(size_t i = 0; i < n; i++) {
new_list[i] = nixlBlobDesc(buffer(i, 0), buffer(i, 1), buffer(i, 2), "");
}
}
return new_list;
}))
.def(py::init([](nixl_mem_t mem, py::list descs, bool sorted) {
nixl_reg_dlist_t new_list(mem, sorted, descs.size());
for(long unsigned int i = 0; i<descs.size(); i++)
new_list[i] = nixlBlobDesc(descs[i][0].cast<uintptr_t>(), descs[i][1].cast<size_t>(), descs[i][2].cast<uint64_t>(), descs[i][3].cast<std::string>());
if (sorted) new_list.verifySorted();
for(size_t i = 0; i < descs.size(); i++) {
if (!py::isinstance<py::tuple>(descs[i])) {
throw py::type_error("Each descriptor must be a tuple when provided as a list");
}
auto desc = descs[i].cast<py::tuple>();
if (desc.size() != 4) {
throw py::value_error("Each descriptor tuple must have exactly 4 elements");
}
new_list[i] = nixlBlobDesc(desc[0].cast<uintptr_t>(), desc[1].cast<size_t>(), desc[2].cast<uint64_t>(), desc[3].cast<std::string>());
}
assert(!sorted || new_list.verifySorted());
return new_list;
}), py::arg("type"), py::arg("descs"), py::arg("sorted")=false)
.def("getType", &nixl_reg_dlist_t::getType)
.def("descCount", &nixl_reg_dlist_t::descCount)
.def("isEmpty", &nixl_reg_dlist_t::isEmpty)
.def("isSorted", &nixl_reg_dlist_t::isSorted)
.def(py::self == py::self)
.def("__getitem__", [](nixl_reg_dlist_t &list, unsigned int i) ->
std::tuple<uintptr_t, size_t, uint64_t, std::string> {
std::tuple<uintptr_t, size_t, uint64_t, std::string> ret;
.def("__getitem__", [](nixl_reg_dlist_t &list, unsigned int i) -> py::tuple {
nixlBlobDesc desc = list[i];
std::get<0>(ret) = desc.addr;
std::get<1>(ret) = desc.len;
std::get<2>(ret) = desc.devId;
std::get<3>(ret) = desc.metaInfo;
return ret;
return py::make_tuple(desc.addr, desc.len, desc.devId, py::bytes(desc.metaInfo));
})
.def("__setitem__", [](nixl_reg_dlist_t &list, unsigned int i, const py::tuple &desc) {
list[i] = nixlBlobDesc(desc[0].cast<uintptr_t>(), desc[1].cast<size_t>(), desc[2].cast<uint64_t>(), desc[3].cast<std::string>());
Expand Down Expand Up @@ -377,9 +424,9 @@ PYBIND11_MODULE(_bindings, m) {
.def("makeXferReq", [](nixlAgent &agent,
const nixl_xfer_op_t &operation,
uintptr_t local_side,
const std::vector<int> &local_indices,
py::object local_indices,
uintptr_t remote_side,
const std::vector<int> &remote_indices,
py::object remote_indices,
const std::string &notif_msg,
std::vector<uintptr_t> backends,
bool skip_desc_merge) -> uintptr_t {
Expand All @@ -394,9 +441,33 @@ PYBIND11_MODULE(_bindings, m) {
extra_params.hasNotif = true;
}
extra_params.skipDescMerge = skip_desc_merge;
std::vector<int> local_indices_vec;
std::vector<int> remote_indices_vec;

auto init_indices_lambda = [](py::object &indices) -> std::vector<int> {
if (py::isinstance<py::array>(indices)) {
auto indices_array = indices.cast<py::array_t<uint32_t>>();
if (indices_array.ndim() != 1)
throw std::invalid_argument("indices numpy array must be 1D");
if (!py::dtype::of<uint32_t>().equal(indices_array.dtype()) && !py::dtype::of<int32_t>().equal(indices_array.dtype()))
throw std::invalid_argument("indices numpy array must be 1D of uint32 or int32");
if (!(indices_array.flags() & py::array::c_style))
throw std::invalid_argument("indices numpy array must be C-contiguous");
// We assume that the indices array matches the nixlBasicDesc layout so we can simply memcpy
std::vector<int> ret(indices_array.size());
std::memcpy(ret.data(), indices_array.data(), indices_array.size() * sizeof(uint32_t));
return ret;
} else {
return indices.cast<std::vector<int>>();
}
};

local_indices_vec = init_indices_lambda(local_indices);
remote_indices_vec = init_indices_lambda(remote_indices);

throw_nixl_exception(agent.makeXferReq(operation,
(nixlDlistH*) local_side, local_indices,
(nixlDlistH*) remote_side, remote_indices,
(nixlDlistH*)local_side, local_indices_vec,
(nixlDlistH*)remote_side, remote_indices_vec,
handle, &extra_params));

return (uintptr_t) handle;
Expand Down
Loading