Skip to content

Commit c2d3a28

Browse files
mkhazraeemkhazraee
authored
Descriptors: changed devID from 32-bits to 64-bits (#171)
* In serialized form of nixlBasicDesc, it was using 24 bytes anyways on 64-bit systems, so no memory savings, while going to u64 can be useful for some backends. * Also used constexpr as we moved to c++17, and now empty copyMeta can be removed from nixlBasicDesc. Signed-off-by: Moein Khazraee <[email protected]>
1 parent bd0ab7e commit c2d3a28

File tree

8 files changed

+108
-115
lines changed

8 files changed

+108
-115
lines changed

‎src/api/cpp/nixl_descriptors.h‎

Lines changed: 3 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ class nixlBasicDesc {
3636
/** @var Buffer Length */
3737
size_t len;
3838
/** @var deviceID/blockID/fileID */
39-
uint32_t devId;
39+
uint64_t devId;
4040

4141
/**
4242
* @brief Default constructor for nixlBasicDesc
@@ -52,7 +52,7 @@ class nixlBasicDesc {
5252
*/
5353
nixlBasicDesc(const uintptr_t &addr,
5454
const size_t &len,
55-
const uint32_t &dev_id);
55+
const uint64_t &dev_id);
5656
/**
5757
* @brief Deserializer constructor for nixlBasicDesc with
5858
* serialized blob of another nixlBasicDesc
@@ -110,13 +110,6 @@ class nixlBasicDesc {
110110
* @param query nixlBasicDesc Object
111111
*/
112112
bool overlaps (const nixlBasicDesc &query) const;
113-
/**
114-
* @brief Copy Metadata from one descriptor to another.
115-
* No meta info in BasicDesc, so empty implementation
116-
*
117-
* @param desc nixlBasicDesc Object
118-
*/
119-
void copyMeta (const nixlBasicDesc &desc) {};
120113
/**
121114
* @brief Serialize descriptor into a blob
122115
*/
@@ -151,7 +144,7 @@ class nixlBlobDesc : public nixlBasicDesc {
151144
* @param meta_info Metadata blob
152145
*/
153146
nixlBlobDesc(const uintptr_t &addr, const size_t &len,
154-
const uint32_t &dev_id, const nixl_blob_t &meta_info);
147+
const uint64_t &dev_id, const nixl_blob_t &meta_info);
155148
/**
156149
* @brief Constructor for nixlBlobDesc from nixlBasicDesc and metadata blob
157150
*

‎src/bindings/python/nixl_bindings.cpp‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,7 @@ PYBIND11_MODULE(_bindings, m) {
163163
.def(py::init([](nixl_mem_t mem, std::vector<py::tuple> descs, bool sorted) {
164164
nixl_xfer_dlist_t new_list(mem, sorted, descs.size());
165165
for(long unsigned int i = 0; i<descs.size(); i++)
166-
new_list[i] = nixlBasicDesc(descs[i][0].cast<uintptr_t>(), descs[i][1].cast<size_t>(), descs[i][2].cast<uint32_t>());
166+
new_list[i] = nixlBasicDesc(descs[i][0].cast<uintptr_t>(), descs[i][1].cast<size_t>(), descs[i][2].cast<uint64_t>());
167167
if (sorted) new_list.verifySorted();
168168
return new_list;
169169
}), py::arg("type"), py::arg("descs"), py::arg("sorted")=false)
@@ -173,26 +173,26 @@ PYBIND11_MODULE(_bindings, m) {
173173
.def("isSorted", &nixl_xfer_dlist_t::isSorted)
174174
.def(py::self == py::self)
175175
.def("__getitem__", [](nixl_xfer_dlist_t &list, unsigned int i) ->
176-
std::tuple<uintptr_t, size_t, uint32_t> {
177-
std::tuple<uintptr_t, size_t, uint32_t> ret;
176+
std::tuple<uintptr_t, size_t, uint64_t> {
177+
std::tuple<uintptr_t, size_t, uint64_t> ret;
178178
nixlBasicDesc desc = list[i];
179179
std::get<0>(ret) = desc.addr;
180180
std::get<1>(ret) = desc.len;
181181
std::get<2>(ret) = desc.devId;
182182
return ret;
183183
})
184184
.def("__setitem__", [](nixl_xfer_dlist_t &list, unsigned int i, const py::tuple &desc) {
185-
list[i] = nixlBasicDesc(desc[0].cast<uintptr_t>(), desc[1].cast<size_t>(), desc[2].cast<uint32_t>());
185+
list[i] = nixlBasicDesc(desc[0].cast<uintptr_t>(), desc[1].cast<size_t>(), desc[2].cast<uint64_t>());
186186
})
187187
.def("addDesc", [](nixl_xfer_dlist_t &list, const py::tuple &desc) {
188-
list.addDesc(nixlBasicDesc(desc[0].cast<uintptr_t>(), desc[1].cast<size_t>(), desc[2].cast<uint32_t>()));
188+
list.addDesc(nixlBasicDesc(desc[0].cast<uintptr_t>(), desc[1].cast<size_t>(), desc[2].cast<uint64_t>()));
189189
})
190190
.def("append", [](nixl_xfer_dlist_t &list, const py::tuple &desc) {
191-
list.addDesc(nixlBasicDesc(desc[0].cast<uintptr_t>(), desc[1].cast<size_t>(), desc[2].cast<uint32_t>()));
191+
list.addDesc(nixlBasicDesc(desc[0].cast<uintptr_t>(), desc[1].cast<size_t>(), desc[2].cast<uint64_t>()));
192192
})
193193
.def("index", [](nixl_xfer_dlist_t &list, const py::tuple &desc) {
194194
int ret = (nixl_status_t) list.getIndex(nixlBasicDesc(desc[0].cast<uintptr_t>(), desc[1].cast<size_t>(),
195-
desc[2].cast<uint32_t>()));
195+
desc[2].cast<uint64_t>()));
196196
if(ret < 0) throw_nixl_exception((nixl_status_t) ret);
197197
return (int) ret;
198198
})
@@ -220,7 +220,7 @@ PYBIND11_MODULE(_bindings, m) {
220220
.def(py::init([](nixl_mem_t mem, std::vector<py::tuple> descs, bool sorted) {
221221
nixl_reg_dlist_t new_list(mem, sorted, descs.size());
222222
for(long unsigned int i = 0; i<descs.size(); i++)
223-
new_list[i] = nixlBlobDesc(descs[i][0].cast<uintptr_t>(), descs[i][1].cast<size_t>(), descs[i][2].cast<uint32_t>(), descs[i][3].cast<std::string>());
223+
new_list[i] = nixlBlobDesc(descs[i][0].cast<uintptr_t>(), descs[i][1].cast<size_t>(), descs[i][2].cast<uint64_t>(), descs[i][3].cast<std::string>());
224224
if (sorted) new_list.verifySorted();
225225
return new_list;
226226
}), py::arg("type"), py::arg("descs"), py::arg("sorted")=false)
@@ -230,8 +230,8 @@ PYBIND11_MODULE(_bindings, m) {
230230
.def("isSorted", &nixl_reg_dlist_t::isSorted)
231231
.def(py::self == py::self)
232232
.def("__getitem__", [](nixl_reg_dlist_t &list, unsigned int i) ->
233-
std::tuple<uintptr_t, size_t, uint32_t, std::string> {
234-
std::tuple<uintptr_t, size_t, uint32_t, std::string> ret;
233+
std::tuple<uintptr_t, size_t, uint64_t, std::string> {
234+
std::tuple<uintptr_t, size_t, uint64_t, std::string> ret;
235235
nixlBlobDesc desc = list[i];
236236
std::get<0>(ret) = desc.addr;
237237
std::get<1>(ret) = desc.len;
@@ -240,19 +240,19 @@ PYBIND11_MODULE(_bindings, m) {
240240
return ret;
241241
})
242242
.def("__setitem__", [](nixl_reg_dlist_t &list, unsigned int i, const py::tuple &desc) {
243-
list[i] = nixlBlobDesc(desc[0].cast<uintptr_t>(), desc[1].cast<size_t>(), desc[2].cast<uint32_t>(), desc[3].cast<std::string>());
243+
list[i] = nixlBlobDesc(desc[0].cast<uintptr_t>(), desc[1].cast<size_t>(), desc[2].cast<uint64_t>(), desc[3].cast<std::string>());
244244
})
245245
.def("addDesc", [](nixl_reg_dlist_t &list, const py::tuple &desc) {
246246
list.addDesc(nixlBlobDesc(desc[0].cast<uintptr_t>(), desc[1].cast<size_t>(),
247-
desc[2].cast<uint32_t>(),desc[3].cast<std::string>()));
247+
desc[2].cast<uint64_t>(),desc[3].cast<std::string>()));
248248
})
249249
.def("append", [](nixl_reg_dlist_t &list, const py::tuple &desc) {
250250
list.addDesc(nixlBlobDesc(desc[0].cast<uintptr_t>(), desc[1].cast<size_t>(),
251-
desc[2].cast<uint32_t>(),desc[3].cast<std::string>()));
251+
desc[2].cast<uint64_t>(),desc[3].cast<std::string>()));
252252
})
253253
.def("index", [](nixl_reg_dlist_t &list, const py::tuple &desc) {
254254
int ret = list.getIndex(nixlBlobDesc(desc[0].cast<uintptr_t>(), desc[1].cast<size_t>(),
255-
desc[2].cast<uint32_t>(),desc[3].cast<std::string>()));
255+
desc[2].cast<uint64_t>(),desc[3].cast<std::string>()));
256256
if(ret < 0) throw_nixl_exception((nixl_status_t) ret);
257257
return ret;
258258
})

‎src/infra/nixl_descriptors.cpp‎

Lines changed: 87 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030

3131
nixlBasicDesc::nixlBasicDesc(const uintptr_t &addr,
3232
const size_t &len,
33-
const uint32_t &dev_id) {
33+
const uint64_t &dev_id) {
3434
this->addr = addr;
3535
this->len = len;
3636
this->devId = dev_id;
@@ -97,7 +97,7 @@ void nixlBasicDesc::print(const std::string &suffix) const {
9797

9898
nixlBlobDesc::nixlBlobDesc(const uintptr_t &addr,
9999
const size_t &len,
100-
const uint32_t &dev_id,
100+
const uint64_t &dev_id,
101101
const nixl_blob_t &meta_info) :
102102
nixlBasicDesc(addr, len, dev_id) {
103103
this->metaInfo = meta_info;
@@ -323,102 +323,103 @@ template <class T>
323323
nixl_status_t nixlDescList<T>::populate (const nixlDescList<nixlBasicDesc> &query,
324324
nixlDescList<T> &resp) const {
325325
// Populate only makes sense when there is extra metadata
326-
if (std::is_same<nixlBasicDesc, T>::value)
327-
return NIXL_ERR_INVALID_PARAM;
328-
329-
if ((type != query.getType()) || (type != resp.type))
326+
if constexpr (std::is_same<nixlBasicDesc, T>::value) {
330327
return NIXL_ERR_INVALID_PARAM;
331-
332-
// 1-to-1 mapping cannot hold
333-
if (query.isSorted() != resp.sorted)
334-
return NIXL_ERR_INVALID_PARAM;
335-
336-
T new_elm;
337-
nixlBasicDesc *p = &new_elm;
338-
int count = 0, last_found = 0;
339-
int s_index, q_index, size;
340-
bool found, q_sorted = query.isSorted();
341-
const nixlBasicDesc *q, *s;
342-
343-
resp.resize(query.descCount());
344-
345-
if (!sorted) {
346-
for (int i=0; i<query.descCount(); ++i)
347-
for (auto & elm : descs)
348-
if (elm.covers(query[i])){
349-
*p = query[i];
350-
new_elm.copyMeta(elm);
351-
resp.descs[i]=new_elm;
352-
count++;
353-
break;
354-
}
355-
356-
if (query.descCount()==count) {
357-
return NIXL_SUCCESS;
358-
} else {
359-
resp.clear();
360-
return NIXL_ERR_UNKNOWN;
361-
}
362328
} else {
363-
if (q_sorted) {
364-
size = (int) descs.size();
365-
s_index = 0;
366-
q_index = 0;
367-
368-
while (q_index<query.descCount()){
369-
s = &descs[s_index];
370-
q = &query[q_index];
371-
if ((*s).covers(*q)) {
372-
*p = *q;
373-
new_elm.copyMeta(descs[s_index]); // needs const nixlBasicDesc&
374-
resp.descs[q_index] = new_elm;
375-
q_index++;
376-
} else {
377-
s_index++;
378-
// if (*q < descs[s_index]) ||
379-
if (s_index==size) {
380-
resp.clear();
381-
return NIXL_ERR_UNKNOWN;
329+
if ((type != query.getType()) || (type != resp.type))
330+
return NIXL_ERR_INVALID_PARAM;
331+
332+
// 1-to-1 mapping cannot hold
333+
if (query.isSorted() != resp.sorted)
334+
return NIXL_ERR_INVALID_PARAM;
335+
336+
T new_elm;
337+
nixlBasicDesc *p = &new_elm;
338+
int count = 0, last_found = 0;
339+
int s_index, q_index, size;
340+
bool found, q_sorted = query.isSorted();
341+
const nixlBasicDesc *q, *s;
342+
343+
resp.resize(query.descCount());
344+
345+
if (!sorted) {
346+
for (int i=0; i<query.descCount(); ++i)
347+
for (auto & elm : descs)
348+
if (elm.covers(query[i])){
349+
*p = query[i];
350+
new_elm.copyMeta(elm);
351+
resp.descs[i]=new_elm;
352+
count++;
353+
break;
382354
}
383-
}
384-
}
385-
386-
resp.sorted = true; // Should be redundant
387-
return NIXL_SUCCESS;
388355

356+
if (query.descCount()==count) {
357+
return NIXL_SUCCESS;
358+
} else {
359+
resp.clear();
360+
return NIXL_ERR_UNKNOWN;
361+
}
389362
} else {
390-
for (int i=0; i<query.descCount(); ++i) {
391-
found = false;
392-
q = &query[i];
393-
auto itr = std::lower_bound(descs.begin() + last_found,
394-
descs.end(), *q);
395-
396-
// Same start address case
397-
if (itr != descs.end()){
398-
if ((*itr).covers(*q)) {
399-
found = true;
363+
if (q_sorted) {
364+
size = (int) descs.size();
365+
s_index = 0;
366+
q_index = 0;
367+
368+
while (q_index<query.descCount()){
369+
s = &descs[s_index];
370+
q = &query[q_index];
371+
if ((*s).covers(*q)) {
372+
*p = *q;
373+
new_elm.copyMeta(descs[s_index]); // needs const nixlBasicDesc&
374+
resp.descs[q_index] = new_elm;
375+
q_index++;
376+
} else {
377+
s_index++;
378+
// if (*q < descs[s_index]) ||
379+
if (s_index==size) {
380+
resp.clear();
381+
return NIXL_ERR_UNKNOWN;
382+
}
400383
}
401384
}
402385

403-
// query starts starts later, try previous entry
404-
if ((!found) && (itr != descs.begin())){
405-
itr = std::prev(itr , 1);
406-
if ((*itr).covers(*q)) {
407-
found = true;
386+
resp.sorted = true; // Should be redundant
387+
return NIXL_SUCCESS;
388+
389+
} else {
390+
for (int i=0; i<query.descCount(); ++i) {
391+
found = false;
392+
q = &query[i];
393+
auto itr = std::lower_bound(descs.begin() + last_found,
394+
descs.end(), *q);
395+
396+
// Same start address case
397+
if (itr != descs.end()){
398+
if ((*itr).covers(*q)) {
399+
found = true;
400+
}
401+
}
402+
403+
// query starts starts later, try previous entry
404+
if ((!found) && (itr != descs.begin())){
405+
itr = std::prev(itr , 1);
406+
if ((*itr).covers(*q)) {
407+
found = true;
408+
}
408409
}
409-
}
410410

411-
if (found) {
412-
*p = *q;
413-
new_elm.copyMeta(*itr);
414-
resp.descs[i] = new_elm;
415-
} else {
416-
resp.clear();
417-
return NIXL_ERR_UNKNOWN;
411+
if (found) {
412+
*p = *q;
413+
new_elm.copyMeta(*itr);
414+
resp.descs[i] = new_elm;
415+
} else {
416+
resp.clear();
417+
return NIXL_ERR_UNKNOWN;
418+
}
418419
}
420+
resp.sorted = query.isSorted(); // Update as resize resets it
421+
return NIXL_SUCCESS;
419422
}
420-
resp.sorted = query.isSorted(); // Update as resize resets it
421-
return NIXL_SUCCESS;
422423
}
423424
}
424425
}

‎src/plugins/ucx/ucx_backend.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,7 @@ void nixlUcxEngine::vramInitCtx()
152152
cudaCtx = new nixlUcxCudaCtx;
153153
}
154154

155-
int nixlUcxEngine::vramUpdateCtx(void *address, uint32_t devId, bool &restart_reqd)
155+
int nixlUcxEngine::vramUpdateCtx(void *address, uint64_t devId, bool &restart_reqd)
156156
{
157157
int ret;
158158
bool was_updated;

‎src/plugins/ucx/ucx_backend.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ class nixlUcxEngine : public nixlBackendEngine {
142142

143143
void vramInitCtx();
144144
void vramFiniCtx();
145-
int vramUpdateCtx(void *address, uint32_t devId, bool &restart_reqd);
145+
int vramUpdateCtx(void *address, uint64_t devId, bool &restart_reqd);
146146
int vramApplyCtx();
147147

148148
// Threading infrastructure

‎src/plugins/ucx_mo/ucx_mo_backend.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ nixlUcxMoEngine::getEngCnt()
7575
}
7676

7777
int32_t
78-
nixlUcxMoEngine::getEngIdx(nixl_mem_t type, uint32_t devId)
78+
nixlUcxMoEngine::getEngIdx(nixl_mem_t type, uint64_t devId)
7979
{
8080
switch (type) {
8181
case VRAM_SEG:

‎src/plugins/ucx_mo/ucx_mo_backend.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ class nixlUcxMoEngine : public nixlBackendEngine {
128128
uint32_t _gpuCnt;
129129
int setEngCnt(uint32_t host_engines);
130130
uint32_t getEngCnt();
131-
int32_t getEngIdx(nixl_mem_t type, uint32_t devId);
131+
int32_t getEngIdx(nixl_mem_t type, uint64_t devId);
132132
std::string getEngName(const std::string &baseName, uint32_t eidx);
133133
std::string getEngBase(const std::string &engName);
134134
bool pthrOn;

‎test/nixl/desc_example.cpp‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,6 @@ int main()
8181
nixlBasicDesc buff3 (buff2);
8282
nixlBasicDesc buff4;
8383
buff4 = buff1;
84-
buff4.copyMeta (buff2);
8584
nixlBasicDesc buff5 (1980,21,3);
8685
nixlBasicDesc buff6 (1010,30,4);
8786
nixlBasicDesc buff7 (1010,30,0);

0 commit comments

Comments
 (0)