diff --git a/benchmark/nixlbench/src/utils/utils.cpp b/benchmark/nixlbench/src/utils/utils.cpp index 0801ae94b4..32942ea0b8 100644 --- a/benchmark/nixlbench/src/utils/utils.cpp +++ b/benchmark/nixlbench/src/utils/utils.cpp @@ -88,6 +88,9 @@ NB_ARG_BOOL(recreate_xfer, false, "Recreate xfer each iteration (default: false for all backends, true for GUSLI)"); NB_ARG_BOOL(reregister_mem, false, "Register and deregister memory on every iteration"); +NB_ARG_BOOL(prepared_xfer, + false, + "Use prepared transfer API (prepare+make), incompatible with reregister_mem"); NB_ARG_INT32(pipeline_depth, 1, "Number of transfer requests in flight simultaneously"); NB_ARG_INT32(large_blk_iter_ftr, 16, @@ -283,6 +286,7 @@ std::string xferBenchConfig::filepath = ""; std::string xferBenchConfig::filenames = ""; bool xferBenchConfig::storage_enable_direct = false; bool xferBenchConfig::reregister_mem = false; +bool xferBenchConfig::prepared_xfer = false; int xferBenchConfig::pipeline_depth = 1; long xferBenchConfig::page_size = sysconf(_SC_PAGESIZE); std::string xferBenchConfig::obj_access_key = ""; @@ -510,6 +514,7 @@ xferBenchConfig::loadParams(void) { storage_enable_direct = NB_ARG(storage_enable_direct); recreate_xfer = NB_ARG(recreate_xfer); reregister_mem = NB_ARG(reregister_mem); + prepared_xfer = NB_ARG(prepared_xfer); pipeline_depth = NB_ARG(pipeline_depth); if (pipeline_depth < 1) { std::cerr << "pipeline_depth must be >= 1" << std::endl; @@ -532,6 +537,12 @@ xferBenchConfig::loadParams(void) { << " Setting recreate_xfer to true." << std::endl; recreate_xfer = true; } + if (prepared_xfer && reregister_mem) { + std::cerr << "prepared_xfer is incompatible with reregister_mem: the prepared " + "descriptor list handles pin the registration." + << std::endl; + return -1; + } // Validate runtime configuration if (!isStorageBackend()) { @@ -678,6 +689,8 @@ xferBenchConfig::printConfig() { std::to_string(recreate_xfer)); printOption("Re-register memory each iteration (--reregister_mem=[0,1])", std::to_string(reregister_mem)); + printOption("Prepared xfer (prep+make) (--prepared_xfer=[0,1])", + std::to_string(prepared_xfer)); printOption("Pipeline depth (--pipeline_depth=N)", std::to_string(pipeline_depth)); printOption("Use hugepages (--use_hugepages=[0,1])", std::to_string(use_hugepages)); diff --git a/benchmark/nixlbench/src/utils/utils.h b/benchmark/nixlbench/src/utils/utils.h index 425b923144..a0ed72ece8 100644 --- a/benchmark/nixlbench/src/utils/utils.h +++ b/benchmark/nixlbench/src/utils/utils.h @@ -188,6 +188,7 @@ class xferBenchConfig { static int posix_kernel_queue_size; static bool storage_enable_direct; static bool reregister_mem; + static bool prepared_xfer; static int pipeline_depth; static int gds_batch_pool_size; static int gds_batch_limit; diff --git a/benchmark/nixlbench/src/worker/nixl/nixl_worker.cpp b/benchmark/nixlbench/src/worker/nixl/nixl_worker.cpp index 4a6607b659..425f1cb445 100644 --- a/benchmark/nixlbench/src/worker/nixl/nixl_worker.cpp +++ b/benchmark/nixlbench/src/worker/nixl/nixl_worker.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include "utils/neuron.h" @@ -1474,6 +1475,9 @@ struct slotState { bool in_flight = false; bool registered = false; nixlTime::us_t post_ts = 0; + nixlDlistH *prep_local_dlist = nullptr; + nixlDlistH *prep_remote_dlist = nullptr; + std::vector indices; }; // Register memory (if --reregister_mem) and create the XferReq for a slot @@ -1500,10 +1504,36 @@ prepareSlot(nixlAgent *agent, } if (!slot.req) { - nixl_xfer_dlist_t ld(GET_SEG_TYPE(true)); - nixl_xfer_dlist_t rd(GET_SEG_TYPE(false)); - prepareTransferDescriptors(ld, rd, slot.local_iov, slot.remote_iov); - nixl_status_t rc = agent->createXferReq(op, ld, rd, target, slot.req, ¶ms); + nixl_status_t rc; + if (xferBenchConfig::prepared_xfer) { + if (!slot.prep_local_dlist) { + nixl_xfer_dlist_t ld(GET_SEG_TYPE(true)); + nixl_xfer_dlist_t rd(GET_SEG_TYPE(false)); + prepareTransferDescriptors(ld, rd, slot.local_iov, slot.remote_iov); + rc = agent->prepXferDlist(NIXL_INIT_AGENT, ld, slot.prep_local_dlist, ¶ms); + if (rc != NIXL_SUCCESS) { + return rc; + } + rc = agent->prepXferDlist(target, rd, slot.prep_remote_dlist, ¶ms); + if (rc != NIXL_SUCCESS) { + return rc; + } + slot.indices.resize(ld.descCount()); + std::iota(slot.indices.begin(), slot.indices.end(), 0); + } + rc = agent->makeXferReq(op, + slot.prep_local_dlist, + slot.indices, + slot.prep_remote_dlist, + slot.indices, + slot.req, + ¶ms); + } else { + nixl_xfer_dlist_t ld(GET_SEG_TYPE(true)); + nixl_xfer_dlist_t rd(GET_SEG_TYPE(false)); + prepareTransferDescriptors(ld, rd, slot.local_iov, slot.remote_iov); + rc = agent->createXferReq(op, ld, rd, target, slot.req, ¶ms); + } if (rc != NIXL_SUCCESS) { return rc; } @@ -1536,6 +1566,17 @@ recycleSlot(nixlAgent *agent, nixlBackendH *backend_engine, slotState &slot) { agent->releaseXferReq(slot.req); slot.req = nullptr; } + if (xferBenchConfig::prepared_xfer) { + if (slot.prep_local_dlist) { + agent->releasedDlistH(slot.prep_local_dlist); + slot.prep_local_dlist = nullptr; + } + if (slot.prep_remote_dlist) { + agent->releasedDlistH(slot.prep_remote_dlist); + slot.prep_remote_dlist = nullptr; + } + slot.indices.clear(); + } if (xferBenchConfig::reregister_mem && slot.registered) { nixl_status_t rc = deregisterIterationMem(agent, slot.local_iov, slot.remote_iov, backend_engine); @@ -1555,6 +1596,14 @@ cleanupSlots(nixlAgent *agent, nixlBackendH *backend_engine, std::vectorreleaseXferReq(slot.req); slot.req = nullptr; } + if (slot.prep_local_dlist) { + agent->releasedDlistH(slot.prep_local_dlist); + slot.prep_local_dlist = nullptr; + } + if (slot.prep_remote_dlist) { + agent->releasedDlistH(slot.prep_remote_dlist); + slot.prep_remote_dlist = nullptr; + } if (xferBenchConfig::reregister_mem && slot.registered) { deregisterIterationMem(agent, slot.local_iov, slot.remote_iov, backend_engine); slot.registered = false;