Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 53 additions & 2 deletions src/plugins/libfabric/libfabric_backend.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -962,6 +962,40 @@ nixlLibfabricEngine::estimateXferCost(const nixl_xfer_op_t &operation,
return NIXL_SUCCESS;
}

int
nixlLibfabricEngine::batchingRail(const nixl_meta_dlist_t &local,
int desc_idx,
size_t xfer_base_offset) const {
auto *md = static_cast<nixlLibfabricPrivateMetadata *>(local[desc_idx].metadataP);
if (!md || md->selected_rails_.empty()) {
return -1;
}
// Striped descriptors are split across their rails and posted without FI_MORE, so they
// are not part of the single-rail batching tracked here.
if (rail_manager.usesStriping(local[desc_idx].len, md->selected_rails_.size())) {
return -1;
}
return (int)md->selected_rails_[nixlLibfabricRailManager::railSelectionIndex(
xfer_base_offset, desc_idx, /*batch_write=*/true, md->selected_rails_.size())];
}

bool
nixlLibfabricEngine::useFiMore(int desc_idx,
int rail_id,
const std::vector<int> &last_desc_idx_per_rail,
std::vector<int> &posts_since_flush) const {
if (rail_id < 0) {
return false;
}
if (desc_idx == last_desc_idx_per_rail[rail_id] ||
posts_since_flush[rail_id] == NIXL_LIBFABRIC_FI_MORE_BATCH_SIZE - 1) {
posts_since_flush[rail_id] = 0;
return false;
}
++posts_since_flush[rail_id];
return true;
}

nixl_status_t
nixlLibfabricEngine::postXfer(const nixl_xfer_op_t &operation,
const nixl_meta_dlist_t &local,
Expand Down Expand Up @@ -1031,7 +1065,24 @@ nixlLibfabricEngine::postXfer(const nixl_xfer_op_t &operation,
// Core transfer submission to process each descriptor with direct submission
// Reserve base_offset once per transfer so all descriptors see a stable rail assignment
const size_t xfer_base_offset = rail_manager.reserveBaseOffset();

const bool batch_writes = op_type == nixlLibfabricReq::WRITE;
std::vector<int> last_desc_idx_per_rail(rail_manager.getNumRails(), -1);
std::vector<int> posts_since_flush(rail_manager.getNumRails(), 0);
if (batch_writes) {
for (int desc_idx = 0; desc_idx < desc_count; ++desc_idx) {
const int rail_id = batchingRail(local, desc_idx, xfer_base_offset);
if (rail_id >= 0) {
last_desc_idx_per_rail[rail_id] = desc_idx;
}
}
}

for (int desc_idx = 0; desc_idx < desc_count; ++desc_idx) {
const int rail_id = batch_writes ? batchingRail(local, desc_idx, xfer_base_offset) : -1;
const bool apply_fi_more =
useFiMore(desc_idx, rail_id, last_desc_idx_per_rail, posts_since_flush);

auto *local_md = static_cast<nixlLibfabricPrivateMetadata *>(local[desc_idx].metadataP);
auto *remote_md = static_cast<nixlLibfabricPublicMetadata *>(remote[desc_idx].metadataP);
if (!local_md || !remote_md || !remote_md->conn_) {
Expand Down Expand Up @@ -1081,8 +1132,8 @@ nixlLibfabricEngine::postXfer(const nixl_xfer_op_t &operation,
}, // Completion callback
submitted_count,
desc_idx,
desc_count,
xfer_base_offset);
xfer_base_offset,
apply_fi_more);

if (status != NIXL_SUCCESS) {
NIXL_ERROR << "prepareAndSubmitTransfer failed for descriptor " << desc_idx
Expand Down
13 changes: 13 additions & 0 deletions src/plugins/libfabric/libfabric_backend.h
Original file line number Diff line number Diff line change
Expand Up @@ -291,6 +291,19 @@ class nixlLibfabricEngine : public nixlBackendEngine {
void *buffer,
std::shared_ptr<nixlLibfabricConnection> conn,
nixlBackendMD *&output);
/** Rail a WRITE descriptor posts on, or -1 if it does not participate in FI_MORE batching
* (no metadata, no selected rails, or striped across multiple rails). */
int
batchingRail(const nixl_meta_dlist_t &local, int desc_idx, size_t xfer_base_offset) const;

/** Whether descriptor desc_idx should carry FI_MORE: false (flush) for a rail's last post
* in [start,end) and when the rail's batch reaches NIXL_LIBFABRIC_FI_MORE_BATCH_SIZE.
* Updates posts_since_flush. */
bool
useFiMore(int desc_idx,
int rail_id,
const std::vector<int> &last_desc_idx_per_rail,
std::vector<int> &posts_since_flush) const;

#ifdef HAVE_CUDA
// CUDA context management methods
Expand Down
3 changes: 3 additions & 0 deletions src/utils/libfabric/libfabric_common.h
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,9 @@
#define NIXL_LIBFABRIC_DEFAULT_STRIPING_THRESHOLD (128 * 1024) // 128KB
#define LF_EP_NAME_MAX_LEN 56

// Number of consecutive WRITE descriptors batched onto one rail with FI_MORE before flushing.
#define NIXL_LIBFABRIC_FI_MORE_BATCH_SIZE 16

// Request pool configuration constants
#define NIXL_LIBFABRIC_CONTROL_REQUESTS_PER_RAIL 4096 // SEND/RECV operations (for notifications)
#define NIXL_LIBFABRIC_DATA_REQUESTS_PER_RAIL 1024 // WRITE/READ operations
Expand Down
35 changes: 15 additions & 20 deletions src/utils/libfabric/libfabric_rail_manager.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -353,11 +353,6 @@ nixlLibfabricRailManager::createRails(const std::vector<std::string> &efa_device
return NIXL_SUCCESS;
}

bool
nixlLibfabricRailManager::shouldUseStriping(size_t transfer_size) const {
return transfer_size >= striping_threshold_;
}

size_t
nixlLibfabricRailManager::reserveBaseOffset() {
return round_robin_counter.fetch_add(1);
Expand All @@ -380,8 +375,8 @@ nixlLibfabricRailManager::prepareAndSubmitTransfer(
std::function<void()> completion_callback,
size_t &submitted_count_out,
int desc_idx,
int desc_count,
size_t base_offset) {
size_t base_offset,
bool apply_fi_more) {
// Initialize output parameter
submitted_count_out = 0;

Expand All @@ -391,22 +386,22 @@ nixlLibfabricRailManager::prepareAndSubmitTransfer(
}

// Determine striping strategy
bool use_striping = shouldUseStriping(transfer_size) && selected_rails.size() > 1;
bool use_striping = usesStriping(transfer_size, selected_rails.size());
NIXL_DEBUG << "use_striping=" << use_striping;
if (!use_striping) {
// WRITE: group 16 consecutive descs to same rail for FI_MORE batching.
// READ: per-descriptor round-robin (FI_MORE has no benefit for reads).
constexpr int FI_MORE_BATCH_SIZE = 16;
// WRITE: group NIXL_LIBFABRIC_FI_MORE_BATCH_SIZE consecutive descriptors on one rail
// for FI_MORE batching. READ: per-descriptor round-robin (FI_MORE has no benefit).
// apply_fi_more is precomputed by the caller, which leaves it false for the last post
// on each rail so that every rail's FI_MORE batch is flushed.
const bool batch_write = (op_type == nixlLibfabricReq::WRITE);
const size_t rr_idx =
batch_write ? (base_offset + desc_idx / FI_MORE_BATCH_SIZE) : (base_offset + desc_idx);
const size_t rail_id = selected_rails[rr_idx % selected_rails.size()];
const size_t remote_ep_id =
remote_selected_endpoints[rr_idx % remote_selected_endpoints.size()];
const int pos_in_group = desc_idx % FI_MORE_BATCH_SIZE;
const bool is_last_in_group =
(pos_in_group == FI_MORE_BATCH_SIZE - 1) || (desc_idx == desc_count - 1);
const uint64_t fi_flags = (batch_write && !is_last_in_group) ? FI_MORE : 0;
const size_t rail_sel_idx =
railSelectionIndex(base_offset, desc_idx, batch_write, selected_rails.size());
const size_t rail_id = selected_rails[rail_sel_idx];
// The remote endpoint round-robins over its own count, which can differ from the
// local rail count.
const size_t remote_ep_id = remote_selected_endpoints[railSelectionIndex(
base_offset, desc_idx, batch_write, remote_selected_endpoints.size())];
const uint64_t fi_flags = (batch_write && apply_fi_more) ? FI_MORE : 0;
NIXL_DEBUG << "rail " << rail_id << ", remote_ep_id " << remote_ep_id;
// Allocate request
nixlLibfabricReq *req = rails_[rail_id]->allocateDataRequest(op_type, xfer_id);
Expand Down
30 changes: 24 additions & 6 deletions src/utils/libfabric/libfabric_rail_manager.h
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,17 @@ class nixlLibfabricRailManager {
return rails_.size();
}

/** Round-robin index of a descriptor into a selection array of `count` entries.
* batch_write groups NIXL_LIBFABRIC_FI_MORE_BATCH_SIZE consecutive descriptors per entry.
* The FI_MORE flush precompute and the rail selection must agree, so both call this. */
static size_t
railSelectionIndex(size_t base_offset, int desc_idx, bool batch_write, size_t count) {
const size_t rr_idx = batch_write ?
(base_offset + static_cast<size_t>(desc_idx) / NIXL_LIBFABRIC_FI_MORE_BATCH_SIZE) :
(base_offset + static_cast<size_t>(desc_idx));
return rr_idx % count;
}

// Memory registration management
/** Register memory with topology-aware rail selection based on memory type and location
* @param buffer Memory buffer to register
Expand Down Expand Up @@ -198,8 +209,11 @@ class nixlLibfabricRailManager {
* @param completion_callback Callback for completion notification
* @param submitted_count_out Number of requests successfully submitted
* @param desc_idx Index of current descriptor within the transfer
* @param desc_count Total number of descriptors in the transfer
* @param base_offset Pre-reserved round-robin offset from reserveBaseOffset()
* @param apply_fi_more Precomputed by the caller: true keeps a non-striped WRITE batched
* (posted with FI_MORE); false flushes the rail's batch. The caller passes false for
* a rail's last post and when a batch reaches NIXL_LIBFABRIC_FI_MORE_BATCH_SIZE.
* Defaults to false so an unmarked descriptor flushes safely.
* @return NIXL_SUCCESS on success, error code on failure
*/
nixl_status_t
Expand All @@ -218,19 +232,23 @@ class nixlLibfabricRailManager {
std::function<void()> completion_callback,
size_t &submitted_count_out,
int desc_idx,
int desc_count,
size_t base_offset);
size_t base_offset,
bool apply_fi_more = false);

/** Reserve a base offset for a transfer to ensure stable rail assignment
* across all descriptors in the transfer. Call once per postXfer. */
size_t
reserveBaseOffset();
/** Determine if striping should be used for given transfer size

/** True when a transfer of this size across this many rails is striped (split across the
* rails) rather than posted on a single rail.
* @param transfer_size Size of the transfer in bytes
* @return true if striping should be used, false for round-robin
* @param rail_count Number of rails selected for the transfer
*/
bool
shouldUseStriping(size_t transfer_size) const;
usesStriping(size_t transfer_size, size_t rail_count) const {
return transfer_size >= striping_threshold_ && rail_count > 1;
}

// Control Message APIs
/** Control message types for rail communication */
Expand Down
Loading