From 9dce5cb6615e45cfd71a45fcb6f42cfa5b28dfa9 Mon Sep 17 00:00:00 2001 From: Arun Karthik Date: Mon, 20 Jul 2026 10:44:21 +0000 Subject: [PATCH] libfabric: flush FI_MORE per rail, not per group FI_MORE WRITE batching (#1626) flushes the doorbell positionally (last of each 16-group / last of the transfer), which is only correct when every descriptor in a group is on the same rail. But each descriptor's rail comes from its own buffer's selected_rails, so a transfer spanning buffers with different rail lists (e.g. dynamo's many DRAM registrations) alternates physical rails within a group. The positional flush then rings only one rail's doorbell; the other rail's batch is never submitted and the transfer hangs. nixlbench doesn't hit this: one buffer per device = one rail list. Fix: postXfer precomputes the flush decision per physical rail (walking descriptors backwards, the first rail seen is its last post), and prepareAndSubmitTransfer takes a bool apply_fi_more instead of deciding positionally. Batching is opt-in, so unmarked descriptors (READ, striped, rail-less) flush. Also: pair the remote endpoint with the rail by the same index; hoist FI_MORE_BATCH_SIZE to libfabric_common.h; drop the unused desc_count param. Signed-off-by: Arun Karthik (cherry picked from commit 78ece0c9897f6751052514c7cb37fde6ab7228ec) --- src/plugins/libfabric/libfabric_backend.cpp | 55 ++++++++++++++++++- src/plugins/libfabric/libfabric_backend.h | 13 +++++ src/utils/libfabric/libfabric_common.h | 3 + .../libfabric/libfabric_rail_manager.cpp | 35 +++++------- src/utils/libfabric/libfabric_rail_manager.h | 30 ++++++++-- 5 files changed, 108 insertions(+), 28 deletions(-) diff --git a/src/plugins/libfabric/libfabric_backend.cpp b/src/plugins/libfabric/libfabric_backend.cpp index 09a3b73a46..6475b4aa5f 100644 --- a/src/plugins/libfabric/libfabric_backend.cpp +++ b/src/plugins/libfabric/libfabric_backend.cpp @@ -962,6 +962,40 @@ nixlLibfabricEngine::estimateXferCost(const nixl_xfer_op_t &operation, return NIXL_SUCCESS; } +int +nixlLibfabricEngine::batchingRail(const nixl_meta_dlist_t &local, + int desc_idx, + size_t xfer_base_offset) const { + auto *md = static_cast(local[desc_idx].metadataP); + if (!md || md->selected_rails_.empty()) { + return -1; + } + // Striped descriptors are split across their rails and posted without FI_MORE, so they + // are not part of the single-rail batching tracked here. + if (rail_manager.usesStriping(local[desc_idx].len, md->selected_rails_.size())) { + return -1; + } + return (int)md->selected_rails_[nixlLibfabricRailManager::railSelectionIndex( + xfer_base_offset, desc_idx, /*batch_write=*/true, md->selected_rails_.size())]; +} + +bool +nixlLibfabricEngine::useFiMore(int desc_idx, + int rail_id, + const std::vector &last_desc_idx_per_rail, + std::vector &posts_since_flush) const { + if (rail_id < 0) { + return false; + } + if (desc_idx == last_desc_idx_per_rail[rail_id] || + posts_since_flush[rail_id] == NIXL_LIBFABRIC_FI_MORE_BATCH_SIZE - 1) { + posts_since_flush[rail_id] = 0; + return false; + } + ++posts_since_flush[rail_id]; + return true; +} + nixl_status_t nixlLibfabricEngine::postXfer(const nixl_xfer_op_t &operation, const nixl_meta_dlist_t &local, @@ -1031,7 +1065,24 @@ nixlLibfabricEngine::postXfer(const nixl_xfer_op_t &operation, // Core transfer submission to process each descriptor with direct submission // Reserve base_offset once per transfer so all descriptors see a stable rail assignment const size_t xfer_base_offset = rail_manager.reserveBaseOffset(); + + const bool batch_writes = op_type == nixlLibfabricReq::WRITE; + std::vector last_desc_idx_per_rail(rail_manager.getNumRails(), -1); + std::vector posts_since_flush(rail_manager.getNumRails(), 0); + if (batch_writes) { + for (int desc_idx = 0; desc_idx < desc_count; ++desc_idx) { + const int rail_id = batchingRail(local, desc_idx, xfer_base_offset); + if (rail_id >= 0) { + last_desc_idx_per_rail[rail_id] = desc_idx; + } + } + } + for (int desc_idx = 0; desc_idx < desc_count; ++desc_idx) { + const int rail_id = batch_writes ? batchingRail(local, desc_idx, xfer_base_offset) : -1; + const bool apply_fi_more = + useFiMore(desc_idx, rail_id, last_desc_idx_per_rail, posts_since_flush); + auto *local_md = static_cast(local[desc_idx].metadataP); auto *remote_md = static_cast(remote[desc_idx].metadataP); if (!local_md || !remote_md || !remote_md->conn_) { @@ -1081,8 +1132,8 @@ nixlLibfabricEngine::postXfer(const nixl_xfer_op_t &operation, }, // Completion callback submitted_count, desc_idx, - desc_count, - xfer_base_offset); + xfer_base_offset, + apply_fi_more); if (status != NIXL_SUCCESS) { NIXL_ERROR << "prepareAndSubmitTransfer failed for descriptor " << desc_idx diff --git a/src/plugins/libfabric/libfabric_backend.h b/src/plugins/libfabric/libfabric_backend.h index dcb88f91e8..9548e6ddaa 100644 --- a/src/plugins/libfabric/libfabric_backend.h +++ b/src/plugins/libfabric/libfabric_backend.h @@ -291,6 +291,19 @@ class nixlLibfabricEngine : public nixlBackendEngine { void *buffer, std::shared_ptr conn, nixlBackendMD *&output); + /** Rail a WRITE descriptor posts on, or -1 if it does not participate in FI_MORE batching + * (no metadata, no selected rails, or striped across multiple rails). */ + int + batchingRail(const nixl_meta_dlist_t &local, int desc_idx, size_t xfer_base_offset) const; + + /** Whether descriptor desc_idx should carry FI_MORE: false (flush) for a rail's last post + * in [start,end) and when the rail's batch reaches NIXL_LIBFABRIC_FI_MORE_BATCH_SIZE. + * Updates posts_since_flush. */ + bool + useFiMore(int desc_idx, + int rail_id, + const std::vector &last_desc_idx_per_rail, + std::vector &posts_since_flush) const; #ifdef HAVE_CUDA // CUDA context management methods diff --git a/src/utils/libfabric/libfabric_common.h b/src/utils/libfabric/libfabric_common.h index bd6110855c..1c35bc9816 100644 --- a/src/utils/libfabric/libfabric_common.h +++ b/src/utils/libfabric/libfabric_common.h @@ -42,6 +42,9 @@ #define NIXL_LIBFABRIC_DEFAULT_STRIPING_THRESHOLD (128 * 1024) // 128KB #define LF_EP_NAME_MAX_LEN 56 +// Number of consecutive WRITE descriptors batched onto one rail with FI_MORE before flushing. +#define NIXL_LIBFABRIC_FI_MORE_BATCH_SIZE 16 + // Request pool configuration constants #define NIXL_LIBFABRIC_CONTROL_REQUESTS_PER_RAIL 4096 // SEND/RECV operations (for notifications) #define NIXL_LIBFABRIC_DATA_REQUESTS_PER_RAIL 1024 // WRITE/READ operations diff --git a/src/utils/libfabric/libfabric_rail_manager.cpp b/src/utils/libfabric/libfabric_rail_manager.cpp index 5312bb9395..94552e2416 100644 --- a/src/utils/libfabric/libfabric_rail_manager.cpp +++ b/src/utils/libfabric/libfabric_rail_manager.cpp @@ -353,11 +353,6 @@ nixlLibfabricRailManager::createRails(const std::vector &efa_device return NIXL_SUCCESS; } -bool -nixlLibfabricRailManager::shouldUseStriping(size_t transfer_size) const { - return transfer_size >= striping_threshold_; -} - size_t nixlLibfabricRailManager::reserveBaseOffset() { return round_robin_counter.fetch_add(1); @@ -380,8 +375,8 @@ nixlLibfabricRailManager::prepareAndSubmitTransfer( std::function completion_callback, size_t &submitted_count_out, int desc_idx, - int desc_count, - size_t base_offset) { + size_t base_offset, + bool apply_fi_more) { // Initialize output parameter submitted_count_out = 0; @@ -391,22 +386,22 @@ nixlLibfabricRailManager::prepareAndSubmitTransfer( } // Determine striping strategy - bool use_striping = shouldUseStriping(transfer_size) && selected_rails.size() > 1; + bool use_striping = usesStriping(transfer_size, selected_rails.size()); NIXL_DEBUG << "use_striping=" << use_striping; if (!use_striping) { - // WRITE: group 16 consecutive descs to same rail for FI_MORE batching. - // READ: per-descriptor round-robin (FI_MORE has no benefit for reads). - constexpr int FI_MORE_BATCH_SIZE = 16; + // WRITE: group NIXL_LIBFABRIC_FI_MORE_BATCH_SIZE consecutive descriptors on one rail + // for FI_MORE batching. READ: per-descriptor round-robin (FI_MORE has no benefit). + // apply_fi_more is precomputed by the caller, which leaves it false for the last post + // on each rail so that every rail's FI_MORE batch is flushed. const bool batch_write = (op_type == nixlLibfabricReq::WRITE); - const size_t rr_idx = - batch_write ? (base_offset + desc_idx / FI_MORE_BATCH_SIZE) : (base_offset + desc_idx); - const size_t rail_id = selected_rails[rr_idx % selected_rails.size()]; - const size_t remote_ep_id = - remote_selected_endpoints[rr_idx % remote_selected_endpoints.size()]; - const int pos_in_group = desc_idx % FI_MORE_BATCH_SIZE; - const bool is_last_in_group = - (pos_in_group == FI_MORE_BATCH_SIZE - 1) || (desc_idx == desc_count - 1); - const uint64_t fi_flags = (batch_write && !is_last_in_group) ? FI_MORE : 0; + const size_t rail_sel_idx = + railSelectionIndex(base_offset, desc_idx, batch_write, selected_rails.size()); + const size_t rail_id = selected_rails[rail_sel_idx]; + // The remote endpoint round-robins over its own count, which can differ from the + // local rail count. + const size_t remote_ep_id = remote_selected_endpoints[railSelectionIndex( + base_offset, desc_idx, batch_write, remote_selected_endpoints.size())]; + const uint64_t fi_flags = (batch_write && apply_fi_more) ? FI_MORE : 0; NIXL_DEBUG << "rail " << rail_id << ", remote_ep_id " << remote_ep_id; // Allocate request nixlLibfabricReq *req = rails_[rail_id]->allocateDataRequest(op_type, xfer_id); diff --git a/src/utils/libfabric/libfabric_rail_manager.h b/src/utils/libfabric/libfabric_rail_manager.h index 10ce041cc7..13c80a7352 100644 --- a/src/utils/libfabric/libfabric_rail_manager.h +++ b/src/utils/libfabric/libfabric_rail_manager.h @@ -132,6 +132,17 @@ class nixlLibfabricRailManager { return rails_.size(); } + /** Round-robin index of a descriptor into a selection array of `count` entries. + * batch_write groups NIXL_LIBFABRIC_FI_MORE_BATCH_SIZE consecutive descriptors per entry. + * The FI_MORE flush precompute and the rail selection must agree, so both call this. */ + static size_t + railSelectionIndex(size_t base_offset, int desc_idx, bool batch_write, size_t count) { + const size_t rr_idx = batch_write ? + (base_offset + static_cast(desc_idx) / NIXL_LIBFABRIC_FI_MORE_BATCH_SIZE) : + (base_offset + static_cast(desc_idx)); + return rr_idx % count; + } + // Memory registration management /** Register memory with topology-aware rail selection based on memory type and location * @param buffer Memory buffer to register @@ -198,8 +209,11 @@ class nixlLibfabricRailManager { * @param completion_callback Callback for completion notification * @param submitted_count_out Number of requests successfully submitted * @param desc_idx Index of current descriptor within the transfer - * @param desc_count Total number of descriptors in the transfer * @param base_offset Pre-reserved round-robin offset from reserveBaseOffset() + * @param apply_fi_more Precomputed by the caller: true keeps a non-striped WRITE batched + * (posted with FI_MORE); false flushes the rail's batch. The caller passes false for + * a rail's last post and when a batch reaches NIXL_LIBFABRIC_FI_MORE_BATCH_SIZE. + * Defaults to false so an unmarked descriptor flushes safely. * @return NIXL_SUCCESS on success, error code on failure */ nixl_status_t @@ -218,19 +232,23 @@ class nixlLibfabricRailManager { std::function completion_callback, size_t &submitted_count_out, int desc_idx, - int desc_count, - size_t base_offset); + size_t base_offset, + bool apply_fi_more = false); /** Reserve a base offset for a transfer to ensure stable rail assignment * across all descriptors in the transfer. Call once per postXfer. */ size_t reserveBaseOffset(); - /** Determine if striping should be used for given transfer size + + /** True when a transfer of this size across this many rails is striped (split across the + * rails) rather than posted on a single rail. * @param transfer_size Size of the transfer in bytes - * @return true if striping should be used, false for round-robin + * @param rail_count Number of rails selected for the transfer */ bool - shouldUseStriping(size_t transfer_size) const; + usesStriping(size_t transfer_size, size_t rail_count) const { + return transfer_size >= striping_threshold_ && rail_count > 1; + } // Control Message APIs /** Control message types for rail communication */