diff --git a/src/ucs/arch/aarch64/cpu.c b/src/ucs/arch/aarch64/cpu.c index 980bb1e818b..ba90b8655bb 100644 --- a/src/ucs/arch/aarch64/cpu.c +++ b/src/ucs/arch/aarch64/cpu.c @@ -13,6 +13,18 @@ #include #include +#if HAVE_DECL_GETAUXVAL +# include + +/* Older userspace headers may not expose Arm64 HWCAP3 definitions. */ +#ifndef AT_HWCAP3 +# define AT_HWCAP3 29 +#endif + +#ifndef HWCAP3_LS64 +# define HWCAP3_LS64 (1UL << 3) +#endif +#endif static void ucs_aarch64_cpuid_from_proc(ucs_aarch64_cpuid_t *cpuid) { @@ -68,4 +80,23 @@ void ucs_aarch64_cpuid(ucs_aarch64_cpuid_t *cpuid) *cpuid = cached_cpuid; } +ucs_cpu_flag_t ucs_arch_get_cpu_flag() +{ + static ucs_cpu_flag_t cpu_flag = UCS_CPU_FLAG_UNKNOWN; + + if (cpu_flag == UCS_CPU_FLAG_UNKNOWN) { + ucs_cpu_flag_t result = 0; + +#if HAVE_DECL_GETAUXVAL + if (getauxval(AT_HWCAP3) & HWCAP3_LS64) { + result |= UCS_CPU_FLAG_LS64; + } +#endif + + cpu_flag = result; + } + + return cpu_flag; +} + #endif diff --git a/src/ucs/arch/aarch64/cpu.h b/src/ucs/arch/aarch64/cpu.h index b590a1452f9..7419df95722 100644 --- a/src/ucs/arch/aarch64/cpu.h +++ b/src/ucs/arch/aarch64/cpu.h @@ -169,10 +169,7 @@ static inline ucs_cpu_model_t ucs_arch_get_cpu_model() return UCS_CPU_MODEL_ARM_AARCH64; } -static inline int ucs_arch_get_cpu_flag() -{ - return UCS_CPU_FLAG_UNKNOWN; -} +ucs_cpu_flag_t ucs_arch_get_cpu_flag(); static inline void ucs_cpu_init() { diff --git a/src/ucs/arch/cpu.h b/src/ucs/arch/cpu.h index 62575f492ba..9a03bfdba02 100644 --- a/src/ucs/arch/cpu.h +++ b/src/ucs/arch/cpu.h @@ -65,7 +65,8 @@ typedef enum ucs_cpu_flag { UCS_CPU_FLAG_SSE41 = UCS_BIT(7), UCS_CPU_FLAG_SSE42 = UCS_BIT(8), UCS_CPU_FLAG_AVX = UCS_BIT(9), - UCS_CPU_FLAG_AVX2 = UCS_BIT(10) + UCS_CPU_FLAG_AVX2 = UCS_BIT(10), + UCS_CPU_FLAG_LS64 = UCS_BIT(11) } ucs_cpu_flag_t; @@ -161,6 +162,25 @@ static inline void ucs_clear_cache(void *start, void *end) } +/** + * Check whether the CPU supports a given feature flag. + * + * Architectures which cannot detect CPU flags report @ref + * UCS_CPU_FLAG_UNKNOWN, which has all bits set, so the result of @ref + * ucs_arch_get_cpu_flag must not be tested by a plain bitwise AND. + * + * @param flag Feature flag to check. + * + * @return Nonzero if the CPU is known to support the flag. + */ +static inline int ucs_cpu_has_flag(ucs_cpu_flag_t flag) +{ + int flags = ucs_arch_get_cpu_flag(); + + return (flags != UCS_CPU_FLAG_UNKNOWN) && (flags & flag); +} + + static inline int ucs_cpu_model_prefer_relaxed_order( ucs_cpu_vendor_t cpu_vendor, ucs_cpu_model_t cpu_model) { diff --git a/src/uct/ib/mlx5/configure.m4 b/src/uct/ib/mlx5/configure.m4 index ed62c325f66..8ae89834d67 100644 --- a/src/uct/ib/mlx5/configure.m4 +++ b/src/uct/ib/mlx5/configure.m4 @@ -8,6 +8,32 @@ # Add IB mlx5 provider support # +AC_CACHE_CHECK([for AArch64 ST64B assembler support], + [ucx_cv_have_aarch64_st64b_asm], + [AC_COMPILE_IFELSE([AC_LANG_SOURCE([[ +#ifndef __aarch64__ +#error "ST64B is only supported on AArch64" +#endif + +void test_st64b(void *dst, void *src) +{ + register void *src_reg asm("x8") = src; + register void *dst_reg asm("x9") = dst; + + asm volatile(".arch_extension ls64\n" + "st64b x8, [x9]" + : + : "r"(src_reg), "r"(dst_reg) + : "memory"); +} +]])], + [ucx_cv_have_aarch64_st64b_asm=yes], + [ucx_cv_have_aarch64_st64b_asm=no])]) + +AS_IF([test "x$ucx_cv_have_aarch64_st64b_asm" = "xyes"], + [AC_DEFINE([HAVE_AARCH64_ST64B_ASM], [1], + [Define to 1 if AArch64 ST64B assembler is supported])]) + AC_CONFIG_FILES([src/uct/ib/mlx5/Makefile src/uct/ib/mlx5/ucx-ib-mlx5.pc]) m4_include([src/uct/ib/mlx5/gdaki/configure.m4]) diff --git a/src/uct/ib/mlx5/dc/dc_mlx5.c b/src/uct/ib/mlx5/dc/dc_mlx5.c index eadf92ccfea..efa79d0bcb5 100644 --- a/src/uct/ib/mlx5/dc/dc_mlx5.c +++ b/src/uct/ib/mlx5/dc/dc_mlx5.c @@ -545,7 +545,8 @@ ucs_status_t uct_dc_mlx5_iface_create_dci(uct_dc_mlx5_iface_t *iface, if (dci->txwq.super.type == UCT_IB_MLX5_OBJ_TYPE_VERBS) { status = uct_ib_mlx5_txwq_init(iface->super.super.super.super.worker, - iface->super.tx.mmio_mode, &dci->txwq, + iface->super.tx.mmio_mode, + iface->super.tx.bf_copy_mode, &dci->txwq, dci->txwq.super.verbs.qp); if (status != UCS_OK) { goto err; diff --git a/src/uct/ib/mlx5/dv/ib_mlx5_dv.c b/src/uct/ib/mlx5/dv/ib_mlx5_dv.c index 19072bb1015..7b55d3ecb1d 100644 --- a/src/uct/ib/mlx5/dv/ib_mlx5_dv.c +++ b/src/uct/ib/mlx5/dv/ib_mlx5_dv.c @@ -155,6 +155,13 @@ ucs_status_t uct_ib_mlx5_devx_create_qp_common(uct_ib_iface_t *iface, goto err; } + if (tx != NULL) { + status = uct_ib_mlx5_txwq_init_bf_copy(tx, attr->bf_copy_mode); + if (status != UCS_OK) { + goto err; + } + } + uar = uct_worker_tl_data_get(iface->super.worker, UCT_IB_MLX5_DEVX_UAR_KEY, uct_ib_mlx5_devx_uar_t, diff --git a/src/uct/ib/mlx5/gdaki/gdaki.c b/src/uct/ib/mlx5/gdaki/gdaki.c index 1cc703cd2af..27799941912 100644 --- a/src/uct/ib/mlx5/gdaki/gdaki.c +++ b/src/uct/ib/mlx5/gdaki/gdaki.c @@ -487,6 +487,7 @@ uct_rc_gdaki_init_channel_chunk(uct_rc_gdaki_iface_t *iface, cq_attr.flags |= UCT_IB_MLX5_CQ_IGNORE_OVERRUN; qp_attr.mmio_mode = UCT_IB_MLX5_MMIO_MODE_DB; + qp_attr.bf_copy_mode = UCT_IB_MLX5_BF_COPY_MODE_GENERIC; qp_attr.super.srq_num = 0; qp_attr.super.max_inl_cqe[UCT_IB_DIR_TX] = 0; uct_ib_mlx5_wq_calc_sizes(&qp_attr); diff --git a/src/uct/ib/mlx5/ib_mlx5.c b/src/uct/ib/mlx5/ib_mlx5.c index b8bba6e9679..8801aebbdc2 100644 --- a/src/uct/ib/mlx5/ib_mlx5.c +++ b/src/uct/ib/mlx5/ib_mlx5.c @@ -33,6 +33,13 @@ static const char *uct_ib_mlx5_mmio_modes[] = { [UCT_IB_MLX5_MMIO_MODE_LAST] = NULL }; +static const char *uct_ib_mlx5_bf_copy_modes[] = { + [UCT_IB_MLX5_BF_COPY_MODE_AUTO] = "auto", + [UCT_IB_MLX5_BF_COPY_MODE_GENERIC] = "generic", + [UCT_IB_MLX5_BF_COPY_MODE_ST64B] = "st64b", + [UCT_IB_MLX5_BF_COPY_MODE_LAST] = NULL +}; + ucs_config_field_t uct_ib_mlx5_iface_config_table[] = { #if HAVE_IBV_DM {"DM_SIZE", "2k", @@ -55,6 +62,15 @@ ucs_config_field_t uct_ib_mlx5_iface_config_table[] = { ucs_offsetof(uct_ib_mlx5_iface_config_t, mmio_mode), UCS_CONFIG_TYPE_ENUM(uct_ib_mlx5_mmio_modes)}, + {"BF_COPY_MODE", "auto", + "How to copy WQE building blocks to BlueFlame MMIO register. One of " + "the following:\n" + " auto - Select best according to runtime CPU capabilities.\n" + " generic - Use portable scalar stores.\n" + " st64b - Use AArch64 ST64B store, fail if LS64 is not supported.", + ucs_offsetof(uct_ib_mlx5_iface_config_t, bf_copy_mode), + UCS_CONFIG_TYPE_ENUM(uct_ib_mlx5_bf_copy_modes)}, + {"AR_ENABLE", "auto", "Enable Adaptive Routing (out of order) feature on SL that supports it.\n" "On RoCE devices, this is done by modifying the QP directly.\n" @@ -783,8 +799,46 @@ uct_ib_mlx5_get_mmio_mode(uct_priv_worker_t *worker, return UCS_OK; } +ucs_status_t +uct_ib_mlx5_txwq_init_bf_copy(uct_ib_mlx5_txwq_t *txwq, + uct_ib_mlx5_bf_copy_mode_t bf_copy_mode) +{ + ucs_assert(bf_copy_mode < UCT_IB_MLX5_BF_COPY_MODE_LAST); + +#if UCT_IB_MLX5_HAVE_ST64B + txwq->bf_copy_mode = UCT_IB_MLX5_BF_COPY_MODE_GENERIC; + if (bf_copy_mode == UCT_IB_MLX5_BF_COPY_MODE_GENERIC) { + return UCS_OK; + } + + if (ucs_cpu_has_flag(UCS_CPU_FLAG_LS64)) { + txwq->bf_copy_mode = UCT_IB_MLX5_BF_COPY_MODE_ST64B; + return UCS_OK; + } + + if (bf_copy_mode == UCT_IB_MLX5_BF_COPY_MODE_ST64B) { + ucs_error("mlx5 BlueFlame ST64B copy was requested but CPU does " + "not report LS64 support"); + return UCS_ERR_UNSUPPORTED; + } +#else + if (bf_copy_mode == UCT_IB_MLX5_BF_COPY_MODE_ST64B) { +#if defined(__aarch64__) + ucs_error("mlx5 BlueFlame ST64B copy was requested but UCX was built " + "without assembler support for ST64B"); +#else + ucs_error("mlx5 BlueFlame ST64B copy is supported only on AArch64"); +#endif + return UCS_ERR_UNSUPPORTED; + } +#endif + + return UCS_OK; +} + ucs_status_t uct_ib_mlx5_txwq_init(uct_priv_worker_t *worker, uct_ib_mlx5_mmio_mode_t cfg_mmio_mode, + uct_ib_mlx5_bf_copy_mode_t bf_copy_mode, uct_ib_mlx5_txwq_t *txwq, struct ibv_qp *verbs_qp) { @@ -818,6 +872,11 @@ ucs_status_t uct_ib_mlx5_txwq_init(uct_priv_worker_t *worker, return status; } + status = uct_ib_mlx5_txwq_init_bf_copy(txwq, bf_copy_mode); + if (status != UCS_OK) { + return status; + } + ucs_debug("tx wq %d bytes [bb=%d, nwqe=%d] mmio_mode %s", qp_info.dv.sq.stride * qp_info.dv.sq.wqe_cnt, qp_info.dv.sq.stride, qp_info.dv.sq.wqe_cnt, diff --git a/src/uct/ib/mlx5/ib_mlx5.h b/src/uct/ib/mlx5/ib_mlx5.h index 9467d939fcd..5f7877db0b3 100644 --- a/src/uct/ib/mlx5/ib_mlx5.h +++ b/src/uct/ib/mlx5/ib_mlx5.h @@ -92,6 +92,13 @@ # define UCT_IB_MLX5_UAR_ALLOC_TYPE_NC (1U << 31) #endif +/* BlueFlame copy by AArch64 ST64B is compiled in */ +#if defined(__aarch64__) && HAVE_AARCH64_ST64B_ASM +# define UCT_IB_MLX5_HAVE_ST64B 1 +#else +# define UCT_IB_MLX5_HAVE_ST64B 0 +#endif + #define UCT_IB_MLX5_OPMOD_EXT_ATOMIC(_log_arg_size) \ ((8) | ((_log_arg_size) - 2)) @@ -470,6 +477,14 @@ typedef enum { } uct_ib_mlx5_mmio_mode_t; +typedef enum { + UCT_IB_MLX5_BF_COPY_MODE_AUTO, + UCT_IB_MLX5_BF_COPY_MODE_GENERIC, + UCT_IB_MLX5_BF_COPY_MODE_ST64B, + UCT_IB_MLX5_BF_COPY_MODE_LAST +} uct_ib_mlx5_bf_copy_mode_t; + + typedef struct uct_ib_mlx5_iface_config { #if HAVE_IBV_DM struct { @@ -477,9 +492,10 @@ typedef struct uct_ib_mlx5_iface_config { unsigned count; } dm; #endif - uct_ib_mlx5_mmio_mode_t mmio_mode; - ucs_ternary_auto_value_t ar_enable; - int cqe_zip_enable[UCT_IB_DIR_LAST]; + uct_ib_mlx5_mmio_mode_t mmio_mode; + uct_ib_mlx5_bf_copy_mode_t bf_copy_mode; + ucs_ternary_auto_value_t ar_enable; + int cqe_zip_enable[UCT_IB_DIR_LAST]; } uct_ib_mlx5_iface_config_t; @@ -645,6 +661,7 @@ typedef struct uct_ib_mlx5_res_domain { typedef struct uct_ib_mlx5_qp_attr { uct_ib_qp_attr_t super; uct_ib_mlx5_mmio_mode_t mmio_mode; + uct_ib_mlx5_bf_copy_mode_t bf_copy_mode; uint32_t uidx; int full_handshake; int rdma_wr_disabled; @@ -701,6 +718,9 @@ typedef struct uct_ib_mlx5_txwq { uint16_t ft_ci; /* First BB index of last ft completed WQE */ uint16_t path_mtu_mask; /* Path MTU in bytes - 1 */ uint8_t path_mtu_shift; /* log2(path MTU in bytes) */ +#if UCT_IB_MLX5_HAVE_ST64B + uint8_t bf_copy_mode; +#endif #if UCS_ENABLE_ASSERT uint8_t flags; /* Debug flags */ #endif @@ -914,7 +934,13 @@ uct_ib_mlx5_get_mmio_mode(uct_priv_worker_t *worker, */ ucs_status_t uct_ib_mlx5_txwq_init(uct_priv_worker_t *worker, uct_ib_mlx5_mmio_mode_t cfg_mmio_mode, - uct_ib_mlx5_txwq_t *txwq, struct ibv_qp *verbs_qp); + uct_ib_mlx5_bf_copy_mode_t bf_copy_mode, + uct_ib_mlx5_txwq_t *txwq, + struct ibv_qp *verbs_qp); + +ucs_status_t +uct_ib_mlx5_txwq_init_bf_copy(uct_ib_mlx5_txwq_t *txwq, + uct_ib_mlx5_bf_copy_mode_t bf_copy_mode); /* Get pointer to a WQE by producer index */ void *uct_ib_mlx5_txwq_get_wqe(const uct_ib_mlx5_txwq_t *txwq, uint16_t pi); diff --git a/src/uct/ib/mlx5/ib_mlx5.inl b/src/uct/ib/mlx5/ib_mlx5.inl index a7eeeb13251..60287044bd4 100644 --- a/src/uct/ib/mlx5/ib_mlx5.inl +++ b/src/uct/ib/mlx5/ib_mlx5.inl @@ -580,21 +580,55 @@ static UCS_F_ALWAYS_INLINE void uct_ib_mlx5_bf_copy_bb(void * restrict dst, #endif } +#if UCT_IB_MLX5_HAVE_ST64B +static UCS_F_ALWAYS_INLINE void +uct_ib_mlx5_bf_copy_bb_st64b(void *restrict dst, void *restrict src) +{ + ucs_assert(((uintptr_t)src % MLX5_SEND_WQE_BB) == 0); + ucs_assert(((uintptr_t)dst % MLX5_SEND_WQE_BB) == 0); + + asm volatile(".arch_extension ls64\n" + "ldp x8, x9, [%[src], #0]\n" + "ldp x10, x11, [%[src], #16]\n" + "ldp x12, x13, [%[src], #32]\n" + "ldp x14, x15, [%[src], #48]\n" + "st64b x8, [%[dst]]" + : + : [src] "r"(src), [dst] "r"(dst) + : "x8", "x9", "x10", "x11", "x12", "x13", "x14", + "x15", "memory"); +} +#endif + +#define UCT_IB_MLX5_BF_COPY(_dst, _src, _num_bb, _wq, _copy_bb) \ + ({ \ + uint16_t _bb_num = (_num_bb); \ + uint16_t _n; \ + \ + for (_n = 0; _n < _bb_num; ++_n) { \ + _copy_bb(_dst, _src); \ + (_dst) = UCS_PTR_BYTE_OFFSET(_dst, MLX5_SEND_WQE_BB); \ + (_src) = UCS_PTR_BYTE_OFFSET(_src, MLX5_SEND_WQE_BB); \ + if (ucs_unlikely((_src) == (_wq)->qend)) { \ + (_src) = (_wq)->qstart; \ + } \ + } \ + \ + (_src); \ + }) + static UCS_F_ALWAYS_INLINE void *uct_ib_mlx5_bf_copy(void *dst, void *src, uint16_t num_bb, const uct_ib_mlx5_txwq_t *wq) { - uint16_t n; - - for (n = 0; n < num_bb; ++n) { - uct_ib_mlx5_bf_copy_bb(dst, src); - dst = UCS_PTR_BYTE_OFFSET(dst, MLX5_SEND_WQE_BB); - src = UCS_PTR_BYTE_OFFSET(src, MLX5_SEND_WQE_BB); - if (ucs_unlikely(src == wq->qend)) { - src = wq->qstart; - } +#if UCT_IB_MLX5_HAVE_ST64B + if (wq->bf_copy_mode == UCT_IB_MLX5_BF_COPY_MODE_ST64B) { + return UCT_IB_MLX5_BF_COPY(dst, src, num_bb, wq, + uct_ib_mlx5_bf_copy_bb_st64b); } - return src; +#endif + + return UCT_IB_MLX5_BF_COPY(dst, src, num_bb, wq, uct_ib_mlx5_bf_copy_bb); } static UCS_F_ALWAYS_INLINE void * diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_common.c b/src/uct/ib/mlx5/rc/rc_mlx5_common.c index 0b29cba8b95..c4c782efaa9 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_common.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_common.c @@ -289,6 +289,7 @@ ucs_status_t uct_rc_mlx5_devx_create_cmd_qp(uct_rc_mlx5_iface_common_t *iface) attr.super.srq_num = iface->rx.srq.srq_num; attr.super.port = dev->first_port; attr.mmio_mode = iface->tx.mmio_mode; + attr.bf_copy_mode = iface->tx.bf_copy_mode; status = uct_ib_mlx5_devx_create_qp(&iface->super.super, &iface->cq[UCT_IB_DIR_RX], &iface->cq[UCT_IB_DIR_RX], @@ -414,6 +415,7 @@ uct_rc_mlx5_get_cmd_qp(uct_rc_mlx5_iface_common_t *iface) iface->tm.cmd_wq.super.super.qp_num = qp->qp_num; return uct_ib_mlx5_txwq_init(iface->super.super.super.worker, iface->tx.mmio_mode, + iface->tx.bf_copy_mode, &iface->tm.cmd_wq.super, qp); } #endif @@ -539,6 +541,7 @@ void uct_rc_mlx5_iface_fill_attr(uct_rc_mlx5_iface_common_t *iface, break; } + qp_attr->bf_copy_mode = iface->tx.bf_copy_mode; qp_attr->mmio_mode = iface->tx.mmio_mode; qp_attr->super.srq_num = srq->srq_num; } diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_common.h b/src/uct/ib/mlx5/rc/rc_mlx5_common.h index c6283abdc59..b6404cf6bfe 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_common.h +++ b/src/uct/ib/mlx5/rc/rc_mlx5_common.h @@ -347,6 +347,7 @@ typedef struct uct_rc_mlx5_iface_common { struct { ucs_mpool_t atomic_desc_mp; uct_ib_mlx5_mmio_mode_t mmio_mode; + uct_ib_mlx5_bf_copy_mode_t bf_copy_mode; uint16_t bb_max; /* limit number of outstanding WQE BBs */ } tx; struct { diff --git a/src/uct/ib/mlx5/rc/rc_mlx5_iface.c b/src/uct/ib/mlx5/rc/rc_mlx5_iface.c index 8f2c00d3466..b9c0680fefb 100644 --- a/src/uct/ib/mlx5/rc/rc_mlx5_iface.c +++ b/src/uct/ib/mlx5/rc/rc_mlx5_iface.c @@ -350,7 +350,8 @@ ucs_status_t uct_rc_mlx5_iface_create_qp(uct_rc_mlx5_iface_common_t *iface, if (attr->super.cap.max_send_wr) { status = uct_ib_mlx5_txwq_init(iface->super.super.super.worker, - iface->tx.mmio_mode, txwq, + iface->tx.mmio_mode, + iface->tx.bf_copy_mode, txwq, qp->verbs.qp); if (status != UCS_OK) { ucs_error("Failed to get mlx5 QP information"); @@ -865,6 +866,7 @@ UCS_CLASS_INIT_FUNC(uct_rc_mlx5_iface_common_t, uct_iface_ops_t *tl_ops, dev = uct_ib_iface_device(&self->super.super); self->tx.mmio_mode = mlx5_config->super.mmio_mode; + self->tx.bf_copy_mode = mlx5_config->super.bf_copy_mode; self->tx.bb_max = ucs_min(mlx5_config->tx_max_bb, UINT16_MAX); self->tm.am_desc.super.cb = uct_rc_mlx5_release_desc; diff --git a/src/uct/ib/mlx5/ud/ud_mlx5.c b/src/uct/ib/mlx5/ud/ud_mlx5.c index 543a696b99a..2421a584e77 100644 --- a/src/uct/ib/mlx5/ud/ud_mlx5.c +++ b/src/uct/ib/mlx5/ud/ud_mlx5.c @@ -828,7 +828,8 @@ static ucs_status_t uct_ud_mlx5_iface_create_qp(uct_ib_iface_t *ib_iface, } status = uct_ib_mlx5_txwq_init(iface->super.super.super.worker, - iface->tx.mmio_mode, &iface->tx.wq, + iface->tx.mmio_mode, + iface->tx.bf_copy_mode, &iface->tx.wq, qp->verbs.qp); if (status != UCS_OK) { goto err_destroy_qp; @@ -1017,8 +1018,9 @@ static UCS_CLASS_INIT_FUNC(uct_ud_mlx5_iface_t, uct_md_h tl_md, uct_ib_mlx5_parse_cqe_zipping(md, &config->mlx5_common, &init_attr); - self->tx.mmio_mode = config->mlx5_common.mmio_mode; - self->tx.wq.super.type = UCT_IB_MLX5_OBJ_TYPE_LAST; + self->tx.mmio_mode = config->mlx5_common.mmio_mode; + self->tx.bf_copy_mode = config->mlx5_common.bf_copy_mode; + self->tx.wq.super.type = UCT_IB_MLX5_OBJ_TYPE_LAST; UCS_CLASS_CALL_SUPER_INIT(uct_ud_iface_t, &uct_ud_mlx5_iface_ops, &uct_ud_mlx5_iface_tl_ops, tl_md, worker, params, diff --git a/src/uct/ib/mlx5/ud/ud_mlx5.h b/src/uct/ib/mlx5/ud/ud_mlx5.h index 4649b35048d..b05975b2779 100644 --- a/src/uct/ib/mlx5/ud/ud_mlx5.h +++ b/src/uct/ib/mlx5/ud/ud_mlx5.h @@ -37,6 +37,7 @@ typedef struct { struct { uct_ib_mlx5_txwq_t wq; uct_ib_mlx5_mmio_mode_t mmio_mode; + uct_ib_mlx5_bf_copy_mode_t bf_copy_mode; } tx; struct { uct_ib_mlx5_rxwq_t wq; @@ -60,4 +61,3 @@ uct_ud_mlx5_tx_moderation(uct_ud_mlx5_iface_t *iface, uint8_t ce_se) } #endif - diff --git a/test/gtest/uct/ib/test_rc.cc b/test/gtest/uct/ib/test_rc.cc index afb7d52a00e..bc042047067 100644 --- a/test/gtest/uct/ib/test_rc.cc +++ b/test/gtest/uct/ib/test_rc.cc @@ -9,6 +9,10 @@ #include #include +extern "C" { +#include +} + #ifdef HAVE_MLX5_DV extern "C" { #include @@ -311,6 +315,156 @@ UCS_TEST_SKIP_COND_P(test_rc_max_wr, send_limit, UCT_INSTANTIATE_RC_TEST_CASE(test_rc_max_wr) +#ifdef HAVE_MLX5_DV +class test_rc_mlx5_bf_copy : public test_rc { +public: + virtual void init() + { + test_rc::init(); + + if (!check_bf_caps()) { + cleanup(); + UCS_TEST_SKIP_R("write-combining UAR is not supported"); + } + + m_rx_count = 0; + m_rx_errors = 0; + uct_iface_set_am_handler(m_e2->iface(), AM_ID, am_handler, this, 0); + } + + /* BlueFlame writes the full WQE to the MMIO register, which is only valid + * when the UAR is write-combining. */ + bool check_bf_caps() + { + FOR_EACH_ENTITY(entity) { + uct_ib_mlx5_md_t *md = ucs_derived_of((*entity)->md(), + uct_ib_mlx5_md_t); + if (!(md->flags & UCT_IB_MLX5_MD_FLAG_UAR_USE_WC)) { + return false; + } + } + return true; + } + +protected: + enum { + AM_ID = 0, + NUM_ITERS = 512, + MAX_PAYLOAD = UCT_IB_MLX5_BF_REG_SIZE + }; + + /* Post AM shorts of varying size, so that single-BB and multi-BB BlueFlame + * copies and the send queue wrap-around are all exercised. */ + void test_bf_copy() + { + uct_rc_mlx5_base_ep_t *ep; + uint8_t payload[MAX_PAYLOAD]; + size_t max_len, len, i; + ucs_status_t status; + unsigned iter; + + ep = ucs_derived_of(m_e1->ep(0), uct_rc_mlx5_base_ep_t); + ASSERT_TRUE(ep->tx.wq.reg != NULL); + ASSERT_EQ(UCT_IB_MLX5_MMIO_MODE_BF_POST, ep->tx.wq.reg->mode); + + max_len = m_e1->iface_attr().cap.am.max_short - sizeof(uint64_t); + if (max_len > sizeof(payload)) { + max_len = sizeof(payload); + } + + for (iter = 0; iter < NUM_ITERS; ++iter) { + len = iter % (max_len + 1); + for (i = 0; i < len; ++i) { + payload[i] = iter + i; + } + + do { + status = uct_ep_am_short(m_e1->ep(0), AM_ID, iter, payload, + len); + if (status == UCS_ERR_NO_RESOURCE) { + progress_loop(); + } + } while (status == UCS_ERR_NO_RESOURCE); + + ASSERT_UCS_OK(status); + } + + flush(); + wait_for_value(&m_rx_count, (unsigned)NUM_ITERS, true); + EXPECT_EQ((unsigned)NUM_ITERS, m_rx_count); + EXPECT_EQ(0u, m_rx_errors); + } + + static ucs_status_t am_handler(void *arg, void *data, size_t length, + unsigned flags) + { + test_rc_mlx5_bf_copy *self = reinterpret_cast( + arg); + const uint8_t *payload; + uint64_t hdr; + size_t i; + + if (length < sizeof(hdr)) { + ++self->m_rx_errors; + return UCS_OK; + } + + memcpy(&hdr, data, sizeof(hdr)); + payload = reinterpret_cast(data) + sizeof(hdr); + + for (i = 0; i < (length - sizeof(hdr)); ++i) { + if (payload[i] != (uint8_t)(hdr + i)) { + ++self->m_rx_errors; + break; + } + } + + ++self->m_rx_count; + return UCS_OK; + } + + volatile unsigned m_rx_count; + volatile unsigned m_rx_errors; +}; + +UCS_TEST_SKIP_COND_P(test_rc_mlx5_bf_copy, generic, + !check_caps(UCT_IFACE_FLAG_AM_SHORT), + "RC_MLX5_MMIO_MODE=bf_post", + "RC_MLX5_BF_COPY_MODE=generic", + "RC_FC_ENABLE?=n") +{ + test_bf_copy(); +} + +_UCT_INSTANTIATE_TEST_CASE(test_rc_mlx5_bf_copy, rc_mlx5) + +#if UCT_IB_MLX5_HAVE_ST64B +class test_rc_mlx5_bf_copy_st64b : public test_rc_mlx5_bf_copy { +public: + virtual void init() + { + if (!ucs_cpu_has_flag(UCS_CPU_FLAG_LS64)) { + UCS_TEST_SKIP_R("CPU does not report LS64 support"); + } + + test_rc_mlx5_bf_copy::init(); + } +}; + +UCS_TEST_SKIP_COND_P(test_rc_mlx5_bf_copy_st64b, st64b, + !check_caps(UCT_IFACE_FLAG_AM_SHORT), + "RC_MLX5_MMIO_MODE=bf_post", + "RC_MLX5_BF_COPY_MODE=st64b", + "RC_FC_ENABLE?=n") +{ + test_bf_copy(); +} + +_UCT_INSTANTIATE_TEST_CASE(test_rc_mlx5_bf_copy_st64b, rc_mlx5) +#endif + +#endif + class test_rc_iface_flush_remote : public uct_test { protected: