Skip to content

UCT/IB/MLX5: DV UAR alloc type NC support #6082

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yosefe merged 1 commit into from
Jan 5, 2021
Merged

UCT/IB/MLX5: DV UAR alloc type NC support #6082

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions buildlib/azure-pipelines-pr.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,11 @@ stages:
name: new
demands: ucx_new -equals yes
test_perf: 1
- template: tests.yml
parameters:
name: hwi
demands: ucx_hwi -equals yes
test_perf: 0

- stage: io_demo
dependsOn: [Codestyle]
Expand Down
4 changes: 3 additions & 1 deletion src/uct/ib/configure.m4
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,9 @@ AS_IF([test "x$with_ib" = "xyes"],
mlx5dv_is_supported,
mlx5dv_devx_subscribe_devx_event,
MLX5DV_CQ_INIT_ATTR_MASK_CQE_SIZE,
MLX5DV_QP_CREATE_ALLOW_SCATTER_TO_CQE],
MLX5DV_QP_CREATE_ALLOW_SCATTER_TO_CQE,
MLX5DV_UAR_ALLOC_TYPE_BF,
MLX5DV_UAR_ALLOC_TYPE_NC],
[], [], [[#include <infiniband/mlx5dv.h>]])
AC_CHECK_MEMBERS([struct mlx5dv_cq.cq_uar],
[], [], [[#include <infiniband/mlx5dv.h>]])
Expand Down
49 changes: 45 additions & 4 deletions src/uct/ib/mlx5/ib_mlx5.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -400,15 +400,56 @@ int uct_ib_mlx5_devx_uar_cmp(uct_ib_mlx5_devx_uar_t *uar,
return uar->ctx == md->super.dev.ibv_context;
}

#if HAVE_DEVX
static ucs_status_t
uct_ib_mlx5_devx_alloc_uar(uct_ib_mlx5_md_t *md, unsigned flags, int log_level,
char *title, char *fallback,
struct mlx5dv_devx_uar **uar_p)
{
struct mlx5dv_devx_uar *uar;
char buf[512];

uar = mlx5dv_devx_alloc_uar(md->super.dev.ibv_context, flags);
if (uar == NULL) {
sprintf(buf, "mlx5dv_devx_alloc_uar(device=%s, flags=0x%x(%s)) "
"failed: %m", uct_ib_device_name(&md->super.dev), flags, title);
if (fallback == NULL) {
ucs_log(log_level, "%s", buf);
} else {
ucs_log(log_level, "%s, fallback to %s", buf, fallback);
}

Comment on lines +414 to +421

@yosefe yosefe Jan 3, 2021

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what was wrong with the previous version?

@Artemy-Mellanox Artemy-Mellanox Jan 3, 2021

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

coverity

return UCS_ERR_NO_MEMORY;
}

*uar_p = uar;
return UCS_OK;
}
#endif

ucs_status_t uct_ib_mlx5_devx_uar_init(uct_ib_mlx5_devx_uar_t *uar,
uct_ib_mlx5_md_t *md,
uct_ib_mlx5_mmio_mode_t mmio_mode)
{
#if HAVE_DEVX
uar->uar = mlx5dv_devx_alloc_uar(md->super.dev.ibv_context, 0);
if (uar->uar == NULL) {
ucs_error("mlx5dv_devx_alloc_uar() failed: %m");
return UCS_ERR_NO_MEMORY;
ucs_status_t status;

#if HAVE_DECL_MLX5DV_UAR_ALLOC_TYPE_NC
status = uct_ib_mlx5_devx_alloc_uar(md, UCT_IB_MLX5_UAR_ALLOC_TYPE_WC,
UCS_LOG_LEVEL_DEBUG, "WC", "NC",
&uar->uar);
if (status != UCS_OK) {
status = uct_ib_mlx5_devx_alloc_uar(md, UCT_IB_MLX5_UAR_ALLOC_TYPE_NC,
UCS_LOG_LEVEL_ERROR, "NC", NULL,
&uar->uar);
}
#else
status = uct_ib_mlx5_devx_alloc_uar(md, UCT_IB_MLX5_UAR_ALLOC_TYPE_WC,
UCS_LOG_LEVEL_ERROR, "WC", NULL,
&uar->uar);
#endif
if (status != UCS_OK) {
return status;
}

uar->super.addr.ptr = uar->uar->reg_addr;
Expand Down
9 changes: 9 additions & 0 deletions src/uct/ib/mlx5/ib_mlx5.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,15 @@
#define UCT_IB_MLX5_MP_RQ_LAST_MSG_FLAG UCS_BIT(30) /* MP last packet indication */
#define UCT_IB_MLX5_MP_RQ_FILLER_FLAG UCS_BIT(31) /* Filler CQE indicator */

#if HAVE_DECL_MLX5DV_UAR_ALLOC_TYPE_BF
# define UCT_IB_MLX5_UAR_ALLOC_TYPE_WC MLX5DV_UAR_ALLOC_TYPE_BF
#else
# define UCT_IB_MLX5_UAR_ALLOC_TYPE_WC 0
#endif

#if HAVE_DECL_MLX5DV_UAR_ALLOC_TYPE_NC
# define UCT_IB_MLX5_UAR_ALLOC_TYPE_NC MLX5DV_UAR_ALLOC_TYPE_NC
#endif

#define UCT_IB_MLX5_OPMOD_EXT_ATOMIC(_log_arg_size) \
((8) | ((_log_arg_size) - 2))
Expand Down