From 01d716433a69138a2084d75f5935d630550a6b44 Mon Sep 17 00:00:00 2001 From: Sergey Oblomov Date: Tue, 9 Mar 2021 12:21:02 +0200 Subject: [PATCH] SPML/UCX: removed direct dependency to SPML UCX - added synchronise_quiet parameter to local context object Signed-off-by: Sergey Oblomov --- oshmem/mca/spml/ucx/spml_ucx.c | 11 ++++++----- oshmem/mca/spml/ucx/spml_ucx.h | 3 ++- oshmem/mca/spml/ucx/spml_ucx_component.c | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/oshmem/mca/spml/ucx/spml_ucx.c b/oshmem/mca/spml/ucx/spml_ucx.c index 1c226c1fdeb..fba5b2cb806 100644 --- a/oshmem/mca/spml/ucx/spml_ucx.c +++ b/oshmem/mca/spml/ucx/spml_ucx.c @@ -78,14 +78,14 @@ mca_spml_ucx_t mca_spml_ucx = { .num_disconnect = 1, .heap_reg_nb = 0, .enabled = 0, - .get_mkey_slow = NULL, - .synchronized_quiet = false + .get_mkey_slow = NULL }; mca_spml_ucx_ctx_t mca_spml_ucx_ctx_default = { - .ucp_worker = NULL, - .ucp_peers = NULL, - .options = 0 + .ucp_worker = NULL, + .ucp_peers = NULL, + .options = 0, + .synchronized_quiet = false }; #if HAVE_DECL_UCP_ATOMIC_OP_NBX @@ -673,6 +673,7 @@ static int mca_spml_ucx_ctx_create_common(long options, mca_spml_ucx_ctx_t **ucx ucx_ctx->options = options; ucx_ctx->ucp_worker = calloc(1, sizeof(ucp_worker_h)); ucx_ctx->ucp_workers = 1; + ucx_ctx->synchronized_quiet = mca_spml_ucx_ctx_default.synchronized_quiet; params.field_mask = UCP_WORKER_PARAM_FIELD_THREAD_MODE; if (oshmem_mpi_thread_provided == SHMEM_THREAD_SINGLE || options & SHMEM_CTX_PRIVATE || options & SHMEM_CTX_SERIALIZED) { diff --git a/oshmem/mca/spml/ucx/spml_ucx.h b/oshmem/mca/spml/ucx/spml_ucx.h index 0afd932563a..b2802e7161a 100644 --- a/oshmem/mca/spml/ucx/spml_ucx.h +++ b/oshmem/mca/spml/ucx/spml_ucx.h @@ -75,6 +75,7 @@ struct mca_spml_ucx_ctx { unsigned int ucp_workers; int *put_proc_indexes; unsigned put_proc_count; + bool synchronized_quiet; }; typedef struct mca_spml_ucx_ctx mca_spml_ucx_ctx_t; @@ -256,7 +257,7 @@ static inline int ucx_status_to_oshmem_nb(ucs_status_t status) static inline void mca_spml_ucx_remote_op_posted(mca_spml_ucx_ctx_t *ctx, int dst) { - if (OPAL_UNLIKELY(mca_spml_ucx.synchronized_quiet)) { + if (OPAL_UNLIKELY(ctx->synchronized_quiet)) { if (!opal_bitmap_is_set_bit(&ctx->put_op_bitmap, dst)) { ctx->put_proc_indexes[ctx->put_proc_count++] = dst; opal_bitmap_set_bit(&ctx->put_op_bitmap, dst); diff --git a/oshmem/mca/spml/ucx/spml_ucx_component.c b/oshmem/mca/spml/ucx/spml_ucx_component.c index 877c0d70891..eb40076a706 100644 --- a/oshmem/mca/spml/ucx/spml_ucx_component.c +++ b/oshmem/mca/spml/ucx/spml_ucx_component.c @@ -159,7 +159,7 @@ static int mca_spml_ucx_component_register(void) mca_spml_ucx_param_register_bool("synchronized_quiet", 0, "Use synchronized quiet on shmem_quiet or shmem_barrier_all operations", - &mca_spml_ucx.synchronized_quiet); + &mca_spml_ucx_ctx_default.synchronized_quiet); mca_spml_ucx_param_register_ulong("nb_progress_thresh_global", 0, "Number of nb_put or nb_get operations before ucx progress is triggered. Disabled by default (0). Setting this value will override nb_put/get_progress_thresh.",