diff --git a/src/uct/rocm/copy/rocm_copy_iface.c b/src/uct/rocm/copy/rocm_copy_iface.c index 2fb9266f349..58fb0aa0b78 100644 --- a/src/uct/rocm/copy/rocm_copy_iface.c +++ b/src/uct/rocm/copy/rocm_copy_iface.c @@ -32,18 +32,14 @@ static ucs_config_field_t uct_rocm_copy_iface_config_table[] = { ucs_offsetof(uct_rocm_copy_iface_config_t, h2d_thresh), UCS_CONFIG_TYPE_MEMUNITS}, - {"ENABLE_ASYNC_ZCOPY", "y", - "Enable asynchronous zcopy operations", + {"ENABLE_ASYNC_ZCOPY", "y", "Enable asynchronous zcopy operations", ucs_offsetof(uct_rocm_copy_iface_config_t, enable_async_zcopy), UCS_CONFIG_TYPE_BOOL}, - {"LAT", "2e-7", - "Latency", - ucs_offsetof(uct_rocm_copy_iface_config_t, latency), - UCS_CONFIG_TYPE_TIME}, + {"LAT", "1e-7", "Latency", + ucs_offsetof(uct_rocm_copy_iface_config_t, latency), UCS_CONFIG_TYPE_TIME}, - {"SIGPOOL_MAX_ELEMS", "1024", - "Maximum number of elements in signal pool", + {"SIGPOOL_MAX_ELEMS", "1024", "Maximum number of elements in signal pool", ucs_offsetof(uct_rocm_copy_iface_config_t, sigpool_max_elems), UCS_CONFIG_TYPE_UINT}, diff --git a/src/uct/rocm/ipc/rocm_ipc_iface.c b/src/uct/rocm/ipc/rocm_ipc_iface.c index 85f39c509e8..7c87e240138 100644 --- a/src/uct/rocm/ipc/rocm_ipc_iface.c +++ b/src/uct/rocm/ipc/rocm_ipc_iface.c @@ -17,10 +17,16 @@ static ucs_config_field_t uct_rocm_ipc_iface_config_table[] = { - {"", "", NULL, - ucs_offsetof(uct_rocm_ipc_iface_config_t, super), + {"", "", NULL, ucs_offsetof(uct_rocm_ipc_iface_config_t, super), UCS_CONFIG_TYPE_TABLE(uct_iface_config_table)}, + {"MIN_ZCOPY", "128", "Minimum data size for ROCm/IPC zcopy protocols", + ucs_offsetof(uct_rocm_ipc_iface_config_t, min_zcopy), + UCS_CONFIG_TYPE_MEMUNITS}, + + {"LAT", "1e-7", "Latency", + ucs_offsetof(uct_rocm_ipc_iface_config_t, latency), UCS_CONFIG_TYPE_TIME}, + {NULL} }; @@ -77,13 +83,13 @@ static ucs_status_t uct_rocm_ipc_iface_query(uct_iface_h tl_iface, uct_base_iface_query(&iface->super, iface_attr); - iface_attr->cap.put.min_zcopy = 0; + iface_attr->cap.put.min_zcopy = iface->config.min_zcopy; iface_attr->cap.put.max_zcopy = SIZE_MAX; iface_attr->cap.put.opt_zcopy_align = sizeof(uint32_t); iface_attr->cap.put.align_mtu = iface_attr->cap.put.opt_zcopy_align; iface_attr->cap.put.max_iov = 1; - iface_attr->cap.get.min_zcopy = 0; + iface_attr->cap.get.min_zcopy = iface->config.min_zcopy; iface_attr->cap.get.max_zcopy = SIZE_MAX; iface_attr->cap.get.opt_zcopy_align = sizeof(uint32_t); iface_attr->cap.get.align_mtu = iface_attr->cap.get.opt_zcopy_align; @@ -98,7 +104,7 @@ static ucs_status_t uct_rocm_ipc_iface_query(uct_iface_h tl_iface, UCT_IFACE_FLAG_PENDING | UCT_IFACE_FLAG_CONNECT_TO_IFACE; - iface_attr->latency = ucs_linear_func_make(1e-9, 0); + iface_attr->latency = ucs_linear_func_make(iface->config.latency, 0); iface_attr->bandwidth.dedicated = 0; iface_attr->bandwidth.shared = uct_rocm_ipc_iface_get_bw(); iface_attr->overhead = 0; @@ -170,6 +176,8 @@ static UCS_CLASS_INIT_FUNC(uct_rocm_ipc_iface_t, uct_md_h md, uct_worker_h worke const uct_iface_params_t *params, const uct_iface_config_t *tl_config) { + uct_rocm_ipc_iface_config_t *config = + ucs_derived_of(tl_config, uct_rocm_ipc_iface_config_t); ucs_status_t status; ucs_mpool_params_t mp_params; @@ -179,6 +187,9 @@ static UCS_CLASS_INIT_FUNC(uct_rocm_ipc_iface_t, uct_md_h md, uct_worker_h worke tl_config UCS_STATS_ARG(params->stats_root) UCS_STATS_ARG(UCT_ROCM_IPC_TL_NAME)); + self->config.min_zcopy = config->min_zcopy; + self->config.latency = config->latency; + ucs_mpool_params_reset(&mp_params); mp_params.elem_size = sizeof(uct_rocm_base_signal_desc_t); mp_params.elems_per_chunk = 128; diff --git a/src/uct/rocm/ipc/rocm_ipc_iface.h b/src/uct/rocm/ipc/rocm_ipc_iface.h index f604ba83405..9afada6e643 100644 --- a/src/uct/rocm/ipc/rocm_ipc_iface.h +++ b/src/uct/rocm/ipc/rocm_ipc_iface.h @@ -17,10 +17,17 @@ typedef struct uct_rocm_ipc_iface { uct_base_iface_t super; ucs_mpool_t signal_pool; ucs_queue_head_t signal_queue; + struct { + size_t min_zcopy; + double latency; + } config; + } uct_rocm_ipc_iface_t; typedef struct uct_rocm_ipc_iface_config { uct_iface_config_t super; + size_t min_zcopy; + double latency; } uct_rocm_ipc_iface_config_t; #endif