Skip to content

UCT/ROCM: adjust rocm/ipc performance parameters #9568

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yosefe merged 1 commit into from
Jan 3, 2024
Merged

UCT/ROCM: adjust rocm/ipc performance parameters #9568

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 4 additions & 8 deletions src/uct/rocm/copy/rocm_copy_iface.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,18 +32,14 @@ static ucs_config_field_t uct_rocm_copy_iface_config_table[] = {
ucs_offsetof(uct_rocm_copy_iface_config_t, h2d_thresh),
UCS_CONFIG_TYPE_MEMUNITS},

{"ENABLE_ASYNC_ZCOPY", "y",
"Enable asynchronous zcopy operations",
{"ENABLE_ASYNC_ZCOPY", "y", "Enable asynchronous zcopy operations",
ucs_offsetof(uct_rocm_copy_iface_config_t, enable_async_zcopy),
UCS_CONFIG_TYPE_BOOL},

{"LAT", "2e-7",
"Latency",
ucs_offsetof(uct_rocm_copy_iface_config_t, latency),
UCS_CONFIG_TYPE_TIME},
{"LAT", "1e-7", "Latency",
ucs_offsetof(uct_rocm_copy_iface_config_t, latency), UCS_CONFIG_TYPE_TIME},

{"SIGPOOL_MAX_ELEMS", "1024",
"Maximum number of elements in signal pool",
{"SIGPOOL_MAX_ELEMS", "1024", "Maximum number of elements in signal pool",
ucs_offsetof(uct_rocm_copy_iface_config_t, sigpool_max_elems),
UCS_CONFIG_TYPE_UINT},

Expand Down
21 changes: 16 additions & 5 deletions src/uct/rocm/ipc/rocm_ipc_iface.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,16 @@

static ucs_config_field_t uct_rocm_ipc_iface_config_table[] = {

{"", "", NULL,
ucs_offsetof(uct_rocm_ipc_iface_config_t, super),
{"", "", NULL, ucs_offsetof(uct_rocm_ipc_iface_config_t, super),
UCS_CONFIG_TYPE_TABLE(uct_iface_config_table)},

{"MIN_ZCOPY", "128", "Minimum data size for ROCm/IPC zcopy protocols",
ucs_offsetof(uct_rocm_ipc_iface_config_t, min_zcopy),
UCS_CONFIG_TYPE_MEMUNITS},

{"LAT", "1e-7", "Latency",
ucs_offsetof(uct_rocm_ipc_iface_config_t, latency), UCS_CONFIG_TYPE_TIME},

{NULL}
};

Expand Down Expand Up @@ -77,13 +83,13 @@ static ucs_status_t uct_rocm_ipc_iface_query(uct_iface_h tl_iface,

uct_base_iface_query(&iface->super, iface_attr);

iface_attr->cap.put.min_zcopy = 0;
iface_attr->cap.put.min_zcopy = iface->config.min_zcopy;
iface_attr->cap.put.max_zcopy = SIZE_MAX;
iface_attr->cap.put.opt_zcopy_align = sizeof(uint32_t);
iface_attr->cap.put.align_mtu = iface_attr->cap.put.opt_zcopy_align;
iface_attr->cap.put.max_iov = 1;

iface_attr->cap.get.min_zcopy = 0;
iface_attr->cap.get.min_zcopy = iface->config.min_zcopy;
iface_attr->cap.get.max_zcopy = SIZE_MAX;
iface_attr->cap.get.opt_zcopy_align = sizeof(uint32_t);
iface_attr->cap.get.align_mtu = iface_attr->cap.get.opt_zcopy_align;
Expand All @@ -98,7 +104,7 @@ static ucs_status_t uct_rocm_ipc_iface_query(uct_iface_h tl_iface,
UCT_IFACE_FLAG_PENDING |
UCT_IFACE_FLAG_CONNECT_TO_IFACE;

iface_attr->latency = ucs_linear_func_make(1e-9, 0);
iface_attr->latency = ucs_linear_func_make(iface->config.latency, 0);
iface_attr->bandwidth.dedicated = 0;
iface_attr->bandwidth.shared = uct_rocm_ipc_iface_get_bw();
iface_attr->overhead = 0;
Expand Down Expand Up @@ -170,6 +176,8 @@ static UCS_CLASS_INIT_FUNC(uct_rocm_ipc_iface_t, uct_md_h md, uct_worker_h worke
const uct_iface_params_t *params,
const uct_iface_config_t *tl_config)
{
uct_rocm_ipc_iface_config_t *config =
ucs_derived_of(tl_config, uct_rocm_ipc_iface_config_t);
ucs_status_t status;
ucs_mpool_params_t mp_params;

Expand All @@ -179,6 +187,9 @@ static UCS_CLASS_INIT_FUNC(uct_rocm_ipc_iface_t, uct_md_h md, uct_worker_h worke
tl_config UCS_STATS_ARG(params->stats_root)
UCS_STATS_ARG(UCT_ROCM_IPC_TL_NAME));

self->config.min_zcopy = config->min_zcopy;
self->config.latency = config->latency;

ucs_mpool_params_reset(&mp_params);
mp_params.elem_size = sizeof(uct_rocm_base_signal_desc_t);
mp_params.elems_per_chunk = 128;
Expand Down
7 changes: 7 additions & 0 deletions src/uct/rocm/ipc/rocm_ipc_iface.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,17 @@ typedef struct uct_rocm_ipc_iface {
uct_base_iface_t super;
ucs_mpool_t signal_pool;
ucs_queue_head_t signal_queue;
struct {
size_t min_zcopy;
double latency;
} config;

} uct_rocm_ipc_iface_t;

typedef struct uct_rocm_ipc_iface_config {
uct_iface_config_t super;
size_t min_zcopy;
double latency;
} uct_rocm_ipc_iface_config_t;

#endif