Skip to content

Fix preference treatment in coll/base #8220

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rajachan merged 4 commits into from
Nov 20, 2020
Merged

Fix preference treatment in coll/base #8220

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
dd54af9
coll/base: Fix collective module selection preference treatment
devreal Nov 16, 2020
09c2f4a
coll/[sm|han|adapt]: don't disqualify on priority 0
devreal Nov 19, 2020
971d58c
coll/han: remove references to experimental solo and shared collectiv…
devreal Nov 19, 2020
1cdc855
coll/han: reduce default segment size for reduce/allreduce to 64k
devreal Nov 19, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions ompi/mca/coll/adapt/coll_adapt_module.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@
* of Tennessee Research Foundation. All rights
* reserved.
* $COPYRIGHT$
*
*
* Additional copyrights may follow
*
*
* $HEADER$
*/

Expand Down Expand Up @@ -146,7 +146,7 @@ mca_coll_base_module_t *ompi_coll_adapt_comm_query(struct ompi_communicator_t *
/* Get the priority level attached to this module.
If priority is less than or equal to 0, then the module is unavailable. */
*priority = mca_coll_adapt_component.adapt_priority;
if (mca_coll_adapt_component.adapt_priority <= 0) {
if (mca_coll_adapt_component.adapt_priority < 0) {
opal_output_verbose(10, ompi_coll_base_framework.framework_output,
"coll:adapt:comm_query (%d/%s): priority too low; "
"disqualifying myself",
Expand Down
30 changes: 13 additions & 17 deletions ompi/mca/coll/base/coll_base_comm_select.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -337,6 +337,7 @@ static opal_list_t *check_components(opal_list_t * components,
ompi_communicator_t * comm)
{
int priority, flag;
int count_include = 0;
const mca_base_component_t *component;
mca_base_component_list_item_t *cli;
mca_coll_base_module_2_3_0_t *module;
Expand All @@ -363,7 +364,8 @@ static opal_list_t *check_components(opal_list_t * components,
if(NULL == coll_argv) {
goto proceed_to_select;
}
int idx2, count_include = opal_argv_count(coll_argv);
int idx2;
count_include = opal_argv_count(coll_argv);
/* Allocate the coll_include argv */
coll_include = (char**)malloc((count_include + 1) * sizeof(char*));
coll_include[count_include] = NULL; /* NULL terminated array */
Expand All @@ -385,15 +387,6 @@ static opal_list_t *check_components(opal_list_t * components,
}
coll_include[idx] = coll_argv[idx];
}
/* Reverse the order of the coll_inclide argv to faciliate the ordering of
* the selected components reverse.
*/
for( idx2 = 0; idx2 < (count_include - 1); idx2++ ) {
char* temp = coll_include[idx2];
coll_include[idx2] = coll_include[count_include - 1];
coll_include[count_include - 1] = temp;
count_include--;
}
}
proceed_to_select:
/* Make a list of the components that query successfully */
Expand Down Expand Up @@ -453,14 +446,17 @@ static opal_list_t *check_components(opal_list_t * components,

/* For all valid component reorder them not on their provided priorities but on
* the order requested in the info key. As at this point the coll_include is
* already ordered backward we can simply prepend the components.
* already ordered backward we can simply append the components.
* Note that the last element in selectable will have the highest priorty.
*/
mca_coll_base_avail_coll_t *item, *item_next;
OPAL_LIST_FOREACH_SAFE(item, item_next,
selectable, mca_coll_base_avail_coll_t) {
if( component_in_argv(coll_include, item->ac_component_name) ) {
opal_list_remove_item(selectable, &item->super);
opal_list_prepend(selectable, &item->super);
for (int idx = count_include-1; idx >= 0; --idx) {
mca_coll_base_avail_coll_t *item;
OPAL_LIST_FOREACH(item, selectable, mca_coll_base_avail_coll_t) {
if (0 == strcmp(item->ac_component_name, coll_include[idx])) {
opal_list_remove_item(selectable, &item->super);
opal_list_append(selectable, &item->super);
break;
}
}
}

Expand Down
17 changes: 8 additions & 9 deletions ompi/mca/coll/han/coll_han_component.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@ ompi_coll_han_components available_components[COMPONENTS_COUNT] = {
{ LIBNBC, "libnbc", NULL },
{ TUNED, "tuned", NULL },
{ SM, "sm", NULL },
{ SHARED, "shared", NULL },
{ ADAPT, "adapt", NULL },
{ HAN, "han", NULL }
};
Expand Down Expand Up @@ -179,12 +178,12 @@ static int han_register(void)

cs->han_bcast_low_module = 0;
(void) mca_base_component_var_register(c, "bcast_low_module",
"low level module for bcast, 0 sm, 1 solo",
"low level module for bcast, 0 tuned, 1 sm",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY, &cs->han_bcast_low_module);

cs->han_reduce_segsize = 524288;
cs->han_reduce_segsize = 65536;
(void) mca_base_component_var_register(c, "reduce_segsize",
"segment size for reduce",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
Expand All @@ -200,11 +199,11 @@ static int han_register(void)

cs->han_reduce_low_module = 0;
(void) mca_base_component_var_register(c, "reduce_low_module",
"low level module for allreduce, 0 sm, 1 shared",
"low level module for allreduce, 0 tuned, 1 sm",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY, &cs->han_reduce_low_module);
cs->han_allreduce_segsize = 524288;
cs->han_allreduce_segsize = 65536;
(void) mca_base_component_var_register(c, "allreduce_segsize",
"segment size for allreduce",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
Expand All @@ -220,7 +219,7 @@ static int han_register(void)

cs->han_allreduce_low_module = 0;
(void) mca_base_component_var_register(c, "allreduce_low_module",
"low level module for allreduce, 0 sm, 1 shared",
"low level module for allreduce, 0 tuned, 1 sm",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY, &cs->han_allreduce_low_module);
Expand All @@ -234,7 +233,7 @@ static int han_register(void)

cs->han_allgather_low_module = 0;
(void) mca_base_component_var_register(c, "allgather_low_module",
"low level module for allgather, 0 sm, 1 shared",
"low level module for allgather, 0 tuned, 1 sm",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY, &cs->han_allgather_low_module);
Expand All @@ -248,7 +247,7 @@ static int han_register(void)

cs->han_gather_low_module = 0;
(void) mca_base_component_var_register(c, "gather_low_module",
"low level module for gather, 0 sm, 1 shared",
"low level module for gather, 0 tuned, 1 sm",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY, &cs->han_gather_low_module);
Expand All @@ -262,7 +261,7 @@ static int han_register(void)

cs->han_scatter_low_module = 0;
(void) mca_base_component_var_register(c, "scatter_low_module",
"low level module for scatter, 0 sm, 1 shared",
"low level module for scatter, 0 tuned, 1 sm",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY, &cs->han_scatter_low_module);
Expand Down
1 change: 0 additions & 1 deletion ompi/mca/coll/han/coll_han_dynamic.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,6 @@ typedef enum COMPONENTS {
LIBNBC,
TUNED,
SM,
SHARED,
ADAPT,
HAN,
COMPONENTS_COUNT
Expand Down
2 changes: 1 addition & 1 deletion ompi/mca/coll/han/coll_han_module.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,7 @@ mca_coll_han_comm_query(struct ompi_communicator_t * comm, int *priority)
/* Get the priority level attached to this module. If priority is less
* than or equal to 0, then the module is unavailable. */
*priority = mca_coll_han_component.han_priority;
if (mca_coll_han_component.han_priority <= 0) {
if (mca_coll_han_component.han_priority < 0) {
opal_output_verbose(10, ompi_coll_base_framework.framework_output,
"coll:han:comm_query (%d/%s): priority too low; disqualifying myself",
comm->c_contextid, comm->c_name);
Expand Down
4 changes: 2 additions & 2 deletions ompi/mca/coll/han/coll_han_subcomms.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,7 @@ int mca_coll_han_comm_create(struct ompi_communicator_t *comm,
* Upgrade sm module priority to set up low_comms[0] with sm module
* This sub-communicator contains the ranks that share my node.
*/
opal_info_set(&comm_info, "ompi_comm_coll_preference", "sm,^han");
opal_info_set(&comm_info, "ompi_comm_coll_preference", "tuned,^han");
ompi_comm_split_type(comm, MPI_COMM_TYPE_SHARED, 0,
&comm_info, &(low_comms[0]));

Expand All @@ -272,7 +272,7 @@ int mca_coll_han_comm_create(struct ompi_communicator_t *comm,
* Upgrade shared module priority to set up low_comms[1] with shared module
* This sub-communicator contains the ranks that share my node.
*/
opal_info_set(&comm_info, "ompi_comm_coll_preference", "shared,^han");
opal_info_set(&comm_info, "ompi_comm_coll_preference", "sm,^han");
ompi_comm_split_type(comm, MPI_COMM_TYPE_SHARED, 0,
&comm_info, &(low_comms[1]));

Expand Down
4 changes: 2 additions & 2 deletions ompi/mca/coll/sm/coll_sm_module.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -182,10 +182,10 @@ mca_coll_sm_comm_query(struct ompi_communicator_t *comm, int *priority)
/* Get the priority level attached to this module. If priority is less
* than or equal to 0, then the module is unavailable. */
*priority = mca_coll_sm_component.sm_priority;
if (mca_coll_sm_component.sm_priority <= 0) {
if (mca_coll_sm_component.sm_priority < 0) {
opal_output_verbose(10, ompi_coll_base_framework.framework_output,
"coll:sm:comm_query (%d/%s): priority too low; disqualifying myself", comm->c_contextid, comm->c_name);
return NULL;
return NULL;
}

sm_module = OBJ_NEW(mca_coll_sm_module_t);
Expand Down