diff --git a/src/ucp/core/ucp_context.c b/src/ucp/core/ucp_context.c index 73864058fe6..81bd0220d3a 100644 --- a/src/ucp/core/ucp_context.c +++ b/src/ucp/core/ucp_context.c @@ -110,6 +110,15 @@ static const char *ucp_fence_modes[] = { [UCP_FENCE_MODE_LAST] = NULL }; +static const char *ucp_gpu_nic_assignment_modes[] = { + [UCP_GPU_NIC_ASSIGNMENT_MODE_AUTO] = "auto", + [UCP_GPU_NIC_ASSIGNMENT_MODE_OFF] = "off", + [UCP_GPU_NIC_ASSIGNMENT_MODE_FLIP] = "flip", + [UCP_GPU_NIC_ASSIGNMENT_MODE_ROUND_ROBIN] = "round_robin", + [UCP_GPU_NIC_ASSIGNMENT_MODE_SHARED] = "shared", + [UCP_GPU_NIC_ASSIGNMENT_MODE_LAST] = NULL +}; + static const char *ucp_rndv_modes[] = { [UCP_RNDV_MODE_AUTO] = "auto", [UCP_RNDV_MODE_GET_ZCOPY] = "get_zcopy", @@ -617,6 +626,26 @@ static ucs_config_field_t ucp_context_config_table[] = { ucs_offsetof(ucp_context_config_t, connect_all_to_all), UCS_CONFIG_TYPE_BOOL}, + {"GPU_NIC_ASSIGNMENT_MODE", "auto", + "Assign NICs to GPUs within each topology group, and restrict the lanes\n" + "for a GPU's memory to the NICs assigned to that GPU.\n" + "All ports of a NIC are assigned together.\n" + "The 'flip', 'round_robin' and 'shared' modes apply on any hardware.\n" + "With 'flip' and 'round_robin', a group with fewer NICs than GPUs leaves\n" + "some GPUs without any NICs.\n" + " - auto : use 'flip' on hardware with a known GPU-NIC topology,\n" + " otherwise 'off'.\n" + " - off : do not assign; select lanes from all NICs.\n" + " - flip : assign each NIC to a single GPU, walking the N GPUs of\n" + " the group forward then backward:\n" + " 0, 1, .., N-1, N-1, .., 1, 0, 0, 1, ..\n" + " - round_robin : assign each NIC to a single GPU, walking the N GPUs of\n" + " the group in ascending order:\n" + " 0, 1, .., N-1, 0, 1, .., N-1, 0, ..\n" + " - shared : assign all NICs of a group to every GPU of that group.", + ucs_offsetof(ucp_context_config_t, gpu_nic_assignment_mode), + UCS_CONFIG_TYPE_ENUM(ucp_gpu_nic_assignment_modes)}, + {"SINGLE_NET_DEVICE", "n", "Restrict each protocol's lanes to one network device.\n" "The device is picked among the closest eligible network devices using \n" @@ -2755,7 +2784,8 @@ ucp_version_check(unsigned api_major_version, unsigned api_minor_version) } static ucs_status_t -ucp_context_gpu_nic_assignment_init(ucp_gpu_nic_assignment_t **assignment_p) +ucp_context_gpu_nic_assignment_init(ucp_gpu_nic_assignment_mode_t mode, + ucp_gpu_nic_assignment_t **assignment_p) { ucp_gpu_nic_assignment_t *assignment; ucs_topo_groups_t groups; @@ -2763,14 +2793,23 @@ ucp_context_gpu_nic_assignment_init(ucp_gpu_nic_assignment_t **assignment_p) *assignment_p = NULL; - /* TODO: Improve Vera Rubin detection by checking NICs/GPUs models. */ - if (ucs_arch_get_cpu_model() != UCS_CPU_MODEL_NVIDIA_VERA) { - ucs_debug("gpu-nic assignment is not supported on %s architecture, " - "skipping", - ucs_cpu_model_name()); + if (mode == UCP_GPU_NIC_ASSIGNMENT_MODE_AUTO) { + /* TODO: Improve Vera Rubin detection by checking NICs/GPUs models. */ + if (ucs_arch_get_cpu_model() != UCS_CPU_MODEL_NVIDIA_VERA) { + ucs_debug("gpu-nic assignment is not supported on %s " + "architecture, skipping", + ucs_cpu_model_name()); + return UCS_OK; + } + + mode = UCP_GPU_NIC_ASSIGNMENT_MODE_FLIP; + } else if (mode == UCP_GPU_NIC_ASSIGNMENT_MODE_OFF) { + ucs_debug("gpu-nic assignment is disabled by configuration"); return UCS_OK; } + ucs_debug("gpu-nic assignment mode %s", ucp_gpu_nic_assignment_modes[mode]); + status = ucs_topo_build_groups(&groups); if (status != UCS_OK) { return status; @@ -2788,9 +2827,7 @@ ucp_context_gpu_nic_assignment_init(ucp_gpu_nic_assignment_t **assignment_p) goto out_release_groups; } - status = ucp_gpu_nic_assignment_build(&groups, - UCP_GPU_NIC_ASSIGNMENT_POLICY_FLIP, - assignment); + status = ucp_gpu_nic_assignment_build(&groups, mode, assignment); if (status != UCS_OK) { ucs_free(assignment); goto out_release_groups; @@ -2857,7 +2894,9 @@ ucs_status_t ucp_init_version(unsigned api_major_version, unsigned api_minor_ver goto err_thread_lock_finalize; } - status = ucp_context_gpu_nic_assignment_init(&context->gpu_nic_assignment); + status = ucp_context_gpu_nic_assignment_init( + context->config.ext.gpu_nic_assignment_mode, + &context->gpu_nic_assignment); if (status != UCS_OK) { goto err_free_res; } diff --git a/src/ucp/core/ucp_context.h b/src/ucp/core/ucp_context.h index 242edfcae7f..f8b8e145785 100644 --- a/src/ucp/core/ucp_context.h +++ b/src/ucp/core/ucp_context.h @@ -255,6 +255,8 @@ typedef struct ucp_context_config { /** Extend endpoint lanes connections of each local device to all remote * devices */ int connect_all_to_all; + /** GPU-to-NIC assignment mode */ + ucp_gpu_nic_assignment_mode_t gpu_nic_assignment_mode; /** Restrict lanes to one network device per protocol */ int proto_use_single_net_device; /** Max HCAs for GPU memory registration: auto=closest, N=limit, inf=all */ diff --git a/src/ucp/core/ucp_gpu_nic_assignment.c b/src/ucp/core/ucp_gpu_nic_assignment.c index 728f4cf843f..682fc9f770e 100644 --- a/src/ucp/core/ucp_gpu_nic_assignment.c +++ b/src/ucp/core/ucp_gpu_nic_assignment.c @@ -34,7 +34,7 @@ typedef struct { static size_t -ucp_gpu_nic_assignment_get_gpu_idx(ucp_gpu_nic_assignment_policy_t policy, +ucp_gpu_nic_assignment_get_gpu_idx(ucp_gpu_nic_assignment_mode_t mode, size_t num_gpus, size_t nic_idx) { size_t gpu_idx, leg; @@ -42,12 +42,12 @@ ucp_gpu_nic_assignment_get_gpu_idx(ucp_gpu_nic_assignment_policy_t policy, ucs_assert(num_gpus > 0); gpu_idx = nic_idx % num_gpus; - if (policy == UCP_GPU_NIC_ASSIGNMENT_POLICY_ROUND_ROBIN) { + if (mode == UCP_GPU_NIC_ASSIGNMENT_MODE_ROUND_ROBIN) { return gpu_idx; } - ucs_assertv(policy == UCP_GPU_NIC_ASSIGNMENT_POLICY_FLIP, - "invalid policy: %d", (int)policy); + ucs_assertv(mode == UCP_GPU_NIC_ASSIGNMENT_MODE_FLIP, "invalid mode: %d", + (int)mode); leg = nic_idx / num_gpus; if ((leg % 2) != 0) { gpu_idx = num_gpus - 1 - gpu_idx; @@ -327,36 +327,50 @@ ucp_gpu_nic_assignment_init(ucp_gpu_nic_assignment_t *assignment, } static void -ucp_gpu_nic_assignment_add_group(ucp_gpu_nic_assignment_t *assignment, - const ucs_topo_group_t *group, - ucp_gpu_nic_assignment_policy_t policy) +ucp_gpu_nic_assignment_add_nic_to_gpu(ucp_gpu_nic_assignment_t *assignment, + const ucs_topo_group_element_t *gpu, + const ucs_topo_group_element_t *nic) { - size_t num_gpus = ucs_array_length(&group->gpus); ucp_gpu_nic_sys_dev_bitmap_t *nic_sys_dev_bitmap; - const ucs_topo_group_element_t *nic; - const ucs_topo_group_element_t *gpu; - size_t nic_idx; - size_t gpu_idx; - ucs_array_for_each_index(nic, nic_idx, &group->nics) { - gpu_idx = ucp_gpu_nic_assignment_get_gpu_idx(policy, num_gpus, nic_idx); - gpu = &ucs_array_elem(&group->gpus, gpu_idx); + ucs_assert((gpu->num_sys_devs > 0) && + (gpu->sys_devs[0] != UCS_SYS_DEVICE_ID_UNKNOWN)); - ucs_assert((gpu->num_sys_devs > 0) && - (gpu->sys_devs[0] != UCS_SYS_DEVICE_ID_UNKNOWN)); + nic_sys_dev_bitmap = ucs_const_cast( + ucp_gpu_nic_sys_dev_bitmap_t*, + ucp_gpu_nic_assignment_lookup(assignment, gpu->sys_devs[0])); + ucs_assert(nic_sys_dev_bitmap != NULL); - nic_sys_dev_bitmap = ucs_const_cast( - ucp_gpu_nic_sys_dev_bitmap_t*, - ucp_gpu_nic_assignment_lookup(assignment, gpu->sys_devs[0])); - ucs_assert(nic_sys_dev_bitmap != NULL); + ucp_gpu_nic_bitmap_add_nic(nic_sys_dev_bitmap, nic); +} - ucp_gpu_nic_bitmap_add_nic(nic_sys_dev_bitmap, nic); +static void +ucp_gpu_nic_assignment_add_group(ucp_gpu_nic_assignment_t *assignment, + const ucs_topo_group_t *group, + ucp_gpu_nic_assignment_mode_t mode) +{ + size_t num_gpus = ucs_array_length(&group->gpus); + const ucs_topo_group_element_t *gpu, *nic; + size_t nic_idx, gpu_idx; + + ucs_array_for_each_index(nic, nic_idx, &group->nics) { + if (mode == UCP_GPU_NIC_ASSIGNMENT_MODE_SHARED) { + /* Assign the NIC to every GPU in the group. */ + ucs_array_for_each(gpu, &group->gpus) { + ucp_gpu_nic_assignment_add_nic_to_gpu(assignment, gpu, nic); + } + } else { + gpu_idx = ucp_gpu_nic_assignment_get_gpu_idx(mode, num_gpus, + nic_idx); + gpu = &ucs_array_elem(&group->gpus, gpu_idx); + ucp_gpu_nic_assignment_add_nic_to_gpu(assignment, gpu, nic); + } } } ucs_status_t ucp_gpu_nic_assignment_build(const ucs_topo_groups_t *groups, - ucp_gpu_nic_assignment_policy_t policy, + ucp_gpu_nic_assignment_mode_t mode, ucp_gpu_nic_assignment_t *assignment_p) { ucp_gpu_nic_assignment_t assignment; @@ -364,8 +378,10 @@ ucp_gpu_nic_assignment_build(const ucs_topo_groups_t *groups, ucs_status_t status; ucs_assert(groups != NULL); - ucs_assertv(policy < UCP_GPU_NIC_ASSIGNMENT_POLICY_LAST, - "invalid gpu-nic assignment policy %d", (int)policy); + ucs_assertv((mode == UCP_GPU_NIC_ASSIGNMENT_MODE_FLIP) || + (mode == UCP_GPU_NIC_ASSIGNMENT_MODE_ROUND_ROBIN) || + (mode == UCP_GPU_NIC_ASSIGNMENT_MODE_SHARED), + "invalid gpu-nic assignment mode %d", (int)mode); status = ucp_gpu_nic_assignment_init(&assignment, groups); if (status != UCS_OK) { @@ -377,7 +393,7 @@ ucp_gpu_nic_assignment_build(const ucs_topo_groups_t *groups, continue; } - ucp_gpu_nic_assignment_add_group(&assignment, group, policy); + ucp_gpu_nic_assignment_add_group(&assignment, group, mode); } ucp_gpu_nic_assignment_log(&assignment, groups); diff --git a/src/ucp/core/ucp_gpu_nic_assignment.h b/src/ucp/core/ucp_gpu_nic_assignment.h index 1da6a3f24e6..ff39dbe476a 100644 --- a/src/ucp/core/ucp_gpu_nic_assignment.h +++ b/src/ucp/core/ucp_gpu_nic_assignment.h @@ -37,27 +37,6 @@ struct ucp_gpu_nic_assignment { }; -typedef enum { - /** - * Assign each NIC, with all of its ports, going forward and then backward - * across the GPUs. - * The group's NICs are assigned to the following GPU indices in order: - * 0, 1, ..., num_gpus-1, num_gpus-1, ..., 1, 0, 0, 1, ... - */ - UCP_GPU_NIC_ASSIGNMENT_POLICY_FLIP, - - /** - * Assign each NIC, with all of its ports, to GPUs repeatedly in ascending - * order. - * The group's NICs are assigned to the following GPU indices in order: - * 0, 1, ..., num_gpus-1, 0, 1, ..., num_gpus-1, 0, 1, ... - */ - UCP_GPU_NIC_ASSIGNMENT_POLICY_ROUND_ROBIN, - - UCP_GPU_NIC_ASSIGNMENT_POLICY_LAST -} ucp_gpu_nic_assignment_policy_t; - - /** * Build GPU-to-NIC assignments from topology groups. * @@ -65,7 +44,7 @@ typedef enum { * topology group. * * @param [in] groups Topology groups to build assignments from. - * @param [in] policy Assignment policy. + * @param [in] mode Assignment mode: flip, round-robin or shared. * @param [out] assignment_p Completed assignment. Updated only on success. * * @return UCS_OK on success, or an error status if assignment construction @@ -73,7 +52,7 @@ typedef enum { */ ucs_status_t ucp_gpu_nic_assignment_build(const ucs_topo_groups_t *groups, - ucp_gpu_nic_assignment_policy_t policy, + ucp_gpu_nic_assignment_mode_t mode, ucp_gpu_nic_assignment_t *assignment_p); diff --git a/src/ucp/core/ucp_types.h b/src/ucp/core/ucp_types.h index ff791a217fe..161ddb4601b 100644 --- a/src/ucp/core/ucp_types.h +++ b/src/ucp/core/ucp_types.h @@ -1,5 +1,5 @@ /** - * Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2017. ALL RIGHTS RESERVED. + * Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2026. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ @@ -234,6 +234,39 @@ typedef enum { } ucp_fence_mode_t; +/** + * GPU-to-NIC assignment mode. + */ +typedef enum { + /** Use flip on hardware with a known GPU-NIC topology, otherwise off */ + UCP_GPU_NIC_ASSIGNMENT_MODE_AUTO, + + /** Do not assign NICs to GPUs, select lanes from all NICs */ + UCP_GPU_NIC_ASSIGNMENT_MODE_OFF, + + /** + * Assign each NIC, with all of its ports, going forward and then backward + * across the N GPUs of the group. + * The group's NICs are assigned to the following GPU indices in order: + * 0, 1, ..., N-1, N-1, ..., 1, 0, 0, 1, ... + */ + UCP_GPU_NIC_ASSIGNMENT_MODE_FLIP, + + /** + * Assign each NIC, with all of its ports, to the N GPUs of the group + * repeatedly in ascending order. + * The group's NICs are assigned to the following GPU indices in order: + * 0, 1, ..., N-1, 0, 1, ..., N-1, 0, 1, ... + */ + UCP_GPU_NIC_ASSIGNMENT_MODE_ROUND_ROBIN, + + /** Assign every NIC, with all of its ports, to every GPU in the group */ + UCP_GPU_NIC_ASSIGNMENT_MODE_SHARED, + + UCP_GPU_NIC_ASSIGNMENT_MODE_LAST +} ucp_gpu_nic_assignment_mode_t; + + /** * Communication scheme in RNDV protocol. */ diff --git a/test/gtest/ucp/test_ucp_gpu_nic_assignment.cc b/test/gtest/ucp/test_ucp_gpu_nic_assignment.cc index f22e76d2ea9..712bcccb39d 100644 --- a/test/gtest/ucp/test_ucp_gpu_nic_assignment.cc +++ b/test/gtest/ucp/test_ucp_gpu_nic_assignment.cc @@ -118,12 +118,12 @@ class test_ucp_gpu_nic_assignment : public ucs::test { } } - void build_assignment(ucp_gpu_nic_assignment_policy_t policy) + void build_assignment(ucp_gpu_nic_assignment_mode_t mode) { ucs_status_t status; ucp_gpu_nic_assignment_release(&m_assignment); - status = ucp_gpu_nic_assignment_build(&m_groups, policy, &m_assignment); + status = ucp_gpu_nic_assignment_build(&m_groups, mode, &m_assignment); ASSERT_UCS_OK(status); /* Unknown device lookup */ @@ -161,6 +161,13 @@ class test_ucp_gpu_nic_assignment : public ucs::test { void check_nic_owners(const topology_shape_t &config, const std::vector &expected_owners) { + /* Validate the expected owners vector. */ + ASSERT_EQ(config.num_nics(), expected_owners.size()); + for (size_t owner_idx = 0; owner_idx < expected_owners.size(); + ++owner_idx) { + ASSERT_LT(expected_owners[owner_idx], config.num_gpus()); + } + /* Expected number of assigned NIC ports per physical GPU. */ std::vector expected_port_counts(config.num_gpus(), 0); @@ -219,8 +226,48 @@ class test_ucp_gpu_nic_assignment : public ucs::test { } } + void check_nic_shared(const topology_shape_t &config) + { + /* Verify that each GPU owns all ports of every NIC in its group. */ + for (size_t group_idx = 0; group_idx < config.num_groups; ++group_idx) { + const size_t first_nic_idx = group_idx * config.num_nics_per_group; + const size_t end_nic_idx = first_nic_idx + + config.num_nics_per_group; + + for (size_t local_idx = 0; local_idx < config.num_gpus_per_group; + ++local_idx) { + const size_t gpu_idx = (group_idx * config.num_gpus_per_group) + + local_idx; + const ucp_gpu_nic_sys_dev_bitmap_t *nic_sys_dev_bitmap = + ucp_gpu_nic_assignment_lookup(&m_assignment, + gpu_sys_dev(config, + gpu_idx, 0)); + ASSERT_NE(nullptr, nic_sys_dev_bitmap); + + for (size_t nic_idx = 0; nic_idx < config.num_nics(); + ++nic_idx) { + const bool same_group = (nic_idx >= first_nic_idx) && + (nic_idx < end_nic_idx); + + for (size_t port_idx = 0; port_idx < config.num_nic_ports; + ++port_idx) { + EXPECT_EQ(same_group, + ucp_gpu_nic_bitmap_get( + nic_sys_dev_bitmap, + nic_port_sys_dev(config, nic_idx, + port_idx))); + } + } + + EXPECT_EQ(config.num_nics_per_group * config.num_nic_ports, + static_cast(UCS_STATIC_BITMAP_POPCOUNT( + *nic_sys_dev_bitmap))); + } + } + } + void check_assignment(const topology_shape_t &config, - ucp_gpu_nic_assignment_policy_t policy, + ucp_gpu_nic_assignment_mode_t mode, const std::vector &expected_owners) { ASSERT_NE(config.num_groups, 0); @@ -233,32 +280,28 @@ class test_ucp_gpu_nic_assignment : public ucs::test { /* Leave (UCS_SYS_DEVICE_ID_UNKNOWN - 1) unmapped for the ungrouped lookup. */ ASSERT_LE(config.num_sys_devs(), UCS_SYS_DEVICE_ID_COUNT - 1); - if (config.num_gpus() == 0) { - ASSERT_TRUE(expected_owners.empty()); - } else { - ASSERT_EQ(config.num_nics(), expected_owners.size()); - for (size_t owner_idx = 0; owner_idx < expected_owners.size(); - ++owner_idx) { - ASSERT_LT(expected_owners[owner_idx], config.num_gpus()); - } - } - build_groups(config); - build_assignment(policy); + build_assignment(mode); if (config.num_gpus() == 0) { + ASSERT_TRUE(expected_owners.empty()); EXPECT_EQ(nullptr, m_assignment.nic_sys_dev_bitmaps); EXPECT_EQ(0ul, m_assignment.num_bitmaps); } else { check_gpu_device_aliases(config); - check_nic_owners(config, expected_owners); + if (mode == UCP_GPU_NIC_ASSIGNMENT_MODE_SHARED) { + ASSERT_TRUE(expected_owners.empty()); + check_nic_shared(config); + } else { + check_nic_owners(config, expected_owners); + } } } - void check_clique_assignment(size_t num_groups, size_t num_gpus_per_group, - size_t num_nics_per_group, - ucp_gpu_nic_assignment_policy_t policy, - const std::vector &expected_owners) + void check_assignment(size_t num_groups, size_t num_gpus_per_group, + size_t num_nics_per_group, + ucp_gpu_nic_assignment_mode_t mode, + const std::vector &expected_owners) { topology_shape_t config; @@ -274,7 +317,7 @@ class test_ucp_gpu_nic_assignment : public ucs::test { ++num_nic_ports) { config.num_gpu_devices = num_gpu_devices; config.num_nic_ports = num_nic_ports; - check_assignment(config, policy, expected_owners); + check_assignment(config, mode, expected_owners); } } } @@ -285,48 +328,54 @@ class test_ucp_gpu_nic_assignment : public ucs::test { }; UCS_TEST_F(test_ucp_gpu_nic_assignment, no_nics) { - check_clique_assignment(2, 3, 0, UCP_GPU_NIC_ASSIGNMENT_POLICY_FLIP, {}); + check_assignment(2, 3, 0, UCP_GPU_NIC_ASSIGNMENT_MODE_FLIP, {}); } UCS_TEST_F(test_ucp_gpu_nic_assignment, no_gpus) { - check_clique_assignment(2, 0, 3, UCP_GPU_NIC_ASSIGNMENT_POLICY_FLIP, {}); + check_assignment(2, 0, 3, UCP_GPU_NIC_ASSIGNMENT_MODE_FLIP, {}); } -UCS_TEST_F(test_ucp_gpu_nic_assignment, clique_flip_divisible) { +UCS_TEST_F(test_ucp_gpu_nic_assignment, flip_divisible) { /* Number of NICs is divisible by the number of GPUs. */ - check_clique_assignment(3, 2, 4, UCP_GPU_NIC_ASSIGNMENT_POLICY_FLIP, - {0, 1, 1, 0, /**/ - 2, 3, 3, 2, /**/ - 4, 5, 5, 4}); - check_clique_assignment(2, 3, 6, UCP_GPU_NIC_ASSIGNMENT_POLICY_FLIP, - {0, 1, 2, 2, 1, 0, /**/ - 3, 4, 5, 5, 4, 3}); + check_assignment(3, 2, 4, UCP_GPU_NIC_ASSIGNMENT_MODE_FLIP, + {0, 1, 1, 0, /**/ + 2, 3, 3, 2, /**/ + 4, 5, 5, 4}); + check_assignment(2, 3, 6, UCP_GPU_NIC_ASSIGNMENT_MODE_FLIP, + {0, 1, 2, 2, 1, 0, /**/ + 3, 4, 5, 5, 4, 3}); +} + +UCS_TEST_F(test_ucp_gpu_nic_assignment, flip_not_divisible) { + check_assignment(2, 2, 5, UCP_GPU_NIC_ASSIGNMENT_MODE_FLIP, + {0, 1, 1, 0, 0, /**/ + 2, 3, 3, 2, 2}); + check_assignment(2, 3, 2, UCP_GPU_NIC_ASSIGNMENT_MODE_FLIP, + {0, 1, /**/ + 3, 4}); } -UCS_TEST_F(test_ucp_gpu_nic_assignment, clique_flip_not_divisible) { - check_clique_assignment(2, 2, 5, UCP_GPU_NIC_ASSIGNMENT_POLICY_FLIP, - {0, 1, 1, 0, 0, /**/ - 2, 3, 3, 2, 2}); - check_clique_assignment(2, 3, 2, UCP_GPU_NIC_ASSIGNMENT_POLICY_FLIP, - {0, 1, /**/ - 3, 4}); +UCS_TEST_F(test_ucp_gpu_nic_assignment, round_robin_divisible) { + check_assignment(3, 2, 4, UCP_GPU_NIC_ASSIGNMENT_MODE_ROUND_ROBIN, + {0, 1, 0, 1, /**/ + 2, 3, 2, 3, /**/ + 4, 5, 4, 5}); + check_assignment(2, 3, 6, UCP_GPU_NIC_ASSIGNMENT_MODE_ROUND_ROBIN, + {0, 1, 2, 0, 1, 2, /**/ + 3, 4, 5, 3, 4, 5}); } -UCS_TEST_F(test_ucp_gpu_nic_assignment, clique_round_robin_divisible) { - check_clique_assignment(3, 2, 4, UCP_GPU_NIC_ASSIGNMENT_POLICY_ROUND_ROBIN, - {0, 1, 0, 1, /**/ - 2, 3, 2, 3, /**/ - 4, 5, 4, 5}); - check_clique_assignment(2, 3, 6, UCP_GPU_NIC_ASSIGNMENT_POLICY_ROUND_ROBIN, - {0, 1, 2, 0, 1, 2, /**/ - 3, 4, 5, 3, 4, 5}); +UCS_TEST_F(test_ucp_gpu_nic_assignment, round_robin_not_divisible) { + check_assignment(2, 2, 5, UCP_GPU_NIC_ASSIGNMENT_MODE_ROUND_ROBIN, + {0, 1, 0, 1, 0, /**/ + 2, 3, 2, 3, 2}); + check_assignment(2, 3, 2, UCP_GPU_NIC_ASSIGNMENT_MODE_ROUND_ROBIN, + {0, 1, /**/ + 3, 4}); } -UCS_TEST_F(test_ucp_gpu_nic_assignment, clique_round_robin_not_divisible) { - check_clique_assignment(2, 2, 5, UCP_GPU_NIC_ASSIGNMENT_POLICY_ROUND_ROBIN, - {0, 1, 0, 1, 0, /**/ - 2, 3, 2, 3, 2}); - check_clique_assignment(2, 3, 2, UCP_GPU_NIC_ASSIGNMENT_POLICY_ROUND_ROBIN, - {0, 1, /**/ - 3, 4}); +UCS_TEST_F(test_ucp_gpu_nic_assignment, shared) { + check_assignment(3, 2, 4, UCP_GPU_NIC_ASSIGNMENT_MODE_SHARED, {}); + /* More GPUs than NICs. */ + check_assignment(2, 3, 2, UCP_GPU_NIC_ASSIGNMENT_MODE_SHARED, {}); }