Skip to content

IB/ADDRESS: pack MTU value for non 4K value - v1.10 #6277

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yosefe merged 1 commit into from
Feb 5, 2021
Merged

IB/ADDRESS: pack MTU value for non 4K value - v1.10 #6277

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion src/uct/ib/base/ib_iface.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -403,6 +403,10 @@ unsigned uct_ib_iface_address_pack_flags(uct_ib_iface_t *iface)
pack_flags |= UCT_IB_ADDRESS_PACK_FLAG_SUBNET_PREFIX;
}

if (iface->config.path_mtu != IBV_MTU_4096) {
pack_flags |= UCT_IB_ADDRESS_PACK_FLAG_PATH_MTU;
}

return pack_flags;
}

Expand All @@ -424,8 +428,8 @@ void uct_ib_iface_address_pack(uct_ib_iface_t *iface, uct_ib_address_t *ib_addr)
params.gid = iface->gid_info.gid;
params.lid = uct_ib_iface_port_attr(iface)->lid;
params.roce_info = iface->gid_info.roce_info;
params.path_mtu = iface->config.path_mtu;
/* to suppress gcc 4.3.4 warning */
params.path_mtu = UCT_IB_ADDRESS_INVALID_PATH_MTU;
params.gid_index = UCT_IB_ADDRESS_INVALID_GID_INDEX;
params.pkey = iface->pkey;
uct_ib_address_pack(&params, ib_addr);
Expand Down
101 changes: 94 additions & 7 deletions test/gtest/uct/ib/test_ib.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ test_uct_ib::test_uct_ib() : m_e1(NULL), m_e2(NULL) { }
void test_uct_ib::create_connected_entities() {
m_e1 = uct_test::create_entity(0);
m_e2 = uct_test::create_entity(0);

m_entities.push_back(m_e1);
m_entities.push_back(m_e2);

Expand Down Expand Up @@ -99,8 +99,7 @@ class test_uct_ib_addr : public test_uct_ib {
pack_params.gid = gid_in;
pack_params.lid = lid_in;
pack_params.roce_info = iface->gid_info.roce_info;
/* to suppress gcc 4.3.4 warning */
pack_params.path_mtu = (enum ibv_mtu)0;
pack_params.path_mtu = iface->config.path_mtu;
pack_params.gid_index = std::numeric_limits<uint8_t>::max();
pack_params.pkey = iface->pkey;
address_size = uct_ib_address_size(&pack_params);
Expand Down Expand Up @@ -130,8 +129,15 @@ class test_uct_ib_addr : public test_uct_ib {
EXPECT_EQ(gid_in.global.interface_id, unpack_params.gid.global.interface_id);
}

EXPECT_TRUE(!(unpack_params.flags & UCT_IB_ADDRESS_PACK_FLAG_PATH_MTU));
EXPECT_EQ(UCT_IB_ADDRESS_INVALID_PATH_MTU, unpack_params.path_mtu);
if (iface->config.path_mtu == IBV_MTU_4096) {
EXPECT_FALSE(unpack_params.flags &
UCT_IB_ADDRESS_PACK_FLAG_PATH_MTU);
EXPECT_EQ(UCT_IB_ADDRESS_INVALID_PATH_MTU, unpack_params.path_mtu);
} else {
EXPECT_TRUE(unpack_params.flags &
UCT_IB_ADDRESS_PACK_FLAG_PATH_MTU);
EXPECT_EQ(iface->config.path_mtu, unpack_params.path_mtu);
}

EXPECT_TRUE(!(unpack_params.flags & UCT_IB_ADDRESS_PACK_FLAG_GID_INDEX));
EXPECT_EQ(UCT_IB_ADDRESS_INVALID_GID_INDEX, unpack_params.gid_index);
Expand Down Expand Up @@ -180,6 +186,20 @@ UCS_TEST_P(test_uct_ib_addr, address_pack) {
test_address_pack(0xdeadfeedbeefa880ul);
}

UCS_TEST_P(test_uct_ib_addr, address_pack_path_mtu, "IB_PATH_MTU=2048")
{
uct_ib_iface_t *iface = ucs_derived_of(m_entities.front()->iface(),
uct_ib_iface_t);
size_t addr_len = uct_ib_iface_address_size(iface);
std::vector<char> buffer(addr_len);
uct_ib_address_t *addr = (uct_ib_address_t*)&buffer[0];
uct_ib_iface_address_pack(iface, addr);
uct_ib_address_pack_params_t params;
uct_ib_address_unpack(addr, &params);
EXPECT_TRUE(params.flags & UCT_IB_ADDRESS_PACK_FLAG_PATH_MTU);
EXPECT_EQ(IBV_MTU_2048, params.path_mtu);
}

UCS_TEST_P(test_uct_ib_addr, fill_ah_attr) {
test_fill_ah_attr(UCT_IB_LINK_LOCAL_PREFIX);
test_fill_ah_attr(UCT_IB_SITE_LOCAL_PREFIX | htobe64(0x7200));
Expand Down Expand Up @@ -494,7 +514,7 @@ class test_uct_ib_sl_utils : public test_uct_ib_utils {
sls_without_ar = 0;
}

status = ib_select_sl(ar_enable, ooo_sl_mask, config, sl);
status = ib_select_sl(ar_enable, ooo_sl_mask, config, sl);
if ((ooo_sl_mask == 0) || (ar_enable == UCS_AUTO)) {
if (config_sl == UCS_ULUNITS_AUTO) {
EXPECT_EQ(m_default_sl, sl);
Expand Down Expand Up @@ -908,5 +928,72 @@ UCS_TEST_SKIP_COND_P(test_uct_event_ib, txrx_cq,
m_e2->flush();
}


UCT_INSTANTIATE_IB_TEST_CASE(test_uct_event_ib);

class test_uct_ib_mtu : public test_uct_ib {
public:
void create_connected_entities()
{
modify_config("IB_PATH_MTU", "4096");
m_e1 = uct_test::create_entity(0);
modify_config("IB_PATH_MTU", "2048");
m_e2 = uct_test::create_entity(0);

m_entities.push_back(m_e1);
m_entities.push_back(m_e2);

m_e1->connect(0, *m_e2, 0);
m_e2->connect(0, *m_e1, 0);
}

static ucs_status_t
ib_am_bcopy_handler(void *arg, void *data, size_t length, unsigned flags)
{
EXPECT_EQ((size_t)arg, length);
++test_uct_ib::m_ib_am_handler_counter;
return UCS_OK;
}

static size_t ib_am_bcopy_pack_cb(void *dest, void *arg)
{
size_t length = (size_t)arg;

memset(dest, 0, length);
return length;
}

void send_recv_bcopy(uct_ep_h ep, entity *ent, size_t length)
{
size_t start_am_counter = test_uct_ib::m_ib_am_handler_counter;
uct_ib_iface_t *iface = ucs_derived_of(ep->iface, uct_ib_iface_t);
uct_iface_attr_t attr;
ucs_status_t status;
size_t len;

status = uct_iface_query(&iface->super.super, &attr);
ASSERT_UCS_OK(status);
ASSERT_TRUE(attr.cap.flags & UCT_IFACE_FLAG_AM_BCOPY);
ASSERT_LE(length, attr.cap.am.max_bcopy);

/* set a callback for the uct to invoke for receiving the data */
uct_iface_set_am_handler(ent->iface(), 0, ib_am_bcopy_handler,
(void*)length, 0);

/* send the data */
len = uct_ep_am_bcopy(ep, 0, ib_am_bcopy_pack_cb, (void*)length, 0);
ASSERT_EQ(length, len);

flush();
wait_for_value(&test_uct_ib::m_ib_am_handler_counter,
start_am_counter + 1, true);
}
};

UCS_TEST_SKIP_COND_P(test_uct_ib_mtu, non_equal_mtu,
!check_caps(UCT_IFACE_FLAG_AM_BCOPY))
{
send_recv_bcopy(m_e1->ep(0), m_e2, m_e1->iface_attr().cap.am.max_bcopy);
send_recv_bcopy(m_e2->ep(0), m_e1, m_e2->iface_attr().cap.am.max_bcopy);
}

UCT_INSTANTIATE_RC_TEST_CASE(test_uct_ib_mtu);
2 changes: 1 addition & 1 deletion test/gtest/uct/ib/test_ib_pkey.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ class test_uct_ib_pkey : public test_uct_ib_with_specific_port {

uct_ib_iface_address_pack(iface, ib_addr);
uct_ib_address_unpack(ib_addr, &params);
EXPECT_TRUE((params.flags & UCT_IB_ADDRESS_PACK_FLAG_PKEY) != 0);
EXPECT_TRUE(params.flags & UCT_IB_ADDRESS_PACK_FLAG_PKEY);
EXPECT_EQ(m_pkey[0], params.pkey);

return params.pkey;
Expand Down
16 changes: 12 additions & 4 deletions test/gtest/uct/ib/test_ud_ds.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,16 @@ class test_ud_ds : public uct_test {
uct_ud_iface_addr_t *if_addr,
uct_ud_ep_conn_sn_t conn_sn, uct_ud_ep_t *ep);

void check_mtu(uct_ib_address_pack_params_t *unpack_params)
{
if (unpack_params->flags & UCT_IB_ADDRESS_PACK_FLAG_PATH_MTU) {
EXPECT_NE(UCT_IB_ADDRESS_INVALID_PATH_MTU, unpack_params->path_mtu);
EXPECT_NE(IBV_MTU_4096, unpack_params->path_mtu);
} else {
EXPECT_EQ(UCT_IB_ADDRESS_INVALID_PATH_MTU, unpack_params->path_mtu);
}
}

protected:
entity *m_e1, *m_e2;
uct_ib_address_t *ib_adr1, *ib_adr2;
Expand All @@ -89,10 +99,8 @@ UCS_TEST_P(test_ud_ds, if_addr) {
EXPECT_NE(uct_ib_unpack_uint24(if_adr1.qp_num),
uct_ib_unpack_uint24(if_adr2.qp_num));

EXPECT_TRUE(!(unpack_params1.flags & UCT_IB_ADDRESS_PACK_FLAG_PATH_MTU));
EXPECT_EQ(UCT_IB_ADDRESS_INVALID_PATH_MTU, unpack_params1.path_mtu);
EXPECT_TRUE(!(unpack_params2.flags & UCT_IB_ADDRESS_PACK_FLAG_PATH_MTU));
EXPECT_EQ(UCT_IB_ADDRESS_INVALID_PATH_MTU, unpack_params2.path_mtu);
check_mtu(&unpack_params1);
check_mtu(&unpack_params2);

EXPECT_TRUE(!(unpack_params1.flags & UCT_IB_ADDRESS_PACK_FLAG_GID_INDEX));
EXPECT_EQ(UCT_IB_ADDRESS_INVALID_GID_INDEX, unpack_params1.gid_index);
Expand Down