From 5727aeeb53f4569c807108e135da1ade61e47905 Mon Sep 17 00:00:00 2001 From: Anatoly Vildemanov Date: Tue, 1 Dec 2020 11:11:34 +0300 Subject: [PATCH 001/107] AZP: add warning for azure --- buildlib/tests.yml | 1 + contrib/test_jenkins.sh | 40 ++++++++++++++++++++++++++++++---------- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/buildlib/tests.yml b/buildlib/tests.yml index ecf7e671afd..711ce53cb14 100644 --- a/buildlib/tests.yml +++ b/buildlib/tests.yml @@ -38,4 +38,5 @@ jobs: RUN_TESTS: yes JENKINS_TEST_PERF: ${{ parameters.test_perf }} JENKINS_NO_VALGRIND: ${{ parameters.valgrind_disable }} + RUNNING_IN_AZURE: yes diff --git a/contrib/test_jenkins.sh b/contrib/test_jenkins.sh index d8159a381da..04653eb46d6 100755 --- a/contrib/test_jenkins.sh +++ b/contrib/test_jenkins.sh @@ -92,6 +92,26 @@ then fi echo "==== Running on $(hostname), worker $worker / $nworkers ====" +# Report an warning message to Azure pipeline +log_warning() { + msg=$1 + azp_tag="" + set +x + test "x$RUNNING_IN_AZURE" = "xyes" && azp_tag="##vso[task.logissue type=warning]" + echo "${azp_tag}${msg}" + set -x +} + +# Report an error message to Azure pipeline +log_error() { + msg=$1 + azp_tag="" + set +x + test "x$RUNNING_IN_AZURE" = "xyes" && azp_tag="##vso[task.logissue type=error]" + echo "${azp_tag}${msg}" + set -x +} + # # cleanup ucx # @@ -300,7 +320,7 @@ build_docs() { then doxy_ready=1 else - echo " doxygen was not found" + log_warning "Doxygen was not found" fi else doxy_ready=1 @@ -328,7 +348,7 @@ build_java_docs() { module unload dev/jdk module unload dev/mvn else - echo "No jdk and mvn module, failed to build docs". + log_warning "No jdk and mvn module, failed to build docs". fi } @@ -396,7 +416,7 @@ build_release_pkg() { # extract version from configure.ac and convert to MAJOR.MINOR.PATCH representation version=$(grep -P "define\S+ucx_ver" configure.ac | awk '{print $2}' | sed 's,),,' | xargs echo | tr ' ' '.') if ! grep -q "$version" ucx.spec.in; then - echo "Current UCX version ($version) is not present in ucx.spec.in changelog" + log_error "Current UCX version ($version) is not present in ucx.spec.in changelog" exit 1 fi cd - @@ -423,7 +443,7 @@ build_icc() { make_clean distclean echo "ok 1 - build successful " >> build_icc.tap else - echo "==== Not building with Intel compiler ====" + log_warning "==== Not building with Intel compiler ====" echo "ok 1 - # SKIP because Intel compiler not installed" >> build_icc.tap fi module_unload intel/ics-19.1.1 @@ -450,7 +470,7 @@ build_pgi() { make_clean distclean echo "ok 1 - build successful " >> build_pgi.tap else - echo "==== Not building with PGI compiler ====" + log_warning "==== Not building with PGI compiler ====" echo "ok 1 - # SKIP because PGI compiler not installed" >> build_pgi.tap fi @@ -595,12 +615,12 @@ build_gcc_latest() { echo "ok 1 - build successful " >> build_gcc_latest.tap module unload dev/gcc-latest else - echo "==== Not building with latest gcc compiler ====" + log_warning "==== Not building with latest gcc compiler ====" echo "ok 1 - # SKIP because dev/gcc-latest module is not available" >> build_gcc_latest.tap fi else - echo "==== Not building with gcc compiler ====" - echo "Required glibc version is too old ($ldd_ver)" + log_warning "==== Not building with gcc compiler ====" + log_warning "Required glibc version is too old ($ldd_ver)" echo "ok 1 - # SKIP because glibc version is older than 2.14" >> build_gcc_latest.tap fi } @@ -679,7 +699,7 @@ check_make_distcheck() { ../contrib/configure-release --prefix=$PWD/install $MAKEP DISTCHECK_CONFIGURE_FLAGS="--enable-gtest" distcheck else - echo "Not testing make distcheck: GCC version is too old ($(gcc --version|head -1))" + log_warning "Not testing make distcheck: GCC version is too old ($(gcc --version|head -1))" fi } @@ -697,7 +717,7 @@ check_config_h() { then echo "ok 1 - check successful " >> check_config_h.tap else - echo "Error: missing include config.h in files: $missing" + log_error "Missing include config.h in files: $missing" exit 1 fi } From 434b9fd9affce70a377c3904092bb51ea31ed90a Mon Sep 17 00:00:00 2001 From: dmitrygx Date: Wed, 25 Nov 2020 11:10:31 +0200 Subject: [PATCH 002/107] UCP/WIREUP/GTEST: Fix dead code in CM disconnect --- src/ucp/wireup/wireup_cm.c | 19 ++++++++++-------- test/gtest/ucp/test_ucp_sockaddr.cc | 30 ++++++++++++++++++++++++----- 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/src/ucp/wireup/wireup_cm.c b/src/ucp/wireup/wireup_cm.c index f0d1eea1405..a648d8c72be 100644 --- a/src/ucp/wireup/wireup_cm.c +++ b/src/ucp/wireup/wireup_cm.c @@ -638,10 +638,9 @@ static void ucp_ep_cm_disconnect_flushed_cb(ucp_request_t *req) UCS_ASYNC_UNBLOCK(async); } -static unsigned ucp_ep_cm_remote_disconnect_progress(void *arg) +static unsigned ucp_ep_cm_remote_disconnect_progress(ucp_ep_h ucp_ep) { ucs_status_t status = UCS_ERR_CONNECTION_RESET; - ucp_ep_h ucp_ep = arg; void *req; ucs_trace("ep %p: flags 0x%x cm_remote_disconnect_progress", ucp_ep, @@ -663,15 +662,18 @@ static unsigned ucp_ep_cm_remote_disconnect_progress(void *arg) * @ref ucp_ep_close_flushed_callback */ ucs_debug("ep %p: ep closed but request is not set, waiting for" " the flush callback", ucp_ep); - goto err; + return 1; } if (!(ucp_ep->flags & UCP_EP_FLAG_REMOTE_CONNECTED)) { /* CM disconnect happens during WIREUP MSGs exchange phase, when EP * is locally connected to the peer */ - goto err; + goto set_ep_failed; } + /* if the EP is local and remote connected, need to flush it from main + * thread first */ + /* * TODO: set the ucp_ep to error state to prevent user from sending more * ops. @@ -689,12 +691,13 @@ static unsigned ucp_ep_cm_remote_disconnect_progress(void *arg) status = UCS_PTR_STATUS(req); ucs_error("ucp_ep_flush_internal completed with error: %s", ucs_status_string(status)); - goto err; + goto set_ep_failed; } + ucp_ep_invoke_err_cb(ucp_ep, UCS_ERR_CONNECTION_RESET); return 1; -err: +set_ep_failed: ucp_worker_set_ep_failed(ucp_ep->worker, ucp_ep, ucp_ep_get_cm_uct_ep(ucp_ep), ucp_ep_get_cm_lane(ucp_ep), status); @@ -714,8 +717,6 @@ static unsigned ucp_ep_cm_disconnect_progress(void *arg) uct_cm_ep, ucp_ep->flags); ucs_assert(ucp_ep_get_cm_uct_ep(ucp_ep) == uct_cm_ep); - ucp_ep->flags &= ~UCP_EP_FLAG_REMOTE_CONNECTED; - if (ucp_ep->flags & UCP_EP_FLAG_FAILED) { /* - ignore close event on failed ep, since all lanes are destroyed in generic err flow @@ -738,6 +739,8 @@ static unsigned ucp_ep_cm_disconnect_progress(void *arg) ucp_ep, ucp_ep->flags); } + ucp_ep->flags &= ~UCP_EP_FLAG_REMOTE_CONNECTED; + UCS_ASYNC_UNBLOCK(async); return 1; } diff --git a/test/gtest/ucp/test_ucp_sockaddr.cc b/test/gtest/ucp/test_ucp_sockaddr.cc index ec01a41adaf..8fd5882b797 100644 --- a/test/gtest/ucp/test_ucp_sockaddr.cc +++ b/test/gtest/ucp/test_ucp_sockaddr.cc @@ -61,6 +61,8 @@ class test_ucp_sockaddr : public ucp_test { ucs::sock_addr_storage m_test_addr; void init() { + m_err_count = 0; + if (get_variant_value() & TEST_MODIFIER_CM) { modify_config("SOCKADDR_CM_ENABLE", "yes"); } @@ -398,14 +400,14 @@ class test_ucp_sockaddr : public ucp_test { virtual ucp_ep_params_t get_ep_params() { ucp_ep_params_t ep_params = ucp_test::get_ep_params(); - ep_params.field_mask |= UCP_EP_PARAM_FIELD_ERR_HANDLING_MODE | - UCP_EP_PARAM_FIELD_ERR_HANDLER; + ep_params.field_mask |= UCP_EP_PARAM_FIELD_ERR_HANDLING_MODE | + UCP_EP_PARAM_FIELD_ERR_HANDLER; /* The error handling requirement is needed since we need to take * care of a case where the client gets an error. In case ucp needs to * handle a large worker address but neither ud nor ud_x are present */ - ep_params.err_mode = UCP_ERR_HANDLING_MODE_PEER; - ep_params.err_handler.cb = err_handler_cb; - ep_params.err_handler.arg = NULL; + ep_params.err_mode = UCP_ERR_HANDLING_MODE_PEER; + ep_params.err_handler.cb = err_handler_cb; + ep_params.err_handler.arg = NULL; return ep_params; } @@ -512,6 +514,8 @@ class test_ucp_sockaddr : public ucp_test { static void err_handler_cb(void *arg, ucp_ep_h ep, ucs_status_t status) { ucp_test::err_handler_cb(arg, ep, status); + ++m_err_count; + /* The current expected errors are only from the err_handle test * and from transports where the worker address is too long but ud/ud_x * are not present, or ud/ud_x are present but their addresses are too @@ -566,8 +570,14 @@ class test_ucp_sockaddr : public ucp_test { ((lane1 != UCP_NULL_LANE) && (lane2 != UCP_NULL_LANE) && ucp_ep_config_lane_is_peer_equal(key1, lane1, key2, lane2))); } + +protected: + static unsigned m_err_count; }; +unsigned test_ucp_sockaddr::m_err_count = 0; + + UCS_TEST_SKIP_COND_P(test_ucp_sockaddr, listen, no_close_protocol()) { listen_and_communicate(false, 0); } @@ -606,6 +616,16 @@ UCS_TEST_P(test_ucp_sockaddr, onesided_disconnect_bidi) { one_sided_disconnect(sender(), UCP_EP_CLOSE_MODE_FLUSH); } +UCS_TEST_SKIP_COND_P(test_ucp_sockaddr, onesided_disconnect_bidi_wait_err_cb, + no_close_protocol()) { + listen_and_communicate(false, SEND_DIRECTION_BIDI); + + one_sided_disconnect(sender(), UCP_EP_CLOSE_MODE_FLUSH); + + wait_for_flag(&m_err_count); + EXPECT_EQ(1, m_err_count); +} + UCS_TEST_SKIP_COND_P(test_ucp_sockaddr, concurrent_disconnect, no_close_protocol()) { listen_and_communicate(false, 0); From 5c04b9ecbbad26d78502c1af80a11cc4ef855b9c Mon Sep 17 00:00:00 2001 From: dmitrygx Date: Wed, 25 Nov 2020 18:12:27 +0200 Subject: [PATCH 003/107] GTEST/UCP: Fix review comments --- src/ucp/wireup/wireup_cm.c | 18 +++++++++++++++--- test/gtest/ucp/test_ucp_sockaddr.cc | 3 +-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/ucp/wireup/wireup_cm.c b/src/ucp/wireup/wireup_cm.c index a648d8c72be..c3e5330b4be 100644 --- a/src/ucp/wireup/wireup_cm.c +++ b/src/ucp/wireup/wireup_cm.c @@ -638,7 +638,8 @@ static void ucp_ep_cm_disconnect_flushed_cb(ucp_request_t *req) UCS_ASYNC_UNBLOCK(async); } -static unsigned ucp_ep_cm_remote_disconnect_progress(ucp_ep_h ucp_ep) +static unsigned ucp_ep_cm_remote_disconnect_progress(ucp_ep_h ucp_ep, + int *invoke_err_cb_p) { ucs_status_t status = UCS_ERR_CONNECTION_RESET; void *req; @@ -694,7 +695,7 @@ static unsigned ucp_ep_cm_remote_disconnect_progress(ucp_ep_h ucp_ep) goto set_ep_failed; } - ucp_ep_invoke_err_cb(ucp_ep, UCS_ERR_CONNECTION_RESET); + *invoke_err_cb_p = 1; return 1; set_ep_failed: @@ -709,6 +710,7 @@ static unsigned ucp_ep_cm_disconnect_progress(void *arg) ucp_ep_h ucp_ep = arg; uct_ep_h uct_cm_ep = ucp_ep_get_cm_uct_ep(ucp_ep); ucs_async_context_t *async = &ucp_ep->worker->async; + int invoke_err_cb = 0; ucp_request_t *close_req; UCS_ASYNC_BLOCK(async); @@ -727,20 +729,30 @@ static unsigned ucp_ep_cm_disconnect_progress(void *arg) ucs_assert(ucp_ep->flags & UCP_EP_FLAG_CLOSED); } } else if (ucp_ep->flags & UCP_EP_FLAG_LOCAL_CONNECTED) { - ucp_ep_cm_remote_disconnect_progress(ucp_ep); + ucp_ep_cm_remote_disconnect_progress(ucp_ep, &invoke_err_cb); } else if (ucp_ep->flags & UCP_EP_FLAG_CLOSE_REQ_VALID) { /* if the EP is not local connected, the EP has been closed and flushed, CM lane is disconnected, complete close request and destroy EP */ ucs_assert(ucp_ep->flags & UCP_EP_FLAG_CLOSED); close_req = ucp_ep_ext_control(ucp_ep)->close_req.req; ucp_ep_local_disconnect_progress(close_req); + /* don't touch UCP EP after local disconnect, since it is not valid + *anymore */ + goto out; } else { ucs_warn("ep %p: unexpected state on disconnect, flags: 0x%u", ucp_ep, ucp_ep->flags); } ucp_ep->flags &= ~UCP_EP_FLAG_REMOTE_CONNECTED; + if (invoke_err_cb) { + /* here invoking error callback if it is asked through the flag set by + * ucp_ep_cm_remote_disconnect_progress(), because a user could close + * UCP EP in the error callback */ + ucp_ep_invoke_err_cb(ucp_ep, UCS_ERR_CONNECTION_RESET); + } +out: UCS_ASYNC_UNBLOCK(async); return 1; } diff --git a/test/gtest/ucp/test_ucp_sockaddr.cc b/test/gtest/ucp/test_ucp_sockaddr.cc index 8fd5882b797..d0a77098e42 100644 --- a/test/gtest/ucp/test_ucp_sockaddr.cc +++ b/test/gtest/ucp/test_ucp_sockaddr.cc @@ -621,9 +621,8 @@ UCS_TEST_SKIP_COND_P(test_ucp_sockaddr, onesided_disconnect_bidi_wait_err_cb, listen_and_communicate(false, SEND_DIRECTION_BIDI); one_sided_disconnect(sender(), UCP_EP_CLOSE_MODE_FLUSH); - wait_for_flag(&m_err_count); - EXPECT_EQ(1, m_err_count); + EXPECT_EQ(1u, m_err_count); } UCS_TEST_SKIP_COND_P(test_ucp_sockaddr, concurrent_disconnect, From 8e10f297978bf1f280e4a7fbe7a57c7e7d7f6da9 Mon Sep 17 00:00:00 2001 From: dmitrygx Date: Thu, 26 Nov 2020 12:44:27 +0200 Subject: [PATCH 004/107] UCP/WIREUP: Fix CM disconnect bugs --- src/ucp/wireup/wireup.c | 10 +++++++++- src/ucp/wireup/wireup_cm.c | 5 +++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/ucp/wireup/wireup.c b/src/ucp/wireup/wireup.c index 7f34412bbf8..17d566f30bb 100644 --- a/src/ucp/wireup/wireup.c +++ b/src/ucp/wireup/wireup.c @@ -337,7 +337,15 @@ void ucp_wireup_remote_connected(ucp_ep_h ep) } ucs_trace("ep %p: remote connected", ep); - ep->flags |= UCP_EP_FLAG_REMOTE_CONNECTED; + if (!(ep->flags & UCP_EP_FLAG_CLOSED)) { + /* set REMOTE_CONNECTED flag if an EP is not closed, otherwise - + * just make UCT EPs are remote conencted to remove WIREUP_EP + * for them and complete flush(LOCAL) oepration in UCP EP close + * procedure (don't set REMOTE_CONNECTED flag to avoid possible + * wrong behavior in ucp_ep_close_flushed_callback() when a peer was + * already disconnected, but we set REMOTE_CONENCTED flag again) */ + ep->flags |= UCP_EP_FLAG_REMOTE_CONNECTED; + } for (lane = 0; lane < ucp_ep_num_lanes(ep); ++lane) { if (ucp_ep_is_lane_p2p(ep, lane)) { diff --git a/src/ucp/wireup/wireup_cm.c b/src/ucp/wireup/wireup_cm.c index c3e5330b4be..06ca81c8988 100644 --- a/src/ucp/wireup/wireup_cm.c +++ b/src/ucp/wireup/wireup_cm.c @@ -739,6 +739,11 @@ static unsigned ucp_ep_cm_disconnect_progress(void *arg) /* don't touch UCP EP after local disconnect, since it is not valid *anymore */ goto out; + } else if (ucp_ep->flags & UCP_EP_FLAG_CLOSED) { + /* if an EP was closed and not local conencted anymore, not failed + * and no CLOSE request is set, it means that an EP was disconnected + * from a peer */ + ucs_assert(ucp_ep->flags & UCP_EP_FLAG_DISCONNECTED_CM_LANE); } else { ucs_warn("ep %p: unexpected state on disconnect, flags: 0x%u", ucp_ep, ucp_ep->flags); From 3ed57e750f23b7ae7090881e535aa877034e26f1 Mon Sep 17 00:00:00 2001 From: dmitrygx Date: Thu, 26 Nov 2020 14:32:55 +0200 Subject: [PATCH 005/107] UCP/WIREUP: Fix code cleanup --- src/ucp/wireup/wireup_cm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ucp/wireup/wireup_cm.c b/src/ucp/wireup/wireup_cm.c index 06ca81c8988..3c154f079f8 100644 --- a/src/ucp/wireup/wireup_cm.c +++ b/src/ucp/wireup/wireup_cm.c @@ -737,7 +737,7 @@ static unsigned ucp_ep_cm_disconnect_progress(void *arg) close_req = ucp_ep_ext_control(ucp_ep)->close_req.req; ucp_ep_local_disconnect_progress(close_req); /* don't touch UCP EP after local disconnect, since it is not valid - *anymore */ + * anymore */ goto out; } else if (ucp_ep->flags & UCP_EP_FLAG_CLOSED) { /* if an EP was closed and not local conencted anymore, not failed From 7c8bd965d55f96d652d68fc7213d8c7b52e79660 Mon Sep 17 00:00:00 2001 From: dmitrygx Date: Fri, 27 Nov 2020 10:16:13 +0300 Subject: [PATCH 006/107] UCP/EP: Fix review comments --- src/ucp/wireup/wireup.c | 6 +++--- src/ucp/wireup/wireup_cm.c | 36 ++++++++++++++++++++---------------- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/src/ucp/wireup/wireup.c b/src/ucp/wireup/wireup.c index 17d566f30bb..0f6de078369 100644 --- a/src/ucp/wireup/wireup.c +++ b/src/ucp/wireup/wireup.c @@ -339,11 +339,11 @@ void ucp_wireup_remote_connected(ucp_ep_h ep) ucs_trace("ep %p: remote connected", ep); if (!(ep->flags & UCP_EP_FLAG_CLOSED)) { /* set REMOTE_CONNECTED flag if an EP is not closed, otherwise - - * just make UCT EPs are remote conencted to remove WIREUP_EP - * for them and complete flush(LOCAL) oepration in UCP EP close + * just make UCT EPs are remote connected to remove WIREUP_EP + * for them and complete flush(LOCAL) operation in UCP EP close * procedure (don't set REMOTE_CONNECTED flag to avoid possible * wrong behavior in ucp_ep_close_flushed_callback() when a peer was - * already disconnected, but we set REMOTE_CONENCTED flag again) */ + * already disconnected, but we set REMOTE_CONNECTED flag again) */ ep->flags |= UCP_EP_FLAG_REMOTE_CONNECTED; } diff --git a/src/ucp/wireup/wireup_cm.c b/src/ucp/wireup/wireup_cm.c index 3c154f079f8..898675b42df 100644 --- a/src/ucp/wireup/wireup_cm.c +++ b/src/ucp/wireup/wireup_cm.c @@ -638,8 +638,9 @@ static void ucp_ep_cm_disconnect_flushed_cb(ucp_request_t *req) UCS_ASYNC_UNBLOCK(async); } -static unsigned ucp_ep_cm_remote_disconnect_progress(ucp_ep_h ucp_ep, - int *invoke_err_cb_p) +/* Returns boolean which specifies whether error callback should be invoked + * or not */ +static int ucp_ep_cm_remote_disconnect_progress(ucp_ep_h ucp_ep) { ucs_status_t status = UCS_ERR_CONNECTION_RESET; void *req; @@ -654,7 +655,7 @@ static unsigned ucp_ep_cm_remote_disconnect_progress(ucp_ep_h ucp_ep, UCP_EP_FLAG_CLOSE_REQ_VALID)) { ucp_request_complete_send(ucp_ep_ext_control(ucp_ep)->close_req.req, UCS_OK); - return 1; + return 0; } if (ucp_ep->flags & UCP_EP_FLAG_CLOSED) { @@ -663,7 +664,7 @@ static unsigned ucp_ep_cm_remote_disconnect_progress(ucp_ep_h ucp_ep, * @ref ucp_ep_close_flushed_callback */ ucs_debug("ep %p: ep closed but request is not set, waiting for" " the flush callback", ucp_ep); - return 1; + return 0; } if (!(ucp_ep->flags & UCP_EP_FLAG_REMOTE_CONNECTED)) { @@ -695,14 +696,13 @@ static unsigned ucp_ep_cm_remote_disconnect_progress(ucp_ep_h ucp_ep, goto set_ep_failed; } - *invoke_err_cb_p = 1; return 1; set_ep_failed: ucp_worker_set_ep_failed(ucp_ep->worker, ucp_ep, ucp_ep_get_cm_uct_ep(ucp_ep), ucp_ep_get_cm_lane(ucp_ep), status); - return 1; + return 0; } static unsigned ucp_ep_cm_disconnect_progress(void *arg) @@ -710,8 +710,8 @@ static unsigned ucp_ep_cm_disconnect_progress(void *arg) ucp_ep_h ucp_ep = arg; uct_ep_h uct_cm_ep = ucp_ep_get_cm_uct_ep(ucp_ep); ucs_async_context_t *async = &ucp_ep->worker->async; - int invoke_err_cb = 0; ucp_request_t *close_req; + int invoke_err_cb; UCS_ASYNC_BLOCK(async); @@ -729,18 +729,26 @@ static unsigned ucp_ep_cm_disconnect_progress(void *arg) ucs_assert(ucp_ep->flags & UCP_EP_FLAG_CLOSED); } } else if (ucp_ep->flags & UCP_EP_FLAG_LOCAL_CONNECTED) { - ucp_ep_cm_remote_disconnect_progress(ucp_ep, &invoke_err_cb); + invoke_err_cb = ucp_ep_cm_remote_disconnect_progress(ucp_ep); + ucp_ep->flags &= ~UCP_EP_FLAG_REMOTE_CONNECTED; + if (invoke_err_cb) { + /* here invoking error callback if it is asked through the flag, + * because a user could destroy UCP EP in the error callback */ + ucp_ep_invoke_err_cb(ucp_ep, UCS_ERR_CONNECTION_RESET); + } + goto out; } else if (ucp_ep->flags & UCP_EP_FLAG_CLOSE_REQ_VALID) { /* if the EP is not local connected, the EP has been closed and flushed, CM lane is disconnected, complete close request and destroy EP */ ucs_assert(ucp_ep->flags & UCP_EP_FLAG_CLOSED); - close_req = ucp_ep_ext_control(ucp_ep)->close_req.req; + ucp_ep->flags &= ~UCP_EP_FLAG_REMOTE_CONNECTED; + close_req = ucp_ep_ext_control(ucp_ep)->close_req.req; ucp_ep_local_disconnect_progress(close_req); /* don't touch UCP EP after local disconnect, since it is not valid * anymore */ goto out; } else if (ucp_ep->flags & UCP_EP_FLAG_CLOSED) { - /* if an EP was closed and not local conencted anymore, not failed + /* if an EP was closed and not local connected anymore, not failed * and no CLOSE request is set, it means that an EP was disconnected * from a peer */ ucs_assert(ucp_ep->flags & UCP_EP_FLAG_DISCONNECTED_CM_LANE); @@ -749,13 +757,9 @@ static unsigned ucp_ep_cm_disconnect_progress(void *arg) ucp_ep, ucp_ep->flags); } + /* don't remove the flag at the beggining of the function, some functions + * may rely on that flag (e.g. ucp_ep_cm_remote_disconnect_progress()) */ ucp_ep->flags &= ~UCP_EP_FLAG_REMOTE_CONNECTED; - if (invoke_err_cb) { - /* here invoking error callback if it is asked through the flag set by - * ucp_ep_cm_remote_disconnect_progress(), because a user could close - * UCP EP in the error callback */ - ucp_ep_invoke_err_cb(ucp_ep, UCS_ERR_CONNECTION_RESET); - } out: UCS_ASYNC_UNBLOCK(async); From 111135836d19d2796ea2fa9200d4b8c5d0c6f32f Mon Sep 17 00:00:00 2001 From: dmitrygx Date: Fri, 27 Nov 2020 17:23:00 +0200 Subject: [PATCH 007/107] UCP/WIREUP: Improve a comment and add useful assertion --- src/ucp/wireup/wireup_cm.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/ucp/wireup/wireup_cm.c b/src/ucp/wireup/wireup_cm.c index 898675b42df..b5b2f0a8c3e 100644 --- a/src/ucp/wireup/wireup_cm.c +++ b/src/ucp/wireup/wireup_cm.c @@ -748,10 +748,12 @@ static unsigned ucp_ep_cm_disconnect_progress(void *arg) * anymore */ goto out; } else if (ucp_ep->flags & UCP_EP_FLAG_CLOSED) { - /* if an EP was closed and not local connected anymore, not failed - * and no CLOSE request is set, it means that an EP was disconnected - * from a peer */ - ucs_assert(ucp_ep->flags & UCP_EP_FLAG_DISCONNECTED_CM_LANE); + /* if an EP was closed and not local connected anymore (i.e. + * ucp_ep_cm_disconnect_cm_lane() was called from ucp_ep_close_nbx()), + * not failed and no CLOSE request is set, it means that an EP was + * disconnected from a peer */ + ucs_assert((ucp_ep->flags & UCP_EP_FLAG_DISCONNECTED_CM_LANE) && + !(ucp_ep->flags & UCP_EP_FLAG_ERR_HANDLER_INVOKED)); } else { ucs_warn("ep %p: unexpected state on disconnect, flags: 0x%u", ucp_ep, ucp_ep->flags); From 5bc27a9b92304dc2eaf73f577c6cf87bb024347d Mon Sep 17 00:00:00 2001 From: dmitrygx Date: Fri, 27 Nov 2020 17:33:21 +0200 Subject: [PATCH 008/107] UCP/WIREUP: Fix typo in a comment --- src/ucp/wireup/wireup_cm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ucp/wireup/wireup_cm.c b/src/ucp/wireup/wireup_cm.c index b5b2f0a8c3e..a2db8961b51 100644 --- a/src/ucp/wireup/wireup_cm.c +++ b/src/ucp/wireup/wireup_cm.c @@ -759,7 +759,7 @@ static unsigned ucp_ep_cm_disconnect_progress(void *arg) ucp_ep, ucp_ep->flags); } - /* don't remove the flag at the beggining of the function, some functions + /* don't remove the flag at the beginning of the function, some functions * may rely on that flag (e.g. ucp_ep_cm_remote_disconnect_progress()) */ ucp_ep->flags &= ~UCP_EP_FLAG_REMOTE_CONNECTED; From 68eb80e0be5df8b0449089157c884c8c80498b94 Mon Sep 17 00:00:00 2001 From: dmitrygx Date: Mon, 30 Nov 2020 09:32:55 +0200 Subject: [PATCH 009/107] UCP/WIREUP: Fix review comments --- src/ucp/wireup/wireup_cm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ucp/wireup/wireup_cm.c b/src/ucp/wireup/wireup_cm.c index a2db8961b51..53be7a01d00 100644 --- a/src/ucp/wireup/wireup_cm.c +++ b/src/ucp/wireup/wireup_cm.c @@ -752,8 +752,8 @@ static unsigned ucp_ep_cm_disconnect_progress(void *arg) * ucp_ep_cm_disconnect_cm_lane() was called from ucp_ep_close_nbx()), * not failed and no CLOSE request is set, it means that an EP was * disconnected from a peer */ - ucs_assert((ucp_ep->flags & UCP_EP_FLAG_DISCONNECTED_CM_LANE) && - !(ucp_ep->flags & UCP_EP_FLAG_ERR_HANDLER_INVOKED)); + ucs_assert(ucp_ep->flags & UCP_EP_FLAG_DISCONNECTED_CM_LANE); + ucs_assert(!(ucp_ep->flags & UCP_EP_FLAG_ERR_HANDLER_INVOKED)); } else { ucs_warn("ep %p: unexpected state on disconnect, flags: 0x%u", ucp_ep, ucp_ep->flags); From 10fd38929cb793b408dadc57de1fc42a5c945dd5 Mon Sep 17 00:00:00 2001 From: dmitrygx Date: Wed, 2 Dec 2020 11:03:34 +0200 Subject: [PATCH 010/107] UCP/WIREUP: Fix case when EP was closed but not remote connected yet --- src/ucp/wireup/wireup_cm.c | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/ucp/wireup/wireup_cm.c b/src/ucp/wireup/wireup_cm.c index 53be7a01d00..1de6cc58367 100644 --- a/src/ucp/wireup/wireup_cm.c +++ b/src/ucp/wireup/wireup_cm.c @@ -658,21 +658,23 @@ static int ucp_ep_cm_remote_disconnect_progress(ucp_ep_h ucp_ep) return 0; } - if (ucp_ep->flags & UCP_EP_FLAG_CLOSED) { - /* the ep is closed by API but close req is not valid yet (checked - * above), it will be set later from scheduled - * @ref ucp_ep_close_flushed_callback */ - ucs_debug("ep %p: ep closed but request is not set, waiting for" - " the flush callback", ucp_ep); - return 0; - } - if (!(ucp_ep->flags & UCP_EP_FLAG_REMOTE_CONNECTED)) { - /* CM disconnect happens during WIREUP MSGs exchange phase, when EP - * is locally connected to the peer */ + /* CM disconnect happens during WIREUP MSGs exchange phase, when EP is + * locally connected to the peer, so UCP EP should not wait for flush + * completion even if it was started from close EP procedure, because + * it won't be never completed due to unreachability of the peer */ goto set_ep_failed; } + if (ucp_ep->flags & UCP_EP_FLAG_CLOSED) { + /* the ep is remote conencted (checked above) and closed by API but + * close req is not valid yet (checked above), it will be set later + * from scheduled @ref ucp_ep_close_flushed_callback */ + ucs_debug("ep %p: ep is remote connected and closed, but request is" + " not set, waiting for the flush callback", ucp_ep); + return 0; + } + /* if the EP is local and remote connected, need to flush it from main * thread first */ From 87f54eace6422419c15327d10538aa50c48716c9 Mon Sep 17 00:00:00 2001 From: dmitrygx Date: Wed, 2 Dec 2020 22:05:15 +0200 Subject: [PATCH 011/107] UCP/WIREUP: Remove flush in disconnect_cb --- src/ucp/core/ucp_ep.c | 5 +++ src/ucp/wireup/wireup_cm.c | 78 ++------------------------------------ 2 files changed, 9 insertions(+), 74 deletions(-) diff --git a/src/ucp/core/ucp_ep.c b/src/ucp/core/ucp_ep.c index e998f258d89..ed791028a22 100644 --- a/src/ucp/core/ucp_ep.c +++ b/src/ucp/core/ucp_ep.c @@ -1032,6 +1032,11 @@ ucs_status_ptr_t ucp_ep_close_nbx(ucp_ep_h ep, const ucp_request_param_t *param) ep->flags |= UCP_EP_FLAG_CLOSED; if (ucp_request_param_flags(param) & UCP_EP_CLOSE_FLAG_FORCE) { + /* FIXME: there is a potential issue with flush completion after an EP + * was forcibly closed from a user's error handling callback after + * disconnect event was received, but some EP flush operation still + * is in-progress, so, the destroyed EP will be touched upon flush + * completion om some transport */ if (!(ep->flags & UCP_EP_FLAG_FAILED)) { ucp_ep_discard_lanes(ep, UCS_ERR_CANCELED); } diff --git a/src/ucp/wireup/wireup_cm.c b/src/ucp/wireup/wireup_cm.c index 1de6cc58367..5124a9b3893 100644 --- a/src/ucp/wireup/wireup_cm.c +++ b/src/ucp/wireup/wireup_cm.c @@ -605,45 +605,9 @@ static void ucp_cm_client_connect_cb(uct_ep_h uct_cm_ep, void *arg, UCS_ASYNC_UNBLOCK(&worker->async); } -/* - * Internal flush completion callback which is a part of close protocol, - * this flush was initiated by remote peer in disconnect callback on CM lane. - */ -static void ucp_ep_cm_disconnect_flushed_cb(ucp_request_t *req) -{ - ucp_ep_h ucp_ep = req->send.ep; - /* the EP can be closed/destroyed from err callback */ - ucs_async_context_t *async = &ucp_ep->worker->async; - - UCS_ASYNC_BLOCK(async); - if (req->status == UCS_OK) { - ucs_assert(ucp_ep_is_cm_local_connected(ucp_ep)); - ucp_ep_cm_disconnect_cm_lane(ucp_ep); - } else if (ucp_ep->flags & UCP_EP_FLAG_FAILED) { - ucs_assert(!ucp_ep_is_cm_local_connected(ucp_ep)); - } else { - /* 1) ucp_ep_close(force) is called from err callback which was invoked - on remote connection reset - TODO: remove this case when IB flush cancel is fixed (#4743), - moving QP to err state should move UCP EP to error state, - then ucp_worker_set_ep_failed disconnects CM lane - 2) transport err is also possible on flush - */ - ucs_assert((req->status == UCS_ERR_CANCELED) || - (req->status == UCS_ERR_ENDPOINT_TIMEOUT)); - } - - ucs_assert(!(req->flags & UCP_REQUEST_FLAG_CALLBACK)); - ucp_request_put(req); - UCS_ASYNC_UNBLOCK(async); -} - -/* Returns boolean which specifies whether error callback should be invoked - * or not */ -static int ucp_ep_cm_remote_disconnect_progress(ucp_ep_h ucp_ep) +static void ucp_ep_cm_remote_disconnect_progress(ucp_ep_h ucp_ep) { ucs_status_t status = UCS_ERR_CONNECTION_RESET; - void *req; ucs_trace("ep %p: flags 0x%x cm_remote_disconnect_progress", ucp_ep, ucp_ep->flags); @@ -655,7 +619,7 @@ static int ucp_ep_cm_remote_disconnect_progress(ucp_ep_h ucp_ep) UCP_EP_FLAG_CLOSE_REQ_VALID)) { ucp_request_complete_send(ucp_ep_ext_control(ucp_ep)->close_req.req, UCS_OK); - return 0; + return; } if (!(ucp_ep->flags & UCP_EP_FLAG_REMOTE_CONNECTED)) { @@ -672,39 +636,13 @@ static int ucp_ep_cm_remote_disconnect_progress(ucp_ep_h ucp_ep) * from scheduled @ref ucp_ep_close_flushed_callback */ ucs_debug("ep %p: ep is remote connected and closed, but request is" " not set, waiting for the flush callback", ucp_ep); - return 0; - } - - /* if the EP is local and remote connected, need to flush it from main - * thread first */ - - /* - * TODO: set the ucp_ep to error state to prevent user from sending more - * ops. - */ - ucs_assert(ucp_ep->flags & UCP_EP_FLAG_FLUSH_STATE_VALID); - ucs_assert(!(ucp_ep->flags & UCP_EP_FLAG_CLOSED)); - req = ucp_ep_flush_internal(ucp_ep, 0, &ucp_request_null_param, NULL, - ucp_ep_cm_disconnect_flushed_cb, - "cm_disconnected_cb"); - if (req == NULL) { - /* flush is successfully completed in place, notify remote peer - * that we are disconnected, the EP will be destroyed from API call */ - ucp_ep_cm_disconnect_cm_lane(ucp_ep); - } else if (UCS_PTR_IS_ERR(req)) { - status = UCS_PTR_STATUS(req); - ucs_error("ucp_ep_flush_internal completed with error: %s", - ucs_status_string(status)); - goto set_ep_failed; + return; } - return 1; - set_ep_failed: ucp_worker_set_ep_failed(ucp_ep->worker, ucp_ep, ucp_ep_get_cm_uct_ep(ucp_ep), ucp_ep_get_cm_lane(ucp_ep), status); - return 0; } static unsigned ucp_ep_cm_disconnect_progress(void *arg) @@ -713,7 +651,6 @@ static unsigned ucp_ep_cm_disconnect_progress(void *arg) uct_ep_h uct_cm_ep = ucp_ep_get_cm_uct_ep(ucp_ep); ucs_async_context_t *async = &ucp_ep->worker->async; ucp_request_t *close_req; - int invoke_err_cb; UCS_ASYNC_BLOCK(async); @@ -731,14 +668,7 @@ static unsigned ucp_ep_cm_disconnect_progress(void *arg) ucs_assert(ucp_ep->flags & UCP_EP_FLAG_CLOSED); } } else if (ucp_ep->flags & UCP_EP_FLAG_LOCAL_CONNECTED) { - invoke_err_cb = ucp_ep_cm_remote_disconnect_progress(ucp_ep); - ucp_ep->flags &= ~UCP_EP_FLAG_REMOTE_CONNECTED; - if (invoke_err_cb) { - /* here invoking error callback if it is asked through the flag, - * because a user could destroy UCP EP in the error callback */ - ucp_ep_invoke_err_cb(ucp_ep, UCS_ERR_CONNECTION_RESET); - } - goto out; + ucp_ep_cm_remote_disconnect_progress(ucp_ep); } else if (ucp_ep->flags & UCP_EP_FLAG_CLOSE_REQ_VALID) { /* if the EP is not local connected, the EP has been closed and flushed, CM lane is disconnected, complete close request and destroy EP */ From fd5d2e88539675e54404e96878aebd464e999b1d Mon Sep 17 00:00:00 2001 From: dmitrygx Date: Thu, 3 Dec 2020 08:59:48 +0200 Subject: [PATCH 012/107] UCP/CORE/WIREUP: Fix review comments --- src/ucp/core/ucp_ep.c | 2 +- src/ucp/wireup/wireup.c | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ucp/core/ucp_ep.c b/src/ucp/core/ucp_ep.c index ed791028a22..6ba4b0d7f23 100644 --- a/src/ucp/core/ucp_ep.c +++ b/src/ucp/core/ucp_ep.c @@ -1036,7 +1036,7 @@ ucs_status_ptr_t ucp_ep_close_nbx(ucp_ep_h ep, const ucp_request_param_t *param) * was forcibly closed from a user's error handling callback after * disconnect event was received, but some EP flush operation still * is in-progress, so, the destroyed EP will be touched upon flush - * completion om some transport */ + * completion on some transport */ if (!(ep->flags & UCP_EP_FLAG_FAILED)) { ucp_ep_discard_lanes(ep, UCS_ERR_CANCELED); } diff --git a/src/ucp/wireup/wireup.c b/src/ucp/wireup/wireup.c index 0f6de078369..07e3a2cc4ed 100644 --- a/src/ucp/wireup/wireup.c +++ b/src/ucp/wireup/wireup.c @@ -339,11 +339,11 @@ void ucp_wireup_remote_connected(ucp_ep_h ep) ucs_trace("ep %p: remote connected", ep); if (!(ep->flags & UCP_EP_FLAG_CLOSED)) { /* set REMOTE_CONNECTED flag if an EP is not closed, otherwise - - * just make UCT EPs are remote connected to remove WIREUP_EP - * for them and complete flush(LOCAL) operation in UCP EP close - * procedure (don't set REMOTE_CONNECTED flag to avoid possible - * wrong behavior in ucp_ep_close_flushed_callback() when a peer was - * already disconnected, but we set REMOTE_CONNECTED flag again) */ + * just make UCT EPs remote connected to remove WIREUP_EP for them + * and complete flush(LOCAL) operation in UCP EP close procedure + * (don't set REMOTE_CONNECTED flag to avoid possible wrong behavior + * in ucp_ep_close_flushed_callback() when a peer was already + * disconnected, but we set REMOTE_CONNECTED flag again) */ ep->flags |= UCP_EP_FLAG_REMOTE_CONNECTED; } From d958ebe57262c9981b0dad9076802c22ba48f70a Mon Sep 17 00:00:00 2001 From: dmitrygx Date: Thu, 3 Dec 2020 12:02:20 +0200 Subject: [PATCH 013/107] UCP/WIREUP: Fix review comments --- src/ucp/wireup/wireup_cm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ucp/wireup/wireup_cm.c b/src/ucp/wireup/wireup_cm.c index 5124a9b3893..136c0c5c14c 100644 --- a/src/ucp/wireup/wireup_cm.c +++ b/src/ucp/wireup/wireup_cm.c @@ -631,7 +631,7 @@ static void ucp_ep_cm_remote_disconnect_progress(ucp_ep_h ucp_ep) } if (ucp_ep->flags & UCP_EP_FLAG_CLOSED) { - /* the ep is remote conencted (checked above) and closed by API but + /* the ep is remote connected (checked above) and closed by API but * close req is not valid yet (checked above), it will be set later * from scheduled @ref ucp_ep_close_flushed_callback */ ucs_debug("ep %p: ep is remote connected and closed, but request is" From c49224510007cc73b89ac473afe285943119c7bd Mon Sep 17 00:00:00 2001 From: Yossi Itigin Date: Thu, 3 Dec 2020 20:39:35 +0200 Subject: [PATCH 014/107] UCP/CM: Fix leak of listener requests If a listener is destroyed while it has a pending connect request which has not been dispatched yet, we must release it. Otherwise, rdma_destroy_id() will hang while waiting for all events to be released. --- src/ucp/core/ucp_listener.c | 8 ++++++++ src/ucp/wireup/wireup_cm.c | 19 +++++++++++++++++++ src/ucp/wireup/wireup_cm.h | 3 +++ 3 files changed, 30 insertions(+) diff --git a/src/ucp/core/ucp_listener.c b/src/ucp/core/ucp_listener.c index 69556de1083..28ae19425ad 100644 --- a/src/ucp/core/ucp_listener.c +++ b/src/ucp/core/ucp_listener.c @@ -523,6 +523,12 @@ void ucp_listener_destroy(ucp_listener_h listener) { ucs_trace("listener %p: destroying", listener); + UCS_ASYNC_BLOCK(&listener->worker->async); + ucs_callbackq_remove_if(&listener->worker->uct->progress_q, + ucp_cm_server_conn_request_progress_cb_pred, + listener); + UCS_ASYNC_UNBLOCK(&listener->worker->async); + if (ucp_worker_sockaddr_is_cm_proto(listener->worker)) { ucp_listener_close_uct_listeners(listener); } else { @@ -537,6 +543,8 @@ ucs_status_t ucp_listener_reject(ucp_listener_h listener, { ucp_worker_h worker = listener->worker; + ucs_trace("listener %p: free conn_request %p", listener, conn_request); + UCS_ASYNC_BLOCK(&worker->async); if (ucp_worker_sockaddr_is_cm_proto(worker)) { diff --git a/src/ucp/wireup/wireup_cm.c b/src/ucp/wireup/wireup_cm.c index f0d1eea1405..ab0d639ef28 100644 --- a/src/ucp/wireup/wireup_cm.c +++ b/src/ucp/wireup/wireup_cm.c @@ -872,6 +872,25 @@ static ucp_rsc_index_t ucp_listener_get_cm_index(uct_listener_h listener, return UCP_NULL_RESOURCE; } +int ucp_cm_server_conn_request_progress_cb_pred(const ucs_callbackq_elem_t *elem, + void *arg) +{ + ucp_listener_h listener = arg; + ucp_conn_request_h conn_request; + + if (elem->cb != ucp_cm_server_conn_request_progress) { + return 0; + } + + conn_request = elem->arg; + if (conn_request->listener != listener) { + return 0; + } + + ucp_listener_reject(listener, conn_request); + return 1; +} + void ucp_cm_server_conn_request_cb(uct_listener_h listener, void *arg, const uct_cm_listener_conn_request_args_t *conn_req_args) diff --git a/src/ucp/wireup/wireup_cm.h b/src/ucp/wireup/wireup_cm.h index edd2648c385..891b53f70e6 100644 --- a/src/ucp/wireup/wireup_cm.h +++ b/src/ucp/wireup/wireup_cm.h @@ -34,6 +34,9 @@ ucs_status_t ucp_ep_client_cm_connect_start(ucp_ep_h ucp_ep, ucs_status_t ucp_ep_client_cm_create_uct_ep(ucp_ep_h ucp_ep); +int ucp_cm_server_conn_request_progress_cb_pred(const ucs_callbackq_elem_t *elem, + void *arg); + void ucp_cm_server_conn_request_cb(uct_listener_h listener, void *arg, const uct_cm_listener_conn_request_args_t *conn_req_args); From ae06c8eda84ac9ed9562eda3f4b1a99027717df1 Mon Sep 17 00:00:00 2001 From: Sergey Oblomov Date: Mon, 7 Dec 2020 13:09:32 +0200 Subject: [PATCH 015/107] API/UCT/IFACE: added keepalive_timeout value - added keepalive_interval value to uct_iface_params structure --- src/uct/api/uct.h | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/src/uct/api/uct.h b/src/uct/api/uct.h index 7ff5a085693..5b6a673b7c0 100644 --- a/src/uct/api/uct.h +++ b/src/uct/api/uct.h @@ -597,51 +597,54 @@ enum uct_iface_open_mode { */ enum uct_iface_params_field { /** Enables @ref uct_iface_params_t::cpu_mask */ - UCT_IFACE_PARAM_FIELD_CPU_MASK = UCS_BIT(0), + UCT_IFACE_PARAM_FIELD_CPU_MASK = UCS_BIT(0), /** Enables @ref uct_iface_params_t::open_mode */ - UCT_IFACE_PARAM_FIELD_OPEN_MODE = UCS_BIT(1), + UCT_IFACE_PARAM_FIELD_OPEN_MODE = UCS_BIT(1), /** Enables @ref uct_iface_params_t_mode_device * "uct_iface_params_t::mode::device" */ - UCT_IFACE_PARAM_FIELD_DEVICE = UCS_BIT(2), + UCT_IFACE_PARAM_FIELD_DEVICE = UCS_BIT(2), /** Enables @ref uct_iface_params_t_mode_sockaddr * "uct_iface_params_t::mode::sockaddr" */ - UCT_IFACE_PARAM_FIELD_SOCKADDR = UCS_BIT(3), + UCT_IFACE_PARAM_FIELD_SOCKADDR = UCS_BIT(3), /** Enables @ref uct_iface_params_t::stats_root */ - UCT_IFACE_PARAM_FIELD_STATS_ROOT = UCS_BIT(4), + UCT_IFACE_PARAM_FIELD_STATS_ROOT = UCS_BIT(4), /** Enables @ref uct_iface_params_t::rx_headroom */ - UCT_IFACE_PARAM_FIELD_RX_HEADROOM = UCS_BIT(5), + UCT_IFACE_PARAM_FIELD_RX_HEADROOM = UCS_BIT(5), /** Enables @ref uct_iface_params_t::err_handler_arg */ - UCT_IFACE_PARAM_FIELD_ERR_HANDLER_ARG = UCS_BIT(6), + UCT_IFACE_PARAM_FIELD_ERR_HANDLER_ARG = UCS_BIT(6), /** Enables @ref uct_iface_params_t::err_handler */ - UCT_IFACE_PARAM_FIELD_ERR_HANDLER = UCS_BIT(7), + UCT_IFACE_PARAM_FIELD_ERR_HANDLER = UCS_BIT(7), /** Enables @ref uct_iface_params_t::err_handler_flags */ - UCT_IFACE_PARAM_FIELD_ERR_HANDLER_FLAGS = UCS_BIT(8), + UCT_IFACE_PARAM_FIELD_ERR_HANDLER_FLAGS = UCS_BIT(8), /** Enables @ref uct_iface_params_t::eager_arg */ - UCT_IFACE_PARAM_FIELD_HW_TM_EAGER_ARG = UCS_BIT(9), + UCT_IFACE_PARAM_FIELD_HW_TM_EAGER_ARG = UCS_BIT(9), /** Enables @ref uct_iface_params_t::eager_cb */ - UCT_IFACE_PARAM_FIELD_HW_TM_EAGER_CB = UCS_BIT(10), + UCT_IFACE_PARAM_FIELD_HW_TM_EAGER_CB = UCS_BIT(10), /** Enables @ref uct_iface_params_t::rndv_arg */ - UCT_IFACE_PARAM_FIELD_HW_TM_RNDV_ARG = UCS_BIT(11), + UCT_IFACE_PARAM_FIELD_HW_TM_RNDV_ARG = UCS_BIT(11), /** Enables @ref uct_iface_params_t::rndv_cb */ - UCT_IFACE_PARAM_FIELD_HW_TM_RNDV_CB = UCS_BIT(12), + UCT_IFACE_PARAM_FIELD_HW_TM_RNDV_CB = UCS_BIT(12), /** Enables @ref uct_iface_params_t::async_event_arg */ - UCT_IFACE_PARAM_FIELD_ASYNC_EVENT_ARG = UCS_BIT(13), + UCT_IFACE_PARAM_FIELD_ASYNC_EVENT_ARG = UCS_BIT(13), /** Enables @ref uct_iface_params_t::async_event_cb */ - UCT_IFACE_PARAM_FIELD_ASYNC_EVENT_CB = UCS_BIT(14) + UCT_IFACE_PARAM_FIELD_ASYNC_EVENT_CB = UCS_BIT(14), + + /** Enables @ref uct_iface_params_t::keepalive_interval */ + UCT_IFACE_PARAM_FIELD_KEEPALIVE_INTERVAL = UCS_BIT(15) }; /** @@ -1052,6 +1055,9 @@ struct uct_iface_params { * read by user if the iface has @ref UCT_IFACE_FLAG_EVENT_ASYNC_CB * capability */ uct_async_event_cb_t async_event_cb; + + /* Time period between keepalive rounds */ + ucs_time_t keepalive_interval; }; From 8346f5e2b8c7acaec79cce4026c9908225ccd0e8 Mon Sep 17 00:00:00 2001 From: Devendar Bureddy Date: Mon, 7 Dec 2020 21:49:05 +0200 Subject: [PATCH 016/107] JENKINS: update cuda module version to 11.1.1 --- contrib/test_jenkins.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/test_jenkins.sh b/contrib/test_jenkins.sh index 5471d6c483e..631cd09de44 100755 --- a/contrib/test_jenkins.sh +++ b/contrib/test_jenkins.sh @@ -26,8 +26,8 @@ WORKSPACE=${WORKSPACE:=$PWD} ucx_inst=${WORKSPACE}/install -CUDA_MODULE="dev/cuda11.0" -GDRCOPY_MODULE="dev/gdrcopy2.0_cuda11.0" +CUDA_MODULE="dev/cuda11.1.1" +GDRCOPY_MODULE="dev/gdrcopy2.1_cuda11.1.1" if [ -z "$BUILD_NUMBER" ]; then echo "Running interactive" From 3e300a2ed5993fa3e17da66ca53d6eb879d51885 Mon Sep 17 00:00:00 2001 From: Peter Rudenko Date: Sat, 28 Nov 2020 17:00:45 +0200 Subject: [PATCH 017/107] UCP: ucp_tag_msg_recv_nbx routine. UCP: implement ucp_tag_msg_recv_nb as ucp_tag_msg_recv_nbx. UCP: copy user data only if flag is present. DOC: updated news. --- NEWS | 3 +- src/ucp/api/ucp.h | 38 ++++++++++++++++++++++-- src/ucp/tag/tag_recv.c | 43 +++++++++++++++++++--------- test/gtest/ucp/test_ucp_tag_probe.cc | 11 +++++-- 4 files changed, 75 insertions(+), 20 deletions(-) diff --git a/NEWS b/NEWS index 720173b1e94..bd959726173 100644 --- a/NEWS +++ b/NEWS @@ -9,7 +9,8 @@ ## Current ### Features: TBD -#### UCX Core TBD +#### UCX Core +- Added ucp_tag_msg_recv_nbx routine. #### UCX Java (API Preview) TBD ### Bugfixes: TBD diff --git a/src/ucp/api/ucp.h b/src/ucp/api/ucp.h index 598c46a6c03..70620ab3f88 100644 --- a/src/ucp/api/ucp.h +++ b/src/ucp/api/ucp.h @@ -3460,7 +3460,7 @@ ucp_tag_message_h ucp_tag_probe_nb(ucp_worker_h worker, ucp_tag_t tag, * This routine receives a message that is described by the local address @a * buffer, size @a count, @a message handle, and @a datatype object on the @a * worker. The @a message handle can be obtained by calling the @ref - * ucp_tag_probe_nb "ucp_tag_probe_nb()" routine. @ref ucp_tag_msg_recv_nb + * ucp_tag_probe_nb "ucp_tag_probe_nb()" routine. The @ref ucp_tag_msg_recv_nb * "ucp_tag_msg_recv_nb()" routine is non-blocking and therefore returns * immediately. The receive operation is considered completed when the message * is delivered to the @a buffer. In order to notify the application about @@ -3470,7 +3470,7 @@ ucp_tag_message_h ucp_tag_probe_nb(ucp_worker_h worker, ucp_tag_t tag, * routine returns an error. * * @param [in] worker UCP worker that is used for the receive operation. - * @param [in] buffer Pointer to the buffer to receive the data to. + * @param [in] buffer Pointer to the buffer that will receive the data. * @param [in] count Number of elements to receive * @param [in] datatype Datatype descriptor for the elements in the buffer. * @param [in] message Message handle. @@ -3492,6 +3492,40 @@ ucs_status_ptr_t ucp_tag_msg_recv_nb(ucp_worker_h worker, void *buffer, ucp_tag_recv_callback_t cb); +/** + * @ingroup UCP_COMM + * @brief Non-blocking receive operation for a probed message. + * + * This routine receives a message that is described by the local address @a + * buffer, size @a count, and @a message handle on the @a worker. + * The @a message handle can be obtained by calling the @ref + * ucp_tag_probe_nb "ucp_tag_probe_nb()" routine. The @ref ucp_tag_msg_recv_nbx + * "ucp_tag_msg_recv_nbx()" routine is non-blocking and therefore returns + * immediately. The receive operation is considered completed when the message + * is delivered to the @a buffer. In order to notify the application about + * completion of the receive operation the UCP library will invoke the + * call-back @a cb when the received message is in the receive buffer and ready + * for application access. If the receive operation cannot be started the + * routine returns an error. + * + * @param [in] worker UCP worker that is used for the receive operation. + * @param [in] buffer Pointer to the buffer that will receive the data. + * @param [in] count Number of elements to receive + * @param [in] message Message handle. + * @param [in] param Operation parameters, see @ref ucp_request_param_t + * + * @return UCS_PTR_IS_ERR(_ptr) - The receive operation failed. + * @return otherwise - Operation was scheduled for receive. The request + * handle is returned to the application in order + * to track progress of the operation. The + * application is responsible for releasing the + * handle using @ref ucp_request_free + * "ucp_request_free()" routine. + */ +ucs_status_ptr_t ucp_tag_msg_recv_nbx(ucp_worker_h worker, void *buffer, + size_t count, ucp_tag_message_h message, + const ucp_request_param_t *param); + /** * @ingroup UCP_COMM * @brief Non-blocking implicit remote memory put operation. diff --git a/src/ucp/tag/tag_recv.c b/src/ucp/tag/tag_recv.c index c0a19d1aa18..ba6292e52f2 100644 --- a/src/ucp/tag/tag_recv.c +++ b/src/ucp/tag/tag_recv.c @@ -101,7 +101,12 @@ ucp_tag_recv_common(ucp_worker_h worker, void *buffer, size_t count, req->recv.tag.tag_mask = tag_mask; if (param->op_attr_mask & UCP_OP_ATTR_FIELD_CALLBACK) { req->recv.tag.cb = param->cb.recv; - req->user_data = param->user_data; + + if (param->op_attr_mask & UCP_OP_ATTR_FIELD_USER_DATA) { + req->user_data = param->user_data; + } else { + req->user_data = NULL; + } } if (ucs_log_is_enabled(UCS_LOG_LEVEL_TRACE_REQ)) { @@ -221,34 +226,44 @@ UCS_PROFILE_FUNC(ucs_status_ptr_t, ucp_tag_recv_nbx, return ret; } -UCS_PROFILE_FUNC(ucs_status_ptr_t, ucp_tag_msg_recv_nb, - (worker, buffer, count, datatype, message, cb), - ucp_worker_h worker, void *buffer, size_t count, - ucp_datatype_t datatype, ucp_tag_message_h message, - ucp_tag_recv_callback_t cb) +ucs_status_ptr_t ucp_tag_msg_recv_nb(ucp_worker_h worker, void *buffer, size_t count, + ucp_datatype_t datatype, ucp_tag_message_h message, + ucp_tag_recv_callback_t cb) { ucp_request_param_t param = { .op_attr_mask = UCP_OP_ATTR_FIELD_CALLBACK | + UCP_OP_ATTR_FIELD_DATATYPE | UCP_OP_ATTR_FLAG_NO_IMM_CMPL, + .datatype = datatype, .cb.recv = (ucp_tag_recv_nbx_callback_t)cb }; + + return ucp_tag_msg_recv_nbx(worker, buffer, count, message, ¶m); +} + +UCS_PROFILE_FUNC(ucs_status_ptr_t, ucp_tag_msg_recv_nbx, + (worker, buffer, count, message, param), + ucp_worker_h worker, void *buffer, size_t count, + ucp_tag_message_h message, const ucp_request_param_t *param) +{ ucp_recv_desc_t *rdesc = message; ucs_status_ptr_t ret; ucp_request_t *req; + ucp_datatype_t datatype; UCP_CONTEXT_CHECK_FEATURE_FLAGS(worker->context, UCP_FEATURE_TAG, return UCS_STATUS_PTR(UCS_ERR_INVALID_PARAM)); UCP_WORKER_THREAD_CS_ENTER_CONDITIONAL(worker); - req = ucp_request_get(worker); - if (ucs_likely(req != NULL)) { - ret = ucp_tag_recv_common(worker, buffer, count, datatype, - ucp_rdesc_get_tag(rdesc), UCP_TAG_MASK_FULL, - req, rdesc, ¶m, "msg_recv_nb"); - } else { - ret = UCS_STATUS_PTR(UCS_ERR_NO_MEMORY); - } + req = ucp_request_get_param(worker, param, + {ret = UCS_STATUS_PTR(UCS_ERR_NO_MEMORY); + goto out;}); + datatype = ucp_request_param_datatype(param); + ret = ucp_tag_recv_common(worker, buffer, count, datatype, + ucp_rdesc_get_tag(rdesc), UCP_TAG_MASK_FULL, + req, rdesc, param, "msg_recv_nbx"); +out: UCP_WORKER_THREAD_CS_EXIT_CONDITIONAL(worker); return ret; } diff --git a/test/gtest/ucp/test_ucp_tag_probe.cc b/test/gtest/ucp/test_ucp_tag_probe.cc index cfb355518c3..63137bfe94b 100644 --- a/test/gtest/ucp/test_ucp_tag_probe.cc +++ b/test/gtest/ucp/test_ucp_tag_probe.cc @@ -166,9 +166,14 @@ UCS_TEST_P(test_ucp_tag_probe, send_rndv_msg_probe, "RNDV_THRESH=1048576") { EXPECT_EQ((ucp_tag_t)0x111337, info.sender_tag); /* receiver - process the rts and schedule a get operation */ - my_recv_req = (request*)ucp_tag_msg_recv_nb(receiver().worker(), &recvbuf[0], - recvbuf.size(), DATATYPE, message, - recv_callback); + ucp_request_param_t param; + param.op_attr_mask = UCP_OP_ATTR_FIELD_CALLBACK | UCP_OP_ATTR_FIELD_DATATYPE | + UCP_OP_ATTR_FLAG_NO_IMM_CMPL; + param.cb.recv = recv_callback; + param.datatype = DATATYPE; + + my_recv_req = (request*)ucp_tag_msg_recv_nbx(receiver().worker(), &recvbuf[0], + recvbuf.size(), message, ¶m); ASSERT_TRUE(!UCS_PTR_IS_ERR(my_recv_req)); /* receiver - perform rndv get and send the ATS */ From 3fd1a31ad27851c76f5c550d63d5bca2de1d3ea1 Mon Sep 17 00:00:00 2001 From: Yossi Itigin Date: Thu, 3 Dec 2020 20:39:54 +0200 Subject: [PATCH 018/107] TEST/IODEMO: Fix test destructor flow - Destroy all connections to flush requests before destroying memory pools - Avoid removing elements from connections map while iterating over it --- test/apps/iodemo/io_demo.cc | 9 ++++++++- test/apps/iodemo/ucx_wrapper.cc | 20 ++++++++------------ test/apps/iodemo/ucx_wrapper.h | 6 ++---- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/test/apps/iodemo/io_demo.cc b/test/apps/iodemo/io_demo.cc index 6f16013c032..d82460d7cd3 100644 --- a/test/apps/iodemo/io_demo.cc +++ b/test/apps/iodemo/io_demo.cc @@ -638,6 +638,10 @@ class DemoServer : public P2pDemoCommon { save_prev_state(); } + virtual ~DemoServer() { + destroy_connections(); + } + void run() { struct sockaddr_in listen_addr; memset(&listen_addr, 0, sizeof(listen_addr)); @@ -926,6 +930,10 @@ class DemoClient : public P2pDemoCommon { _read_callback_pool(opts().iomsg_size, "read callbacks") { } + virtual ~DemoClient() { + destroy_connections(); + } + typedef enum { OK, CONN_RETRIES_EXCEEDED, @@ -1519,7 +1527,6 @@ class DemoClient : public P2pDemoCommon { } } - private: std::vector _server_info; std::vector _active_servers; diff --git a/test/apps/iodemo/ucx_wrapper.cc b/test/apps/iodemo/ucx_wrapper.cc index 7590b07c462..06bfa0103b8 100644 --- a/test/apps/iodemo/ucx_wrapper.cc +++ b/test/apps/iodemo/ucx_wrapper.cc @@ -434,11 +434,11 @@ void UcxContext::destroy_connections() _conn_requests.pop_front(); } - for (conn_map_t::iterator iter = _conns.begin(); iter != _conns.end(); ++iter) { - delete iter->second; + UCX_LOG << "destroy_connections"; + while (!_conns.empty()) { + // UcxConnection destructor removes itself from _conns map + delete _conns.begin()->second; } - - _conns.clear(); } void UcxContext::destroy_listener() @@ -524,9 +524,12 @@ UcxConnection::~UcxConnection() { UCX_CONN_LOG << "destroying, ep is " << _ep; + _context.remove_connection(this); + cancel_all(); + // if _ep is NULL, connection was closed and removed by error handler if (_ep != NULL) { - disconnect(UCP_EP_CLOSE_MODE_FORCE); + ep_close(UCP_EP_CLOSE_MODE_FORCE); } if (_close_request) { @@ -859,13 +862,6 @@ void UcxConnection::handle_connection_error(ucs_status_t status) } } -void UcxConnection::disconnect(enum ucp_ep_close_mode mode) -{ - _context.remove_connection(this); - cancel_all(); - ep_close(mode); -} - void UcxConnection::ep_close(enum ucp_ep_close_mode mode) { static const char *mode_str[] = {"force", "flush"}; diff --git a/test/apps/iodemo/ucx_wrapper.h b/test/apps/iodemo/ucx_wrapper.h index 1658dd1dc50..17ba37ba4cc 100644 --- a/test/apps/iodemo/ucx_wrapper.h +++ b/test/apps/iodemo/ucx_wrapper.h @@ -89,6 +89,8 @@ class UcxContext { static const std::string sockaddr_str(const struct sockaddr* saddr, size_t addrlen); + void destroy_connections(); + protected: // Called when new IO message is received @@ -164,8 +166,6 @@ class UcxContext { void handle_connection_error(UcxConnection *conn); - void destroy_connections(); - void destroy_listener(); void destroy_worker(); @@ -260,8 +260,6 @@ class UcxConnection { void handle_connection_error(ucs_status_t status); - void disconnect(enum ucp_ep_close_mode mode); - void ep_close(enum ucp_ep_close_mode mode); bool process_request(const char *what, ucs_status_ptr_t ptr_status, From b34727b653f7844ecbd588a8e130624a3b9b599f Mon Sep 17 00:00:00 2001 From: Anatoly Vildemanov Date: Tue, 8 Dec 2020 17:42:16 +0300 Subject: [PATCH 019/107] JENKINS: change log --- buildlib/tests.yml | 1 + contrib/test_jenkins.sh | 12 ++---------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/buildlib/tests.yml b/buildlib/tests.yml index 711ce53cb14..87e513c7fb0 100644 --- a/buildlib/tests.yml +++ b/buildlib/tests.yml @@ -24,6 +24,7 @@ jobs: clean: true - bash: | + source ./buildlib/az-helpers.sh ./contrib/test_jenkins.sh displayName: Run ./contrib/test_jenkins.sh env: diff --git a/contrib/test_jenkins.sh b/contrib/test_jenkins.sh index 04653eb46d6..dffafc42ea0 100755 --- a/contrib/test_jenkins.sh +++ b/contrib/test_jenkins.sh @@ -95,21 +95,13 @@ echo "==== Running on $(hostname), worker $worker / $nworkers ====" # Report an warning message to Azure pipeline log_warning() { msg=$1 - azp_tag="" - set +x - test "x$RUNNING_IN_AZURE" = "xyes" && azp_tag="##vso[task.logissue type=warning]" - echo "${azp_tag}${msg}" - set -x + test "x$RUNNING_IN_AZURE" = "xyes" && { azure_log_warning "${msg}" ; set -x; } || echo "${msg}" } # Report an error message to Azure pipeline log_error() { msg=$1 - azp_tag="" - set +x - test "x$RUNNING_IN_AZURE" = "xyes" && azp_tag="##vso[task.logissue type=error]" - echo "${azp_tag}${msg}" - set -x + test "x$RUNNING_IN_AZURE" = "xyes" && { azure_log_error "${msg}" ; set -x; } || echo "${msg}" } # From 1d33c7f2fc8261f21eae4106115412f288be0bdf Mon Sep 17 00:00:00 2001 From: Akshay Venkatesh Date: Wed, 11 Nov 2020 13:23:43 -0800 Subject: [PATCH 020/107] UCP/WIREUP: use ucs topo to compare with mem_type md and adjust latency --- src/ucp/core/ucp_context.c | 20 +++++++++++++++++-- src/ucp/core/ucp_context.h | 5 +++++ src/ucp/core/ucp_worker.c | 36 ++++++++++++++++++++++++++++++++++ src/uct/base/uct_iface.c | 3 ++- src/uct/base/uct_iface.h | 1 + src/uct/cuda/base/cuda_iface.c | 11 +++++++++-- src/uct/cuda/base/cuda_iface.h | 3 +++ src/uct/cuda/base/cuda_md.c | 5 +++-- src/uct/rocm/base/rocm_base.c | 5 +++-- src/uct/sm/base/sm_iface.c | 5 +++-- src/uct/sm/self/self.c | 1 + 11 files changed, 84 insertions(+), 11 deletions(-) diff --git a/src/ucp/core/ucp_context.c b/src/ucp/core/ucp_context.c index 5caeb9dc50b..f5dfd8052df 100644 --- a/src/ucp/core/ucp_context.c +++ b/src/ucp/core/ucp_context.c @@ -117,6 +117,10 @@ static ucs_config_field_t ucp_config_table[] = { "establishing client/server connection. ", ucs_offsetof(ucp_config_t, sockaddr_aux_tls), UCS_CONFIG_TYPE_STRING_ARRAY}, + {"SELECT_DISTANCE_MD", "cuda_cpy", + "MD whose distance is queried when evaluating transport selection score", + ucs_offsetof(ucp_config_t, selection_cmp), UCS_CONFIG_TYPE_STRING}, + {"WARN_INVALID_CONFIG", "y", "Issue a warning in case of invalid device and/or transport configuration.", ucs_offsetof(ucp_config_t, warn_invalid_config), UCS_CONFIG_TYPE_BOOL}, @@ -706,7 +710,8 @@ static ucs_status_t ucp_add_tl_resources(ucp_context_h context, /* add sockaddr dummy resource, if md supports it */ if (md->attr.cap.flags & UCT_MD_FLAG_SOCKADDR) { - sa_rsc.dev_type = UCT_DEVICE_TYPE_NET; + sa_rsc.dev_type = UCT_DEVICE_TYPE_NET; + sa_rsc.sys_device = UCS_SYS_DEVICE_ID_UNKNOWN; ucs_snprintf_zero(sa_rsc.tl_name, UCT_TL_NAME_MAX, "%s", md->rsc.md_name); ucs_snprintf_zero(sa_rsc.dev_name, UCT_DEVICE_NAME_MAX, "sockaddr"); ucp_add_tl_resource_if_enabled(context, md, md_index, config, &sa_rsc, @@ -1450,11 +1455,19 @@ static ucs_status_t ucp_fill_config(ucp_context_h context, * routines */ UCP_THREAD_LOCK_INIT(&context->mt_lock); + /* save comparison MD for iface_attr adjustment */ + context->config.selection_cmp = ucs_strdup(config->selection_cmp, + "selection cmp"); + if (context->config.selection_cmp == NULL) { + status = UCS_ERR_NO_MEMORY; + goto err; + } + /* save environment prefix to later notify user for unused variables */ context->config.env_prefix = ucs_strdup(config->env_prefix, "ucp config"); if (context->config.env_prefix == NULL) { status = UCS_ERR_NO_MEMORY; - goto err; + goto err_free_selection_cmp; } /* Get allocation alignment from configuration, make sure it's valid */ @@ -1535,6 +1548,8 @@ static ucs_status_t ucp_fill_config(ucp_context_h context, ucs_free(context->config.alloc_methods); err_free_env_prefix: ucs_free(context->config.env_prefix); +err_free_selection_cmp: + ucs_free(context->config.selection_cmp); err: UCP_THREAD_LOCK_FINALIZE(&context->mt_lock); return status; @@ -1544,6 +1559,7 @@ static void ucp_free_config(ucp_context_h context) { ucs_free(context->config.alloc_methods); ucs_free(context->config.env_prefix); + ucs_free(context->config.selection_cmp); } ucs_status_t ucp_init_version(unsigned api_major_version, unsigned api_minor_version, diff --git a/src/ucp/core/ucp_context.h b/src/ucp/core/ucp_context.h index 6c7dc31df9a..5bd0ebc9aca 100644 --- a/src/ucp/core/ucp_context.h +++ b/src/ucp/core/ucp_context.h @@ -134,6 +134,8 @@ struct ucp_config { int warn_invalid_config; /** This config environment prefix */ char *env_prefix; + /** MD to compare for transport selection scores */ + char *selection_cmp; /** Configuration saved directly in the context */ ucp_context_config_t ctx; }; @@ -256,6 +258,9 @@ typedef struct ucp_context { /* Config environment prefix used to create the context */ char *env_prefix; + /* MD to compare for transport selection scores */ + char *selection_cmp; + } config; /* All configurations about multithreading support */ diff --git a/src/ucp/core/ucp_worker.c b/src/ucp/core/ucp_worker.c index f0cf53a8bb2..25c6ffc72bc 100644 --- a/src/ucp/core/ucp_worker.c +++ b/src/ucp/core/ucp_worker.c @@ -1151,6 +1151,31 @@ static void ucp_worker_close_ifaces(ucp_worker_h worker) UCS_ASYNC_UNBLOCK(&worker->async); } +static ucs_status_t +ucp_worker_get_sys_device_distance(ucp_context_h context, + ucp_rsc_index_t rsc_index, + ucs_sys_dev_distance_t *distance) +{ + ucs_sys_device_t device = UCS_SYS_DEVICE_ID_UNKNOWN; + ucs_sys_device_t cmp_device = UCS_SYS_DEVICE_ID_UNKNOWN; + ucp_rsc_index_t md_index, i; + + for (i = 0; i < context->num_tls; i++) { + md_index = context->tl_rscs[i].md_index; + if (strcmp(context->tl_mds[md_index].rsc.md_name, + context->config.selection_cmp)) { + continue; + } + + device = context->tl_rscs[rsc_index].tl_rsc.sys_device; + cmp_device = context->tl_rscs[i].tl_rsc.sys_device; + + return ucs_topo_get_distance(device, cmp_device, distance); + } + + return UCS_ERR_NO_RESOURCE; +} + ucs_status_t ucp_worker_iface_open(ucp_worker_h worker, ucp_rsc_index_t tl_id, uct_iface_params_t *iface_params, ucp_worker_iface_t **wiface_p) @@ -1158,6 +1183,7 @@ ucs_status_t ucp_worker_iface_open(ucp_worker_h worker, ucp_rsc_index_t tl_id, ucp_context_h context = worker->context; ucp_tl_resource_desc_t *resource = &context->tl_rscs[tl_id]; uct_md_h md = context->tl_mds[resource->md_index].md; + ucs_sys_dev_distance_t distance = {.latency = 0, .bandwidth = 0}; uct_iface_config_t *iface_config; const char *cfg_tl_name; ucp_worker_iface_t *wiface; @@ -1235,6 +1261,16 @@ ucs_status_t ucp_worker_iface_open(ucp_worker_h worker, ucp_rsc_index_t tl_id, goto err_close_iface; } + status = ucp_worker_get_sys_device_distance(context, wiface->rsc_index, + &distance); + if (status == UCS_OK) { + wiface->attr.latency.c += distance.latency; + wiface->attr.bandwidth.shared = + ucs_min(wiface->attr.bandwidth.shared, distance.bandwidth); + wiface->attr.bandwidth.dedicated = + ucs_min(wiface->attr.bandwidth.dedicated, distance.bandwidth); + } + ucs_debug("created interface[%d]=%p using "UCT_TL_RESOURCE_DESC_FMT" on worker %p", tl_id, wiface->iface, UCT_TL_RESOURCE_DESC_ARG(&resource->tl_rsc), worker); diff --git a/src/uct/base/uct_iface.c b/src/uct/base/uct_iface.c index 6b44e395ead..abe1a826a93 100644 --- a/src/uct/base/uct_iface.c +++ b/src/uct/base/uct_iface.c @@ -328,6 +328,7 @@ void uct_base_iface_query(uct_base_iface_t *iface, uct_iface_attr_t *iface_attr) ucs_status_t uct_single_device_resource(uct_md_h md, const char *dev_name, uct_device_type_t dev_type, + ucs_sys_device_t sys_device, uct_tl_device_resource_t **tl_devices_p, unsigned *num_tl_devices_p) { @@ -341,7 +342,7 @@ ucs_status_t uct_single_device_resource(uct_md_h md, const char *dev_name, ucs_snprintf_zero(device->name, sizeof(device->name), "%s", dev_name); device->type = dev_type; - device->sys_device = UCS_SYS_DEVICE_ID_UNKNOWN; + device->sys_device = sys_device; *num_tl_devices_p = 1; *tl_devices_p = device; diff --git a/src/uct/base/uct_iface.h b/src/uct/base/uct_iface.h index e30ee9e86af..6c2753c96b7 100644 --- a/src/uct/base/uct_iface.h +++ b/src/uct/base/uct_iface.h @@ -616,6 +616,7 @@ void uct_base_iface_query(uct_base_iface_t *iface, uct_iface_attr_t *iface_attr) ucs_status_t uct_single_device_resource(uct_md_h md, const char *dev_name, uct_device_type_t dev_type, + ucs_sys_device_t sys_device, uct_tl_device_resource_t **tl_devices_p, unsigned *num_tl_devices_p); diff --git a/src/uct/cuda/base/cuda_iface.c b/src/uct/cuda/base/cuda_iface.c index 3babb7ef423..f4e7bf9bde9 100644 --- a/src/uct/cuda/base/cuda_iface.c +++ b/src/uct/cuda/base/cuda_iface.c @@ -15,7 +15,14 @@ ucs_status_t uct_cuda_base_query_devices(uct_md_h md, uct_tl_device_resource_t **tl_devices_p, unsigned *num_tl_devices_p) { - return uct_single_device_resource(md, UCT_CUDA_DEV_NAME, UCT_DEVICE_TYPE_ACC, + ucs_sys_device_t sys_device = UCS_SYS_DEVICE_ID_UNKNOWN; + CUdevice cuda_device; + + if (cuCtxGetDevice(&cuda_device) == CUDA_SUCCESS) { + uct_cuda_base_get_sys_dev(cuda_device, &sys_device); + } + + return uct_single_device_resource(md, UCT_CUDA_DEV_NAME, + UCT_DEVICE_TYPE_ACC, sys_device, tl_devices_p, num_tl_devices_p); } - diff --git a/src/uct/cuda/base/cuda_iface.h b/src/uct/cuda/base/cuda_iface.h index b612205954d..e25cf7d04ca 100644 --- a/src/uct/cuda/base/cuda_iface.h +++ b/src/uct/cuda/base/cuda_iface.h @@ -86,4 +86,7 @@ ucs_status_t uct_cuda_base_query_devices(uct_md_h md, uct_tl_device_resource_t **tl_devices_p, unsigned *num_tl_devices_p); +ucs_status_t +uct_cuda_base_get_sys_dev(CUdevice cuda_device, ucs_sys_device_t *sys_dev_p); + #endif diff --git a/src/uct/cuda/base/cuda_md.c b/src/uct/cuda/base/cuda_md.c index c239b48bcc8..5eee0e15b4b 100644 --- a/src/uct/cuda/base/cuda_md.c +++ b/src/uct/cuda/base/cuda_md.c @@ -9,6 +9,7 @@ #endif #include "cuda_md.h" +#include "cuda_iface.h" #include #include @@ -17,8 +18,8 @@ #include -static ucs_status_t uct_cuda_base_get_sys_dev(CUdevice cuda_device, - ucs_sys_device_t *sys_dev_p) +ucs_status_t uct_cuda_base_get_sys_dev(CUdevice cuda_device, + ucs_sys_device_t *sys_dev_p) { ucs_sys_bus_id_t bus_id; CUresult cu_err; diff --git a/src/uct/rocm/base/rocm_base.c b/src/uct/rocm/base/rocm_base.c index 18a872bd30d..f1b07178ba8 100644 --- a/src/uct/rocm/base/rocm_base.c +++ b/src/uct/rocm/base/rocm_base.c @@ -110,8 +110,9 @@ ucs_status_t uct_rocm_base_query_devices(uct_md_h md, unsigned *num_tl_devices_p) { return uct_single_device_resource(md, md->component->name, - UCT_DEVICE_TYPE_ACC, tl_devices_p, - num_tl_devices_p); + UCT_DEVICE_TYPE_ACC, + UCS_SYS_DEVICE_ID_UNKNOWN, + tl_devices_p, num_tl_devices_p); } hsa_agent_t uct_rocm_base_get_dev_agent(int dev_num) diff --git a/src/uct/sm/base/sm_iface.c b/src/uct/sm/base/sm_iface.c index feb5a01af9c..c1eda85c834 100644 --- a/src/uct/sm/base/sm_iface.c +++ b/src/uct/sm/base/sm_iface.c @@ -48,8 +48,9 @@ uct_sm_base_query_tl_devices(uct_md_h md, uct_tl_device_resource_t **tl_devices_ unsigned *num_tl_devices_p) { return uct_single_device_resource(md, UCT_SM_DEVICE_NAME, - UCT_DEVICE_TYPE_SHM, tl_devices_p, - num_tl_devices_p); + UCT_DEVICE_TYPE_SHM, + UCS_SYS_DEVICE_ID_UNKNOWN, + tl_devices_p, num_tl_devices_p); } diff --git a/src/uct/sm/self/self.c b/src/uct/sm/self/self.c index 6cfa8359dfb..eae999f6196 100644 --- a/src/uct/sm/self/self.c +++ b/src/uct/sm/self/self.c @@ -222,6 +222,7 @@ uct_self_query_tl_devices(uct_md_h md, uct_tl_device_resource_t **tl_devices_p, { return uct_single_device_resource(md, UCT_SM_DEVICE_NAME, UCT_DEVICE_TYPE_SELF, + UCS_SYS_DEVICE_ID_UNKNOWN, tl_devices_p, num_tl_devices_p); } From 73830be3eac7bcea1f72b0045cb463e7974ac150 Mon Sep 17 00:00:00 2001 From: Sergey Oblomov Date: Mon, 30 Nov 2020 12:46:18 +0200 Subject: [PATCH 021/107] UCP/KEEPALIVE: removed incorrect assert - suppressed incorrect assert on keepalive remove ep - renamed flag UCP_EP_FLAG_TEMPORARY to UCP_EP_FLAG_INTERNAL - mem-type EP's are marked by UCP_EP_FLAG_INTERNAL flag - fixed ep-destroy flow for memtype EPs --- src/ucp/core/ucp_ep.c | 25 +++++++++++++++++++------ src/ucp/core/ucp_ep.h | 5 +++-- src/ucp/wireup/wireup_cm.c | 2 +- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/ucp/core/ucp_ep.c b/src/ucp/core/ucp_ep.c index e998f258d89..b54ba1bd4ab 100644 --- a/src/ucp/core/ucp_ep.c +++ b/src/ucp/core/ucp_ep.c @@ -188,7 +188,9 @@ ucs_status_t ucp_worker_create_ep(ucp_worker_h worker, unsigned ep_init_flags, goto err_destroy_ep_base; } - if (!(ep_init_flags & UCP_EP_INIT_FLAG_MEM_TYPE)) { + if (ep_init_flags & UCP_EP_INIT_FLAG_MEM_TYPE) { + ep->flags |= UCP_EP_FLAG_INTERNAL; + } else { ucs_list_add_tail(&worker->all_eps, &ucp_ep_ext_gen(ep)->ep_list); } @@ -208,7 +210,10 @@ void ucp_ep_delete(ucp_ep_h ep) ucs_callbackq_remove_if(&ep->worker->uct->progress_q, ucp_wireup_msg_ack_cb_pred, ep); - ucp_worker_keepalive_remove_ep(ep); + if (!(ep->flags & UCP_EP_FLAG_INTERNAL)) { + ucp_worker_keepalive_remove_ep(ep); + } + ucs_list_del(&ucp_ep_ext_gen(ep)->ep_list); status = ucs_ptr_map_del(&ep->worker->ptr_map, ucp_ep_local_id(ep)); if (status != UCS_OK) { @@ -357,12 +362,20 @@ ucs_status_t ucp_worker_create_mem_type_endpoints(ucp_worker_h worker) void ucp_worker_destroy_mem_type_endpoints(ucp_worker_h worker) { ucs_memory_type_t mem_type; + ucp_ep_h ep; ucs_memory_type_for_each(mem_type) { - if (worker->mem_type_ep[mem_type] != NULL) { - ucp_ep_destroy_internal(worker->mem_type_ep[mem_type]); - worker->mem_type_ep[mem_type] = NULL; + ep = worker->mem_type_ep[mem_type]; + if (ep == NULL) { + continue; } + + ucs_debug("memtype ep %p: destroy", ep); + ucs_assert(ep->flags & UCP_EP_FLAG_INTERNAL); + + ucp_ep_cleanup_lanes(ep); + ucp_ep_delete(ep); + worker->mem_type_ep[mem_type] = NULL; } } @@ -792,7 +805,7 @@ void ucp_ep_destroy_internal(ucp_ep_h ep) { ucs_debug("ep %p: destroy", ep); ucp_ep_cleanup_lanes(ep); - if (ep->flags & UCP_EP_FLAG_TEMPORARY) { + if (ep->flags & UCP_EP_FLAG_INTERNAL) { /* it's failed tmp ep of main ep */ ucs_assert(ucp_ep_ext_control(ep)->local_ep_id == UCP_EP_ID_INVALID); ucp_ep_destroy_base(ep); diff --git a/src/ucp/core/ucp_ep.h b/src/ucp/core/ucp_ep.h index 4146643e5a5..e5004914b53 100644 --- a/src/ucp/core/ucp_ep.h +++ b/src/ucp/core/ucp_ep.h @@ -60,8 +60,9 @@ enum { UCP_EP_FLAG_CLOSE_REQ_VALID = UCS_BIT(11),/* close protocol is started and close_req is valid */ UCP_EP_FLAG_ERR_HANDLER_INVOKED = UCS_BIT(12),/* error handler was called */ - UCP_EP_FLAG_TEMPORARY = UCS_BIT(13),/* the temporary EP which holds - temporary wireup configuration */ + UCP_EP_FLAG_INTERNAL = UCS_BIT(13),/* the internal EP which holds + temporary wireup configuration or + mem-type EP */ UCP_EP_FLAG_INDIRECT_ID = UCS_BIT(14),/* protocols on this endpoint will send indirect endpoint id instead of pointer, can be replaced with looking at local ID */ diff --git a/src/ucp/wireup/wireup_cm.c b/src/ucp/wireup/wireup_cm.c index f0d1eea1405..87e6b59be63 100644 --- a/src/ucp/wireup/wireup_cm.c +++ b/src/ucp/wireup/wireup_cm.c @@ -320,7 +320,7 @@ static ssize_t ucp_cm_client_priv_pack_cb(void *arg, goto out_check_err; } - cm_wireup_ep->tmp_ep->flags |= UCP_EP_FLAG_TEMPORARY; + cm_wireup_ep->tmp_ep->flags |= UCP_EP_FLAG_INTERNAL; status = ucp_worker_get_ep_config(worker, &key, 0, &cm_wireup_ep->tmp_ep->cfg_index); From 5c003ae57137a5be8e59327609c41c3d1f0e0889 Mon Sep 17 00:00:00 2001 From: Leonid Genkin Date: Thu, 26 Nov 2020 09:48:30 +0200 Subject: [PATCH 022/107] UCS/DATASTRUCT: Add bitmap data struct --- src/ucs/Makefile.am | 1 + src/ucs/datastruct/bitmap.h | 209 ++++++++++++++++++++++++++++++++++ test/gtest/Makefile.am | 1 + test/gtest/ucs/test_bitmap.cc | 156 +++++++++++++++++++++++++ 4 files changed, 367 insertions(+) create mode 100644 src/ucs/datastruct/bitmap.h create mode 100644 test/gtest/ucs/test_bitmap.cc diff --git a/src/ucs/Makefile.am b/src/ucs/Makefile.am index 688ffce0f5e..1c01d4ab5d7 100644 --- a/src/ucs/Makefile.am +++ b/src/ucs/Makefile.am @@ -28,6 +28,7 @@ nobase_dist_libucs_la_HEADERS = \ config/types.h \ datastruct/array.h \ datastruct/array.inl \ + datastruct/bitmap.h \ datastruct/callbackq.h \ datastruct/hlist.h \ datastruct/khash.h \ diff --git a/src/ucs/datastruct/bitmap.h b/src/ucs/datastruct/bitmap.h new file mode 100644 index 00000000000..20c0af1ca16 --- /dev/null +++ b/src/ucs/datastruct/bitmap.h @@ -0,0 +1,209 @@ +/** + * Copyright (C) Mellanox Technologies Ltd. 2020. ALL RIGHTS RESERVED. + * + * See file LICENSE for terms. + */ + +#ifndef UCS_BITMAP_H_ +#define UCS_BITMAP_H_ + +#include +#include + +BEGIN_C_DECLS + + +typedef uint64_t ucs_bitmap_word_t; + + +/* + * Bits number in a single bitmap word + */ +#define UCS_BITMAP_BITS_IN_WORD \ + (sizeof(ucs_bitmap_word_t) * 8) + + +/* + * Word index of a bit in bitmap + * + * @param _bit_index Index of this bit relative to the bitmap + * + * @return Index of the word this bit belongs to + */ +#define UCS_BITMAP_WORD_INDEX(_bit_index) \ + ((_bit_index) / UCS_BITMAP_BITS_IN_WORD) + + +#define _UCS_BITMAP_BIT_IN_WORD_INDEX(_bit_index) \ + ((_bit_index) % UCS_BITMAP_BITS_IN_WORD) + + +#define _UCS_BITMAP_BITS_TO_WORDS(_length) \ + ((((_length) + (UCS_BITMAP_BITS_IN_WORD - 1)) / UCS_BITMAP_BITS_IN_WORD)) + + +#define _UCS_BITMAP_WORD(_bitmap, _index) \ + ((_bitmap)->bits[UCS_BITMAP_WORD_INDEX(_index)]) + + +#define _UCS_BITMAP_WORD_INDEX0(_bit_index) \ + ((_bit_index) & ~(UCS_BITMAP_BITS_IN_WORD - 1)) + + +#define _UCS_BITMAP_GET_NEXT_BIT(_index) \ + (-2ull << (uint64_t)((_index) & (UCS_BITMAP_BITS_IN_WORD - 1))) + + +/* + * Represents an n-bit bitmap, by using an array + * of 64-bit unsigned long integers. + * + * @param _length Number of bits in the bitmap + */ +#define _UCS_BITMAP_DECLARE_TYPE(_length) \ + typedef struct { \ + ucs_bitmap_word_t bits[_UCS_BITMAP_BITS_TO_WORDS(_length)]; \ + } ucs_bitmap_t(_length); + + +/** + * Expands to bitmap type definition + * + * @param _length Number of bits (as passed to _UCS_BITMAP_DECLARE_TYPE) + * + * Example: + * + * @code{.c} + * ucs_bitmap_t(64) my_bitmap; + * @endcode + */ +#define ucs_bitmap_t(_length) ucs_bitmap_##_length##_suffix + + +_UCS_BITMAP_DECLARE_TYPE(64) +_UCS_BITMAP_DECLARE_TYPE(128) +_UCS_BITMAP_DECLARE_TYPE(256) + + +/** + * Get the value of a bit in the bitmap + * + * @param _bitmap Read value from this bitmap + * @param _index Bit index to read + * + * @return Bit value (0 or 1) + */ +#define UCS_BITMAP_GET(_bitmap, _index) \ + (!!(_UCS_BITMAP_WORD(_bitmap, _index) & \ + UCS_BIT(_UCS_BITMAP_BIT_IN_WORD_INDEX(_index)))) + + +/** + * Set the value of a bit in the bitmap + * + * @param _bitmap Set value in this bitmap + * @param _index Bit index to set + */ + #define UCS_BITMAP_SET(_bitmap, _index) \ + (_UCS_BITMAP_WORD(_bitmap, _index) |= \ + (UCS_BIT(_UCS_BITMAP_BIT_IN_WORD_INDEX(_index)))); + + +/** + * Unset (clear) the value of a bit in the bitmap + * + * @param _bitmap Unset value in this bitmap + * @param _index Bit index to unset + */ + #define UCS_BITMAP_UNSET(_bitmap, _index) \ + ((_bitmap)->bits)[UCS_BITMAP_WORD_INDEX(_index)] &= \ + ~((UCS_BIT(_UCS_BITMAP_BIT_IN_WORD_INDEX(_index)))); + + +/** + * Clear a bitmap by setting all its bits to zero + * + * @param _bitmap Clear all bits in this bitmap + */ +#define UCS_BITMAP_CLEAR(_bitmap) \ + memset((_bitmap)->bits, 0, sizeof((_bitmap)->bits)) + + +/** + * Iterate over all set (1) bits of a given bitmap + * + * @param _bitmap Iterate over bits of this bitmap + * @param _index Bit index (global offset - relative to the whole bitmap) + */ +#define UCS_BITMAP_FOR_EACH_BIT(_bitmap, _index) \ + for (_index = ucs_ffs64_safe((_bitmap)->bits[0]); \ + _index < ucs_static_array_size((_bitmap)->bits) * UCS_BITMAP_BITS_IN_WORD; \ + _index = _UCS_BITMAP_WORD_INDEX0(_index) + \ + ucs_ffs64_safe(_UCS_BITMAP_WORD((_bitmap), (_index)) & \ + _UCS_BITMAP_GET_NEXT_BIT(_index))) \ + if (UCS_BITMAP_GET((_bitmap), (_index))) + + +/** + * Perform bitwise NOT of a bitmap + * + * @param _bitmap Negate this bitmap + */ +#define UCS_BITMAP_NOT(_bitmap) \ + ({ \ + int _word; \ + for (_word = 0; _word < ucs_static_array_size((_bitmap)->bits); _word++) \ + (_bitmap)->bits[_word] = ~(_bitmap)->bits[_word]; \ + }) + + +#define _UCS_BITMAP_OP(_bitmap1, _bitmap2, _op) \ + ({ \ + int _word; \ + for (_word = 0; _word < ucs_static_array_size((_bitmap1)->bits); _word++) \ + (_bitmap1)->bits[_word] = (_bitmap1)->bits[_word] _op \ + (_bitmap2)->bits[_word]; \ + }) + + +/** + * Perform bitwise AND of 2 bitmaps, storing the result in the first one + * + * @param _bitmap1 First operand + * @param _bitmap1 Second operand + */ +#define UCS_BITMAP_AND(_bitmap1, _bitmap2) _UCS_BITMAP_OP(_bitmap1, _bitmap2, &) + + +/** + * Perform bitwise OR of 2 bitmaps, storing the result in the first one + * + * @param _bitmap1 First operand + * @param _bitmap1 Second operand + */ +#define UCS_BITMAP_OR(_bitmap1, _bitmap2) _UCS_BITMAP_OP(_bitmap1, _bitmap2, |) + + +/** + * Perform bitwise XOR of 2 bitmaps, storing the result in the first one + * + * @param _bitmap1 First operand + * @param _bitmap1 Second operand + */ +#define UCS_BITMAP_XOR(_bitmap1, _bitmap2) _UCS_BITMAP_OP(_bitmap1, _bitmap2, ^) + + +/** + * Copy the whole contents of a bitmap + * + * @param _dest_bitmap Copy bits to this bitmap + * @param _src_bitmap Copy bits from this bitmap + */ +#define UCS_BITMAP_COPY(_dest_bitmap, _src_bitmap) \ + memcpy((_dest_bitmap)->bits, (_src_bitmap)->bits, \ + ucs_static_array_size((_src_bitmap)->bits)); + + +END_C_DECLS + +#endif /* BITMAP_H_ */ diff --git a/test/gtest/Makefile.am b/test/gtest/Makefile.am index 9c45bf384bb..8f70164b2b0 100644 --- a/test/gtest/Makefile.am +++ b/test/gtest/Makefile.am @@ -148,6 +148,7 @@ gtest_SOURCES = \ ucs/test_algorithm.cc \ ucs/test_arbiter.cc \ ucs/test_async.cc \ + ucs/test_bitmap.cc \ ucs/test_callbackq.cc \ ucs/test_class.cc \ ucs/test_config.cc \ diff --git a/test/gtest/ucs/test_bitmap.cc b/test/gtest/ucs/test_bitmap.cc new file mode 100644 index 00000000000..15791e1be5e --- /dev/null +++ b/test/gtest/ucs/test_bitmap.cc @@ -0,0 +1,156 @@ +/** + * Copyright (C) Mellanox Technologies Ltd. 2020. ALL RIGHTS RESERVED. + * + * See file LICENSE for terms. + */ + +#include +#include + +class test_ucs_bitmap : public ucs::test { +public: + virtual void init() + { + UCS_BITMAP_CLEAR(&bitmap); + } + +protected: + void copy_bitmap(ucs_bitmap_t(128) *bitmap, uint64_t *dest) + { + int i; + + UCS_BITMAP_FOR_EACH_BIT(bitmap, i) { + dest[UCS_BITMAP_WORD_INDEX(i)] |= UCS_BIT(i % UCS_BITMAP_BITS_IN_WORD); + } + } + +protected: + ucs_bitmap_t(128) bitmap; +}; + +void test_set_get_unset(ucs_bitmap_t(128) *bitmap, uint64_t offset) +{ + UCS_BITMAP_SET(bitmap, offset); + EXPECT_EQ(UCS_BITMAP_GET(bitmap, offset), 1); + EXPECT_EQ(bitmap->bits[offset >= UCS_BITMAP_BITS_IN_WORD], UCS_BIT(offset % 64)); + EXPECT_EQ(bitmap->bits[offset < UCS_BITMAP_BITS_IN_WORD], 0); + + UCS_BITMAP_UNSET(bitmap, offset); + EXPECT_EQ(bitmap->bits[0], 0); + EXPECT_EQ(bitmap->bits[1], 0); + EXPECT_EQ(UCS_BITMAP_GET(bitmap, offset), 0); +} + +UCS_TEST_F(test_ucs_bitmap, test_get_set_clear) +{ + const uint64_t offset = 15; + + EXPECT_EQ(bitmap.bits[0], 0); + EXPECT_EQ(bitmap.bits[1], 0); + EXPECT_EQ(UCS_BITMAP_GET(&bitmap, offset), 0); + + test_set_get_unset(&bitmap, offset); + test_set_get_unset(&bitmap, offset + 64); + + UCS_BITMAP_CLEAR(&bitmap); + for (int i = 0; i < 128; i++) { + EXPECT_EQ(UCS_BITMAP_GET(&bitmap, i), 0); + } +} + +UCS_TEST_F(test_ucs_bitmap, test_foreach) +{ + uint64_t bitmap_words[2] = {}; + + UCS_BITMAP_SET(&bitmap, 1); + UCS_BITMAP_SET(&bitmap, 25); + UCS_BITMAP_SET(&bitmap, 61); + + UCS_BITMAP_SET(&bitmap, UCS_BITMAP_BITS_IN_WORD + 0); + UCS_BITMAP_SET(&bitmap, UCS_BITMAP_BITS_IN_WORD + 37); + UCS_BITMAP_SET(&bitmap, UCS_BITMAP_BITS_IN_WORD + 58); + + copy_bitmap(&bitmap, bitmap_words); + + EXPECT_EQ(bitmap_words[0], UCS_BIT(1) | UCS_BIT(25) | UCS_BIT(61)); + EXPECT_EQ(bitmap_words[1], UCS_BIT(0) | UCS_BIT(37) | UCS_BIT(58)); +} + +UCS_TEST_F(test_ucs_bitmap, test_not) +{ + UCS_BITMAP_SET(&bitmap, 1); + UCS_BITMAP_NOT(&bitmap); + + EXPECT_EQ(bitmap.bits[0], -3ull); + EXPECT_EQ(bitmap.bits[1], -1); +} + +UCS_TEST_F(test_ucs_bitmap, test_and) +{ + ucs_bitmap_t(128) bitmap2; + + UCS_BITMAP_CLEAR(&bitmap2); + UCS_BITMAP_SET(&bitmap, 1); + UCS_BITMAP_SET(&bitmap, UCS_BITMAP_BITS_IN_WORD + 1); + UCS_BITMAP_SET(&bitmap, UCS_BITMAP_BITS_IN_WORD + 16); + + UCS_BITMAP_SET(&bitmap2, 25); + UCS_BITMAP_SET(&bitmap2, UCS_BITMAP_BITS_IN_WORD + 1); + UCS_BITMAP_SET(&bitmap2, UCS_BITMAP_BITS_IN_WORD + 30); + UCS_BITMAP_AND(&bitmap, &bitmap2); + + EXPECT_EQ(bitmap.bits[0], 0); + EXPECT_EQ(bitmap.bits[1], UCS_BIT(1)); +} + +UCS_TEST_F(test_ucs_bitmap, test_or) +{ + ucs_bitmap_t(128) bitmap2; + + UCS_BITMAP_CLEAR(&bitmap2); + UCS_BITMAP_SET(&bitmap, 1); + UCS_BITMAP_SET(&bitmap, UCS_BITMAP_BITS_IN_WORD + 1); + UCS_BITMAP_SET(&bitmap, UCS_BITMAP_BITS_IN_WORD + 16); + + UCS_BITMAP_SET(&bitmap2, 25); + UCS_BITMAP_SET(&bitmap2, UCS_BITMAP_BITS_IN_WORD + 1); + UCS_BITMAP_SET(&bitmap2, UCS_BITMAP_BITS_IN_WORD + 30); + UCS_BITMAP_OR(&bitmap, &bitmap2); + + EXPECT_EQ(bitmap.bits[0], UCS_BIT(1) | UCS_BIT(25)); + EXPECT_EQ(bitmap.bits[1], UCS_BIT(1) | UCS_BIT(16) | UCS_BIT(30)); +} + + +UCS_TEST_F(test_ucs_bitmap, test_xor) +{ + ucs_bitmap_t(128) bitmap2; + + UCS_BITMAP_CLEAR(&bitmap2); + bitmap.bits[0] = 1; + bitmap.bits[1] = -1; + bitmap2.bits[0] = -1; + bitmap2.bits[1] = 1; + UCS_BITMAP_XOR(&bitmap, &bitmap2); + + EXPECT_EQ(bitmap.bits[0], -2); + EXPECT_EQ(bitmap.bits[1], -2); +} + +UCS_TEST_F(test_ucs_bitmap, test_copy) +{ + ucs_bitmap_t(128) bitmap2; + + UCS_BITMAP_SET(&bitmap, 1); + UCS_BITMAP_SET(&bitmap, 25); + UCS_BITMAP_SET(&bitmap, 61); + + UCS_BITMAP_SET(&bitmap, UCS_BITMAP_BITS_IN_WORD + 0); + UCS_BITMAP_SET(&bitmap, UCS_BITMAP_BITS_IN_WORD + 37); + UCS_BITMAP_SET(&bitmap, UCS_BITMAP_BITS_IN_WORD + 58); + + UCS_BITMAP_COPY(&bitmap2, &bitmap); + + EXPECT_EQ(bitmap.bits[0], UCS_BIT(1) | UCS_BIT(25) | UCS_BIT(61)); + EXPECT_EQ(bitmap.bits[1], UCS_BIT(0) | UCS_BIT(37) | UCS_BIT(58)); +} From 9d87dcbd650c2eecaf1ccb10ca28eba153cdc3cd Mon Sep 17 00:00:00 2001 From: Sergey Oblomov Date: Fri, 4 Dec 2020 10:50:14 +0200 Subject: [PATCH 023/107] UCS/CONFIG: fixed crash on incorrect value set - there was possible double free of string when user tried to set incorrect value using ucs_config_parser_set_value function - added gtest --- src/ucs/config/parser.c | 11 +++++++++++ test/gtest/ucs/test_config.cc | 36 +++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/src/ucs/config/parser.c b/src/ucs/config/parser.c index 732e73be076..a4bbf08af71 100644 --- a/src/ucs/config/parser.c +++ b/src/ucs/config/parser.c @@ -1033,9 +1033,12 @@ ucs_config_parser_set_value_internal(void *opts, ucs_config_field_t *fields, const char *name, const char *value, const char *table_prefix, int recurse) { + char value_buf[256] = ""; ucs_config_field_t *field, *sub_fields; size_t prefix_len; ucs_status_t status; + ucs_status_t UCS_V_UNUSED status_restore; + int UCS_V_UNUSED ret; unsigned count; void *var; @@ -1079,9 +1082,17 @@ ucs_config_parser_set_value_internal(void *opts, ucs_config_field_t *fields, return UCS_ERR_NO_ELEM; } + /* backup current value to restore it in case if new value + * is not accepted */ + ret = field->parser.write(value_buf, sizeof(value_buf) - 1, var, + field->parser.arg); + ucs_assert(ret != 0); /* write success */ ucs_config_parser_release_field(field, var); status = ucs_config_parser_parse_field(field, value, var); if (status != UCS_OK) { + status_restore = ucs_config_parser_parse_field(field, value_buf, var); + /* current value must be valid */ + ucs_assert(status_restore == UCS_OK); return status; } ++count; diff --git a/test/gtest/ucs/test_config.cc b/test/gtest/ucs/test_config.cc index 3bcf5e9e074..a3fcff757e6 100644 --- a/test/gtest/ucs/test_config.cc +++ b/test/gtest/ucs/test_config.cc @@ -224,7 +224,14 @@ ucs_config_field_t car_opts_table[] = { static std::vector config_err_exp_str; class test_config : public ucs::test { +public: + test_config() { + m_num_errors = 0; + } + protected: + static int m_num_errors; + static ucs_log_func_rc_t config_error_handler(const char *file, unsigned line, const char *function, ucs_log_level_t level, @@ -246,6 +253,22 @@ class test_config : public ucs::test { return UCS_LOG_FUNC_RC_CONTINUE; } + static ucs_log_func_rc_t + config_error_suppress(const char *file, unsigned line, const char *function, + ucs_log_level_t level, + const ucs_log_component_config_t *comp_conf, + const char *message, va_list ap) + { + // Ignore errors that invalid input parameters as it is expected + if (level == UCS_LOG_LEVEL_ERROR) { + m_num_errors++; + return wrap_errors_logger(file, line, function, level, comp_conf, + message, ap); + } + + return UCS_LOG_FUNC_RC_CONTINUE; + } + /* * Wrapper class for car options parser. */ @@ -377,6 +400,8 @@ class test_config : public ucs::test { } }; +int test_config::m_num_errors; + UCS_TEST_F(test_config, parse_default) { car_opts opts(UCS_DEFAULT_ENV_PREFIX, "TEST"); @@ -457,6 +482,17 @@ UCS_TEST_F(test_config, set_get) { opts.set("VIN", "123456"); EXPECT_EQ(123456UL, opts->vin); + + /* try to set incorrect value - color should not be updated */ + { + scoped_log_handler log_handler_vars(config_error_suppress); + opts.set("COLOR", "magenta"); + } + + EXPECT_EQ(COLOR_WHITE, opts->color); + EXPECT_EQ(std::string(color_names[COLOR_WHITE]), + std::string(opts.get("COLOR"))); + EXPECT_EQ(1, m_num_errors); } UCS_TEST_F(test_config, set_get_with_table_prefix) { From 93cc94a03fe20ce7a94583448fc6ac41ef638a04 Mon Sep 17 00:00:00 2001 From: dmitrygx Date: Wed, 9 Dec 2020 17:50:21 +0200 Subject: [PATCH 024/107] UCP/AMO: Use remote_addr from AMO part of request instead of RMA --- src/ucp/rma/amo_sw.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ucp/rma/amo_sw.c b/src/ucp/rma/amo_sw.c index 12f01dd5e64..37d74238859 100644 --- a/src/ucp/rma/amo_sw.c +++ b/src/ucp/rma/amo_sw.c @@ -23,7 +23,7 @@ static size_t ucp_amo_sw_pack(void *dest, void *arg, uint8_t fetch) size_t size = req->send.length; size_t length; - atomich->address = req->send.rma.remote_addr; + atomich->address = req->send.amo.remote_addr; atomich->req.ep_id = ucp_ep_remote_id(ep); atomich->req.req_id = fetch ? ucp_send_request_get_id(req) : UCP_REQUEST_ID_INVALID; From 8d4752fd48e9b9501bdf6687928a0ece238b705c Mon Sep 17 00:00:00 2001 From: Mikhail Brinskii Date: Thu, 10 Dec 2020 09:56:33 +0200 Subject: [PATCH 025/107] UCP: Fix address unpacking error --- src/ucp/wireup/address.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ucp/wireup/address.c b/src/ucp/wireup/address.c index d61bfe2430c..b3aa5652bb5 100644 --- a/src/ucp/wireup/address.c +++ b/src/ucp/wireup/address.c @@ -1003,7 +1003,7 @@ ucs_status_t ucp_address_unpack(ucp_worker_t *worker, const void *buffer, if (address >= &address_list[UCP_MAX_RESOURCES]) { if (!(unpack_flags & UCP_ADDRESS_PACK_FLAG_NO_TRACE)) { ucs_error("failed to parse address: number of addresses" - "exceeds %d", UCP_MAX_RESOURCES); + " exceeds %d", UCP_MAX_RESOURCES); } goto err_free; } From 0e5d50f2d775515b9fa5cc9e7c71dfda57d1735d Mon Sep 17 00:00:00 2001 From: dmitrygx Date: Thu, 10 Dec 2020 15:16:45 +0200 Subject: [PATCH 026/107] GTEST/UCP: Skip wireup 1sided disconnect test for TCP (workaround) --- test/gtest/ucp/test_ucp_wireup.cc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/gtest/ucp/test_ucp_wireup.cc b/test/gtest/ucp/test_ucp_wireup.cc index 257d60b1768..b90fcd876de 100644 --- a/test/gtest/ucp/test_ucp_wireup.cc +++ b/test/gtest/ucp/test_ucp_wireup.cc @@ -630,7 +630,15 @@ UCS_TEST_P(test_ucp_wireup_1sided, send_disconnect_reply1) { recv_b(sender().worker(), sender().ep(), 8, 1); } -UCS_TEST_P(test_ucp_wireup_1sided, send_disconnect_reply2) { +UCS_TEST_SKIP_COND_P(test_ucp_wireup_1sided, send_disconnect_reply2, + /* skip the test for TCP, because it fails from time to + * time: the sender re-uses a socket fd from the already + * accepted connection from the receiver, but then the + * socket fd is closed, since the receiver closed the + * connection and the underlying TCP EP isn't able to + * receive the data on the failed socket. + * TODO: fix the bug on TCP level */ + has_transport("tcp")) { sender().connect(&receiver(), get_ep_params()); send_b(sender().ep(), 8, 1); From 83d6fe267d865e20c1c300a90268698090542788 Mon Sep 17 00:00:00 2001 From: dmitrygx Date: Thu, 10 Dec 2020 14:00:24 +0200 Subject: [PATCH 027/107] UCP/CORE/GTEST: Rearrange fields in a UCP request to reduce its size --- src/ucp/core/ucp_request.h | 21 +++++++++++---------- src/ucp/proto/proto_common.inl | 20 +++++++++++--------- src/ucp/proto/proto_multi.inl | 8 +++++--- src/ucp/proto/proto_single.inl | 5 +++-- src/ucp/rma/get_am.c | 6 +++--- src/ucp/rma/get_offload.c | 13 ++++++++----- src/ucp/rma/put_am.c | 3 ++- src/ucp/rma/put_offload.c | 16 ++++++++++------ src/ucp/rma/rma_sw.c | 10 +++++----- src/ucp/tag/eager_multi.c | 10 +++++----- src/ucp/tag/eager_single.c | 11 ++++++----- test/gtest/common/test_obj_size.cc | 2 +- 12 files changed, 70 insertions(+), 55 deletions(-) diff --git a/src/ucp/core/ucp_request.h b/src/ucp/core/ucp_request.h index 86b97f0db4b..3c994f65611 100644 --- a/src/ucp/core/ucp_request.h +++ b/src/ucp/core/ucp_request.h @@ -129,7 +129,17 @@ struct ucp_request { ucp_send_nbx_callback_t cb; /* Completion callback */ const ucp_proto_config_t *proto_config; /* Selected protocol for the request */ - ucp_datatype_iter_t dt_iter; /* Send buffer state */ + + /* This structure holds all mutable fields, and everything else + * except common send/recv fields 'status' and 'flags' is immutable + * TODO: rework RMA case where length is used instead of dt.offset */ + struct { + union { + ucp_datatype_iter_t dt_iter; /* Send buffer state */ + ucp_dt_state_t dt; /* Position in the send buffer */ + }; + uct_completion_t uct_comp; /* UCT completion used by flush */ + } state; union { ucp_wireup_msg_t wireup; @@ -260,15 +270,6 @@ struct ucp_request { } atomic_reply; }; - /* This structure holds all mutable fields, and everything else - * except common send/recv fields 'status' and 'flags' is - * immutable - * TODO: rework RMA case where length is used instead of dt.offset */ - struct { - ucp_dt_state_t dt; /* Position in the send buffer */ - uct_completion_t uct_comp; /* UCT completion */ - } state; - union { ucp_lane_index_t am_bw_index; /* AM BW lane index */ ucp_lane_map_t lanes_map_avail; /* Used lanes map */ diff --git a/src/ucp/proto/proto_common.inl b/src/ucp/proto/proto_common.inl index 2293064227c..543995cfff2 100644 --- a/src/ucp/proto/proto_common.inl +++ b/src/ucp/proto/proto_common.inl @@ -17,7 +17,7 @@ static UCS_F_ALWAYS_INLINE void ucp_proto_request_bcopy_complete(ucp_request_t *req, ucs_status_t status) { - ucp_datatype_iter_cleanup(&req->send.dt_iter, UINT_MAX); + ucp_datatype_iter_cleanup(&req->send.state.dt_iter, UINT_MAX); ucp_request_complete_send(req, status); } @@ -43,14 +43,15 @@ ucp_proto_request_zcopy_init(ucp_request_t *req, ucp_md_map_t md_map, ucp_proto_request_completion_init(req, comp_func); - status = ucp_datatype_iter_mem_reg(ep->worker->context, &req->send.dt_iter, + status = ucp_datatype_iter_mem_reg(ep->worker->context, + &req->send.state.dt_iter, md_map); if (status != UCS_OK) { return status; } ucp_trace_req(req, "registered md_map 0x%"PRIx64"/0x%"PRIx64, - req->send.dt_iter.type.contig.reg.md_map, md_map); + req->send.state.dt_iter.type.contig.reg.md_map, md_map); /* We expect the registration to happen on all desired memory domains, since * the protocol initialization code would already disqualify any memory @@ -58,7 +59,7 @@ ucp_proto_request_zcopy_init(ucp_request_t *req, ucp_md_map_t md_map, * memory key for zero-copy operations. This assumption simplifies memory * key lookups during protocol progress. */ - ucs_assert(req->send.dt_iter.type.contig.reg.md_map == md_map); + ucs_assert(req->send.state.dt_iter.type.contig.reg.md_map == md_map); return UCS_OK; } @@ -67,8 +68,9 @@ static UCS_F_ALWAYS_INLINE void ucp_proto_request_zcopy_cleanup(ucp_request_t *req) { ucp_datatype_iter_mem_dereg(req->send.ep->worker->context, - &req->send.dt_iter); - ucp_datatype_iter_cleanup(&req->send.dt_iter, UCS_BIT(UCP_DATATYPE_CONTIG)); + &req->send.state.dt_iter); + ucp_datatype_iter_cleanup(&req->send.state.dt_iter, + UCS_BIT(UCP_DATATYPE_CONTIG)); } static UCS_F_ALWAYS_INLINE void @@ -128,11 +130,11 @@ ucp_proto_request_send_op(ucp_ep_h ep, ucp_proto_select_t *proto_select, req->send.ep = ep; ucp_datatype_iter_init(worker->context, (void*)buffer, count, datatype, - contig_length, &req->send.dt_iter, &sg_count); + contig_length, &req->send.state.dt_iter, &sg_count); ucp_proto_select_param_init(&sel_param, op_id, param->op_attr_mask, - req->send.dt_iter.dt_class, - req->send.dt_iter.mem_type, + req->send.state.dt_iter.dt_class, + req->send.state.dt_iter.mem_type, sg_count); status = ucp_proto_request_set_proto(worker, ep, req, proto_select, rkey_cfg_index, diff --git a/src/ucp/proto/proto_multi.inl b/src/ucp/proto/proto_multi.inl index c5c0e84516e..0e67f48167a 100644 --- a/src/ucp/proto/proto_multi.inl +++ b/src/ucp/proto/proto_multi.inl @@ -34,7 +34,8 @@ ucp_proto_multi_data_pack(ucp_proto_multi_pack_ctx_t *pack_ctx, void *dest) { ucp_request_t *req = pack_ctx->req; - return ucp_datatype_iter_next_pack(&req->send.dt_iter, req->send.ep->worker, + return ucp_datatype_iter_next_pack(&req->send.state.dt_iter, + req->send.ep->worker, pack_ctx->max_payload, pack_ctx->next_iter, dest); } @@ -92,8 +93,9 @@ ucp_proto_multi_progress(ucp_request_t *req, ucp_proto_send_multi_cb_t send_func } /* advance position in send buffer */ - ucp_datatype_iter_copy_from_next(&req->send.dt_iter, &next_iter, dt_mask); - if (ucp_datatype_iter_is_end(&req->send.dt_iter)) { + ucp_datatype_iter_copy_from_next(&req->send.state.dt_iter, &next_iter, + dt_mask); + if (ucp_datatype_iter_is_end(&req->send.state.dt_iter)) { complete_func(req, UCS_OK); return UCS_OK; } diff --git a/src/ucp/proto/proto_single.inl b/src/ucp/proto/proto_single.inl index 0f94037face..b85f8fd800c 100644 --- a/src/ucp/proto/proto_single.inl +++ b/src/ucp/proto/proto_single.inl @@ -84,7 +84,7 @@ ucp_proto_am_zcopy_single_progress(ucp_request_t *req, ucp_am_id_t am_id, ucp_md_map_t md_map; uct_iov_t iov; - ucs_assert(req->send.dt_iter.offset == 0); + ucs_assert(req->send.state.dt_iter.offset == 0); if (!(req->flags & UCP_REQUEST_FLAG_PROTO_INITIALIZED)) { md_map = (spriv->reg_md == UCP_NULL_RESOURCE) ? 0 : UCS_BIT(spriv->reg_md); @@ -98,7 +98,8 @@ ucp_proto_am_zcopy_single_progress(ucp_request_t *req, ucp_am_id_t am_id, req->flags |= UCP_REQUEST_FLAG_PROTO_INITIALIZED; } - ucp_datatype_iter_next_iov(&req->send.dt_iter, spriv->super.memh_index, + ucp_datatype_iter_next_iov(&req->send.state.dt_iter, + spriv->super.memh_index, SIZE_MAX, &next_iter, &iov); status = uct_ep_am_zcopy(ep->uct_eps[spriv->super.lane], am_id, hdr, hdr_size, &iov, 1, 0, &req->send.state.uct_comp); diff --git a/src/ucp/rma/get_am.c b/src/ucp/rma/get_am.c index 79541f6b7f0..7347be81c2c 100644 --- a/src/ucp/rma/get_am.c +++ b/src/ucp/rma/get_am.c @@ -22,7 +22,7 @@ static size_t ucp_proto_get_am_bcopy_pack(void *dest, void *arg) ucp_get_req_hdr_t *getreqh = dest; getreqh->address = req->send.rma.remote_addr; - getreqh->length = req->send.dt_iter.length; + getreqh->length = req->send.state.dt_iter.length; getreqh->req.ep_id = ucp_send_request_get_ep_remote_id(req); getreqh->req.req_id = ucp_send_request_get_id(req); getreqh->mem_type = req->send.rma.rkey->mem_type; @@ -60,8 +60,8 @@ static ucs_status_t ucp_proto_get_am_bcopy_progress(uct_pending_req_t *self) /* initialize some request fields, for compatibility of get_reply * processing */ - req->send.buffer = req->send.dt_iter.type.contig.buffer; - req->send.length = req->send.dt_iter.length; + req->send.buffer = req->send.state.dt_iter.type.contig.buffer; + req->send.length = req->send.state.dt_iter.length; req->flags |= UCP_REQUEST_FLAG_PROTO_INITIALIZED; } diff --git a/src/ucp/rma/get_offload.c b/src/ucp/rma/get_offload.c index bbbc29059e6..3206fb37e26 100644 --- a/src/ucp/rma/get_offload.c +++ b/src/ucp/rma/get_offload.c @@ -33,11 +33,12 @@ ucp_proto_get_offload_bcopy_send_func(ucp_request_t *req, void *dest; max_length = ucp_proto_multi_max_payload(req, lpriv, 0); - length = ucp_datatype_iter_next_ptr(&req->send.dt_iter, max_length, - next_iter, &dest); + length = ucp_datatype_iter_next_ptr(&req->send.state.dt_iter, + max_length, next_iter, &dest); return uct_ep_get_bcopy(req->send.ep->uct_eps[lpriv->super.lane], ucp_proto_get_offload_bcopy_unpack, dest, length, - req->send.rma.remote_addr + req->send.dt_iter.offset, + req->send.rma.remote_addr + + req->send.state.dt_iter.offset, tl_rkey, &req->send.state.uct_comp); } @@ -110,11 +111,13 @@ ucp_proto_get_offload_zcopy_send_func(ucp_request_t *req, lpriv->super.rkey_index); uct_iov_t iov; - ucp_datatype_iter_next_iov(&req->send.dt_iter, lpriv->super.memh_index, + ucp_datatype_iter_next_iov(&req->send.state.dt_iter, + lpriv->super.memh_index, ucp_proto_multi_max_payload(req, lpriv, 0), next_iter, &iov); return uct_ep_get_zcopy(req->send.ep->uct_eps[lpriv->super.lane], &iov, 1, - req->send.rma.remote_addr + req->send.dt_iter.offset, + req->send.rma.remote_addr + + req->send.state.dt_iter.offset, tl_rkey, &req->send.state.uct_comp); } diff --git a/src/ucp/rma/put_am.c b/src/ucp/rma/put_am.c index cd4e1b1ea34..d3c746a0e55 100644 --- a/src/ucp/rma/put_am.c +++ b/src/ucp/rma/put_am.c @@ -22,7 +22,8 @@ static size_t ucp_proto_put_am_bcopy_pack(void *dest, void *arg) ucp_request_t *req = pack_ctx->req; ucp_put_hdr_t *puth = dest; - puth->address = req->send.rma.remote_addr + req->send.dt_iter.offset; + puth->address = req->send.rma.remote_addr + + req->send.state.dt_iter.offset; puth->ep_id = ucp_send_request_get_ep_remote_id(req); puth->mem_type = req->send.rma.rkey->mem_type; diff --git a/src/ucp/rma/put_offload.c b/src/ucp/rma/put_offload.c index d5af9b0ee90..46d5eb424b3 100644 --- a/src/ucp/rma/put_offload.c +++ b/src/ucp/rma/put_offload.c @@ -28,8 +28,8 @@ static ucs_status_t ucp_proto_put_offload_short_progress(uct_pending_req_t *self tl_rkey = ucp_rma_request_get_tl_rkey(req, spriv->super.rkey_index); status = uct_ep_put_short(ep->uct_eps[spriv->super.lane], - req->send.dt_iter.type.contig.buffer, - req->send.dt_iter.length, + req->send.state.dt_iter.type.contig.buffer, + req->send.state.dt_iter.length, req->send.rma.remote_addr, tl_rkey); if (ucs_unlikely(status == UCS_ERR_NO_RESOURCE)) { req->send.lane = spriv->super.lane; /* for pending add */ @@ -39,7 +39,8 @@ static ucs_status_t ucp_proto_put_offload_short_progress(uct_pending_req_t *self /* UCS_INPROGRESS is not expected */ ucs_assert((status == UCS_OK) || UCS_STATUS_IS_ERR(status)); - ucp_datatype_iter_cleanup(&req->send.dt_iter, UCS_BIT(UCP_DATATYPE_CONTIG)); + ucp_datatype_iter_cleanup(&req->send.state.dt_iter, + UCS_BIT(UCP_DATATYPE_CONTIG)); ucp_request_complete_send(req, status); return UCS_OK; } @@ -100,7 +101,8 @@ ucp_proto_put_offload_bcopy_send_func(ucp_request_t *req, tl_rkey = ucp_rma_request_get_tl_rkey(req, lpriv->super.rkey_index); packed_size = uct_ep_put_bcopy(ep->uct_eps[lpriv->super.lane], ucp_proto_put_offload_bcopy_pack, &pack_ctx, - req->send.rma.remote_addr + req->send.dt_iter.offset, + req->send.rma.remote_addr + + req->send.state.dt_iter.offset, tl_rkey); if (ucs_likely(packed_size >= 0)) { return UCS_OK; @@ -167,11 +169,13 @@ ucp_proto_put_offload_zcopy_send_func(ucp_request_t *req, uct_rkey_t tl_rkey = ucp_rma_request_get_tl_rkey(req, lpriv->super.rkey_index); uct_iov_t iov; - ucp_datatype_iter_next_iov(&req->send.dt_iter, lpriv->super.memh_index, + ucp_datatype_iter_next_iov(&req->send.state.dt_iter, + lpriv->super.memh_index, ucp_proto_multi_max_payload(req, lpriv, 0), next_iter, &iov); return uct_ep_put_zcopy(req->send.ep->uct_eps[lpriv->super.lane], &iov, 1, - req->send.rma.remote_addr + req->send.dt_iter.offset, + req->send.rma.remote_addr + + req->send.state.dt_iter.offset, tl_rkey, &req->send.state.uct_comp); } diff --git a/src/ucp/rma/rma_sw.c b/src/ucp/rma/rma_sw.c index 1696d66b259..5ce3d3f875f 100644 --- a/src/ucp/rma/rma_sw.c +++ b/src/ucp/rma/rma_sw.c @@ -246,12 +246,12 @@ UCS_PROFILE_FUNC(ucs_status_t, ucp_get_rep_handler, (arg, data, length, am_flags if (ep->worker->context->config.ext.proto_enable) { // TODO use dt_iter.inl unpack ucp_dt_contig_unpack(ep->worker, - req->send.dt_iter.type.contig.buffer + - req->send.dt_iter.offset, + req->send.state.dt_iter.type.contig.buffer + + req->send.state.dt_iter.offset, getreph + 1, frag_length, - req->send.dt_iter.mem_type); - req->send.dt_iter.offset += frag_length; - if (req->send.dt_iter.offset == req->send.dt_iter.length) { + req->send.state.dt_iter.mem_type); + req->send.state.dt_iter.offset += frag_length; + if (req->send.state.dt_iter.offset == req->send.state.dt_iter.length) { ucp_proto_request_bcopy_complete(req, UCS_OK); ucp_ep_rma_remote_request_completed(ep); } diff --git a/src/ucp/tag/eager_multi.c b/src/ucp/tag/eager_multi.c index b3dbb6f32d9..cb7982e0ef9 100644 --- a/src/ucp/tag/eager_multi.c +++ b/src/ucp/tag/eager_multi.c @@ -24,7 +24,7 @@ static UCS_F_ALWAYS_INLINE void ucp_eager_proto_set_first_hdr(ucp_request_t *req, ucp_eager_first_hdr_t *hdr) { hdr->super.super.tag = req->send.msg_proto.tag.tag; - hdr->total_len = req->send.dt_iter.length; + hdr->total_len = req->send.state.dt_iter.length; hdr->msg_id = req->send.msg_proto.message_id; } @@ -32,7 +32,7 @@ static UCS_F_ALWAYS_INLINE void ucp_eager_proto_set_middle_hdr(ucp_request_t *req, ucp_eager_middle_hdr_t *hdr) { hdr->msg_id = req->send.msg_proto.message_id; - hdr->offset = req->send.dt_iter.offset; + hdr->offset = req->send.state.dt_iter.offset; } static ucs_status_t @@ -104,7 +104,7 @@ ucp_eager_bcopy_multi_send_func(ucp_request_t *req, ucp_am_id_t am_id; size_t hdr_size; - if (req->send.dt_iter.offset == 0) { + if (req->send.state.dt_iter.offset == 0) { am_id = UCP_AM_ID_EAGER_FIRST; pack_cb = ucp_eager_bcopy_pack_first; hdr_size = sizeof(ucp_eager_first_hdr_t); @@ -180,7 +180,7 @@ ucp_proto_eager_zcopy_multi_send_func(ucp_request_t *req, size_t hdr_size; uct_iov_t iov; - if (req->send.dt_iter.offset == 0) { + if (req->send.state.dt_iter.offset == 0) { am_id = UCP_AM_ID_EAGER_FIRST; hdr_size = sizeof(hdr.first); ucp_eager_proto_set_first_hdr(req, &hdr.first); @@ -190,7 +190,7 @@ ucp_proto_eager_zcopy_multi_send_func(ucp_request_t *req, ucp_eager_proto_set_middle_hdr(req, &hdr.middle); } - ucp_datatype_iter_next_iov(&req->send.dt_iter, lpriv->super.memh_index, + ucp_datatype_iter_next_iov(&req->send.state.dt_iter, lpriv->super.memh_index, ucp_proto_multi_max_payload(req, lpriv, hdr_size), next_iter, &iov); return uct_ep_am_zcopy(req->send.ep->uct_eps[lpriv->super.lane], am_id, &hdr, diff --git a/src/ucp/tag/eager_single.c b/src/ucp/tag/eager_single.c index b4d0aac86f9..63929919fb4 100644 --- a/src/ucp/tag/eager_single.c +++ b/src/ucp/tag/eager_single.c @@ -28,14 +28,15 @@ static ucs_status_t ucp_eager_short_progress(uct_pending_req_t *self) status = uct_ep_am_short(req->send.ep->uct_eps[spriv->super.lane], UCP_AM_ID_EAGER_ONLY, req->send.msg_proto.tag.tag, - req->send.dt_iter.type.contig.buffer, - req->send.dt_iter.length); + req->send.state.dt_iter.type.contig.buffer, + req->send.state.dt_iter.length); if (ucs_unlikely(status == UCS_ERR_NO_RESOURCE)) { req->send.lane = spriv->super.lane; /* for pending add */ return status; } - ucp_datatype_iter_cleanup(&req->send.dt_iter, UCS_BIT(UCP_DATATYPE_CONTIG)); + ucp_datatype_iter_cleanup(&req->send.state.dt_iter, + UCS_BIT(UCP_DATATYPE_CONTIG)); ucs_assert(status != UCS_INPROGRESS); ucp_request_complete_send(req, status); @@ -86,9 +87,9 @@ static size_t ucp_eager_single_pack(void *dest, void *arg) ucp_datatype_iter_t next_iter; size_t packed_size; - ucs_assert(req->send.dt_iter.offset == 0); + ucs_assert(req->send.state.dt_iter.offset == 0); hdr->super.tag = req->send.msg_proto.tag.tag; - packed_size = ucp_datatype_iter_next_pack(&req->send.dt_iter, + packed_size = ucp_datatype_iter_next_pack(&req->send.state.dt_iter, req->send.ep->worker, SIZE_MAX, &next_iter, hdr + 1); return sizeof(*hdr) + packed_size; diff --git a/test/gtest/common/test_obj_size.cc b/test/gtest/common/test_obj_size.cc index 9c3ddacd920..5aec7fc95fc 100644 --- a/test/gtest/common/test_obj_size.cc +++ b/test/gtest/common/test_obj_size.cc @@ -48,7 +48,7 @@ UCS_TEST_F(test_obj_size, size) { #else EXPECTED_SIZE(ucp_ep_t, 64); /* TODO reduce request size to 240 or less after removing old protocols state */ - EXPECTED_SIZE(ucp_request_t, 296); + EXPECTED_SIZE(ucp_request_t, 248); EXPECTED_SIZE(ucp_recv_desc_t, 48); EXPECTED_SIZE(uct_ep_t, 8); EXPECTED_SIZE(uct_base_ep_t, 8); From c524eabe438b4ba80be9ad8d9e624ce07a32b17f Mon Sep 17 00:00:00 2001 From: dmitrygx Date: Thu, 10 Dec 2020 17:02:56 +0200 Subject: [PATCH 028/107] TEST/IO_DEMO: Add a window on the client side, per every remote server --- buildlib/io-demo.yml | 2 +- test/apps/iodemo/io_demo.cc | 279 ++++++++++++++++++++++---------- test/apps/iodemo/run_io_demo.sh | 2 +- 3 files changed, 195 insertions(+), 88 deletions(-) diff --git a/buildlib/io-demo.yml b/buildlib/io-demo.yml index 5f012e83f7a..2e1d47e1bdd 100644 --- a/buildlib/io-demo.yml +++ b/buildlib/io-demo.yml @@ -125,7 +125,7 @@ jobs: -w 16 \ -t 10 \ -q \ - -a + -A displayName: Launch with run_io_demo.sh (active message) timeoutInMinutes: 2 diff --git a/test/apps/iodemo/io_demo.cc b/test/apps/iodemo/io_demo.cc index d82460d7cd3..673e99e7464 100644 --- a/test/apps/iodemo/io_demo.cc +++ b/test/apps/iodemo/io_demo.cc @@ -60,6 +60,7 @@ typedef struct { size_t chunk_size; long iter_count; long window_size; + long conn_window_size; std::vector operations; unsigned random_seed; size_t num_offcache_buffers; @@ -855,7 +856,8 @@ class DemoClient : public P2pDemoCommon { public: IoReadResponseCallback(size_t buffer_size, MemoryPool& pool) : - _comp_counter(0), _io_counter(NULL), _server_io_counter(NULL), + _comp_counter(0), _client(NULL), + _server_index(std::numeric_limits::max()), _sn(0), _validate(false), _iov(NULL), _buffer(malloc(buffer_size)), _buffer_size(buffer_size), _meta_comp_counter(0), _pool(pool) { @@ -864,13 +866,13 @@ class DemoClient : public P2pDemoCommon { } } - void init(long *io_counter, long *conn_io_counter, + void init(DemoClient *client, size_t server_index, uint32_t sn, bool validate, BufferIov *iov, int meta_comp_counter = 1) { /* wait for all data chunks and the read response completion */ _comp_counter = iov->size() + meta_comp_counter; - _io_counter = io_counter; - _server_io_counter = conn_io_counter; + _client = client; + _server_index = server_index; _sn = sn; _validate = validate; _iov = iov; @@ -886,8 +888,9 @@ class DemoClient : public P2pDemoCommon { return; } - ++(*_io_counter); - ++(*_server_io_counter); + assert(_server_index != std::numeric_limits::max()); + _client->handle_operation_completion(_server_index, IO_READ); + if (_validate && (status == UCS_OK)) { validate(*_iov, _sn); @@ -912,8 +915,8 @@ class DemoClient : public P2pDemoCommon { private: long _comp_counter; - long* _io_counter; - long* _server_io_counter; + DemoClient* _client; + size_t _server_index; uint32_t _sn; bool _validate; BufferIov* _iov; @@ -924,8 +927,9 @@ class DemoClient : public P2pDemoCommon { }; DemoClient(const options_t& test_opts) : - P2pDemoCommon(test_opts), _prev_connect_time(0), - _num_sent(0), _num_completed(0), + P2pDemoCommon(test_opts), + _num_active_servers_to_use(0), + _prev_connect_time(0), _num_sent(0), _num_completed(0), _status(OK), _start_time(get_time()), _read_callback_pool(opts().iomsg_size, "read callbacks") { } @@ -949,24 +953,48 @@ class DemoClient : public P2pDemoCommon { i->second; } - size_t do_io_read(server_info_t& server_info, uint32_t sn) { - size_t data_size = get_data_size(); - bool validate = opts().validate; + void commit_operation(size_t server_index) { + server_info_t& server_info = _server_info[server_index]; + + assert(get_num_uncompleted(server_info) < opts().conn_window_size); + + ++server_info.num_sent; + ++_num_sent; + if (get_num_uncompleted(server_info) == opts().conn_window_size) { + active_servers_make_unused(server_info.active_index); + } + } + + void handle_operation_completion(size_t server_index, io_op_t op) { + server_info_t& server_info = _server_info[server_index]; + + assert(get_num_uncompleted(server_info) <= opts().conn_window_size); + + if (get_num_uncompleted(server_info) == opts().conn_window_size) { + active_servers_make_used(server_info.active_index); + } + + ++_num_completed; + ++server_info.num_completed[op]; + } + + size_t do_io_read(size_t server_index, uint32_t sn) { + server_info_t& server_info = _server_info[server_index]; + size_t data_size = get_data_size(); + bool validate = opts().validate; if (!send_io_message(server_info.conn, IO_READ, sn, data_size, validate)) { return 0; } - ++server_info.num_sent; - ++_num_sent; + commit_operation(server_index); BufferIov *iov = _data_buffers_pool.get(); IoReadResponseCallback *r = _read_callback_pool.get(); iov->init(data_size, _data_chunks_pool, sn, validate); - r->init(&_num_completed, &server_info.num_completed[IO_READ], sn, - validate, iov); + r->init(this, server_index, sn, validate, iov); recv_data(server_info.conn, *iov, sn, r); server_info.conn->recv_data(r->buffer(), opts().iomsg_size, sn, r); @@ -974,31 +1002,31 @@ class DemoClient : public P2pDemoCommon { return data_size; } - size_t do_io_read_am(server_info_t& server_info, uint32_t sn) { - size_t data_size = get_data_size(); + size_t do_io_read_am(size_t server_index, uint32_t sn) { + server_info_t& server_info = _server_info[server_index]; + size_t data_size = get_data_size(); + + commit_operation(server_index); IoMessage *m = _io_msg_pool.get(); m->init(IO_READ, sn, data_size, opts().validate); server_info.conn->send_am(m->buffer(), opts().iomsg_size, NULL, 0, m); - ++server_info.num_sent; - ++_num_sent; - return data_size; } - size_t do_io_write(server_info_t& server_info, uint32_t sn) { - size_t data_size = get_data_size(); - bool validate = opts().validate; + size_t do_io_write(size_t server_index, uint32_t sn) { + server_info_t& server_info = _server_info[server_index]; + size_t data_size = get_data_size(); + bool validate = opts().validate; if (!send_io_message(server_info.conn, IO_WRITE, sn, data_size, validate)) { return 0; } - ++server_info.num_sent; - ++_num_sent; + commit_operation(server_index); BufferIov *iov = _data_buffers_pool.get(); SendCompleteCallback *cb = _send_callback_pool.get(); @@ -1009,19 +1037,20 @@ class DemoClient : public P2pDemoCommon { VERBOSE_LOG << "sending data " << iov << " size " << data_size << " sn " << sn; send_data(server_info.conn, *iov, sn, cb); + return data_size; } - size_t do_io_write_am(server_info_t& server_info, uint32_t sn) { - size_t data_size = get_data_size(); - bool validate = opts().validate; + size_t do_io_write_am(size_t server_index, uint32_t sn) { + server_info_t& server_info = _server_info[server_index]; + size_t data_size = get_data_size(); + bool validate = opts().validate; + + commit_operation(server_index); IoMessage *m = _io_msg_pool.get(); m->init(IO_WRITE, sn, data_size, validate); - ++server_info.num_sent; - ++_num_sent; - BufferIov *iov = _data_buffers_pool.get(); iov->init(data_size, _data_chunks_pool, sn, validate); @@ -1070,8 +1099,7 @@ class DemoClient : public P2pDemoCommon { size_t server_index = get_server_index(conn); if (server_index < _server_info.size()) { - ++_num_completed; - ++_server_info[server_index].num_completed[IO_WRITE]; + handle_operation_completion(server_index, IO_WRITE); } else { /* do not increment _num_completed here since we decremented * _num_sent on connection termination */ @@ -1097,18 +1125,16 @@ class DemoClient : public P2pDemoCommon { } // Client can receive IO_WRITE_COMP or IO_READ_COMP only + size_t server_index = get_server_index(conn); if (msg->op == IO_WRITE_COMP) { assert(msg->op == IO_WRITE_COMP); - ++_num_completed; - ++_server_info[get_server_index(conn)].num_completed[IO_WRITE]; + handle_operation_completion(server_index, IO_WRITE); } else if (msg->op == IO_READ_COMP) { - BufferIov *iov = _data_buffers_pool.get(); - IoReadResponseCallback *r = _read_callback_pool.get(); - + BufferIov *iov = _data_buffers_pool.get(); iov->init(msg->data_size, _data_chunks_pool, msg->sn, opts().validate); - r->init(&_num_completed, - &_server_info[get_server_index(conn)].num_completed[IO_READ], - msg->sn, opts().validate, iov, 0); + + IoReadResponseCallback *r = _read_callback_pool.get(); + r->init(this, server_index, msg->sn, opts().validate, iov, 0); assert(iov->size() == 1); @@ -1116,10 +1142,14 @@ class DemoClient : public P2pDemoCommon { } } + long get_num_uncompleted(const server_info_t& server_info) const { + return server_info.num_sent - + (server_info.num_completed[IO_READ] + + server_info.num_completed[IO_WRITE]); + } + long get_num_uncompleted(size_t server_index) const { - return _server_info[server_index].num_sent - - (_server_info[server_index].num_completed[IO_READ] + - _server_info[server_index].num_completed[IO_WRITE]); + return get_num_uncompleted(_server_info[server_index]); } static void reset_server_info(server_info_t& server_info) { @@ -1151,20 +1181,9 @@ class DemoClient : public P2pDemoCommon { delete server_info.conn; // Don't wait for any more completions on this connection - _num_sent -= get_num_uncompleted(server_index); + _num_sent -= get_num_uncompleted(server_info); - // Replace in _active_servers by the last element in the vector - size_t active_index = server_info.active_index; - std::swap(_active_servers[active_index], _active_servers.back()); - assert(_active_servers.back() == server_index); - - // Swap the active_index field with the "replacement" server_info - server_info_t& replacement_server_info = - _server_info[_active_servers[active_index]]; - std::swap(replacement_server_info.active_index, server_info.active_index); - assert(server_info.active_index == _active_servers.size() - 1); - - _active_servers.pop_back(); + active_servers_remove(server_info.active_index); reset_server_info(server_info); } @@ -1299,11 +1318,9 @@ class DemoClient : public P2pDemoCommon { continue; } - server_info.retry_count = 0; + server_info.retry_count = 0; _server_index_lookup[server_info.conn] = server_index; - - server_info.active_index = _active_servers.size(); - _active_servers.push_back(server_index); + active_servers_add(server_index); LOG << "Connected to " << server_name(server_index); } @@ -1311,6 +1328,20 @@ class DemoClient : public P2pDemoCommon { _prev_connect_time = curr_time; } + size_t pick_server_index() const { + assert(_num_active_servers_to_use != 0); + + /* Pick a random connected server to which the client has credits + * to send (its conn's window is not full) */ + size_t active_index = IoDemoRandom::rand(size_t(0), + _num_active_servers_to_use - 1); + size_t server_index = _active_servers[active_index]; + assert(get_num_uncompleted(server_index) < opts().conn_window_size); + assert(_server_info[server_index].conn != NULL); + + return server_index; + } + static inline bool is_control_iter(long iter) { return (iter % 10) == 0; } @@ -1333,13 +1364,6 @@ class DemoClient : public P2pDemoCommon { op_info_t op_info[IO_OP_MAX] = {{0,0}}; while ((total_iter < opts().iter_count) && (_status == OK)) { - VERBOSE_LOG << " <<<< iteration " << total_iter << " >>>>"; - - wait_for_responses(opts().window_size - 1); - if (_status != OK) { - break; - } - connect_all(is_control_iter(total_iter)); if (_status != OK) { break; @@ -1353,27 +1377,40 @@ class DemoClient : public P2pDemoCommon { continue; } - /* Pick random connected server */ - size_t active_index = IoDemoRandom::rand(size_t(0), - _active_servers.size() - 1); - size_t server_index = _active_servers[active_index]; - assert(_server_info[server_index].conn != NULL); + VERBOSE_LOG << " <<<< iteration " << total_iter << " >>>>"; + long conns_window_size = opts().conn_window_size * + _active_servers.size(); + long max_outstanding = std::min(opts().window_size, + conns_window_size) - 1; + wait_for_responses(max_outstanding); + if (_status != OK) { + break; + } + + if (_num_active_servers_to_use == 0) { + // It is possible that the number of active servers to use is 0 + // after wait_for_responses(), if some clients were closed in + // UCP Worker progress during handling of remote disconnection + // from servers + continue; + } - io_op_t op = get_op(); + size_t server_index = pick_server_index(); + io_op_t op = get_op(); size_t size; switch (op) { case IO_READ: if (opts().use_am) { - size = do_io_read_am(_server_info[server_index], sn); + size = do_io_read_am(server_index, sn); } else { - size = do_io_read(_server_info[server_index], sn); + size = do_io_read(server_index, sn); } break; case IO_WRITE: if (opts().use_am) { - size = do_io_write_am(_server_info[server_index], sn); + size = do_io_write_am(server_index, sn); } else { - size = do_io_write(_server_info[server_index], sn); + size = do_io_write(server_index, sn); } break; default: @@ -1488,8 +1525,8 @@ class DemoClient : public P2pDemoCommon { for (size_t server_index = 0; server_index < _server_info.size(); ++server_index) { server_info_t& server_info = _server_info[server_index]; - long delta_completed = server_info.num_completed[op_id] - - server_info.prev_completed[op_id]; + long delta_completed = server_info.num_completed[op_id] - + server_info.prev_completed[op_id]; if ((delta_completed < delta_min) || ((delta_completed == delta_min) && (server_info.retry_count > @@ -1527,9 +1564,53 @@ class DemoClient : public P2pDemoCommon { } } + void active_servers_swap(size_t index1, size_t index2) { + size_t& active_server1 = _active_servers[index1]; + size_t& active_server2 = _active_servers[index2]; + + std::swap(_server_info[active_server1].active_index, + _server_info[active_server2].active_index); + std::swap(active_server1, active_server2); + } + + void active_servers_add(size_t server_index) { + assert(_num_active_servers_to_use <= _active_servers.size()); + + _active_servers.push_back(server_index); + _server_info[server_index].active_index = _active_servers.size() - 1; + active_servers_make_used(_server_info[server_index].active_index); + assert(_num_active_servers_to_use <= _active_servers.size()); + } + + void active_servers_remove(size_t active_index) { + assert(active_index < _active_servers.size()); + + if (active_index < _num_active_servers_to_use) { + active_servers_make_unused(active_index); + active_index = _num_active_servers_to_use; + } + + assert(active_index >= _num_active_servers_to_use); + active_servers_swap(active_index, _active_servers.size() - 1); + _active_servers.pop_back(); + } + + void active_servers_make_unused(size_t active_index) { + assert(active_index < _num_active_servers_to_use); + --_num_active_servers_to_use; + active_servers_swap(active_index, _num_active_servers_to_use); + } + + void active_servers_make_used(size_t active_index) { + assert(active_index >= _num_active_servers_to_use); + active_servers_swap(active_index, _num_active_servers_to_use); + ++_num_active_servers_to_use; + } + private: std::vector _server_info; std::vector _active_servers; + size_t _num_active_servers_to_use; std::map _server_index_lookup; double _prev_connect_time; long _num_sent; @@ -1626,6 +1707,21 @@ static void adjust_opts(options_t *test_opts) { test_opts->max_data_size); } +static int parse_window_size(const char *optarg, long &window_size, + const std::string &window_size_str) { + window_size = strtol(optarg, NULL, 0); + if ((window_size <= 0) || + // If the converted value falls out of range of corresponding + // return type, LONG_MAX is returned + (window_size == std::numeric_limits::max())) { + std::cout << "invalid " << window_size_str << " size '" << optarg + << "'" << std::endl; + return -1; + } + + return 0; +} + static int parse_args(int argc, char **argv, options_t *test_opts) { char *str; @@ -1646,12 +1742,13 @@ static int parse_args(int argc, char **argv, options_t *test_opts) test_opts->iomsg_size = 256; test_opts->iter_count = 1000; test_opts->window_size = 1; + test_opts->conn_window_size = 1; test_opts->random_seed = std::time(NULL); test_opts->verbose = false; test_opts->validate = false; test_opts->use_am = false; - while ((c = getopt(argc, argv, "p:c:r:d:b:i:w:k:o:t:n:l:s:y:vqaHP:")) != -1) { + while ((c = getopt(argc, argv, "p:c:r:d:b:i:w:a:k:o:t:n:l:s:y:vqAHP:")) != -1) { switch (c) { case 'p': test_opts->port_num = atoi(optarg); @@ -1692,7 +1789,16 @@ static int parse_args(int argc, char **argv, options_t *test_opts) } break; case 'w': - test_opts->window_size = atoi(optarg); + if (parse_window_size(optarg, test_opts->window_size, + "window") != 0) { + return -1; + } + break; + case 'a': + if (parse_window_size(optarg, test_opts->conn_window_size, + "per connection window") != 0) { + return -1; + } break; case 'k': test_opts->chunk_size = strtol(optarg, NULL, 0); @@ -1754,7 +1860,7 @@ static int parse_args(int argc, char **argv, options_t *test_opts) case 'q': test_opts->validate = true; break; - case 'a': + case 'A': test_opts->use_am = true; break; case 'H': @@ -1779,6 +1885,7 @@ static int parse_args(int argc, char **argv, options_t *test_opts) std::cout << " -b Number of offcache IO buffers" << std::endl; std::cout << " -i Number of iterations to run communication" << std::endl; std::cout << " -w Number of outstanding requests" << std::endl; + std::cout << " -a Number of outstanding requests per connection" << std::endl; std::cout << " -k Split the data transfer to chunks of this size" << std::endl; std::cout << " -r Size of IO request packet" << std::endl; std::cout << " -t Client timeout (or \"inf\")" << std::endl; @@ -1791,7 +1898,7 @@ static int parse_args(int argc, char **argv, options_t *test_opts) std::cout << " -s Random seed to use for randomizing" << std::endl; std::cout << " -v Set verbose mode" << std::endl; std::cout << " -q Enable data integrity and transaction check" << std::endl; - std::cout << " -a Use UCP Active Messages API (use TAG API otherwise)" << std::endl; + std::cout << " -A Use UCP Active Messages API (use TAG API otherwise)" << std::endl; std::cout << " -H Use human-readable timestamps" << std::endl; std::cout << " -P Set report printing interval" << std::endl; std::cout << "" << std::endl; diff --git a/test/apps/iodemo/run_io_demo.sh b/test/apps/iodemo/run_io_demo.sh index 367d705707a..8f4fb0b8924 100755 --- a/test/apps/iodemo/run_io_demo.sh +++ b/test/apps/iodemo/run_io_demo.sh @@ -294,7 +294,7 @@ build_server_args_list() { iodemo_server_args+=" $key $value" shift ;; - -q|-a|-v|-H) + -q|-A|-v|-H) iodemo_server_args+=" $key" ;; *) From 49e61221ece7a6c8a0c3a27b73dba5bd0b80ab00 Mon Sep 17 00:00:00 2001 From: Sergey Oblomov Date: Fri, 13 Nov 2020 15:19:10 +0200 Subject: [PATCH 029/107] UCP/HELLO-WORLD: added error simulations - added sender/receiver/keepalive error simulations to ucp_hello_world application - ucp_failure_mode_t is renamed to failure_mode_t - added keepalive tests --- contrib/test_jenkins.sh | 12 +++- examples/ucp_hello_world.c | 136 ++++++++++++++++++++++++++----------- 2 files changed, 104 insertions(+), 44 deletions(-) diff --git a/contrib/test_jenkins.sh b/contrib/test_jenkins.sh index 81b34b7f4b8..572f314a7d8 100755 --- a/contrib/test_jenkins.sh +++ b/contrib/test_jenkins.sh @@ -802,14 +802,14 @@ run_hello() { fi # set smaller timeouts so the test will complete faster - if [[ ${test_args} == *"-e"* ]] + if [[ ${test_args} =~ "-e" ]] then export UCX_UD_TIMEOUT=15s export UCX_RC_TIMEOUT=1ms export UCX_RC_RETRY_COUNT=4 fi - if [[ ${test_args} == *"-e"* ]] + if [[ ${test_args} =~ "-e" ]] then error_emulation=1 else @@ -842,7 +842,10 @@ run_ucp_hello() { mem_types_list+="cuda cuda-managed " fi - for test_mode in -w -f -b -e + export UCX_KEEPALIVE_TIMEOUT=1s + export UCX_KEEPALIVE_NUM_EPS=10 + + for test_mode in -w -f -b -erecv -esend -ekeepalive do for mem_type in $mem_types_list do @@ -851,6 +854,9 @@ run_ucp_hello() { done done rm -f ./ucp_hello_world + + unset UCX_KEEPALIVE_TIMEOUT + unset UCX_KEEPALIVE_NUM_EPS } # diff --git a/examples/ucp_hello_world.c b/examples/ucp_hello_world.c index 897de0c90a3..f4a2be5e623 100644 --- a/examples/ucp_hello_world.c +++ b/examples/ucp_hello_world.c @@ -67,18 +67,25 @@ enum ucp_test_mode_t { TEST_MODE_EVENTFD } ucp_test_mode = TEST_MODE_PROBE; +typedef enum { + FAILURE_MODE_NONE, + FAILURE_MODE_SEND, /* fail send operation on server */ + FAILURE_MODE_RECV, /* fail receive operation on client */ + FAILURE_MODE_KEEPALIVE /* fail without communication on client */ +} failure_mode_t; + static struct err_handling { ucp_err_handling_mode_t ucp_err_mode; - int failure; + failure_mode_t failure_mode; } err_handling_opt; -static ucs_status_t client_status = UCS_OK; -static uint16_t server_port = 13337; -static long test_string_length = 16; -static const ucp_tag_t tag = 0x1337a880u; -static const ucp_tag_t tag_mask = UINT64_MAX; -static const char *addr_msg_str = "UCX address message"; -static const char *data_msg_str = "UCX data message"; +static ucs_status_t ep_status = UCS_OK; +static uint16_t server_port = 13337; +static long test_string_length = 16; +static const ucp_tag_t tag = 0x1337a880u; +static const ucp_tag_t tag_mask = UINT64_MAX; +static const char *addr_msg_str = "UCX address message"; +static const char *data_msg_str = "UCX data message"; static ucp_address_t *local_addr; static ucp_address_t *peer_addr; @@ -220,9 +227,14 @@ static int run_ucx_client(ucp_worker_h ucp_worker) /* Send client UCX address to server */ ep_params.field_mask = UCP_EP_PARAM_FIELD_REMOTE_ADDRESS | - UCP_EP_PARAM_FIELD_ERR_HANDLING_MODE; + UCP_EP_PARAM_FIELD_ERR_HANDLING_MODE | + UCP_EP_PARAM_FIELD_ERR_HANDLER | + UCP_EP_PARAM_FIELD_USER_DATA; ep_params.address = peer_addr; ep_params.err_mode = err_handling_opt.ucp_err_mode; + ep_params.err_handler.cb = failure_handler; + ep_params.err_handler.arg = NULL; + ep_params.user_data = &ep_status; status = ucp_ep_create(ucp_worker, &ep_params, &server_ep); CHKERR_JUMP(status != UCS_OK, "ucp_ep_create\n", err); @@ -250,14 +262,14 @@ static int run_ucx_client(ucp_worker_h ucp_worker) free(msg); - if (err_handling_opt.failure) { - fprintf(stderr, "Emulating unexpected failure on client side\n"); + if (err_handling_opt.failure_mode == FAILURE_MODE_RECV) { + fprintf(stderr, "Emulating failure before receive operation on client side\n"); raise(SIGKILL); } /* Receive test string from server */ for (;;) { - + CHKERR_JUMP(ep_status != UCS_OK, "receive data: EP disconnected\n", err_ep); /* Probing incoming events in non-block mode */ msg_tag = ucp_tag_probe_nb(ucp_worker, tag, tag_mask, 1, &info_tag); if (msg_tag != NULL) { @@ -281,6 +293,13 @@ static int run_ucx_client(ucp_worker_h ucp_worker) CHKERR_JUMP(status != UCS_OK, "test_poll_wait\n", err_ep); } } + + if (err_handling_opt.failure_mode == FAILURE_MODE_KEEPALIVE) { + fprintf(stderr, "Emulating unexpected failure after receive completion " + "on client side, server should detect error by " + "keepalive mechanism\n"); + raise(SIGKILL); + } msg = mem_type_malloc(info_tag.length); CHKERR_JUMP(msg == NULL, "allocate memory\n", err_ep); @@ -378,6 +397,13 @@ static int run_ucx_server(ucp_worker_h ucp_worker) goto err; } + if (err_handling_opt.failure_mode == FAILURE_MODE_SEND) { + fprintf(stderr, "Emulating unexpected failure on server side, client " + "should detect error by keepalive mechanism\n"); + free(msg); + raise(SIGKILL); + } + peer_addr_len = msg->data_len; peer_addr = malloc(peer_addr_len); if (peer_addr == NULL) { @@ -400,13 +426,13 @@ static int run_ucx_server(ucp_worker_h ucp_worker) ep_params.err_mode = err_handling_opt.ucp_err_mode; ep_params.err_handler.cb = failure_handler; ep_params.err_handler.arg = NULL; - ep_params.user_data = &client_status; + ep_params.user_data = &ep_status; status = ucp_ep_create(ucp_worker, &ep_params, &client_ep); /* If peer failure testing was requested, it could be possible that UCP EP * couldn't be created; in this case set `ret = 0` to report success */ - CHKERR_ACTION(status != UCS_OK, "ucp_ep_create\n", - ret = (err_handling_opt.failure) ? 0 : -1; goto err); + ret = (err_handling_opt.failure_mode != FAILURE_MODE_NONE) ? 0 : -1; + CHKERR_ACTION(status != UCS_OK, "ucp_ep_create\n", goto err); msg_len = sizeof(*msg) + test_string_length; msg = mem_type_malloc(msg_len); @@ -417,12 +443,14 @@ static int run_ucx_server(ucp_worker_h ucp_worker) ret = generate_test_string((char *)(msg + 1), test_string_length); CHKERR_JUMP(ret < 0, "generate test string", err_free_mem_type_msg); - if (err_handling_opt.failure) { - /* Sleep for small amount of time to ensure that server was killed + if (err_handling_opt.failure_mode == FAILURE_MODE_RECV) { + /* Sleep for small amount of time to ensure that client was killed * and peer failure handling is covered */ sleep(5); } + ucp_worker_progress(ucp_worker); + send_param.op_attr_mask = UCP_OP_ATTR_FIELD_CALLBACK | UCP_OP_ATTR_FIELD_USER_DATA | UCP_OP_ATTR_FIELD_MEMORY_TYPE; @@ -434,7 +462,7 @@ static int run_ucx_server(ucp_worker_h ucp_worker) status = ucx_wait(ucp_worker, request, "send", data_msg_str); if (status != UCS_OK) { - if (!err_handling_opt.failure) { + if (err_handling_opt.failure_mode != FAILURE_MODE_NONE) { ret = -1; } else { /* If peer failure testing was requested, set `ret = 0` to report @@ -442,13 +470,20 @@ static int run_ucx_server(ucp_worker_h ucp_worker) ret = 0; /* Make sure that failure_handler was called */ - while (client_status == UCS_OK) { + while (ep_status == UCS_OK) { ucp_worker_progress(ucp_worker); } } goto err_free_mem_type_msg; } + if (err_handling_opt.failure_mode == FAILURE_MODE_KEEPALIVE) { + fprintf(stderr, "Waiting for client is terminated\n"); + while (ep_status == UCS_OK) { + ucp_worker_progress(ucp_worker); + } + } + status = flush_ep(ucp_worker, client_ep); printf("flush_ep completed with status %d (%s)\n", status, ucs_status_string(status)); @@ -564,7 +599,7 @@ int main(int argc, char **argv) ret = run_test(client_target_name, ucp_worker); - if (!ret && !err_handling_opt.failure) { + if (!ret && (err_handling_opt.failure_mode != FAILURE_MODE_NONE)) { /* Make sure remote is disconnected before destroying local worker */ ret = barrier(oob_sock); } @@ -586,15 +621,38 @@ int main(int argc, char **argv) return ret; } +static void print_usage() +{ + fprintf(stderr, "Usage: ucp_hello_world [parameters]\n"); + fprintf(stderr, "UCP hello world client/server example utility\n"); + fprintf(stderr, "\nParameters are:\n"); + fprintf(stderr, " -w Select test mode \"wait\" to test " + "ucp_worker_wait function\n"); + fprintf(stderr, " -f Select test mode \"event fd\" to test " + "ucp_worker_get_efd function with later poll\n"); + fprintf(stderr, " -b Select test mode \"busy polling\" to test " + "ucp_tag_probe_nb and ucp_worker_progress (default)\n"); + fprintf(stderr, " -e Emulate unexpected failure and handle an " + "error with enabled UCP_ERR_HANDLING_MODE_PEER\n"); + fprintf(stderr, " send - send failure on server side " + "before send initiated\n"); + fprintf(stderr, " recv - receive failure on client side " + "before receive completed\n"); + fprintf(stderr, " keepalive - keepalive failure on client side " + "after communication completed\n"); + print_common_help(); + fprintf(stderr, "\n"); +} + ucs_status_t parse_cmd(int argc, char * const argv[], char **server_name) { int c = 0, idx = 0; opterr = 0; - err_handling_opt.ucp_err_mode = UCP_ERR_HANDLING_MODE_NONE; - err_handling_opt.failure = 0; + err_handling_opt.ucp_err_mode = UCP_ERR_HANDLING_MODE_NONE; + err_handling_opt.failure_mode = FAILURE_MODE_NONE; - while ((c = getopt(argc, argv, "wfben:p:s:m:h")) != -1) { + while ((c = getopt(argc, argv, "wfbe:n:p:s:m:h")) != -1) { switch (c) { case 'w': ucp_test_mode = TEST_MODE_WAIT; @@ -606,8 +664,17 @@ ucs_status_t parse_cmd(int argc, char * const argv[], char **server_name) ucp_test_mode = TEST_MODE_PROBE; break; case 'e': - err_handling_opt.ucp_err_mode = UCP_ERR_HANDLING_MODE_PEER; - err_handling_opt.failure = 1; + err_handling_opt.ucp_err_mode = UCP_ERR_HANDLING_MODE_PEER; + if (!strcmp(optarg, "recv")) { + err_handling_opt.failure_mode = FAILURE_MODE_RECV; + } else if (!strcmp(optarg, "send")) { + err_handling_opt.failure_mode = FAILURE_MODE_SEND; + } else if (!strcmp(optarg, "keepalive")) { + err_handling_opt.failure_mode = FAILURE_MODE_KEEPALIVE; + } else { + print_usage(); + return UCS_ERR_UNSUPPORTED; + } break; case 'n': *server_name = optarg; @@ -643,25 +710,12 @@ ucs_status_t parse_cmd(int argc, char * const argv[], char **server_name) /* Fall through */ case 'h': default: - fprintf(stderr, "Usage: ucp_hello_world [parameters]\n"); - fprintf(stderr, "UCP hello world client/server example utility\n"); - fprintf(stderr, "\nParameters are:\n"); - fprintf(stderr, " -w Select test mode \"wait\" to test " - "ucp_worker_wait function\n"); - fprintf(stderr, " -f Select test mode \"event fd\" to test " - "ucp_worker_get_efd function with later poll\n"); - fprintf(stderr, " -b Select test mode \"busy polling\" to test " - "ucp_tag_probe_nb and ucp_worker_progress (default)\n"); - fprintf(stderr, " -e Emulate unexpected failure on server side" - "and handle an error on client side with enabled " - "UCP_ERR_HANDLING_MODE_PEER\n"); - print_common_help(); - fprintf(stderr, "\n"); + print_usage(); return UCS_ERR_UNSUPPORTED; } } - fprintf(stderr, "INFO: UCP_HELLO_WORLD mode = %d server = %s port = %d\n", - ucp_test_mode, *server_name, server_port); + fprintf(stderr, "INFO: UCP_HELLO_WORLD mode = %d server = %s port = %d, pid = %d\n", + ucp_test_mode, *server_name, server_port, getpid()); for (idx = optind; idx < argc; idx++) { fprintf(stderr, "WARNING: Non-option argument %s\n", argv[idx]); From dabd4a19041ce59815823815d202db459bf3946b Mon Sep 17 00:00:00 2001 From: Yossi Itigin Date: Sat, 12 Dec 2020 01:58:59 +0200 Subject: [PATCH 030/107] CONTRIB: Add valgrind suppression for rdma_bind_addr() --- contrib/valgrind.supp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/contrib/valgrind.supp b/contrib/valgrind.supp index f40da7c6e5b..536e1bcda0d 100644 --- a/contrib/valgrind.supp +++ b/contrib/valgrind.supp @@ -293,3 +293,9 @@ ... fun:rdma_create_event_channel } +{ + rdmacm_bind_addr + Memcheck:Leak + ... + fun:rdma_bind_addr +} From 2779ba603ea1317f5f54b22a4a2a14cac6b41afb Mon Sep 17 00:00:00 2001 From: Peter Rudenko Date: Sat, 12 Dec 2020 14:57:11 +0200 Subject: [PATCH 031/107] JUCX: UCS Memory type constants. --- .../java/org/openucx/jucx/ucs/UcsConstants.java | 14 ++++++++++++++ bindings/java/src/main/native/ucs_constants.cc | 10 ++++++++++ 2 files changed, 24 insertions(+) diff --git a/bindings/java/src/main/java/org/openucx/jucx/ucs/UcsConstants.java b/bindings/java/src/main/java/org/openucx/jucx/ucs/UcsConstants.java index e775a3b4daf..77a620fafa1 100644 --- a/bindings/java/src/main/java/org/openucx/jucx/ucs/UcsConstants.java +++ b/bindings/java/src/main/java/org/openucx/jucx/ucs/UcsConstants.java @@ -72,6 +72,20 @@ public static class STATUS { public static int UCS_ERR_LAST; } + public static class MEMORY_TYPE { + static { + load(); + } + + public static int UCS_MEMORY_TYPE_HOST; // Default system memory + public static int UCS_MEMORY_TYPE_CUDA; // NVIDIA CUDA memory + public static int UCS_MEMORY_TYPE_CUDA_MANAGED; // NVIDIA CUDA managed (or unified) memory + public static int UCS_MEMORY_TYPE_ROCM; // AMD ROCM memory + public static int UCS_MEMORY_TYPE_ROCM_MANAGED; // AMD ROCM managed system memory + public static int UCS_MEMORY_TYPE_LAST; + public static int UCS_MEMORY_TYPE_UNKNOWN; + } + private static void load() { NativeLibs.load(); loadConstants(); diff --git a/bindings/java/src/main/native/ucs_constants.cc b/bindings/java/src/main/native/ucs_constants.cc index f255c3a04d7..3461748959f 100644 --- a/bindings/java/src/main/native/ucs_constants.cc +++ b/bindings/java/src/main/native/ucs_constants.cc @@ -54,4 +54,14 @@ Java_org_openucx_jucx_ucs_UcsConstants_loadConstants(JNIEnv *env, jclass ucs_cla JUCX_DEFINE_INT_CONSTANT(UCS_ERR_ENDPOINT_TIMEOUT); JUCX_DEFINE_INT_CONSTANT(UCS_ERR_LAST_ENDPOINT_FAILURE); JUCX_DEFINE_INT_CONSTANT(UCS_ERR_LAST); + + // Memory type + cls = env->FindClass("org/openucx/jucx/ucs/UcsConstants$MEMORY_TYPE"); + JUCX_DEFINE_INT_CONSTANT(UCS_MEMORY_TYPE_HOST); + JUCX_DEFINE_INT_CONSTANT(UCS_MEMORY_TYPE_CUDA); + JUCX_DEFINE_INT_CONSTANT(UCS_MEMORY_TYPE_CUDA_MANAGED); + JUCX_DEFINE_INT_CONSTANT(UCS_MEMORY_TYPE_ROCM); + JUCX_DEFINE_INT_CONSTANT(UCS_MEMORY_TYPE_ROCM_MANAGED); + JUCX_DEFINE_INT_CONSTANT(UCS_MEMORY_TYPE_LAST); + JUCX_DEFINE_INT_CONSTANT(UCS_MEMORY_TYPE_UNKNOWN); } From b7c81a461655a19c6d034780d617276886410721 Mon Sep 17 00:00:00 2001 From: binl Date: Sat, 28 Nov 2020 13:02:45 +0000 Subject: [PATCH 032/107] UCS/UCT/IB: Ignore PFs without RDMA cap on BF2 according to its gid_tlb_len==0 --- src/uct/ib/base/ib_device.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/uct/ib/base/ib_device.c b/src/uct/ib/base/ib_device.c index 33528e80341..5860fa1c8ae 100644 --- a/src/uct/ib/base/ib_device.c +++ b/src/uct/ib/base/ib_device.c @@ -528,10 +528,10 @@ ucs_status_t uct_ib_device_query(uct_ib_device_t *dev, } if (dev->num_ports > UCT_IB_DEV_MAX_PORTS) { - ucs_error("%s has %d ports, but only up to %d are supported", + ucs_debug("%s has %d ports, but only up to %d are supported", ibv_get_device_name(ibv_device), dev->num_ports, UCT_IB_DEV_MAX_PORTS); - return UCS_ERR_UNSUPPORTED; + dev->num_ports = UCT_IB_DEV_MAX_PORTS; } /* Query all ports */ @@ -681,6 +681,12 @@ ucs_status_t uct_ib_device_port_check(uct_ib_device_t *dev, uint8_t port_num, return UCS_ERR_NO_DEVICE; } + if (uct_ib_device_port_attr(dev, port_num)->gid_tbl_len == 0) { + ucs_debug("%s:%d has no gid", uct_ib_device_name(dev), + port_num); + return UCS_ERR_UNSUPPORTED; + } + if (uct_ib_device_port_attr(dev, port_num)->state != IBV_PORT_ACTIVE) { ucs_trace("%s:%d is not active (state: %d)", uct_ib_device_name(dev), port_num, uct_ib_device_port_attr(dev, port_num)->state); From 61beb700263ee5b733ec314904e52507763d2b8a Mon Sep 17 00:00:00 2001 From: Alina Sklarevich Date: Sun, 13 Dec 2020 15:37:09 +0200 Subject: [PATCH 033/107] IB/RDMACM: Add local and remote addresses to the reject error message. --- src/uct/ib/rdmacm/rdmacm_cm.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/uct/ib/rdmacm/rdmacm_cm.c b/src/uct/ib/rdmacm/rdmacm_cm.c index 6ae34442622..e0d284713a1 100644 --- a/src/uct/ib/rdmacm/rdmacm_cm.c +++ b/src/uct/ib/rdmacm/rdmacm_cm.c @@ -44,6 +44,8 @@ ucs_status_t uct_rdmacm_cm_ack_event(struct rdma_cm_event *event) ucs_status_t uct_rdmacm_cm_reject(struct rdma_cm_id *id) { uct_rdmacm_priv_data_hdr_t hdr; + char remote_ip_port_str[UCS_SOCKADDR_STRING_LEN]; + char local_ip_port_str[UCS_SOCKADDR_STRING_LEN]; hdr.length = 0; hdr.status = (uint8_t)UCS_ERR_REJECTED; @@ -51,7 +53,12 @@ ucs_status_t uct_rdmacm_cm_reject(struct rdma_cm_id *id) ucs_trace("reject on cm_id %p", id); if (rdma_reject(id, &hdr, sizeof(hdr))) { - ucs_error("rdma_reject (id=%p) failed with error: %m", id); + ucs_error("rdma_reject (id=%p local addr=%s remote addr=%s) failed " + "with error: %m", id, + ucs_sockaddr_str(rdma_get_local_addr(id), local_ip_port_str, + UCS_SOCKADDR_STRING_LEN), + ucs_sockaddr_str(rdma_get_peer_addr(id), remote_ip_port_str, + UCS_SOCKADDR_STRING_LEN)); return UCS_ERR_IO_ERROR; } From 1795c1feadd5c85527da099df3cecc2b02199273 Mon Sep 17 00:00:00 2001 From: Sergey Oblomov Date: Tue, 8 Dec 2020 18:34:01 +0200 Subject: [PATCH 034/107] UCP/NBX: fixed external request free from CB - fixed crash in completion callback when user is tried to free external request - added gtest - added function wait_for_value for UCP tests --- src/ucp/core/ucp_request.inl | 6 ++-- src/ucp/tag/tag_rndv.c | 3 +- test/gtest/ucp/test_ucp_tag.cc | 66 ++++++++++++++++++++++++++++------ test/gtest/ucp/ucp_test.h | 10 ++++++ 4 files changed, 70 insertions(+), 15 deletions(-) diff --git a/src/ucp/core/ucp_request.inl b/src/ucp/core/ucp_request.inl index 386dc68fd66..5ff91095274 100644 --- a/src/ucp/core/ucp_request.inl +++ b/src/ucp/core/ucp_request.inl @@ -64,12 +64,14 @@ #define ucp_request_complete(_req, _cb, _status, ...) \ { \ + /* NOTE: external request can't have RELEASE flag and we */ \ + /* will never put it into mpool */ \ + uint32_t _flags = ((_req)->flags |= UCP_REQUEST_FLAG_COMPLETED); \ (_req)->status = (_status); \ if (ucs_likely((_req)->flags & UCP_REQUEST_FLAG_CALLBACK)) { \ (_req)->_cb((_req) + 1, (_status), ## __VA_ARGS__); \ } \ - if (ucs_unlikely(((_req)->flags |= UCP_REQUEST_FLAG_COMPLETED) & \ - UCP_REQUEST_FLAG_RELEASED)) { \ + if (ucs_unlikely(_flags & UCP_REQUEST_FLAG_RELEASED)) { \ ucp_request_put(_req); \ } \ } diff --git a/src/ucp/tag/tag_rndv.c b/src/ucp/tag/tag_rndv.c index 618156f0cdd..60d8ffe71ce 100644 --- a/src/ucp/tag/tag_rndv.c +++ b/src/ucp/tag/tag_rndv.c @@ -39,11 +39,10 @@ ucs_status_t ucp_tag_rndv_process_rts(ucp_worker_h worker, rreq = ucp_tag_exp_search(&worker->tm, rts_hdr->tag.tag); if (rreq != NULL) { - ucp_tag_rndv_matched(worker, rreq, rts_hdr); - /* Cancel req in transport if it was offloaded, because it arrived as unexpected */ ucp_tag_offload_try_cancel(worker, rreq, UCP_TAG_OFFLOAD_CANCEL_FORCE); + ucp_tag_rndv_matched(worker, rreq, rts_hdr); UCP_WORKER_STAT_RNDV(worker, EXP, 1); return UCS_OK; diff --git a/test/gtest/ucp/test_ucp_tag.cc b/test/gtest/ucp/test_ucp_tag.cc index eaf0dd2ffc9..8b632603596 100644 --- a/test/gtest/ucp/test_ucp_tag.cc +++ b/test/gtest/ucp/test_ucp_tag.cc @@ -14,6 +14,7 @@ extern "C" { #include #include #include +#include } #include @@ -462,28 +463,39 @@ UCS_TEST_P(test_ucp_tag_limits, check_max_short_zcopy_thresh_zero, "ZCOPY_THRESH UCP_INSTANTIATE_TEST_CASE(test_ucp_tag_limits) -class test_ucp_tag_fallback : public ucp_test { +class test_ucp_tag_nbx : public test_ucp_tag { public: void init() { /* forbid zcopy access because it will always fail due to read-only * memory pages (will fail to register memory) */ modify_config("ZCOPY_THRESH", "inf"); - ucp_test::init(); - sender().connect(&receiver(), get_ep_params()); - receiver().connect(&sender(), get_ep_params()); - } - - static void get_test_variants(std::vector& variants) { - add_variant(variants, UCP_FEATURE_TAG); + test_ucp_tag::init(); + m_completed = 0; } protected: static const size_t MSG_SIZE; + uint32_t m_completed; + + static void send_callback(void *req, ucs_status_t status, + void *user_data) + { + request_free((request*)req); + ucs_atomic_add32((volatile uint32_t*)user_data, 1); + } + + static void recv_callback(void *req, ucs_status_t status, + const ucp_tag_recv_info_t *info, + void *user_data) + { + request_free((request*)req); + ucs_atomic_add32((volatile uint32_t*)user_data, 1); + } }; -const size_t test_ucp_tag_fallback::MSG_SIZE = 4 * 1024 * ucs_get_page_size(); +const size_t test_ucp_tag_nbx::MSG_SIZE = 4 * UCS_KBYTE * ucs_get_page_size(); -UCS_TEST_P(test_ucp_tag_fallback, fallback) +UCS_TEST_P(test_ucp_tag_nbx, fallback) { ucp_request_param_t param = {0}; @@ -509,4 +521,36 @@ UCS_TEST_P(test_ucp_tag_fallback, fallback) munmap(send_buffer, MSG_SIZE); } -UCP_INSTANTIATE_TEST_CASE(test_ucp_tag_fallback) +UCS_TEST_P(test_ucp_tag_nbx, external_request_free) +{ + ucp_request_param_t send_param; + ucp_request_param_t recv_param; + + send_param.op_attr_mask = recv_param.op_attr_mask = UCP_OP_ATTR_FIELD_CALLBACK | + UCP_OP_ATTR_FIELD_REQUEST | + UCP_OP_ATTR_FLAG_NO_IMM_CMPL | + UCP_OP_ATTR_FIELD_USER_DATA; + send_param.user_data = recv_param.user_data = &m_completed; + send_param.request = request_alloc(); + recv_param.request = request_alloc(); + send_param.cb.send = (ucp_send_nbx_callback_t)send_callback; + recv_param.cb.recv = (ucp_tag_recv_nbx_callback_t)recv_callback; + send_param.user_data = &m_completed; + recv_param.user_data = &m_completed; + + std::vector send_buffer(MSG_SIZE); + std::vector recv_buffer(MSG_SIZE); + + ucs_status_ptr_t recv_req = ucp_tag_recv_nbx(receiver().worker(), + &recv_buffer[0], MSG_SIZE, + 0, 0, &recv_param); + ASSERT_TRUE(UCS_PTR_IS_PTR(recv_req)); + + ucs_status_ptr_t send_req = ucp_tag_send_nbx(sender().ep(), &send_buffer[0], + MSG_SIZE, 0, &send_param); + ASSERT_TRUE(UCS_PTR_IS_PTR(send_req)); + + wait_for_value(&m_completed, 2u); +} + +UCP_INSTANTIATE_TEST_CASE(test_ucp_tag_nbx) diff --git a/test/gtest/ucp/ucp_test.h b/test/gtest/ucp/ucp_test.h index 36f0f4811fc..d91b262cceb 100644 --- a/test/gtest/ucp/ucp_test.h +++ b/test/gtest/ucp/ucp_test.h @@ -297,6 +297,16 @@ class ucp_test : public ucp_test_base, } } + template + void wait_for_value(volatile T *var, T value, double timeout = 10.0) const + { + ucs_time_t deadline = ucs_get_time() + + ucs_time_from_sec(timeout) * ucs::test_time_multiplier(); + while ((ucs_get_time() < deadline) && (*var != value)) { + short_progress_loop(); + } + } + static const ucp_datatype_t DATATYPE; static const ucp_datatype_t DATATYPE_IOV; From b81d71096579c89ead6185d568d9ade28a94c744 Mon Sep 17 00:00:00 2001 From: dmitrygx Date: Fri, 11 Dec 2020 17:49:08 +0200 Subject: [PATCH 035/107] UCS/MPOOL: Make elem defined for Valgrind --- src/ucs/datastruct/mpool.c | 1 + test/gtest/ucs/test_mpool.cc | 41 ++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/src/ucs/datastruct/mpool.c b/src/ucs/datastruct/mpool.c index 1b142f8ad23..e8bb703a1ad 100644 --- a/src/ucs/datastruct/mpool.c +++ b/src/ucs/datastruct/mpool.c @@ -38,6 +38,7 @@ static void ucs_mpool_chunk_leak_check(ucs_mpool_t *mp, ucs_mpool_chunk_t *chunk for (i = 0; i < chunk->num_elems; ++i) { elem = ucs_mpool_chunk_elem(mp->data, chunk, i); + VALGRIND_MAKE_MEM_DEFINED(elem, sizeof *elem); if (elem->mpool != NULL) { ucs_warn("object %p was not returned to mpool %s", elem + 1, ucs_mpool_name(mp)); diff --git a/test/gtest/ucs/test_mpool.cc b/test/gtest/ucs/test_mpool.cc index 5d720ab1e31..3e26b7d5056 100644 --- a/test/gtest/ucs/test_mpool.cc +++ b/test/gtest/ucs/test_mpool.cc @@ -44,6 +44,24 @@ class test_mpool : public ucs::test { return UCS_LOG_FUNC_RC_CONTINUE; } + static ucs_log_func_rc_t + mpool_log_leak_handler(const char *file, unsigned line, + const char *function, ucs_log_level_t level, + const ucs_log_component_config_t *comp_conf, + const char *message, va_list ap) + { + if (level == UCS_LOG_LEVEL_WARN) { + std::string err_str = format_message(message, ap); + + if (err_str.find("was not returned to mpool test") != + std::string::npos) { + return UCS_LOG_FUNC_RC_STOP; + } + } + + return UCS_LOG_FUNC_RC_CONTINUE; + } + static const size_t header_size = 30; static const size_t data_size = 152; static const size_t align = 128; @@ -203,3 +221,26 @@ UCS_TEST_F(test_mpool, infinite) { ucs_mpool_cleanup(&mp, 1); } + +UCS_TEST_F(test_mpool, leak_check) { + ucs_mpool_t mp; + ucs_status_t status; + + ucs_mpool_ops_t ops = { + ucs_mpool_chunk_malloc, + ucs_mpool_chunk_free, + NULL, + NULL + }; + + status = ucs_mpool_init(&mp, 0, header_size + data_size, header_size, align, + 6, 18, &ops, "test"); + ASSERT_UCS_OK(status); + + void *obj = ucs_mpool_get(&mp); + EXPECT_TRUE(obj != NULL); + // Do not release allocated object + + scoped_log_handler log_handler(mpool_log_leak_handler); + ucs_mpool_cleanup(&mp, 1); +} From db2b2d3827b918c1c5a3dd2c1ff76fc1880a155e Mon Sep 17 00:00:00 2001 From: Peter Rudenko Date: Mon, 14 Dec 2020 13:25:45 +0200 Subject: [PATCH 036/107] JUCX: Prevent clang of formatting java files. --- .clang-format | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.clang-format b/.clang-format index 2e565377dfa..e490ba75153 100644 --- a/.clang-format +++ b/.clang-format @@ -86,6 +86,10 @@ SpacesBeforeTrailingComments: 1 SpaceAfterTemplateKeyword: false SpacesInContainerLiterals: false +# Java +Language: Java +DisableFormat: true + # Issues: # 1. # Pointer alignment + declaration alignment: From 7d38a900748933f0342911a85763c86e8b3efb78 Mon Sep 17 00:00:00 2001 From: Evgeny Leksikov Date: Mon, 7 Dec 2020 17:55:57 +0200 Subject: [PATCH 037/107] UCT/CM/RDMACM: share dummy CQ per device --- src/uct/ib/rdmacm/rdmacm_cm.c | 50 ++++++++++++++++++++++++ src/uct/ib/rdmacm/rdmacm_cm.h | 18 +++++++-- src/uct/ib/rdmacm/rdmacm_cm_ep.c | 65 ++++++++++---------------------- src/uct/ib/rdmacm/rdmacm_cm_ep.h | 1 - 4 files changed, 85 insertions(+), 49 deletions(-) diff --git a/src/uct/ib/rdmacm/rdmacm_cm.c b/src/uct/ib/rdmacm/rdmacm_cm.c index 6ae34442622..f465805dff0 100644 --- a/src/uct/ib/rdmacm/rdmacm_cm.c +++ b/src/uct/ib/rdmacm/rdmacm_cm.c @@ -58,6 +58,53 @@ ucs_status_t uct_rdmacm_cm_reject(struct rdma_cm_id *id) return UCS_OK; } +ucs_status_t uct_rdmacm_cm_get_cq(uct_rdmacm_cm_t *cm, struct ibv_context *verbs, + uint32_t pd_key, struct ibv_cq **cq_p) +{ + struct ibv_cq *cq; + khiter_t iter; + int ret; + + iter = kh_put(uct_rdmacm_cm_cqs, &cm->cqs, pd_key, &ret); + if (ret == -1) { + ucs_error("cm %p: cannot allocate hash entry for CQ", cm); + return UCS_ERR_NO_MEMORY; + } + + if (ret == 0) { + /* already exists so use it */ + cq = kh_value(&cm->cqs, iter); + } else { + /* Create a dummy completion queue */ + cq = ibv_create_cq(verbs, 1, NULL, NULL, 0); + if (cq == NULL) { + kh_del(uct_rdmacm_cm_cqs, &cm->cqs, iter); + ucs_error("ibv_create_cq() failed: %m"); + return UCS_ERR_IO_ERROR; + } + + kh_value(&cm->cqs, iter) = cq; + } + + *cq_p = cq; + return UCS_OK; +} + +void uct_rdmacm_cm_cqs_cleanup(uct_rdmacm_cm_t *cm) +{ + struct ibv_cq *cq; + int ret; + + kh_foreach_value(&cm->cqs, cq, { + ret = ibv_destroy_cq(cq); + if (ret != 0) { + ucs_warn("ibv_destroy_cq() returned %d: %m", ret); + } + }); + + kh_destroy_inplace(uct_rdmacm_cm_cqs, &cm->cqs); +} + size_t uct_rdmacm_cm_get_max_conn_priv() { return UCT_RDMACM_TCP_PRIV_DATA_LEN - sizeof(uct_rdmacm_priv_data_hdr_t); @@ -577,6 +624,8 @@ UCS_CLASS_INIT_FUNC(uct_rdmacm_cm_t, uct_component_h component, &uct_rdmacm_cm_iface_ops, worker, component, config); + kh_init_inplace(uct_rdmacm_cm_cqs, &self->cqs); + self->ev_ch = rdma_create_event_channel(); if (self->ev_ch == NULL) { ucs_error("rdma_create_event_channel failed: %m"); @@ -624,6 +673,7 @@ UCS_CLASS_CLEANUP_FUNC(uct_rdmacm_cm_t) ucs_trace("destroying event_channel %p on cm %p", self->ev_ch, self); rdma_destroy_event_channel(self->ev_ch); + uct_rdmacm_cm_cqs_cleanup(self); } UCS_CLASS_DEFINE(uct_rdmacm_cm_t, uct_cm_t); diff --git a/src/uct/ib/rdmacm/rdmacm_cm.h b/src/uct/ib/rdmacm/rdmacm_cm.h index 6a236719c79..35ab34bb1c7 100644 --- a/src/uct/ib/rdmacm/rdmacm_cm.h +++ b/src/uct/ib/rdmacm/rdmacm_cm.h @@ -7,18 +7,25 @@ #ifndef UCT_RDMACM_CM_H #define UCT_RDMACM_CM_H -#include #include "rdmacm_def.h" +#include +#include + + +KHASH_MAP_INIT_INT(uct_rdmacm_cm_cqs, struct ibv_cq*); + /** * An rdmacm connection manager */ typedef struct uct_rdmacm_cm { - uct_cm_t super; - struct rdma_event_channel *ev_ch; + uct_cm_t super; + struct rdma_event_channel *ev_ch; + khash_t(uct_rdmacm_cm_cqs) cqs; } uct_rdmacm_cm_t; + UCS_CLASS_DECLARE_NEW_FUNC(uct_rdmacm_cm_t, uct_cm_t, uct_component_h, uct_worker_h, const uct_cm_config_t *); UCS_CLASS_DECLARE_DELETE_FUNC(uct_rdmacm_cm_t, uct_cm_t); @@ -38,4 +45,9 @@ ucs_status_t uct_rdmacm_cm_ack_event(struct rdma_cm_event *event); ucs_status_t uct_rdmacm_cm_reject(struct rdma_cm_id *id); +ucs_status_t uct_rdmacm_cm_get_cq(uct_rdmacm_cm_t *cm, struct ibv_context *verbs, + uint32_t pd_key, struct ibv_cq **cq); + +void uct_rdmacm_cm_cqs_cleanup(uct_rdmacm_cm_t *cm); + #endif diff --git a/src/uct/ib/rdmacm/rdmacm_cm_ep.c b/src/uct/ib/rdmacm/rdmacm_cm_ep.c index db832f01912..e33f10257e2 100644 --- a/src/uct/ib/rdmacm/rdmacm_cm_ep.c +++ b/src/uct/ib/rdmacm/rdmacm_cm_ep.c @@ -147,7 +147,7 @@ ucs_status_t uct_rdmacm_cm_ep_conn_notify(uct_ep_h ep) return cep->status; } -static void uct_rdmacm_cm_ep_destroy_dummy_cq_qp(uct_rdmacm_cm_ep_t *cep) +static void uct_rdmacm_cm_ep_destroy_dummy_qp(uct_rdmacm_cm_ep_t *cep) { int ret; @@ -158,39 +158,20 @@ static void uct_rdmacm_cm_ep_destroy_dummy_cq_qp(uct_rdmacm_cm_ep_t *cep) } } - if (cep->cq != NULL) { - ret = ibv_destroy_cq(cep->cq); - if (ret != 0) { - ucs_warn("ibv_destroy_cq() returned %d: %m", ret); - } - } - cep->qp = NULL; - cep->cq = NULL; } -static ucs_status_t uct_rdmacm_cm_create_dummy_cq_qp(struct rdma_cm_id *id, - struct ibv_cq **cq_p, - struct ibv_qp **qp_p) +static ucs_status_t uct_rdmacm_cm_create_dummy_qp(struct rdma_cm_id *id, + struct ibv_cq *cq, + struct ibv_qp **qp_p) { - struct ibv_qp_init_attr qp_init_attr; - ucs_status_t status; - struct ibv_cq *cq; + struct ibv_qp_init_attr qp_init_attr = {0}; struct ibv_qp *qp; - /* Create a dummy completion queue */ - cq = ibv_create_cq(id->verbs, 1, NULL, NULL, 0); - if (cq == NULL) { - ucs_error("ibv_create_cq() failed: %m"); - status = UCS_ERR_IO_ERROR; - goto err; - } - /* Create a dummy UD qp */ - memset(&qp_init_attr, 0, sizeof(qp_init_attr)); - qp_init_attr.send_cq = cq; - qp_init_attr.recv_cq = cq; - qp_init_attr.qp_type = IBV_QPT_UD; + qp_init_attr.send_cq = cq; + qp_init_attr.recv_cq = cq; + qp_init_attr.qp_type = IBV_QPT_UD; qp_init_attr.cap.max_send_wr = 2; qp_init_attr.cap.max_recv_wr = 2; qp_init_attr.cap.max_send_sge = 1; @@ -199,41 +180,36 @@ static ucs_status_t uct_rdmacm_cm_create_dummy_cq_qp(struct rdma_cm_id *id, qp = ibv_create_qp(id->pd, &qp_init_attr); if (qp == NULL) { ucs_error("failed to create a dummy ud qp. %m"); - status = UCS_ERR_IO_ERROR; - goto err_destroy_cq; + return UCS_ERR_IO_ERROR; } ucs_debug("created ud QP %p with qp_num: 0x%x and cq %p on rdmacm_id %p", qp, qp->qp_num, cq, id); - *cq_p = cq; *qp_p = qp; - return UCS_OK; - -err_destroy_cq: - ibv_destroy_cq(cq); -err: - return status; } ucs_status_t uct_rdamcm_cm_ep_set_qp_num(struct rdma_conn_param *conn_param, uct_rdmacm_cm_ep_t *cep) { - ucs_status_t status; - struct ibv_qp *qp; struct ibv_cq *cq; + ucs_status_t status; + + status = uct_rdmacm_cm_get_cq(uct_rdmacm_cm_ep_get_cm(cep), cep->id->verbs, + cep->id->pd->handle, &cq); + if (status != UCS_OK) { + return status; + } /* create a dummy qp in order to get a unique qp_num to provide to librdmacm */ - status = uct_rdmacm_cm_create_dummy_cq_qp(cep->id, &cq, &qp); + status = uct_rdmacm_cm_create_dummy_qp(cep->id, cq, &cep->qp); if (status != UCS_OK) { return status; } - cep->cq = cq; - cep->qp = qp; - conn_param->qp_num = qp->qp_num; + conn_param->qp_num = cep->qp->qp_num; return UCS_OK; } @@ -386,7 +362,7 @@ static ucs_status_t uct_rdamcm_cm_ep_server_init(uct_rdmacm_cm_ep_t *cep, if (rdma_accept(event->id, &conn_param)) { uct_cm_ep_peer_error(&cep->super, "rdma_accept(on id=%p) failed: %m", event->id); - uct_rdmacm_cm_ep_destroy_dummy_cq_qp(cep); + uct_rdmacm_cm_ep_destroy_dummy_qp(cep); status = UCS_ERR_IO_ERROR; goto err; } @@ -485,7 +461,6 @@ UCS_CLASS_INIT_FUNC(uct_rdmacm_cm_ep_t, const uct_ep_params_t *params) UCS_CLASS_CALL_SUPER_INIT(uct_cm_base_ep_t, params); - self->cq = NULL; self->qp = NULL; self->flags = 0; self->status = UCS_OK; @@ -522,7 +497,7 @@ UCS_CLASS_CLEANUP_FUNC(uct_rdmacm_cm_ep_t) UCS_ASYNC_BLOCK(worker_priv->async); - uct_rdmacm_cm_ep_destroy_dummy_cq_qp(self); + uct_rdmacm_cm_ep_destroy_dummy_qp(self); /* rdma_destroy_id() cleans all events not yet reported on progress thread, * so no events would be reported to the user after destroying the id */ diff --git a/src/uct/ib/rdmacm/rdmacm_cm_ep.h b/src/uct/ib/rdmacm/rdmacm_cm_ep.h index b37ae2d8213..8de9e66d541 100644 --- a/src/uct/ib/rdmacm/rdmacm_cm_ep.h +++ b/src/uct/ib/rdmacm/rdmacm_cm_ep.h @@ -13,7 +13,6 @@ typedef struct uct_rdmacm_cm_ep { uct_cm_base_ep_t super; struct rdma_cm_id *id; /* The rdmacm id that is created per this ep */ - struct ibv_cq *cq; /* Dummy cq used for creating a dummy qp */ struct ibv_qp *qp; /* Dummy qp used for generating a unique qp_num */ uint8_t flags; ucs_status_t status; From 893cf9c23b8f63b197bfd7c3695161a25aab4e48 Mon Sep 17 00:00:00 2001 From: Yossi Itigin Date: Sat, 12 Dec 2020 14:52:40 +0200 Subject: [PATCH 038/107] TEST/APPS: Add test for multi-threaded call to ucp_init() --- contrib/test_jenkins.sh | 10 ++++++++ test/apps/Makefile.am | 8 +++++- test/apps/test_init_mt.c | 55 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 test/apps/test_init_mt.c diff --git a/contrib/test_jenkins.sh b/contrib/test_jenkins.sh index 6a069662081..5f82d47b9e7 100755 --- a/contrib/test_jenkins.sh +++ b/contrib/test_jenkins.sh @@ -1260,6 +1260,15 @@ test_ucp_dlopen() { fi } +test_init_mt() { + echo "==== Running multi-thread init ====" + $MAKEP + for ((i=0;i<50;++i)) + do + $AFFINITY timeout 1m ./test/apps/test_init_mt + done +} + test_memtrack() { ../contrib/configure-devel --prefix=$ucx_inst make_clean @@ -1688,6 +1697,7 @@ run_tests() { do_distributed_task 2 4 test_env_var_aliases do_distributed_task 1 3 test_malloc_hook do_distributed_task 0 4 test_ucp_dlopen + do_distributed_task 1 4 test_init_mt # all are running gtest run_gtest_default diff --git a/test/apps/Makefile.am b/test/apps/Makefile.am index 4e9c7caad13..dd2e7d7d75b 100644 --- a/test/apps/Makefile.am +++ b/test/apps/Makefile.am @@ -20,7 +20,8 @@ noinst_PROGRAMS = \ test_ucp_dlopen \ test_ucs_dlopen \ test_link_map \ - test_dlopen_cfg_print + test_dlopen_cfg_print \ + test_init_mt objdir = $(shell sed -n -e 's/^objdir=\(.*\)$$/\1/p' $(LIBTOOL)) @@ -48,6 +49,11 @@ test_dlopen_cfg_print_CPPFLAGS = $(BASE_CPPFLAGS) -g \ test_dlopen_cfg_print_CFLAGS = $(BASE_CFLAGS) test_dlopen_cfg_print_LDADD = -ldl +test_init_mt_SOURCES = test_init_mt.c +test_init_mt_CPPFLAGS = $(BASE_CPPFLAGS) +test_init_mt_CFLAGS = $(BASE_CFLAGS) $(OPENMP_CFLAGS) +test_init_mt_LDADD = $(top_builddir)/src/ucp/libucp.la + if HAVE_TCMALLOC noinst_PROGRAMS += test_tcmalloc test_tcmalloc_SOURCES = test_tcmalloc.c diff --git a/test/apps/test_init_mt.c b/test/apps/test_init_mt.c new file mode 100644 index 00000000000..612c506cc27 --- /dev/null +++ b/test/apps/test_init_mt.c @@ -0,0 +1,55 @@ +/** + * Copyright (C) Mellanox Technologies Ltd. 2020. ALL RIGHTS RESERVED. + * + * See file LICENSE for terms. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#if _OPENMP +#include +#endif + + +int main(int argc, char **argv) +{ + int count = 0; + +#pragma omp parallel + { + ucs_status_t ctx_status, worker_status; + ucp_context_h context; + ucp_worker_h worker; + ucp_params_t params; + ucp_worker_params_t wparams; + + params.field_mask = UCP_PARAM_FIELD_FEATURES; + params.features = UCP_FEATURE_TAG | UCP_FEATURE_STREAM; + ctx_status = ucp_init(¶ms, NULL, &context); + if (ctx_status == UCS_OK) { + wparams.field_mask = 0; + worker_status = ucp_worker_create(context, &wparams, &worker); + if (worker_status == UCS_OK) { + __sync_add_and_fetch(&count, 1); + } + } + +#pragma omp barrier + + if (ctx_status == UCS_OK) { + if (worker_status == UCS_OK) { + ucp_worker_destroy(worker); + } + ucp_cleanup(context); + } + } + +#pragma omp barrier + + printf("finished %d threads\n", count); + return 0; +} From 73eaf980b105d09f32f8f724a6641378a6b60c94 Mon Sep 17 00:00:00 2001 From: Yossi Itigin Date: Sat, 12 Dec 2020 14:53:33 +0200 Subject: [PATCH 039/107] UCM/LOG: Improve and fix logger + Fix printing of large numbers + Print thread id (tid - pid) --- src/ucm/util/log.c | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/src/ucm/util/log.c b/src/ucm/util/log.c index ec41746a477..66fe58042d1 100644 --- a/src/ucm/util/log.c +++ b/src/ucm/util/log.c @@ -23,6 +23,7 @@ #include #include #include +#include #define UCM_LOG_BUG_SIZE 512 @@ -65,7 +66,7 @@ static char *ucm_log_ltoa(char *p, char *end, long n, int base, int flags, int pad) { static const char digits[] = "0123456789abcdef"; - long divider; + long divider, top_divider; if (((n < 0) || (flags & UCM_LOG_LTOA_FLAG_SIGN)) && (p < end)) { *(p++) = (n < 0 ) ? '-' : '+'; @@ -80,9 +81,11 @@ static char *ucm_log_ltoa(char *p, char *end, long n, int base, int flags, n = labs(n); - divider = 1; - while ((n / divider) != 0) { - divider *= base; + divider = 1; + top_divider = 0; + while ((divider > 0) && ((n / divider) != 0)) { + top_divider = divider; + divider *= base; --pad; } @@ -91,7 +94,7 @@ static char *ucm_log_ltoa(char *p, char *end, long n, int base, int flags, (flags & UCM_LOG_LTOA_FLAG_PAD0) ? '0' : ' '); } - divider /= base; + divider = top_divider; while ((p < end) && (divider > 0)) { *(p++) = digits[(n / divider + base) % base]; divider /= base; @@ -243,6 +246,11 @@ static void ucm_log_vsnprintf(char *buf, size_t max, const char *fmt, va_list ap *pb = '\0'; } +static pid_t ucm_get_tid(void) +{ + return syscall(SYS_gettid); +} + static void ucm_log_snprintf(char *buf, size_t max, const char *fmt, ...) { va_list ap; @@ -260,11 +268,15 @@ void __ucm_log(const char *file, unsigned line, const char *function, va_list ap; struct timeval tv; ssize_t nwrite; + pid_t pid; gettimeofday(&tv, NULL); - ucm_log_snprintf(buf, UCM_LOG_BUG_SIZE - 1, "[%lu.%06lu] [%s:%d] %18s:%-4d UCX %s ", - tv.tv_sec, tv.tv_usec, ucm_log_hostname, getpid(), - ucs_basename(file), line, ucm_log_level_names[level]); + pid = getpid(); + ucm_log_snprintf(buf, UCM_LOG_BUG_SIZE - 1, + "[%lu.%06lu] [%s:%d:%d] %18s:%-4d UCX %s ", + tv.tv_sec, tv.tv_usec, ucm_log_hostname, pid, + ucm_get_tid() - pid, ucs_basename(file), line, + ucm_log_level_names[level]); buf[UCM_LOG_BUG_SIZE - 1] = '\0'; length = strlen(buf); From 3f3e0995569595bc4de97ff83d563a527b4718f5 Mon Sep 17 00:00:00 2001 From: dmitrygx Date: Mon, 14 Dec 2020 18:39:23 +0200 Subject: [PATCH 040/107] UCP/GTEST: Fix and test ucp requests leak from the ptr_map --- src/ucp/core/ucp_am.c | 2 + src/ucp/core/ucp_request.c | 12 ++ src/ucp/core/ucp_request.h | 25 ++-- src/ucp/core/ucp_request.inl | 13 +- src/ucp/core/ucp_worker.c | 1 + src/ucp/core/ucp_worker.inl | 24 +++- src/ucp/rma/amo_sw.c | 28 +++-- src/ucp/rma/get_am.c | 13 +- src/ucp/rma/rma.h | 3 +- src/ucp/rma/rma_basic.c | 6 +- src/ucp/rma/rma_send.c | 6 +- src/ucp/rma/rma_sw.c | 31 +++-- src/ucp/rndv/rndv.c | 63 ++++++---- src/ucp/tag/eager_rcv.c | 4 +- src/ucp/tag/eager_snd.c | 9 +- src/ucp/tag/offload.c | 5 +- src/ucp/tag/offload.h | 6 + src/ucp/tag/tag_match.inl | 3 +- src/ucp/tag/tag_rndv.c | 2 + src/ucs/datastruct/ptr_map.inl | 19 ++- test/gtest/common/test_obj_size.cc | 2 +- test/gtest/ucp/test_ucp_sockaddr.cc | 179 ++++++++++++++++------------ 22 files changed, 308 insertions(+), 148 deletions(-) diff --git a/src/ucp/core/ucp_am.c b/src/ucp/core/ucp_am.c index 326b766e74a..4d097ff34a8 100644 --- a/src/ucp/core/ucp_am.c +++ b/src/ucp/core/ucp_am.c @@ -760,6 +760,8 @@ static ucs_status_t ucp_am_send_start_rndv(ucp_request_t *sreq) sreq->send.length); UCS_PROFILE_REQUEST_EVENT(sreq, "start_rndv", sreq->send.length); + ucp_send_request_set_id(sreq); + /* Note: no need to call ucp_ep_resolve_remote_id() here, because it * was done in ucp_am_send_nbx */ diff --git a/src/ucp/core/ucp_request.c b/src/ucp/core/ucp_request.c index 307b4a77aef..1ce03969a49 100644 --- a/src/ucp/core/ucp_request.c +++ b/src/ucp/core/ucp_request.c @@ -338,6 +338,14 @@ void ucp_request_init_multi_proto(ucp_request_t *req, UCS_PROFILE_REQUEST_EVENT(req, multi_func_str, req->send.length); } +static UCS_F_ALWAYS_INLINE void ucp_send_request_init_id(ucp_request_t *req) +{ + if (req->flags & UCP_REQUEST_FLAG_SYNC) { + ucs_assert(req->flags & UCP_REQUEST_FLAG_SEND_TAG); + ucp_send_request_set_id(req); + } +} + ucs_status_t ucp_request_send_start(ucp_request_t *req, ssize_t max_short, size_t zcopy_thresh, size_t zcopy_max, @@ -366,6 +374,8 @@ ucp_request_send_start(ucp_request_t *req, ssize_t max_short, ucp_request_init_multi_proto(req, proto->bcopy_multi, "start_bcopy_multi"); } + + ucp_send_request_init_id(req); return UCS_OK; } else if (length < zcopy_max) { /* zcopy */ @@ -396,6 +406,8 @@ ucp_request_send_start(ucp_request_t *req, ssize_t max_short, req->send.uct.func = proto->zcopy_single; UCS_PROFILE_REQUEST_EVENT(req, "start_zcopy_single", req->send.length); } + + ucp_send_request_init_id(req); return UCS_OK; } diff --git a/src/ucp/core/ucp_request.h b/src/ucp/core/ucp_request.h index 3c994f65611..a583bbdf284 100644 --- a/src/ucp/core/ucp_request.h +++ b/src/ucp/core/ucp_request.h @@ -55,10 +55,12 @@ enum { UCP_REQUEST_FLAG_RECV_TAG = UCS_BIT(17), #if UCS_ENABLE_ASSERT UCP_REQUEST_FLAG_STREAM_RECV = UCS_BIT(18), - UCP_REQUEST_DEBUG_FLAG_EXTERNAL = UCS_BIT(19) + UCP_REQUEST_DEBUG_FLAG_EXTERNAL = UCS_BIT(19), + UCP_REQUEST_FLAG_IN_PTR_MAP = UCS_BIT(20) #else UCP_REQUEST_FLAG_STREAM_RECV = 0, - UCP_REQUEST_DEBUG_FLAG_EXTERNAL = 0 + UCP_REQUEST_DEBUG_FLAG_EXTERNAL = 0, + UCP_REQUEST_FLAG_IN_PTR_MAP = 0 #endif }; @@ -147,8 +149,13 @@ struct ucp_request { struct { uint64_t message_id; /* used to identify matching parts of a large message */ - ucs_ptr_map_key_t rreq_id; /* receive request ID on the + union { + ucs_ptr_map_key_t sreq_id; /* send request ID on the + sender side */ + ucs_ptr_map_key_t rreq_id; /* receive request ID on the recv side (used in AM rndv) */ + }; + union { struct { ucp_tag_t tag; @@ -171,8 +178,9 @@ struct ucp_request { } msg_proto; struct { - uint64_t remote_addr; /* Remote address */ - ucp_rkey_h rkey; /* Remote memory key */ + ucs_ptr_map_key_t sreq_id; /* Send request ID */ + uint64_t remote_addr; /* Remote address */ + ucp_rkey_h rkey; /* Remote memory key */ } rma; struct { @@ -214,7 +222,7 @@ struct ucp_request { } rkey_ptr; struct { - ucs_ptr_map_key_t req_id; /* the send request ID on receiver side */ + ucs_ptr_map_key_t remote_req_id; /* the send request ID on receiver side */ size_t length; /* the length of the data that should be fetched * from sender side */ size_t offset; /* offset in recv buffer */ @@ -247,6 +255,7 @@ struct ucp_request { } discard_uct_ep; struct { + ucs_ptr_map_key_t sreq_id; /* Send request ID */ uint64_t remote_addr; /* Remote address */ ucp_rkey_h rkey; /* Remote memory key */ uint64_t value; /* Atomic argument */ @@ -274,7 +283,8 @@ struct ucp_request { ucp_lane_index_t am_bw_index; /* AM BW lane index */ ucp_lane_map_t lanes_map_avail; /* Used lanes map */ }; - uint8_t mem_type; /* Memory type */ + uint8_t mem_type; /* Memory type, values are + * ucs_memory_type_t */ ucp_lane_index_t pending_lane; /* Lane on which request was moved * to pending state */ ucp_lane_index_t lane; /* Lane on which this request is being sent */ @@ -295,6 +305,7 @@ struct ucp_request { uct_tag_context_t uct_ctx; /* Transport offload context */ ssize_t remaining; /* How much more data * to be received */ + ucs_ptr_map_key_t rreq_id; /* the receive request ID on receiver side */ union { struct { diff --git a/src/ucp/core/ucp_request.inl b/src/ucp/core/ucp_request.inl index 386dc68fd66..392ff424251 100644 --- a/src/ucp/core/ucp_request.inl +++ b/src/ucp/core/ucp_request.inl @@ -150,6 +150,7 @@ static UCS_F_ALWAYS_INLINE void ucp_request_put(ucp_request_t *req) { ucs_trace_req("put request %p", req); + ucs_assert(!(req->flags & UCP_REQUEST_FLAG_IN_PTR_MAP)); UCS_PROFILE_REQUEST_FREE(req); ucs_mpool_put_inline(req); } @@ -661,7 +662,7 @@ ucp_request_complete_am_recv(ucp_request_t *req, ucs_status_t status) static UCS_F_ALWAYS_INLINE ucs_status_t ucp_request_process_recv_data(ucp_request_t *req, const void *data, size_t length, size_t offset, int is_zcopy, - int is_am) + int is_am, ucs_ptr_map_key_t req_id) { ucs_status_t status; int last; @@ -688,6 +689,10 @@ ucp_request_process_recv_data(ucp_request_t *req, const void *data, ucp_request_recv_buffer_dereg(req); } + if (req_id != UCP_REQUEST_ID_INVALID) { + ucp_worker_del_request_id(req->recv.worker, req, req_id); + } + if (is_am) { ucp_request_complete_am_recv(req, status); } else { @@ -762,6 +767,12 @@ ucp_send_request_get_id(ucp_request_t *req) ucp_ep_use_indirect_id(req->send.ep)); } +static UCS_F_ALWAYS_INLINE void +ucp_send_request_set_id(ucp_request_t *req) +{ + req->send.msg_proto.sreq_id = ucp_send_request_get_id(req); +} + static UCS_F_ALWAYS_INLINE void ucp_request_param_rndv_thresh(ucp_request_t *req, const ucp_request_param_t *param, diff --git a/src/ucp/core/ucp_worker.c b/src/ucp/core/ucp_worker.c index 25c6ffc72bc..3867ab33ee0 100644 --- a/src/ucp/core/ucp_worker.c +++ b/src/ucp/core/ucp_worker.c @@ -2807,6 +2807,7 @@ ucp_worker_discard_tl_uct_ep(ucp_worker_h worker, uct_ep_h uct_ep, kh_value(&worker->discard_uct_ep_hash, iter) = req; ucs_assert(!ucp_wireup_ep_test(uct_ep)); + req->flags = 0; req->send.uct.func = ucp_worker_discard_uct_ep_pending_cb; req->send.state.uct_comp.func = ucp_worker_discard_uct_ep_flush_comp; req->send.state.uct_comp.count = 1; diff --git a/src/ucp/core/ucp_worker.inl b/src/ucp/core/ucp_worker.inl index 8cfe7afcf94..86d83a7d7cb 100644 --- a/src/ucp/core/ucp_worker.inl +++ b/src/ucp/core/ucp_worker.inl @@ -66,9 +66,14 @@ ucp_worker_get_request_id(ucp_worker_h worker, ucp_request_t *req, int indirect) ucs_ptr_map_key_t id; ucs_status_t status; + ucs_assert(!(req->flags & UCP_REQUEST_FLAG_IN_PTR_MAP)); status = ucs_ptr_map_put(&worker->ptr_map, req, indirect, &id); if (ucs_unlikely(indirect)) { - return (status == UCS_OK) ? id : UCP_REQUEST_ID_INVALID; + if (ucs_unlikely(status != UCS_OK)) { + return UCP_REQUEST_ID_INVALID; + } + + req->flags |= UCP_REQUEST_FLAG_IN_PTR_MAP; } ucs_assert(status == UCS_OK); @@ -86,11 +91,23 @@ ucp_worker_get_request_by_id(ucp_worker_h worker, ucs_ptr_map_key_t id) } static UCS_F_ALWAYS_INLINE void -ucp_worker_del_request_id(ucp_worker_h worker, ucs_ptr_map_key_t id) +ucp_worker_request_check_flags(const ucp_request_t *request, + ucs_ptr_map_key_t id) +{ + ucs_assert((request != NULL) && + ((request->flags & UCP_REQUEST_FLAG_IN_PTR_MAP) || + !ucs_ptr_map_key_indirect(id))); +} + +static UCS_F_ALWAYS_INLINE void +ucp_worker_del_request_id(ucp_worker_h worker, ucp_request_t *request, + ucs_ptr_map_key_t id) { ucs_status_t status UCS_V_UNUSED; + ucp_worker_request_check_flags(request, id); status = ucs_ptr_map_del(&worker->ptr_map, id); + request->flags &= ~UCP_REQUEST_FLAG_IN_PTR_MAP; ucs_assert(status == UCS_OK); } @@ -100,7 +117,8 @@ ucp_worker_extract_request_by_id(ucp_worker_h worker, ucs_ptr_map_key_t id) ucp_request_t *request; request = (ucp_request_t*)ucs_ptr_map_extract(&worker->ptr_map, id); - ucs_assert(request != NULL); + ucp_worker_request_check_flags(request, id); + request->flags &= ~UCP_REQUEST_FLAG_IN_PTR_MAP; return request; } diff --git a/src/ucp/rma/amo_sw.c b/src/ucp/rma/amo_sw.c index 37d74238859..35a8f876047 100644 --- a/src/ucp/rma/amo_sw.c +++ b/src/ucp/rma/amo_sw.c @@ -25,8 +25,7 @@ static size_t ucp_amo_sw_pack(void *dest, void *arg, uint8_t fetch) atomich->address = req->send.amo.remote_addr; atomich->req.ep_id = ucp_ep_remote_id(ep); - atomich->req.req_id = fetch ? ucp_send_request_get_id(req) : - UCP_REQUEST_ID_INVALID; + atomich->req.req_id = req->send.amo.sreq_id; atomich->length = size; atomich->opcode = req->send.amo.uct_op; @@ -59,12 +58,24 @@ ucp_amo_sw_progress(uct_pending_req_t *self, uct_pack_callback_t pack_cb, ucp_request_t *req = ucs_container_of(self, ucp_request_t, send.uct); ucs_status_t status; - req->send.lane = ucp_ep_get_am_lane(req->send.ep); - status = ucp_rma_sw_do_am_bcopy(req, UCP_AM_ID_ATOMIC_REQ, - req->send.lane, pack_cb, req, NULL); - if (((status != UCS_ERR_NO_RESOURCE) && (status != UCS_OK)) || - ((status == UCS_OK) && !fetch)) { - ucp_request_complete_send(req, status); + req->send.lane = ucp_ep_get_am_lane(req->send.ep); + req->send.amo.sreq_id = fetch ? ucp_send_request_get_id(req) : + UCP_REQUEST_ID_INVALID; + + status = ucp_rma_sw_do_am_bcopy(req, UCP_AM_ID_ATOMIC_REQ, + req->send.lane, pack_cb, req, NULL); + if ((status != UCS_OK) || ((status == UCS_OK) && !fetch)) { + if (fetch) { + ucp_worker_del_request_id(req->send.ep->worker, req, + req->send.amo.sreq_id); + } + + if (status != UCS_ERR_NO_RESOURCE) { + /* completed with: + * - with error if a fetch operation + * - either with error or with success if a post operation */ + ucp_request_complete_send(req, status); + } } return status; @@ -240,6 +251,7 @@ UCS_PROFILE_FUNC(ucs_status_t, ucp_atomic_req_handler, (arg, data, length, am_fl ucs_fatal("invalid atomic length: %u", atomicreqh->length); } + req->flags = 0; req->send.ep = ep; req->send.atomic_reply.req_id = atomicreqh->req.req_id; req->send.length = atomicreqh->length; diff --git a/src/ucp/rma/get_am.c b/src/ucp/rma/get_am.c index 7347be81c2c..95f70d21157 100644 --- a/src/ucp/rma/get_am.c +++ b/src/ucp/rma/get_am.c @@ -24,7 +24,7 @@ static size_t ucp_proto_get_am_bcopy_pack(void *dest, void *arg) getreqh->address = req->send.rma.remote_addr; getreqh->length = req->send.state.dt_iter.length; getreqh->req.ep_id = ucp_send_request_get_ep_remote_id(req); - getreqh->req.req_id = ucp_send_request_get_id(req); + getreqh->req.req_id = req->send.rma.sreq_id; getreqh->mem_type = req->send.rma.rkey->mem_type; return sizeof(*getreqh); @@ -34,6 +34,8 @@ static UCS_F_ALWAYS_INLINE void ucp_proto_get_am_bcopy_complete(ucp_request_t *req, ucs_status_t status) { ucs_assert(status == UCS_OK); + ucp_worker_del_request_id(req->send.ep->worker, req, + req->send.rma.sreq_id); ucp_ep_rma_remote_request_sent(req->send.ep); } @@ -58,12 +60,13 @@ static ucs_status_t ucp_proto_get_am_bcopy_progress(uct_pending_req_t *self) return status; } - /* initialize some request fields, for compatibility of get_reply + /* initialize some request fields, for compatibility of get_reply * processing */ - req->send.buffer = req->send.state.dt_iter.type.contig.buffer; - req->send.length = req->send.state.dt_iter.length; + req->send.buffer = req->send.state.dt_iter.type.contig.buffer; + req->send.length = req->send.state.dt_iter.length; + req->send.rma.sreq_id = ucp_send_request_get_id(req); - req->flags |= UCP_REQUEST_FLAG_PROTO_INITIALIZED; + req->flags |= UCP_REQUEST_FLAG_PROTO_INITIALIZED; } ucp_worker_flush_ops_count_inc(worker); diff --git a/src/ucp/rma/rma.h b/src/ucp/rma/rma.h index 6f47651d1c7..8f5dbbf8265 100644 --- a/src/ucp/rma/rma.h +++ b/src/ucp/rma/rma.h @@ -83,7 +83,8 @@ extern ucp_amo_proto_t ucp_amo_sw_proto; ucs_status_t ucp_rma_request_advance(ucp_request_t *req, ssize_t frag_length, - ucs_status_t status); + ucs_status_t status, + ucs_ptr_map_key_t req_id); void ucp_ep_flush_remote_completed(ucp_request_t *req); diff --git a/src/ucp/rma/rma_basic.c b/src/ucp/rma/rma_basic.c index cc1ac441d9a..e7fb061de8c 100644 --- a/src/ucp/rma/rma_basic.c +++ b/src/ucp/rma/rma_basic.c @@ -70,7 +70,8 @@ static ucs_status_t ucp_rma_basic_progress_put(uct_pending_req_t *self) status); } - return ucp_rma_request_advance(req, packed_len, status); + return ucp_rma_request_advance(req, packed_len, status, + UCP_REQUEST_ID_INVALID); } static ucs_status_t ucp_rma_basic_progress_get(uct_pending_req_t *self) @@ -117,7 +118,8 @@ static ucs_status_t ucp_rma_basic_progress_get(uct_pending_req_t *self) UCS_INPROGRESS); } - return ucp_rma_request_advance(req, frag_length, status); + return ucp_rma_request_advance(req, frag_length, status, + UCP_REQUEST_ID_INVALID); } ucp_rma_proto_t ucp_rma_basic_proto = { diff --git a/src/ucp/rma/rma_send.c b/src/ucp/rma/rma_send.c index 87d15ca77d6..0e3b0ffcf6c 100644 --- a/src/ucp/rma/rma_send.c +++ b/src/ucp/rma/rma_send.c @@ -76,7 +76,8 @@ * next_partial_send; (oops req already freed) */ ucs_status_t ucp_rma_request_advance(ucp_request_t *req, ssize_t frag_length, - ucs_status_t status) + ucs_status_t status, + ucs_ptr_map_key_t req_id) { ucs_assert(status != UCS_ERR_NOT_IMPLEMENTED); @@ -96,6 +97,9 @@ ucs_status_t ucp_rma_request_advance(ucp_request_t *req, ssize_t frag_length, if (req->send.length == 0) { /* bcopy is the fast path */ if (ucs_likely(req->send.state.uct_comp.count == 0)) { + if (req_id != UCP_REQUEST_ID_INVALID) { + ucp_worker_del_request_id(req->send.ep->worker, req, req_id); + } ucp_request_send_buffer_dereg(req); ucp_request_complete_send(req, UCS_OK); } diff --git a/src/ucp/rma/rma_sw.c b/src/ucp/rma/rma_sw.c index 5ce3d3f875f..0a606c35a75 100644 --- a/src/ucp/rma/rma_sw.c +++ b/src/ucp/rma/rma_sw.c @@ -48,7 +48,7 @@ static ucs_status_t ucp_rma_sw_progress_put(uct_pending_req_t *self) ucp_rma_sw_put_pack_cb, req, &packed_len); return ucp_rma_request_advance(req, packed_len - sizeof(ucp_put_hdr_t), - status); + status, UCP_REQUEST_ID_INVALID); } static size_t ucp_rma_sw_get_req_pack_cb(void *dest, void *arg) @@ -60,7 +60,7 @@ static size_t ucp_rma_sw_get_req_pack_cb(void *dest, void *arg) getreqh->length = req->send.length; getreqh->req.ep_id = ucp_send_request_get_ep_remote_id(req); getreqh->mem_type = req->send.rma.rkey->mem_type; - getreqh->req.req_id = ucp_send_request_get_id(req); + getreqh->req.req_id = req->send.rma.sreq_id; ucs_assert(getreqh->req.ep_id != UCP_EP_ID_INVALID); return sizeof(*getreqh); @@ -72,14 +72,19 @@ static ucs_status_t ucp_rma_sw_progress_get(uct_pending_req_t *self) ssize_t packed_len = 0; ucs_status_t status; - req->send.lane = ucp_ep_get_am_lane(req->send.ep); - status = ucp_rma_sw_do_am_bcopy(req, UCP_AM_ID_GET_REQ, - req->send.lane, - ucp_rma_sw_get_req_pack_cb, req, - &packed_len); - if ((status != UCS_OK) && (status != UCS_ERR_NO_RESOURCE)) { - /* completed with error */ - ucp_request_complete_send(req, status); + req->send.lane = ucp_ep_get_am_lane(req->send.ep); + req->send.rma.sreq_id = ucp_send_request_get_id(req); + + status = ucp_rma_sw_do_am_bcopy(req, UCP_AM_ID_GET_REQ, req->send.lane, + ucp_rma_sw_get_req_pack_cb, req, + &packed_len); + if (status != UCS_OK) { + ucp_worker_del_request_id(req->send.ep->worker, req, + req->send.rma.sreq_id); + if (ucs_unlikely(status != UCS_ERR_NO_RESOURCE)) { + /* completed with error */ + ucp_request_complete_send(req, status); + } } /* If completed with UCS_OK, it means that get request packet sent, @@ -132,6 +137,7 @@ void ucp_rma_sw_send_cmpl(ucp_ep_h ep) return; } + req->flags = 0; req->send.ep = ep; req->send.uct.func = ucp_progress_rma_cmpl; ucp_request_send(req, 0); @@ -218,6 +224,7 @@ UCS_PROFILE_FUNC(ucs_status_t, ucp_get_req_handler, (arg, data, length, am_flags return UCS_OK; } + req->flags = 0; req->send.ep = ep; req->send.buffer = (void*)getreqh->address; req->send.length = getreqh->length; @@ -259,8 +266,8 @@ UCS_PROFILE_FUNC(ucs_status_t, ucp_get_rep_handler, (arg, data, length, am_flags memcpy(req->send.buffer, getreph + 1, frag_length); /* complete get request on last fragment of the reply */ - if (ucp_rma_request_advance(req, frag_length, UCS_OK) == UCS_OK) { - ucp_worker_del_request_id(worker, getreph->req_id); + if (ucp_rma_request_advance(req, frag_length, UCS_OK, + getreph->req_id) == UCS_OK) { ucp_ep_rma_remote_request_completed(ep); } } diff --git a/src/ucp/rndv/rndv.c b/src/ucp/rndv/rndv.c index 70eedda4860..7b879f1121c 100644 --- a/src/ucp/rndv/rndv.c +++ b/src/ucp/rndv/rndv.c @@ -101,7 +101,7 @@ size_t ucp_rndv_rts_pack(ucp_request_t *sreq, ucp_rndv_rts_hdr_t *rndv_rts_hdr, void *rkey_buf; rndv_rts_hdr->sreq.ep_id = ucp_send_request_get_ep_remote_id(sreq); - rndv_rts_hdr->sreq.req_id = ucp_send_request_get_id(sreq); + rndv_rts_hdr->sreq.req_id = sreq->send.msg_proto.sreq_id; rndv_rts_hdr->size = sreq->send.length; rndv_rts_hdr->flags = flags; @@ -139,10 +139,9 @@ static size_t ucp_rndv_rtr_pack(void *dest, void *arg) ucp_ep_h ep = rndv_req->send.ep; ssize_t packed_rkey_size; - rndv_rtr_hdr->sreq_id = rndv_req->send.rndv_rtr.req_id; + rndv_rtr_hdr->sreq_id = rndv_req->send.rndv_rtr.remote_req_id; /* request of receiver side */ - rndv_rtr_hdr->rreq_id = ucp_worker_get_request_id(ep->worker, rreq, - ucp_ep_use_indirect_id(ep)); + rndv_rtr_hdr->rreq_id = rreq->recv.rreq_id; /* Pack remote keys (which can be empty list) */ if (UCP_DT_IS_CONTIG(rreq->recv.datatype)) { @@ -150,7 +149,7 @@ static size_t ucp_rndv_rtr_pack(void *dest, void *arg) rndv_rtr_hdr->size = rndv_req->send.rndv_rtr.length; rndv_rtr_hdr->offset = rndv_req->send.rndv_rtr.offset; - packed_rkey_size = ucp_rkey_pack_uct(rndv_req->send.ep->worker->context, + packed_rkey_size = ucp_rkey_pack_uct(ep->worker->context, rreq->recv.state.dt.contig.md_map, rreq->recv.state.dt.contig.memh, rreq->recv.mem_type, @@ -327,6 +326,8 @@ UCS_PROFILE_FUNC_VOID(ucp_rndv_complete_frag_rma_put_zcopy, (fsreq), /* complete send request after put completions of all fragments */ if (sreq->send.state.dt.offset == sreq->send.length) { + ucp_worker_del_request_id(sreq->send.ep->worker, sreq, + sreq->send.msg_proto.sreq_id); ucp_rndv_complete_rma_put_zcopy(sreq); } } @@ -406,15 +407,21 @@ static void ucp_rndv_req_send_rtr(ucp_request_t *rndv_req, ucp_request_t *rreq, ucs_ptr_map_key_t sender_req_id, size_t recv_length, size_t offset) { + ucp_ep_h ep = rndv_req->send.ep; + ucp_trace_req(rndv_req, "send rtr remote sreq_id 0x%"PRIxPTR" rreq %p", sender_req_id, rreq); - rndv_req->super_req = rreq; - rndv_req->send.lane = ucp_ep_get_am_lane(rndv_req->send.ep); - rndv_req->send.uct.func = ucp_proto_progress_rndv_rtr; - rndv_req->send.rndv_rtr.req_id = sender_req_id; - rndv_req->send.rndv_rtr.length = recv_length; - rndv_req->send.rndv_rtr.offset = offset; + rndv_req->super_req = rreq; + rndv_req->send.lane = ucp_ep_get_am_lane(rndv_req-> + send.ep); + rndv_req->send.uct.func = ucp_proto_progress_rndv_rtr; + rndv_req->send.rndv_rtr.remote_req_id = sender_req_id; + rndv_req->send.rndv_rtr.length = recv_length; + rndv_req->send.rndv_rtr.offset = offset; + rreq->recv.rreq_id = + ucp_worker_get_request_id(ep->worker, rreq, + ucp_ep_use_indirect_id(ep)); ucp_request_send(rndv_req, 0); } @@ -724,7 +731,7 @@ UCS_PROFILE_FUNC_VOID(ucp_rndv_recv_frag_put_completion, (self), ucs_ptr_map_key_t rreq_remote_id = freq->send.rndv_put.rreq_remote_id; int is_put_proto = (rreq_remote_id == UCP_REQUEST_ID_INVALID); ucp_request_t *req = freq->super_req; - ucp_request_t *rndv_req; + ucp_request_t *rndv_req = NULL; ucs_trace_req("freq:%p: recv_frag_put done. rreq:%p ", freq, req); @@ -753,7 +760,8 @@ UCS_PROFILE_FUNC_VOID(ucp_rndv_recv_frag_put_completion, (self), if (req->recv.remaining == 0) { ucp_request_complete_tag_recv(req, UCS_OK); if (!is_put_proto) { - ucp_worker_del_request_id(worker, rreq_remote_id); + ucs_assert(rndv_req != NULL); + ucp_worker_del_request_id(worker, rndv_req, rreq_remote_id); } } @@ -773,6 +781,7 @@ ucp_rndv_init_mem_type_frag_req(ucp_worker_h worker, ucp_request_t *freq, int rn ucp_request_send_state_init(freq, ucp_dt_make_contig(1), 0); ucp_request_send_state_reset(freq, comp_cb, rndv_op); + freq->flags = 0; freq->send.buffer = mdesc + 1; freq->send.length = length; freq->send.datatype = ucp_dt_make_contig(1); @@ -1041,7 +1050,7 @@ static void ucp_rndv_send_frag_rtr(ucp_worker_h worker, ucp_request_t *rndv_req, freq->recv.state.dt.contig.md_map = 0; freq->recv.frag.offset = offset; freq->super_req = rreq; - freq->flags |= UCP_REQUEST_FLAG_RNDV_FRAG; + freq->flags = UCP_REQUEST_FLAG_RNDV_FRAG; memh_index = 0; ucs_for_each_bit(md_index, @@ -1052,6 +1061,7 @@ static void ucp_rndv_send_frag_rtr(ucp_worker_h worker, ucp_request_t *rndv_req, } ucs_assert(memh_index <= UCP_MAX_OP_MDS); + frndv_req->flags = 0; frndv_req->send.ep = rndv_req->send.ep; frndv_req->send.pending_lane = UCP_NULL_LANE; @@ -1608,6 +1618,7 @@ static ucs_status_t ucp_rndv_send_start_put_pipeline(ucp_request_t *sreq, ucp_request_send_state_init(fsreq, ucp_dt_make_contig(1), 0); fsreq->super_req = sreq; + fsreq->flags = 0; fsreq->send.buffer = UCS_PTR_BYTE_OFFSET(sreq->send.buffer, rndv_base_offset); fsreq->send.length = rndv_size; @@ -1638,6 +1649,7 @@ static ucs_status_t ucp_rndv_send_start_put_pipeline(ucp_request_t *sreq, md_index = ucp_ep_md_index(sreq->send.ep, sreq->send.lane); freq->super_req = fsreq; + freq->flags = 0; freq->send.ep = fsreq->send.ep; freq->send.buffer = UCS_PTR_BYTE_OFFSET(fsreq->send.buffer, offset); @@ -1687,7 +1699,7 @@ UCS_PROFILE_FUNC(ucs_status_t, ucp_rndv_atp_handler, req->recv.length, req->recv.frag.offset); } else { UCS_PROFILE_REQUEST_EVENT(req, "rndv_atp_recv", 0); - ucp_worker_del_request_id(arg, rep_hdr->req_id); + ucp_worker_del_request_id(arg, req, rep_hdr->req_id); ucp_rndv_zcopy_recv_req_complete(req, UCS_OK); } @@ -1698,9 +1710,10 @@ UCS_PROFILE_FUNC(ucs_status_t, ucp_rndv_rtr_handler, (arg, data, length, flags), void *arg, void *data, size_t length, unsigned flags) { + ucp_worker_h worker = arg; ucp_rndv_rtr_hdr_t *rndv_rtr_hdr = data; - ucp_request_t *sreq = ucp_worker_get_request_by_id(arg, - rndv_rtr_hdr->sreq_id); + ucp_request_t *sreq = ucp_worker_get_request_by_id(worker, + rndv_rtr_hdr->sreq_id); ucp_ep_h ep = sreq->send.ep; ucp_ep_config_t *ep_config = ucp_ep_config(ep); ucp_context_h context = ep->worker->context; @@ -1715,6 +1728,7 @@ UCS_PROFILE_FUNC(ucs_status_t, ucp_rndv_rtr_handler, /* Do not deregister memory here, because am zcopy rndv may * need it registered (if am and tag is the same lane). */ ucp_tag_offload_cancel_rndv(sreq); + ucs_assert(!ucp_ep_use_indirect_id(ep)); } if (UCP_DT_IS_CONTIG(sreq->send.datatype) && rndv_rtr_hdr->address) { @@ -1800,6 +1814,9 @@ UCS_PROFILE_FUNC(ucs_status_t, ucp_rndv_rtr_handler, } out_send: + /* if it not a PUT pipeline protocol, delete the send request ID */ + ucp_worker_del_request_id(worker, sreq, + sreq->send.msg_proto.sreq_id); ucp_request_send(sreq, 0); return UCS_OK; } @@ -1811,7 +1828,6 @@ UCS_PROFILE_FUNC(ucs_status_t, ucp_rndv_data_handler, ucp_worker_h worker = arg; ucp_rndv_data_hdr_t *rndv_data_hdr = data; ucp_request_t *rreq; - ucs_status_t status; size_t recv_len; rreq = ucp_worker_get_request_by_id(worker, rndv_data_hdr->rreq_id); @@ -1822,12 +1838,11 @@ UCS_PROFILE_FUNC(ucs_status_t, ucp_rndv_data_handler, recv_len = length - sizeof(*rndv_data_hdr); UCS_PROFILE_REQUEST_EVENT(rreq, "rndv_data_recv", recv_len); - status = ucp_request_process_recv_data(rreq, rndv_data_hdr + 1, recv_len, - rndv_data_hdr->offset, 1, - rreq->flags & UCP_REQUEST_FLAG_RECV_AM); - if (status != UCS_INPROGRESS) { - ucp_worker_del_request_id(worker, rndv_data_hdr->rreq_id); - } + ucp_request_process_recv_data(rreq, rndv_data_hdr + 1, recv_len, + rndv_data_hdr->offset, 1, + rreq->flags & UCP_REQUEST_FLAG_RECV_AM, + rndv_data_hdr->rreq_id); + return UCS_OK; } diff --git a/src/ucp/tag/eager_rcv.c b/src/ucp/tag/eager_rcv.c index 905c77b4aeb..86292a104d1 100644 --- a/src/ucp/tag/eager_rcv.c +++ b/src/ucp/tag/eager_rcv.c @@ -259,7 +259,9 @@ UCS_PROFILE_FUNC(ucs_status_t, ucp_eager_offload_sync_ack_handler, ucs_queue_for_each_safe(sreq, iter, queue, send.tag_offload.queue) { if ((sreq->send.tag_offload.ssend_tag == rep_hdr->sender_tag) && (ucp_ep_local_id(sreq->send.ep) == rep_hdr->ep_id)) { - ucp_tag_eager_sync_completion(sreq, UCP_REQUEST_FLAG_REMOTE_COMPLETED, + ucp_tag_offload_request_check_flags(sreq); + ucp_tag_eager_sync_completion(sreq, + UCP_REQUEST_FLAG_REMOTE_COMPLETED, UCS_OK); ucs_queue_del_iter(queue, iter); return UCS_OK; diff --git a/src/ucp/tag/eager_snd.c b/src/ucp/tag/eager_snd.c index 110af2030e7..d764a862a3d 100644 --- a/src/ucp/tag/eager_snd.c +++ b/src/ucp/tag/eager_snd.c @@ -52,7 +52,7 @@ static size_t ucp_tag_pack_eager_sync_only_dt(void *dest, void *arg) hdr->super.super.tag = req->send.msg_proto.tag.tag; hdr->req.ep_id = ucp_send_request_get_ep_remote_id(req); - hdr->req.req_id = ucp_send_request_get_id(req); + hdr->req.req_id = req->send.msg_proto.sreq_id; return ucp_tag_pack_eager_common(req, hdr + 1, req->send.length, sizeof(*hdr), 1); @@ -92,7 +92,7 @@ static size_t ucp_tag_pack_eager_sync_first_dt(void *dest, void *arg) hdr->super.total_len = req->send.length; hdr->req.ep_id = ucp_send_request_get_ep_remote_id(req); hdr->super.msg_id = req->send.msg_proto.message_id; - hdr->req.req_id = ucp_send_request_get_id(req); + hdr->req.req_id = req->send.msg_proto.sreq_id; return ucp_tag_pack_eager_common(req, hdr + 1, length, sizeof(*hdr), 1); } @@ -231,6 +231,7 @@ void ucp_tag_eager_sync_zcopy_req_complete(ucp_request_t *req, ucs_status_t status) { ucs_assert(req->send.state.uct_comp.count == 0); + ucp_request_send_buffer_dereg(req); /* TODO register+lane change */ ucp_tag_eager_sync_completion(req, UCP_REQUEST_FLAG_LOCAL_COMPLETED, status); @@ -256,7 +257,7 @@ static ucs_status_t ucp_tag_eager_sync_zcopy_single(uct_pending_req_t *self) hdr.super.super.tag = req->send.msg_proto.tag.tag; hdr.req.ep_id = ucp_send_request_get_ep_remote_id(req); - hdr.req.req_id = ucp_send_request_get_id(req); + hdr.req.req_id = req->send.msg_proto.sreq_id; return ucp_do_am_zcopy_single(self, UCP_AM_ID_EAGER_SYNC_ONLY, &hdr, sizeof(hdr), NULL, 0ul, @@ -272,7 +273,7 @@ static ucs_status_t ucp_tag_eager_sync_zcopy_multi(uct_pending_req_t *self) first_hdr.super.super.super.tag = req->send.msg_proto.tag.tag; first_hdr.super.total_len = req->send.length; first_hdr.req.ep_id = ucp_send_request_get_ep_remote_id(req); - first_hdr.req.req_id = ucp_send_request_get_id(req); + first_hdr.req.req_id = req->send.msg_proto.sreq_id; first_hdr.super.msg_id = req->send.msg_proto.message_id; middle_hdr.msg_id = req->send.msg_proto.message_id; middle_hdr.offset = req->send.state.dt.offset; diff --git a/src/ucp/tag/offload.c b/src/ucp/tag/offload.c index ff31b2dd52a..1f00f4116c9 100644 --- a/src/ucp/tag/offload.c +++ b/src/ucp/tag/offload.c @@ -565,6 +565,8 @@ static void ucp_tag_offload_rndv_zcopy_completion(uct_completion_t *self) { ucp_request_t *req = ucs_container_of(self, ucp_request_t, send.state.uct_comp); + + ucp_tag_offload_request_check_flags(req); ucp_proto_am_zcopy_req_complete(req, self->status); } @@ -581,10 +583,11 @@ ucs_status_t ucp_tag_offload_rndv_zcopy(uct_pending_req_t *self) ucp_tag_offload_unexp_rndv_hdr_t rndv_hdr = { .ep_id = ucp_send_request_get_ep_remote_id(req), - .req_id = ucp_send_request_get_id(req), + .req_id = req->send.msg_proto.sreq_id, .md_index = ucp_ep_md_index(ep, req->send.lane) }; + ucs_assert(!ucp_ep_use_indirect_id(req->send.ep)); dt_state = req->send.state.dt; UCS_STATIC_ASSERT(sizeof(ucp_rsc_index_t) <= sizeof(rndv_hdr.md_index)); diff --git a/src/ucp/tag/offload.h b/src/ucp/tag/offload.h index ad35f2eca90..3b4999dee83 100644 --- a/src/ucp/tag/offload.h +++ b/src/ucp/tag/offload.h @@ -160,5 +160,11 @@ ucp_tag_offload_unexp(ucp_worker_iface_t *wiface, ucp_tag_t tag, size_t length) } } +static UCS_F_ALWAYS_INLINE void +ucp_tag_offload_request_check_flags(ucp_request_t *req) +{ + ucs_assert(!ucp_ep_use_indirect_id(req->send.ep) && + !(req->flags & UCP_REQUEST_FLAG_IN_PTR_MAP)); +} #endif diff --git a/src/ucp/tag/tag_match.inl b/src/ucp/tag/tag_match.inl index 9d127b779f6..d1369e6c7f5 100644 --- a/src/ucp/tag/tag_match.inl +++ b/src/ucp/tag/tag_match.inl @@ -326,7 +326,8 @@ ucp_tag_request_process_recv_data(ucp_request_t *req, const void *data, return ucp_request_recv_offload_data(req, data, length, recv_flags); } - return ucp_request_process_recv_data(req, data, length, offset, dereg, 0); + return ucp_request_process_recv_data(req, data, length, offset, dereg, 0, + UCP_REQUEST_ID_INVALID); } static UCS_F_ALWAYS_INLINE ucs_status_t diff --git a/src/ucp/tag/tag_rndv.c b/src/ucp/tag/tag_rndv.c index 618156f0cdd..7e048a14945 100644 --- a/src/ucp/tag/tag_rndv.c +++ b/src/ucp/tag/tag_rndv.c @@ -105,6 +105,8 @@ ucs_status_t ucp_tag_send_start_rndv(ucp_request_t *sreq) return status; } + ucp_send_request_set_id(sreq); + if (ucp_ep_is_tag_offload_enabled(ucp_ep_config(ep))) { status = ucp_tag_offload_start_rndv(sreq); } else { diff --git a/src/ucs/datastruct/ptr_map.inl b/src/ucs/datastruct/ptr_map.inl index 0419d62a9b8..eb61b395e68 100644 --- a/src/ucs/datastruct/ptr_map.inl +++ b/src/ucs/datastruct/ptr_map.inl @@ -53,6 +53,19 @@ static inline void ucs_ptr_map_destroy(ucs_ptr_map_t *map) kh_destroy_inplace(ucs_ptr_map_impl, &map->hash); } +/** + * Returns whether the key is indirect or not. + * + * @param [in] key Key to object pointer. + * + * @return 0 - direct, otherside - indirect. + */ +static UCS_F_ALWAYS_INLINE int +ucs_ptr_map_key_indirect(ucs_ptr_map_key_t key) +{ + return key & UCS_PTR_MAP_KEY_INDIRECT_FLAG; +} + /** * Put a pointer into the map. * @@ -79,6 +92,7 @@ ucs_ptr_map_put(ucs_ptr_map_t *map, void *ptr, int indirect, if (ucs_likely(!indirect)) { *key = (uintptr_t)ptr; ucs_assert(!(*key & UCS_PTR_MAP_KEY_MIN_ALIGN)); + ucs_assert(*key != 0); return UCS_OK; } @@ -108,7 +122,7 @@ ucs_ptr_map_get(const ucs_ptr_map_t *map, ucs_ptr_map_key_t key) { khiter_t iter; - if (ucs_likely(!(key & UCS_PTR_MAP_KEY_INDIRECT_FLAG))) { + if (ucs_likely(!ucs_ptr_map_key_indirect(key))) { return (void*)key; } @@ -130,7 +144,8 @@ ucs_ptr_map_extract(ucs_ptr_map_t *map, ucs_ptr_map_key_t key) khiter_t iter; void *value; - if (ucs_likely(!(key & UCS_PTR_MAP_KEY_INDIRECT_FLAG))) { + if (ucs_likely(!ucs_ptr_map_key_indirect(key))) { + ucs_assert(key != 0); return (void*)key; } diff --git a/test/gtest/common/test_obj_size.cc b/test/gtest/common/test_obj_size.cc index 5aec7fc95fc..882cdd16c6b 100644 --- a/test/gtest/common/test_obj_size.cc +++ b/test/gtest/common/test_obj_size.cc @@ -48,7 +48,7 @@ UCS_TEST_F(test_obj_size, size) { #else EXPECTED_SIZE(ucp_ep_t, 64); /* TODO reduce request size to 240 or less after removing old protocols state */ - EXPECTED_SIZE(ucp_request_t, 248); + EXPECTED_SIZE(ucp_request_t, 256); EXPECTED_SIZE(ucp_recv_desc_t, 48); EXPECTED_SIZE(uct_ep_t, 8); EXPECTED_SIZE(uct_base_ep_t, 8); diff --git a/test/gtest/ucp/test_ucp_sockaddr.cc b/test/gtest/ucp/test_ucp_sockaddr.cc index d0a77098e42..73ef845c36b 100644 --- a/test/gtest/ucp/test_ucp_sockaddr.cc +++ b/test/gtest/ucp/test_ucp_sockaddr.cc @@ -1043,34 +1043,41 @@ class test_ucp_sockaddr_protocols : public test_ucp_sockaddr { void test_tag_send_recv(size_t size, bool is_exp, bool is_sync = false) { - std::string send_buf(size, 'x'); - std::string recv_buf(size, 'y'); + /* send multiple messages to test the protocol both before and after + * connection establishment */ + for (int i = 0; i < m_num_iters; i++) { + std::string send_buf(size, 'x'); + std::string recv_buf(size, 'y'); + + void *rreq = NULL, *sreq = NULL; + + if (is_exp) { + rreq = ucp_tag_recv_nb(receiver().worker(), &recv_buf[0], + size, ucp_dt_make_contig(1), 0, 0, + rtag_complete_cb); + } - void *rreq = NULL, *sreq = NULL; + if (is_sync) { + sreq = ucp_tag_send_sync_nb(sender().ep(), &send_buf[0], size, + ucp_dt_make_contig(1), 0, + scomplete_cb); + } else { + sreq = ucp_tag_send_nb(sender().ep(), &send_buf[0], size, + ucp_dt_make_contig(1), 0, scomplete_cb); + } - if (is_exp) { - rreq = ucp_tag_recv_nb(receiver().worker(), &recv_buf[0], size, - ucp_dt_make_contig(1), 0, 0, rtag_complete_cb); - } + if (!is_exp) { + short_progress_loop(); + rreq = ucp_tag_recv_nb(receiver().worker(), &recv_buf[0], size, + ucp_dt_make_contig(1), 0, 0, + rtag_complete_cb); + } - if (is_sync) { - sreq = ucp_tag_send_sync_nb(sender().ep(), &send_buf[0], size, - ucp_dt_make_contig(1), 0, scomplete_cb); - } else { - sreq = ucp_tag_send_nb(sender().ep(), &send_buf[0], size, - ucp_dt_make_contig(1), 0, scomplete_cb); - } + request_wait(sreq); + request_wait(rreq); - if (!is_exp) { - short_progress_loop(); - rreq = ucp_tag_recv_nb(receiver().worker(), &recv_buf[0], size, - ucp_dt_make_contig(1), 0, 0, rtag_complete_cb); + compare_buffers(send_buf, recv_buf); } - - request_wait(sreq); - request_wait(rreq); - - compare_buffers(send_buf, recv_buf); } void wait_for_server_ep() @@ -1082,32 +1089,40 @@ class test_ucp_sockaddr_protocols : public test_ucp_sockaddr { void test_stream_send_recv(size_t size, bool is_exp) { - std::string send_buf(size, 'x'); - std::string recv_buf(size, 'y'); - size_t recv_length; - void *rreq, *sreq; - - if (is_exp) { - wait_for_server_ep(); - rreq = ucp_stream_recv_nb(receiver().ep(), &recv_buf[0], size, - ucp_dt_make_contig(1), rstream_complete_cb, - &recv_length, UCP_STREAM_RECV_FLAG_WAITALL); - sreq = ucp_stream_send_nb(sender().ep(), &send_buf[0], size, - ucp_dt_make_contig(1), scomplete_cb, 0); - } else { - sreq = ucp_stream_send_nb(sender().ep(), &send_buf[0], size, - ucp_dt_make_contig(1), scomplete_cb, 0); - short_progress_loop(); - wait_for_server_ep(); - rreq = ucp_stream_recv_nb(receiver().ep(), &recv_buf[0], size, - ucp_dt_make_contig(1), rstream_complete_cb, - &recv_length, UCP_STREAM_RECV_FLAG_WAITALL); - } + /* send multiple messages to test the protocol both before and after + * connection establishment */ + for (int i = 0; i < m_num_iters; i++) { + std::string send_buf(size, 'x'); + std::string recv_buf(size, 'y'); + size_t recv_length; + void *rreq, *sreq; + + if (is_exp) { + wait_for_server_ep(); + rreq = ucp_stream_recv_nb(receiver().ep(), &recv_buf[0], size, + ucp_dt_make_contig(1), + rstream_complete_cb, &recv_length, + UCP_STREAM_RECV_FLAG_WAITALL); + sreq = ucp_stream_send_nb(sender().ep(), &send_buf[0], size, + ucp_dt_make_contig(1), scomplete_cb, + 0); + } else { + sreq = ucp_stream_send_nb(sender().ep(), &send_buf[0], size, + ucp_dt_make_contig(1), scomplete_cb, + 0); + short_progress_loop(); + wait_for_server_ep(); + rreq = ucp_stream_recv_nb(receiver().ep(), &recv_buf[0], size, + ucp_dt_make_contig(1), + rstream_complete_cb, &recv_length, + UCP_STREAM_RECV_FLAG_WAITALL); + } - request_wait(sreq); - request_wait(rreq); + request_wait(sreq); + request_wait(rreq); - compare_buffers(send_buf, recv_buf); + compare_buffers(send_buf, recv_buf); + } } void register_mem(entity* initiator, entity* target, void *buffer, @@ -1136,47 +1151,56 @@ class test_ucp_sockaddr_protocols : public test_ucp_sockaddr { void test_rma(size_t size, rma_nb_func_t rma_func) { - std::string send_buf(size, 'x'); - std::string recv_buf(size, 'y'); + /* send multiple messages to test the protocol both before and after + * connection establishment */ + for (int i = 0; i < m_num_iters; i++) { + std::string send_buf(size, 'x'); + std::string recv_buf(size, 'y'); - ucp_mem_h memh; - ucp_rkey_h rkey; + ucp_mem_h memh; + ucp_rkey_h rkey; - register_mem(&sender(), &receiver(), &recv_buf[0], size, &memh, &rkey); + register_mem(&sender(), &receiver(), &recv_buf[0], size, &memh, + &rkey); - std::vector reqs; - (this->*rma_func)(send_buf, recv_buf, rkey, reqs); + std::vector reqs; + (this->*rma_func)(send_buf, recv_buf, rkey, reqs); - while (!reqs.empty()) { - request_wait(reqs.back()); - reqs.pop_back(); - } + while (!reqs.empty()) { + request_wait(reqs.back()); + reqs.pop_back(); + } - compare_buffers(send_buf, recv_buf); + compare_buffers(send_buf, recv_buf); - ucp_rkey_destroy(rkey); - ucs_status_t status = ucp_mem_unmap(receiver().ucph(), memh); - ASSERT_UCS_OK(status); + ucp_rkey_destroy(rkey); + ucs_status_t status = ucp_mem_unmap(receiver().ucph(), memh); + ASSERT_UCS_OK(status); + } } void test_am_send_recv(size_t size, size_t hdr_size = 0ul) { - std::string sb(size, 'x'); - std::string hdr(hdr_size, 'x'); + /* send multiple messages to test the protocol both before and after + * connection establishment */ + for (int i = 0; i < m_num_iters; i++) { + std::string sb(size, 'x'); + std::string hdr(hdr_size, 'x'); - bool am_received = false; + bool am_received = false; - set_am_data_handler(receiver(), 0, rx_am_msg_cb, &am_received); + set_am_data_handler(receiver(), 0, rx_am_msg_cb, &am_received); - ucp_request_param_t param = {}; - ucs_status_ptr_t sreq = ucp_am_send_nbx(sender().ep(), 0, &hdr[0], - hdr_size, &sb[0], size, - ¶m); - request_wait(sreq); - wait_for_flag(&am_received); - EXPECT_TRUE(am_received); + ucp_request_param_t param = {}; + ucs_status_ptr_t sreq = ucp_am_send_nbx(sender().ep(), 0, + &hdr[0], hdr_size, + &sb[0], size, ¶m); + request_wait(sreq); + wait_for_flag(&am_received); + EXPECT_TRUE(am_received); - set_am_data_handler(receiver(), 0, NULL, NULL); + set_am_data_handler(receiver(), 0, NULL, NULL); + } } private: @@ -1205,8 +1229,15 @@ class test_ucp_sockaddr_protocols : public test_ucp_sockaddr { param.arg = arg; ASSERT_UCS_OK(ucp_worker_set_am_recv_handler(e.worker(), ¶m)); } + +private: + static const unsigned m_num_iters; }; + +const unsigned test_ucp_sockaddr_protocols::m_num_iters = 10; + + UCS_TEST_P(test_ucp_sockaddr_protocols, stream_short_exp) { test_stream_send_recv(1, true); From ccdc83a94fd27de80796b541fab5a8a3a1e21b19 Mon Sep 17 00:00:00 2001 From: Anatoly Vildemanov Date: Mon, 14 Dec 2020 12:47:25 +0300 Subject: [PATCH 041/107] AZP: io-demo analyzer --- buildlib/az-network-corrupter.sh | 1 + buildlib/azure-pipelines-pr.yml | 6 +- buildlib/io-demo.yml | 152 ---------------- buildlib/io_demo/az-stage-io-demo.yaml | 77 ++++++++ buildlib/io_demo/io-demo.yml | 75 ++++++++ buildlib/io_demo/iodemo_analyzer.py | 234 +++++++++++++++++++++++++ 6 files changed, 392 insertions(+), 153 deletions(-) delete mode 100644 buildlib/io-demo.yml create mode 100644 buildlib/io_demo/az-stage-io-demo.yaml create mode 100644 buildlib/io_demo/io-demo.yml create mode 100644 buildlib/io_demo/iodemo_analyzer.py diff --git a/buildlib/az-network-corrupter.sh b/buildlib/az-network-corrupter.sh index 16ebeef8b72..eb4ffc7054b 100755 --- a/buildlib/az-network-corrupter.sh +++ b/buildlib/az-network-corrupter.sh @@ -14,6 +14,7 @@ manager_script=/hpc/noarch/git_projects/swx_infrastructure/clusters/bin/manage_h if [ "x$reset" = "xyes" ]; then echo "Resetting interface on $(hostname)..." ${manager_script} "$(hostname)" "bond-up" + sleep "$uptime" exit $? fi diff --git a/buildlib/azure-pipelines-pr.yml b/buildlib/azure-pipelines-pr.yml index 645d3ec9c83..f95ae93001a 100644 --- a/buildlib/azure-pipelines-pr.yml +++ b/buildlib/azure-pipelines-pr.yml @@ -203,4 +203,8 @@ stages: name: hwi demands: ucx_hwi -equals yes test_perf: 0 - - template: io-demo.yml + + - stage: io_demo + dependsOn: [Codestyle] + jobs: + - template: io_demo/io-demo.yml diff --git a/buildlib/io-demo.yml b/buildlib/io-demo.yml deleted file mode 100644 index 2e1d47e1bdd..00000000000 --- a/buildlib/io-demo.yml +++ /dev/null @@ -1,152 +0,0 @@ -parameters: - - name: test_duration - type: number - default: 2 - - name: tests - type: object - default: - base: - initial_delay: 20 - cycles: 100 - downtime: 5 - uptime: 20 - -jobs: - - job: io_build - displayName: Build io_demo - - pool: - name: MLNX - demands: - - ucx_iodemo -equals yes - - steps: - # address permissions issue when some files created as read-only - - bash: chmod u+rwx ./ -R - - - checkout: self - clean: true - displayName: Checkout - - - bash: | - set -eEx - - ./autogen.sh - ./contrib/configure-release --prefix=$PWD/__install - make -j`nproc` - - echo "##vso[task.setvariable variable=workspace;isOutput=true]$(Build.Repository.LocalPath)/buildlib" - displayName: Build - name: build - - - job: test - dependsOn: io_build - - pool: - name: MLNX - demands: - - ucx_iodemo -equals yes - - strategy: - matrix: - ${{ each test in parameters.tests }}: - ${{ test.Key }}: - test_name: ${{ test.Key }} - initial_delay: ${{ test.Value.initial_delay }} - cycles: ${{ test.Value.cycles }} - downtime: ${{ test.Value.downtime }} - uptime: ${{ test.Value.uptime }} - maxParallel: 1 - - variables: - workspace: $[ dependencies.io_build.outputs['build.workspace'] ] - - displayName: "io_demo: " - steps: - - checkout: none - - - bash: | - set -eEx - source ./buildlib/az-helpers.sh - ./buildlib/az-network-corrupter.sh \ - initial_delay=$(initial_delay) \ - cycles=$(cycles) \ - downtime=$(downtime) \ - uptime=$(uptime) \ - |& add_timestamp &>corrupter.log & - pgrep -u "$USER" -f 'network-corrupter' - corrupter_pid=$(pgrep -u "$USER" -f 'network-corrupter') - echo "corrupter_pid=$corrupter_pid" - azure_set_variable "corrupter_pid" "$corrupter_pid" - displayName: Start network corrupter - timeoutInMinutes: 5 - - - bash: | - set -eEx - # set UCX environment variables - export UCX_SOCKADDR_CM_ENABLE=y - export UCX_NET_DEVICES=$(ibdev2netdev | sed -ne 's/\(\w*\) port \([0-9]\) ==> '${roce_iface}' .*/\1:\2/p') - $(workspace)/../test/apps/iodemo/run_io_demo.sh \ - -H $(agent_hosts) \ - --tasks-per-node 1 \ - --duration 60 \ - -v \ - --num-clients 1 \ - --num-servers 1 \ - --map-by slot \ - -i $(roce_iface) \ - $(workspace)/../test/apps/iodemo/io_demo \ - -d 512:524288 \ - -o read,write \ - -i 0 \ - -w 16 \ - -t 10 - displayName: Launch with run_io_demo.sh (tag) - timeoutInMinutes: 2 - - - bash: | - set -eEx - # set UCX environment variables - export UCX_SOCKADDR_CM_ENABLE=y - export UCX_NET_DEVICES=$(ibdev2netdev | sed -ne 's/\(\w*\) port \([0-9]\) ==> '${roce_iface}' .*/\1:\2/p') - $(workspace)/../test/apps/iodemo/run_io_demo.sh \ - -H $(agent_hosts) \ - --tasks-per-node 1 \ - --duration 60 \ - -v \ - --num-clients 1 \ - --num-servers 1 \ - --map-by slot \ - -i $(roce_iface) \ - $(workspace)/../test/apps/iodemo/io_demo \ - -d 512:524288 \ - -o read,write \ - -i 0 \ - -w 16 \ - -t 10 \ - -q \ - -A - displayName: Launch with run_io_demo.sh (active message) - timeoutInMinutes: 2 - - - bash: | - set -eEx - pid=$(corrupter_pid) - echo "Stopping corrupter, PID=${pid}" - if ! kill ${pid}; then - echo "##vso[task.logissue type=warning]Can't stop corrupter: process doesn't exist" - echo "##vso[task.complete result=Failed;]" - else - echo "Corrupter stopped successfully" - fi - cat corrupter.log - displayName: Kill corrupter - condition: always() - timeoutInMinutes: 10 - - - bash: | - set -eEx - ./buildlib/az-network-corrupter.sh reset=yes - displayName: Restore port state - condition: always() - timeoutInMinutes: 1 diff --git a/buildlib/io_demo/az-stage-io-demo.yaml b/buildlib/io_demo/az-stage-io-demo.yaml new file mode 100644 index 00000000000..8ddf59fcac7 --- /dev/null +++ b/buildlib/io_demo/az-stage-io-demo.yaml @@ -0,0 +1,77 @@ +parameters: +- name: name # defaults for any parameters that aren't specified + default: 'test' +- name: iodemo_args + default: '' + +steps: +- bash: | + set -eEx + source ./buildlib/az-helpers.sh + ./buildlib/az-network-corrupter.sh \ + initial_delay=$(initial_delay) \ + cycles=$(cycles) \ + downtime=$(downtime) \ + uptime=$(uptime) \ + |& add_timestamp &>corrupter.log & + pgrep -u "$USER" -f 'network-corrupter' + corrupter_pid=$(pgrep -u "$USER" -f 'network-corrupter') + echo "corrupter_pid=$corrupter_pid" + azure_set_variable "corrupter_pid" "$corrupter_pid" + displayName: Start network corrupter + timeoutInMinutes: 15 + +- bash: | + set -eEx + sudo /hpc/local/bin/lshca + mkdir -p $(workspace)/${{ parameters.name }} + # set UCX environment variables + export UCX_SOCKADDR_CM_ENABLE=y + #export UCX_NET_DEVICES=$(ibdev2netdev | sed -ne 's/\(\w*\) port \([0-9]\) ==> '${roce_iface}' .*/\1:\2/p') + $(workspace)/../test/apps/iodemo/run_io_demo.sh \ + -H $(agent_hosts) \ + --tasks-per-node 1 \ + --duration 480 \ + -v \ + --num-clients 1 \ + --num-servers 1 \ + --map-by slot \ + --log-dir $(workspace)/${{ parameters.name }} \ + -i $(roce_iface) \ + $(io_demo_exe) \ + -d 512:524288 \ + -o read,write \ + -i 0 \ + -w 16 \ + -t 10 \ + ${{ parameters.iodemo_args }} + + displayName: Launch with run_io_demo.sh ( ${{ parameters.name }} ) + timeoutInMinutes: 10 + +- bash: | + python buildlib/io_demo/iodemo_analyzer.py -d $(workspace)/${{ parameters.name }} + displayName: Analyze for ${{ parameters.name }} + timeoutInMinutes: 1 + +- bash: | + set -eEx + pid=$(corrupter_pid) + echo "Stopping corrupter, PID=${pid}" + if ! kill ${pid}; then + echo "##vso[task.logissue type=warning]Can't stop corrupter: process doesn't exist" + echo "##vso[task.complete result=Failed;]" + else + echo "Corrupter stopped successfully" + fi + cat corrupter.log + displayName: Kill corrupter + condition: always() + timeoutInMinutes: 10 + +- bash: | + set -eEx + ./buildlib/az-network-corrupter.sh reset=yes + displayName: Restore port state + condition: always() + timeoutInMinutes: 2 diff --git a/buildlib/io_demo/io-demo.yml b/buildlib/io_demo/io-demo.yml new file mode 100644 index 00000000000..80d6a658f6b --- /dev/null +++ b/buildlib/io_demo/io-demo.yml @@ -0,0 +1,75 @@ +parameters: + - name: demands + type: string + default: "ucx_iodemo -equals yes" + - name: test_duration + type: number + default: 2 + - name: tests + type: object + default: + base: + # Disable corrupter for master branch + initial_delay: 2000 + cycles: 100 + downtime: 5 + uptime: 40 + +jobs: + - job: io_build + displayName: Build io_demo + + pool: + name: MLNX + demands: ${{ parameters.demands }} + + steps: + # address permissions issue when some files created as read-only + - bash: chmod u+rwx ./ -R + + - checkout: self + clean: true + displayName: Checkout + - bash: | + set -eEx + ./autogen.sh + ./contrib/configure-release --prefix=$(Build.Repository.LocalPath)/install + make -j`nproc` + make install + echo "##vso[task.setvariable variable=workspace;isOutput=true]$(Build.Repository.LocalPath)/buildlib" + echo "##vso[task.setvariable variable=io_demo_exe;isOutput=true]$(Build.Repository.LocalPath)/install/bin/io_demo" + displayName: Build + name: build + + - job: test + dependsOn: io_build + + pool: + name: MLNX + demands: ${{ parameters.demands }} + + strategy: + matrix: + ${{ each test in parameters.tests }}: + ${{ test.Key }}: + test_name: ${{ test.Key }} + initial_delay: ${{ test.Value.initial_delay }} + cycles: ${{ test.Value.cycles }} + downtime: ${{ test.Value.downtime }} + uptime: ${{ test.Value.uptime }} + maxParallel: 1 + + variables: + workspace: $[ dependencies.io_build.outputs['build.workspace'] ] + io_demo_exe: $[ dependencies.io_build.outputs['build.io_demo_exe'] ] + + displayName: "io_demo: " + steps: + - checkout: none + - template: az-stage-io-demo.yaml + parameters: + name: tag + - template: az-stage-io-demo.yaml + parameters: + name: active + iodemo_args: -q -A diff --git a/buildlib/io_demo/iodemo_analyzer.py b/buildlib/io_demo/iodemo_analyzer.py new file mode 100644 index 00000000000..475e9bdfc97 --- /dev/null +++ b/buildlib/io_demo/iodemo_analyzer.py @@ -0,0 +1,234 @@ +import subprocess +import os +import argparse +import re +import datetime,time +import traceback,sys + +allow_error_list = [ + 'Connection reset by remote peer', + 'UCX-connection.*detected error:', + 'ERROR Remote QP on mlx', + 'UCX ERROR RC QP', + 'ERROR IB Async event on', + 'setting error flag on connection', + 'Operation rejected by remote peer', + 'got error event RDMA_CM_EVENT_ADDR_ERROR', + 'rdma_accept', + 'UCX ERROR Remote access on', + 'UCX ERROR Transport retry count exceeded on', + 'UCX WARN failed to disconnect CM lane', + 'ucp_ep_create\(\) failed: Input/output error', + 'terminate connection.*due to Input/output error', + 'UCX ERROR Local QP operation on', + 'conn_id send request.*failed: Input/output error', + 'deleting connection with status Input/output error', + 'UCX WARN failed to disconnect CM lane .* Operation rejected by remote peer', + 'ucp_ep_query\(\) failed: Endpoint timeout', + 'UCX ERROR rdma_reject.*failed with error: Invalid argument', + 'UCX ERROR rdma_init_qp_attr.*failed: Invalid argument', + 'UCX ERROR rdma_establish on ep.*failed: Invalid argument', + 'UCX ERROR .*client.*failed to process a connect response' + ] + + +re_allow_list = re.compile("|".join(allow_error_list), re.I) +re_timestamp = re.compile(r"\[(\d+\.\d+)\].*") +re_traffic = re.compile(r"\[(\d+\.\d+)\].*read (\d+.\d+).*min:(\d+).*write (\d+.\d+).*min:(\d+).*") +re_traffic_read = re.compile(r"\[(\d+\.\d+)\].*read (\d+.\d+) MB\/s min:(\d+).*") +re_traffic_write = re.compile(r"\[(\d+\.\d+)\].*write (\d+.\d+) MB\/s min:(\d+).*") +re_error = re.compile(r".*(error|assert|backtrace|segmentation).*", re.I) +re_warning = re.compile(r".*warn.*", re.I) + + +def in_allow_list(line, is_allow_list): + if is_allow_list: + s = re_allow_list.search(line) + if s: + return True + return False + + +def process_seek(seek_file): + data = {} + if not seek_file or not os.path.exists(seek_file): + return data + + with open(seek_file) as f: + for line in f.readlines(): + if line: + # [log name] \t [position] \t [previous timestamp] + d = line.split('\t') + ts = datetime.datetime.fromtimestamp(float(d[2])) + rx_ts = float(d[3]) + tx_ts = float(d[4]) + data[d[0]] = { + 'pos': int(d[1]), + 'timestamp': ts, + 'timestamp_rx': rx_ts, + 'timestamp_tx': tx_ts, + } + # Burn After Reading + open(seek_file, 'w').close() + return data + + +def get_logs(directory): + client_list = [] + server_list = [] + for f in os.listdir(directory): + filename = os.path.join(directory, f) + if os.path.isfile(filename) and "_client_" in f: + client_list.append(filename) + if os.path.isfile(filename) and "_server_" in f: + server_list.append(filename) + return client_list, server_list + + +def process_server(files, is_allow_list): + for log in files: + with open(log) as f: + while True: + line = f.readline() + if not line: + break + + m = re_error.match(line, re.IGNORECASE) or re_warning.match(line, re.IGNORECASE) + if m and not in_allow_list(line, is_allow_list): + raise Exception("Contains error: {}\nLog {}:\nLine {}".format(line, log, line)) + + +def process_client(files, threshold, seek_file, is_allow_list): + seek_data = process_seek(seek_file) + for log in files: + with open(log) as f: + curr_ts = 0 + curr_traffic_ts = 0 + cur_traffic_date = "" + prev_traffic_ts = seek_data.get(log, {}).get('timestamp', 0) + zero_rx_ts = seek_data.get(log, {}).get('timestamp_rx', 0) + zero_tx_ts = seek_data.get(log, {}).get('timestamp_tx', 0) + pos_prev = seek_data.get(log, {}).get('pos', 0) + f.seek(pos_prev) + i = 0 + while True: + line = f.readline() + if not line: + if seek_file and cur_traffic_date: + pos = f.tell() + with open(seek_file, 'a+') as s: + s.write("{}\t{}\t{}\t{}\t{}\n".format( + log, pos, cur_traffic_date, zero_rx_ts, zero_tx_ts)) + break + + timestamp_match = re_timestamp.match(line) + if timestamp_match: + date = float(timestamp_match.group(1)) + curr_ts = datetime.datetime.fromtimestamp(date) + if not prev_traffic_ts: + prev_traffic_ts = curr_ts + + i += 1 + read_match = re_traffic_read.match(line) + write_match = re_traffic_write.match(line) + + current_match = None + + if read_match: + current_match = read_match + cur_traffic_date = current_match.group(1) + date_traffic = float(cur_traffic_date) + curr_traffic_ts = datetime.datetime.fromtimestamp(date_traffic) + rx = float(current_match.group(2)) + min_server_rx = int(current_match.group(3)) + + if min_server_rx == 0 and zero_rx_ts: + delta = curr_traffic_ts - datetime.datetime.fromtimestamp(zero_rx_ts) + if delta.total_seconds() > threshold * 60: + raise Exception("Have read min:0 servers {} minutes \ + (more threshold:{})\nlog {}:\nLine {}".format( + delta.total_seconds()/60.0, threshold, log, line)) + else: + zero_rx_ts = date_traffic + + if not rx: + raise Exception("Have read zero speed:\nLog {}:\nLine {}".format(log, line)) + prev_traffic_ts = curr_traffic_ts + + if write_match: + current_match = write_match + cur_traffic_date = current_match.group(1) + date_traffic = float(cur_traffic_date) + curr_traffic_ts = datetime.datetime.fromtimestamp(date_traffic) + tx = float(current_match.group(2)) + min_server_tx=int(current_match.group(3)) + + if min_server_tx == 0 and zero_tx_ts: + delta = curr_traffic_ts - datetime.datetime.fromtimestamp(zero_tx_ts) + if delta.total_seconds() > threshold * 60: + raise Exception("Have write min:0 servers {} minutes \ + (more threshold:{})\nLog {}:\nLine {}".format( + delta.total_seconds()/60.0, threshold, log, line)) + else: + zero_tx_ts = date_traffic + + if not tx: + raise Exception("Have write zero speed:\nLog {}:\nLine {}".format(log, line)) + + prev_traffic_ts = curr_traffic_ts + + + if current_match and prev_traffic_ts: + delta = curr_traffic_ts - prev_traffic_ts + if delta.total_seconds() > threshold * 60: + raise Exception("Have delta {} more {} minutes\nLog {}:\nLine {}".format( + delta.total_seconds()/60.0, threshold, log, line)) + + if not current_match: + current_match = re_error.match(line, re.IGNORECASE) + if current_match: + if not in_allow_list(line, is_allow_list): + raise Exception("contains error: {}\nLog {}:\nLine {}".format(line, log, line)) + else: + current_match = re_warning.match(line, re.IGNORECASE) + if current_match: + print("log {} [{}] contains warning: {}".format(log, i, line)) + + if curr_ts and (curr_ts - prev_traffic_ts).total_seconds() > threshold * 60: + raise Exception("No traffic\n{}\nLog {}".format(line, log)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-f', '--filename', type=str, + help='Log filename') + parser.add_argument('-d', '--directory', type=str, + help='Directory name with Logs') + parser.add_argument('-t', '--no_traffic_in_sec', type=int, default=1, + help='No traffic Threshold in min ') + parser.add_argument('-s', '--seek', type=str, default="", + help='path to seek file') + parser.add_argument('-r', '--role', type=str, default="client", choices=['client', 'server'], + help='choice role if you set filename') + parser.add_argument('--no-allow-list', dest='allow_list', action='store_false') + + args = parser.parse_args() + + clients = [] + servers = [] + if args.filename: + if args.role == "client": + clients.append(args.filename) + elif args.role == "server": + servers.append(args.filename) + + if args.directory: + clients, servers = get_logs(args.directory) + + try: + process_client(clients, args.no_traffic_in_sec, args.seek, args.allow_list) + process_server(servers, args.allow_list) + except Exception as e: + print("Error iodemo analyzer: {}\n".format(e)) + traceback.print_exc(file=sys.stdout) + exit(1) From 122512a8a86970688d2631bd0e44811f50bc94af Mon Sep 17 00:00:00 2001 From: Yossi Itigin Date: Sat, 12 Dec 2020 23:43:53 +0200 Subject: [PATCH 042/107] UCM: Fix heap corruption caused by ucp_set_event_handler() Fix race between sbrk() from testing mmap events while they are being installed, and brk/sbrk() from any other thread in the program as part of normal heap operation. Such race leads to heap corruption and program abort/segfault. In general, we should not call brk/sbrk() directly. Fix method: 1. Initialize bistro hooks during library initialization. This makes sure no other thread would read bad machine instructions. 2. Call brk/sbrk() from events test only when it's from exclusive context (1). For non-exclusive case, make only dummy calls with invalid parameters, which should not have side effects on other threads. 3. Create separate event for brk(), to avoid emulating sbrk() event when the actual call was to brk(). 4. Fix brk() syscall to return full 64-bit value. --- src/ucm/api/ucm.h | 26 ++++- src/ucm/event/event.c | 75 ++++++++------ src/ucm/event/event.h | 5 +- src/ucm/malloc/malloc_hook.c | 2 +- src/ucm/mmap/install.c | 193 +++++++++++++++++++++-------------- src/ucm/mmap/mmap.h | 5 +- src/ucm/util/replace.c | 32 +++--- src/ucm/util/sys.c | 68 +++++++----- src/ucm/util/sys.h | 13 +++ src/ucs/config/ucm_opts.c | 7 +- 10 files changed, 279 insertions(+), 147 deletions(-) diff --git a/src/ucm/api/ucm.h b/src/ucm/api/ucm.h index 6308c369e89..114f167c86e 100644 --- a/src/ucm/api/ucm.h +++ b/src/ucm/api/ucm.h @@ -37,6 +37,7 @@ typedef enum ucm_event_type { UCM_EVENT_SHMDT = UCS_BIT(4), UCM_EVENT_SBRK = UCS_BIT(5), UCM_EVENT_MADVISE = UCS_BIT(6), + UCM_EVENT_BRK = UCS_BIT(7), /* Aggregate events */ UCM_EVENT_VM_MAPPED = UCS_BIT(16), @@ -161,6 +162,15 @@ typedef union ucm_event { int advice; } madvise; + /* + * UCM_EVENT_BRK + * brk() is called. + */ + struct { + int result; + void *addr; + } brk; + /* * UCM_EVENT_VM_MAPPED, UCM_EVENT_VM_UNMAPPED * @@ -211,7 +221,10 @@ typedef struct ucm_global_config { } ucm_global_config_t; -/* Global UCM configuration */ +/* + * Global UCM configuration to be set externally. + * @deprecated replaced by @ref ucm_library_init. + */ extern ucm_global_config_t ucm_global_opts; @@ -253,6 +266,17 @@ typedef void (*ucm_event_callback_t)(ucm_event_type_t event_type, ucm_event_t *event, void *arg); +/** + * Initialize UCM library and set its configuration. + * + * @param [in] ucm_opts UCM library global configuration. If NULL, default + * configuration is applied. + * + * @note Calling this function more than once in the same process has no effect. + */ +void ucm_library_init(const ucm_global_config_t *ucm_opts); + + /** * @brief Install a handler for memory events. * diff --git a/src/ucm/event/event.c b/src/ucm/event/event.c index 1985b9b254f..d525b7a14f2 100644 --- a/src/ucm/event/event.c +++ b/src/ucm/event/event.c @@ -94,6 +94,11 @@ static void ucm_event_call_orig(ucm_event_type_t event_type, ucm_event_t *event, event->shmdt.result = ucm_orig_shmdt(event->shmdt.shmaddr); } break; + case UCM_EVENT_BRK: + if (event->brk.result == -1) { + event->brk.result = ucm_orig_brk(event->brk.addr); + } + break; case UCM_EVENT_SBRK: if (event->sbrk.result == MAP_FAILED) { event->sbrk.result = ucm_orig_sbrk(event->sbrk.increment); @@ -120,8 +125,8 @@ static ucm_event_handler_t ucm_event_orig_handler = { .list = UCS_LIST_INITIALIZER(&ucm_event_handlers, &ucm_event_handlers), .events = UCM_EVENT_MMAP | UCM_EVENT_MUNMAP | UCM_EVENT_MREMAP | UCM_EVENT_SHMAT | UCM_EVENT_SHMDT | UCM_EVENT_SBRK | - UCM_EVENT_MADVISE, /* All events */ - .priority = 0, /* Between negative and positive handlers */ + UCM_EVENT_MADVISE | UCM_EVENT_BRK, /* All events */ + .priority = 0, /* Between negative and positive handlers */ .cb = ucm_event_call_orig }; static ucs_list_link_t ucm_event_handlers = @@ -367,7 +372,8 @@ void *ucm_sbrk(intptr_t increment) ucm_trace("ucm_sbrk(increment=%+ld)", increment); if (increment < 0) { - ucm_dispatch_vm_munmap(UCS_PTR_BYTE_OFFSET(ucm_orig_sbrk(0), increment), + ucm_dispatch_vm_munmap(UCS_PTR_BYTE_OFFSET(ucm_get_current_brk(), + increment), -increment); } @@ -376,7 +382,8 @@ void *ucm_sbrk(intptr_t increment) ucm_event_dispatch(UCM_EVENT_SBRK, &event); if ((increment > 0) && (event.sbrk.result != MAP_FAILED)) { - ucm_dispatch_vm_mmap(UCS_PTR_BYTE_OFFSET(ucm_orig_sbrk(0), -increment), + ucm_dispatch_vm_mmap(UCS_PTR_BYTE_OFFSET(ucm_get_current_brk(), + -increment), increment); } @@ -387,38 +394,36 @@ void *ucm_sbrk(intptr_t increment) int ucm_brk(void *addr) { -#if UCM_BISTRO_HOOKS - void *old_addr; - intptr_t increment; + ptrdiff_t increment; + void *current_brk; ucm_event_t event; - old_addr = ucm_brk_syscall(0); - /* in case if addr == NULL - it just returns current pointer */ - increment = addr ? ((intptr_t)addr - (intptr_t)old_addr) : 0; - ucm_event_enter(); ucm_trace("ucm_brk(addr=%p)", addr); + if (addr == NULL) { + increment = 0; + } else { + current_brk = ucm_get_current_brk(); + increment = UCS_PTR_BYTE_DIFF(current_brk, addr); + } + if (increment < 0) { - ucm_dispatch_vm_munmap(UCS_PTR_BYTE_OFFSET(old_addr, increment), - -increment); + ucm_dispatch_vm_munmap(addr, -increment); } - event.sbrk.result = (void*)-1; - event.sbrk.increment = increment; - ucm_event_dispatch(UCM_EVENT_SBRK, &event); + event.brk.result = -1; + event.brk.addr = addr; + ucm_event_dispatch(UCM_EVENT_BRK, &event); - if ((increment > 0) && (event.sbrk.result != MAP_FAILED)) { - ucm_dispatch_vm_mmap(old_addr, increment); + if ((increment > 0) && (event.brk.result != -1)) { + ucm_dispatch_vm_mmap(current_brk, increment); } ucm_event_leave(); - return event.sbrk.result == MAP_FAILED ? -1 : 0; -#else - return -1; -#endif + return event.brk.result; } int ucm_madvise(void *addr, size_t length, int advice) @@ -455,6 +460,18 @@ int ucm_madvise(void *addr, size_t length, int advice) return event.madvise.result; } +void ucm_library_init(const ucm_global_config_t *ucm_opts) +{ + static ucs_init_once_t init_once = UCS_INIT_ONCE_INITIALIZER; + + UCS_INIT_ONCE(&init_once) { + if (ucm_opts != NULL) { + ucm_global_opts = *ucm_opts; + } + ucm_mmap_init(); + } +} + void ucm_event_handler_add(ucm_event_handler_t *handler) { ucm_event_handler_t *elem; @@ -481,18 +498,15 @@ void ucm_event_handler_remove(ucm_event_handler_t *handler) static ucs_status_t ucm_event_install(int events) { - static ucs_init_once_t init_once = UCS_INIT_ONCE_INITIALIZER; UCS_MODULE_FRAMEWORK_DECLARE(ucm); ucm_event_installer_t *event_installer; int malloc_events; ucs_status_t status; - UCS_INIT_ONCE(&init_once) { - ucm_prevent_dl_unload(); - } + ucm_prevent_dl_unload(); /* TODO lock */ - status = ucm_mmap_install(events); + status = ucm_mmap_install(events, 0); if (status != UCS_OK) { ucm_debug("failed to install mmap events"); goto out_unlock; @@ -523,7 +537,6 @@ static ucs_status_t ucm_event_install(int events) out_unlock: return status; - } ucs_status_t ucm_set_event_handler(int events, int priority, @@ -536,7 +549,7 @@ ucs_status_t ucm_set_event_handler(int events, int priority, if (events & ~(UCM_EVENT_MMAP|UCM_EVENT_MUNMAP|UCM_EVENT_MREMAP| UCM_EVENT_SHMAT|UCM_EVENT_SHMDT| - UCM_EVENT_SBRK| + UCM_EVENT_BRK|UCM_EVENT_SBRK| UCM_EVENT_MADVISE| UCM_EVENT_VM_MAPPED|UCM_EVENT_VM_UNMAPPED| UCM_EVENT_MEM_TYPE_ALLOC|UCM_EVENT_MEM_TYPE_FREE| @@ -549,6 +562,8 @@ ucs_status_t ucm_set_event_handler(int events, int priority, return UCS_ERR_UNSUPPORTED; } + ucm_library_init(NULL); + /* separate event flags from real events */ flags = events & (UCM_EVENT_FLAG_NO_INSTALL | UCM_EVENT_FLAG_EXISTING_ALLOC); @@ -623,11 +638,13 @@ void ucm_unset_event_handler(int events, ucm_event_callback_t cb, void *arg) ucs_status_t ucm_test_events(int events) { + ucm_library_init(NULL); return ucm_mmap_test_installed_events(events); } ucs_status_t ucm_test_external_events(int events) { + ucm_library_init(NULL); return ucm_mmap_test_events(events & ucm_external_events, "external"); } diff --git a/src/ucm/event/event.h b/src/ucm/event/event.h index 763ac3b2098..e7ae14ec6ad 100644 --- a/src/ucm/event/event.h +++ b/src/ucm/event/event.h @@ -13,12 +13,13 @@ #include #define UCM_NATIVE_EVENT_VM_MAPPED (UCM_EVENT_MMAP | UCM_EVENT_MREMAP | \ - UCM_EVENT_SHMAT | UCM_EVENT_SBRK) + UCM_EVENT_SHMAT | UCM_EVENT_SBRK | \ + UCM_EVENT_BRK) #define UCM_NATIVE_EVENT_VM_UNMAPPED (UCM_EVENT_MMAP | UCM_EVENT_MUNMAP | \ UCM_EVENT_MREMAP | UCM_EVENT_SHMDT | \ UCM_EVENT_SHMAT | UCM_EVENT_SBRK | \ - UCM_EVENT_MADVISE) + UCM_EVENT_MADVISE | UCM_EVENT_BRK) typedef struct ucm_event_handler { diff --git a/src/ucm/malloc/malloc_hook.c b/src/ucm/malloc/malloc_hook.c index 50f7b974a64..7446791174e 100644 --- a/src/ucm/malloc/malloc_hook.c +++ b/src/ucm/malloc/malloc_hook.c @@ -550,7 +550,7 @@ static void ucm_malloc_sbrk(ucm_event_type_t event_type, if (ucm_malloc_hook_state.heap_start == (void*)-1) { ucm_malloc_hook_state.heap_start = event->sbrk.result; /* sbrk() returns the previous break */ } - ucm_malloc_hook_state.heap_end = ucm_orig_sbrk(0); + ucm_malloc_hook_state.heap_end = ucm_get_current_brk(); ucm_trace("sbrk(%+ld)=%p - adjusting heap to [%p..%p]", event->sbrk.increment, event->sbrk.result, diff --git a/src/ucm/mmap/install.c b/src/ucm/mmap/install.c index 83d4b7d37e5..368552c2a07 100644 --- a/src/ucm/mmap/install.c +++ b/src/ucm/mmap/install.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -30,8 +31,6 @@ #include #include -#define UCM_IS_HOOK_ENABLED(_entry) \ - ((_entry)->hook_type & UCS_BIT(ucm_mmap_hook_mode())) #define UCM_HOOK_STR \ ((ucm_mmap_hook_mode() == UCM_MMAP_HOOK_RELOC) ? "reloc" : "bistro") @@ -61,17 +60,10 @@ extern const char *ucm_mmap_hook_modes[]; -typedef enum ucm_mmap_hook_type { - UCM_HOOK_RELOC = UCS_BIT(UCM_MMAP_HOOK_RELOC), - UCM_HOOK_BISTRO = UCS_BIT(UCM_MMAP_HOOK_BISTRO), - UCM_HOOK_BOTH = UCM_HOOK_RELOC | UCM_HOOK_BISTRO -} ucm_mmap_hook_type_t; - typedef struct ucm_mmap_func { ucm_reloc_patch_t patch; ucm_event_type_t event_type; ucm_event_type_t deps; - ucm_mmap_hook_type_t hook_type; } ucm_mmap_func_t; typedef struct ucm_mmap_test_events_data { @@ -81,18 +73,16 @@ typedef struct ucm_mmap_test_events_data { } ucm_mmap_test_events_data_t; static ucm_mmap_func_t ucm_mmap_funcs[] = { - { {"mmap", ucm_override_mmap}, UCM_EVENT_MMAP, UCM_EVENT_NONE, UCM_HOOK_BOTH}, - { {"munmap", ucm_override_munmap}, UCM_EVENT_MUNMAP, UCM_EVENT_NONE, UCM_HOOK_BOTH}, + { {"mmap", ucm_override_mmap}, UCM_EVENT_MMAP, UCM_EVENT_NONE}, + { {"munmap", ucm_override_munmap}, UCM_EVENT_MUNMAP, UCM_EVENT_NONE}, #if HAVE_MREMAP - { {"mremap", ucm_override_mremap}, UCM_EVENT_MREMAP, UCM_EVENT_NONE, UCM_HOOK_BOTH}, -#endif - { {"shmat", ucm_override_shmat}, UCM_EVENT_SHMAT, UCM_EVENT_NONE, UCM_HOOK_BOTH}, - { {"shmdt", ucm_override_shmdt}, UCM_EVENT_SHMDT, UCM_EVENT_SHMAT, UCM_HOOK_BOTH}, - { {"sbrk", ucm_override_sbrk}, UCM_EVENT_SBRK, UCM_EVENT_NONE, UCM_HOOK_RELOC}, -#if UCM_BISTRO_HOOKS - { {"brk", ucm_override_brk}, UCM_EVENT_SBRK, UCM_EVENT_NONE, UCM_HOOK_BISTRO}, + { {"mremap", ucm_override_mremap}, UCM_EVENT_MREMAP, UCM_EVENT_NONE}, #endif - { {"madvise", ucm_override_madvise}, UCM_EVENT_MADVISE, UCM_EVENT_NONE, UCM_HOOK_BOTH}, + { {"shmat", ucm_override_shmat}, UCM_EVENT_SHMAT, UCM_EVENT_NONE}, + { {"shmdt", ucm_override_shmdt}, UCM_EVENT_SHMDT, UCM_EVENT_SHMAT}, + { {"sbrk", ucm_override_sbrk}, UCM_EVENT_SBRK, UCM_EVENT_NONE}, + { {"brk", ucm_override_brk}, UCM_EVENT_BRK, UCM_EVENT_NONE}, + { {"madvise", ucm_override_madvise}, UCM_EVENT_MADVISE, UCM_EVENT_NONE}, { {NULL, NULL}, UCM_EVENT_NONE} }; @@ -108,6 +98,7 @@ static const char *ucm_mmap_event_name[] = { UCM_MMAP_EVENT_NAME_ENTRY(SHMDT), UCM_MMAP_EVENT_NAME_ENTRY(SBRK), UCM_MMAP_EVENT_NAME_ENTRY(MADVISE), + UCM_MMAP_EVENT_NAME_ENTRY(BRK), /* Aggregate events */ UCM_MMAP_EVENT_NAME_ENTRY(VM_MAPPED), @@ -129,6 +120,15 @@ static void ucm_mmap_event_test_callback(ucm_event_type_t event_type, } } +/* Call brk() and check return value, to avoid compile error of unused result */ +static void ucm_brk_checked(void *addr) +{ + int ret = brk(addr); + if ((ret != 0) && (addr != NULL)) { + ucm_diag("brk(addr=%p) failed: %m", addr); + } +} + /* Fire events with pre/post action. The problem is in call sequence: we * can't just fire single event - most of the system calls require set of * calls to eliminate resource leaks or data corruption, such sequence @@ -136,10 +136,10 @@ static void ucm_mmap_event_test_callback(ucm_event_type_t event_type, * exclude additional events from processing used pre/post actions where * set of handled events is cleared and evaluated for every system call */ static void -ucm_fire_mmap_events_internal(int events, ucm_mmap_test_events_data_t *data) +ucm_fire_mmap_events_internal(int events, ucm_mmap_test_events_data_t *data, + int exclusive) { size_t sbrk_size; - int sbrk_mask; int shmid; void *p; @@ -183,19 +183,29 @@ ucm_fire_mmap_events_internal(int events, ucm_mmap_test_events_data_t *data) data, shmdt(p)); } - if (events & (UCM_EVENT_SBRK|UCM_EVENT_VM_MAPPED|UCM_EVENT_VM_UNMAPPED)) { - if (RUNNING_ON_VALGRIND) { - /* on valgrind, doing a non-trivial sbrk() causes heap corruption */ - sbrk_size = 0; - sbrk_mask = UCM_EVENT_SBRK; - } else { - sbrk_size = ucm_get_page_size(); - sbrk_mask = UCM_EVENT_SBRK|UCM_EVENT_VM_MAPPED|UCM_EVENT_VM_UNMAPPED; + if (exclusive && !RUNNING_ON_VALGRIND) { + sbrk_size = ucm_get_page_size(); + if (events & (UCM_EVENT_BRK|UCM_EVENT_VM_MAPPED|UCM_EVENT_VM_UNMAPPED)) { + p = ucm_get_current_brk(); + UCM_FIRE_EVENT(events, UCM_EVENT_BRK|UCM_EVENT_VM_MAPPED, data, + ucm_brk_checked(UCS_PTR_BYTE_OFFSET(p, sbrk_size))); + UCM_FIRE_EVENT(events, UCM_EVENT_BRK|UCM_EVENT_VM_UNMAPPED, data, + ucm_brk_checked(p)); + } + if (events & (UCM_EVENT_SBRK|UCM_EVENT_VM_MAPPED|UCM_EVENT_VM_UNMAPPED)) { + UCM_FIRE_EVENT(events, UCM_EVENT_SBRK|UCM_EVENT_VM_MAPPED, + data, (void)sbrk(sbrk_size)); + UCM_FIRE_EVENT(events, UCM_EVENT_SBRK|UCM_EVENT_VM_UNMAPPED, + data, (void)sbrk(-sbrk_size)); + } + } else { + /* To avoid side effects on other threads and valgrind heap corruption, + * pass invalid parameters. We assume that if the natives events are + * delivered, it means VM_MAPPED/UNMAPPED would be delivered as well. + */ + if (events & UCM_EVENT_BRK) { + UCM_FIRE_EVENT(events, UCM_EVENT_BRK, data, ucm_brk_checked(NULL)); } - UCM_FIRE_EVENT(events, (UCM_EVENT_SBRK|UCM_EVENT_VM_MAPPED) & sbrk_mask, - data, (void)sbrk(sbrk_size)); - UCM_FIRE_EVENT(events, (UCM_EVENT_SBRK|UCM_EVENT_VM_UNMAPPED) & sbrk_mask, - data, (void)sbrk(-sbrk_size)); } if (events & (UCM_EVENT_MADVISE|UCM_EVENT_VM_UNMAPPED)) { @@ -217,7 +227,7 @@ void ucm_fire_mmap_events(int events) { ucm_mmap_test_events_data_t data; - ucm_fire_mmap_events_internal(events, &data); + ucm_fire_mmap_events_internal(events, &data, 0); } static void ucm_mmap_event_report_missing(int expected, int actual, @@ -252,7 +262,7 @@ static void ucm_mmap_event_report_missing(int expected, int actual, /* Called with lock held */ static ucs_status_t -ucm_mmap_test_events_nolock(int events, const char *event_type) +ucm_mmap_test_events_nolock(int events, int exclusive, const char *event_type) { ucm_event_handler_t handler; ucm_mmap_test_events_data_t data; @@ -265,7 +275,7 @@ ucm_mmap_test_events_nolock(int events, const char *event_type) data.tid = ucs_get_tid(); ucm_event_handler_add(&handler); - ucm_fire_mmap_events_internal(events, &data); + ucm_fire_mmap_events_internal(events, &data, exclusive); ucm_event_handler_remove(&handler); ucm_debug("mmap test: got 0x%x out of 0x%x", data.out_events, events); @@ -279,6 +289,23 @@ ucm_mmap_test_events_nolock(int events, const char *event_type) return UCS_OK; } +static int ucm_mmap_events_to_native_events(int events) +{ + int native_events; + + native_events = events & ~(UCM_EVENT_MEM_TYPE_ALLOC | + UCM_EVENT_MEM_TYPE_FREE); + + if (events & UCM_EVENT_VM_MAPPED) { + native_events |= UCM_NATIVE_EVENT_VM_MAPPED; + } + if (events & UCM_EVENT_VM_UNMAPPED) { + native_events |= UCM_NATIVE_EVENT_VM_UNMAPPED; + } + + return native_events; +} + ucs_status_t ucm_mmap_test_events(int events, const char *event_type) { ucs_status_t status; @@ -287,7 +314,7 @@ ucs_status_t ucm_mmap_test_events(int events, const char *event_type) * return UCS_OK iff all events are actually working */ pthread_mutex_lock(&ucm_mmap_install_mutex); - status = ucm_mmap_test_events_nolock(events, event_type); + status = ucm_mmap_test_events_nolock(events, 0, event_type); pthread_mutex_unlock(&ucm_mmap_install_mutex); return status; @@ -295,11 +322,14 @@ ucs_status_t ucm_mmap_test_events(int events, const char *event_type) ucs_status_t ucm_mmap_test_installed_events(int events) { + int native_events = ucm_mmap_events_to_native_events(events) & + ucm_mmap_installed_events; + /* * return UCS_OK iff all installed events are actually working * we don't check the status of events which were not successfully installed */ - return ucm_mmap_test_events(events & ucm_mmap_installed_events, "internal"); + return ucm_mmap_test_events(native_events, "internal"); } /* Called with lock held */ @@ -325,47 +355,30 @@ static ucs_status_t ucs_mmap_install_reloc(int events) continue; } - if (UCM_IS_HOOK_ENABLED(entry)) { - ucm_debug("mmap: installing %s hook for %s = %p for event 0x%x", UCM_HOOK_STR, - entry->patch.symbol, entry->patch.value, entry->event_type); - - if (ucm_mmap_hook_mode() == UCM_MMAP_HOOK_RELOC) { - status = ucm_reloc_modify(&entry->patch); - } else { - ucs_assert(ucm_mmap_hook_mode() == UCM_MMAP_HOOK_BISTRO); - status = ucm_bistro_patch(entry->patch.symbol, entry->patch.value, NULL); - } - if (status != UCS_OK) { - ucm_warn("failed to install %s hook for '%s'", - UCM_HOOK_STR, entry->patch.symbol); - return status; - } - - installed_events |= entry->event_type; - } - } - - return UCS_OK; -} - -static int ucm_mmap_events_to_native_events(int events) -{ - int native_events; + ucm_debug("mmap: installing %s hook for %s = %p for event 0x%x", + UCM_HOOK_STR, entry->patch.symbol, entry->patch.value, + entry->event_type); - native_events = events & ~(UCM_EVENT_MEM_TYPE_ALLOC | - UCM_EVENT_MEM_TYPE_FREE); + if (ucm_mmap_hook_mode() == UCM_MMAP_HOOK_RELOC) { + status = ucm_reloc_modify(&entry->patch); + } else { + ucs_assert(ucm_mmap_hook_mode() == UCM_MMAP_HOOK_BISTRO); + status = ucm_bistro_patch(entry->patch.symbol, entry->patch.value, + NULL); + } + if (status != UCS_OK) { + ucm_warn("failed to install %s hook for '%s'", UCM_HOOK_STR, + entry->patch.symbol); + return status; + } - if (events & UCM_EVENT_VM_MAPPED) { - native_events |= UCM_NATIVE_EVENT_VM_MAPPED; - } - if (events & UCM_EVENT_VM_UNMAPPED) { - native_events |= UCM_NATIVE_EVENT_VM_UNMAPPED; + installed_events |= entry->event_type; } - return native_events; + return UCS_OK; } -ucs_status_t ucm_mmap_install(int events) +ucs_status_t ucm_mmap_install(int events, int exclusive) { ucs_status_t status; int native_events; @@ -378,7 +391,8 @@ ucs_status_t ucm_mmap_install(int events) /* if we already installed these events, check that they are still * working, and if not - reinstall them. */ - status = ucm_mmap_test_events_nolock(native_events, 0); + status = ucm_mmap_test_events_nolock(native_events, exclusive, + "previous mmap"); if (status == UCS_OK) { goto out_unlock; } @@ -390,7 +404,7 @@ ucs_status_t ucm_mmap_install(int events) goto out_unlock; } - status = ucm_mmap_test_events_nolock(native_events, 0); + status = ucm_mmap_test_events_nolock(native_events, exclusive, "mmap"); if (status != UCS_OK) { ucm_debug("failed to install mmap events"); goto out_unlock; @@ -404,3 +418,32 @@ ucs_status_t ucm_mmap_install(int events) pthread_mutex_unlock(&ucm_mmap_install_mutex); return status; } + +void ucm_mmap_init() +{ + ucm_event_type_t events; + ucm_mmap_func_t *entry; + + if (!ucm_global_opts.enable_events || + (ucm_mmap_hook_mode() != UCM_MMAP_HOOK_BISTRO)) { + return; + } + + /* We must initialize bistro hooks during startup and not later, before + * other threads could execute the modified functions and fail on invalid + * instructions + */ + events = 0; + for (entry = ucm_mmap_funcs; entry->patch.symbol != NULL; ++entry) { + events |= entry->event_type; + } + if (events & UCM_NATIVE_EVENT_VM_MAPPED) { + events |= UCM_EVENT_VM_MAPPED; + } + if (events & UCM_NATIVE_EVENT_VM_UNMAPPED) { + events |= UCM_EVENT_VM_UNMAPPED; + } + + ucm_prevent_dl_unload(); + ucm_mmap_install(events, 1); +} diff --git a/src/ucm/mmap/mmap.h b/src/ucm/mmap/mmap.h index ed90a801238..1d9b04f4cd2 100644 --- a/src/ucm/mmap/mmap.h +++ b/src/ucm/mmap/mmap.h @@ -21,7 +21,7 @@ # define UCM_DEFAULT_HOOK_MODE_STR UCM_MMAP_HOOK_RELOC_STR #endif -ucs_status_t ucm_mmap_install(int events); +ucs_status_t ucm_mmap_install(int events, int exclusive); void *ucm_override_mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset); int ucm_override_munmap(void *addr, size_t length); @@ -31,11 +31,12 @@ int ucm_override_shmdt(const void *shmaddr); void *ucm_override_sbrk(intptr_t increment); void *ucm_sbrk_select(intptr_t increment); int ucm_override_brk(void *addr); -void *ucm_brk_syscall(void *addr); int ucm_override_madvise(void *addr, size_t length, int advice); +void *ucm_get_current_brk(); void ucm_fire_mmap_events(int events); ucs_status_t ucm_mmap_test_installed_events(int events); ucs_status_t ucm_mmap_test_events(int events, const char *event_type); +void ucm_mmap_init(); static UCS_F_ALWAYS_INLINE ucm_mmap_hook_mode_t ucm_mmap_hook_mode(void) { diff --git a/src/ucm/util/replace.c b/src/ucm/util/replace.c index 6d8abae9405..58a06325778 100644 --- a/src/ucm/util/replace.c +++ b/src/ucm/util/replace.c @@ -17,14 +17,20 @@ #include #include #include +#include #include #include #include + #ifndef MAP_FAILED #define MAP_FAILED ((void*)-1) #endif +#if HAVE___CURBRK +extern void *__curbrk; +#endif + #ifdef PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP pthread_mutex_t ucm_reloc_get_orig_lock = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP; #else @@ -113,17 +119,8 @@ int ucm_orig_shmdt(const void *shmaddr) #endif -#if HAVE___CURBRK -extern void *__curbrk; -#endif - _UCM_DEFINE_DLSYM_FUNC(brk, ucm_orig_dlsym_brk, ucm_override_brk, int, -1, void*) -void *ucm_brk_syscall(void *addr) -{ - return (void*)syscall(SYS_brk, addr); -} - int ucm_orig_brk(void *addr) { void *new_addr; @@ -133,7 +130,7 @@ int ucm_orig_brk(void *addr) #endif new_addr = ucm_brk_syscall(addr); - if (new_addr < addr) { + if (new_addr != addr) { errno = ENOMEM; return -1; } else { @@ -151,15 +148,26 @@ void *ucm_orig_sbrk(intptr_t increment) if (ucm_mmap_hook_mode() == UCM_MMAP_HOOK_RELOC) { return ucm_orig_dlsym_sbrk(increment); } else { - prev = ucm_brk_syscall(0); - return ucm_orig_brk(UCS_PTR_BYTE_OFFSET(prev, increment)) ? (void*)-1 : prev; + prev = ucm_get_current_brk(); + return ucm_orig_brk(UCS_PTR_BYTE_OFFSET(prev, increment)) ? + (void*)-1 : prev; } } #else /* UCM_BISTRO_HOOKS */ +UCM_DEFINE_DLSYM_FUNC(brk, int, -1, void*) UCM_DEFINE_DLSYM_FUNC(sbrk, void*, MAP_FAILED, intptr_t) UCM_DEFINE_DLSYM_FUNC(shmat, void*, MAP_FAILED, int, const void*, int) UCM_DEFINE_DLSYM_FUNC(shmdt, int, -1, const void*) #endif /* UCM_BISTRO_HOOKS */ + +void *ucm_get_current_brk() +{ +#if HAVE___CURBRK + return __curbrk; +#else + return ucm_brk_syscall(0); +#endif +} diff --git a/src/ucm/util/sys.c b/src/ucm/util/sys.c index 92bf834da5c..d588f8d157e 100644 --- a/src/ucm/util/sys.c +++ b/src/ucm/util/sys.c @@ -17,10 +17,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -282,7 +284,8 @@ void ucm_strerror(int eno, char *buf, size_t max) void ucm_prevent_dl_unload() { - int flags = RTLD_LOCAL | RTLD_NODELETE; + static ucs_init_once_t init_once = UCS_INIT_ONCE_INITIALIZER; + int flags = RTLD_LOCAL | RTLD_NODELETE; Dl_info info; void *dl; int ret; @@ -292,32 +295,34 @@ void ucm_prevent_dl_unload() return; } - flags |= (ucm_global_opts.module_unload_prevent_mode == - UCM_UNLOAD_PREVENT_MODE_NOW) ? RTLD_NOW : RTLD_LAZY; + UCS_INIT_ONCE(&init_once) { + flags |= (ucm_global_opts.module_unload_prevent_mode == + UCM_UNLOAD_PREVENT_MODE_NOW) ? RTLD_NOW : RTLD_LAZY; - /* Get the path to current library by current function pointer */ - (void)dlerror(); - ret = dladdr(ucm_prevent_dl_unload, &info); - if (ret == 0) { - ucm_warn("could not find address of current library: %s", dlerror()); - return; - } + /* Get the path to current library by current function pointer */ + (void)dlerror(); + ret = dladdr(ucm_prevent_dl_unload, &info); + if (ret == 0) { + ucm_warn("could not find address of current library: %s", dlerror()); + return; + } - /* Load the current library with NODELETE flag, to prevent it from being - * unloaded. This will create extra reference to the library, but also add - * NODELETE flag to the dynamic link map. - */ - (void)dlerror(); - dl = dlopen(info.dli_fname, flags); - if (dl == NULL) { - ucm_warn("failed to load '%s': %s", info.dli_fname, dlerror()); - return; - } + /* Load the current library with NODELETE flag, to prevent it from being + * unloaded. This will create extra reference to the library, but also add + * NODELETE flag to the dynamic link map. + */ + (void)dlerror(); + dl = dlopen(info.dli_fname, flags); + if (dl == NULL) { + ucm_warn("failed to load '%s': %s", info.dli_fname, dlerror()); + return; + } - ucm_debug("reloaded '%s' at %p with NODELETE flag", info.dli_fname, dl); + ucm_debug("loaded '%s' at %p with NODELETE flag", info.dli_fname, dl); - /* Now we drop our reference to the lib, and it won't be unloaded anymore */ - dlclose(dl); + /* coverity[overwrite_var] */ + dl = NULL; + } } char *ucm_concat_path(char *buffer, size_t max, const char *dir, const char *file) @@ -349,3 +354,20 @@ char *ucm_concat_path(char *buffer, size_t max, const char *dir, const char *fil return buffer; } + +void *ucm_brk_syscall(void *addr) +{ + void *result; + +#ifdef __x86_64__ + asm volatile("mov %1, %%rdi\n\t" + "mov $0xc, %%eax\n\t" + "syscall\n\t" + : "=a"(result) + : "m"(addr)); +#else + /* TODO implement 64-bit syscall for aarch64, ppc64le */ + result = (void*)syscall(SYS_brk, addr); +#endif + return result; +} diff --git a/src/ucm/util/sys.h b/src/ucm/util/sys.h index 37a1d927ef3..1384ef6664f 100644 --- a/src/ucm/util/sys.h +++ b/src/ucm/util/sys.h @@ -88,4 +88,17 @@ void ucm_prevent_dl_unload(); char *ucm_concat_path(char *buffer, size_t max, const char *dir, const char *file); +/** + * Perform brk() syscall + * + * @param addr Address to set as new program break. + * + * @return New program break. + * + * @note If the break could not be changed (for example, parameter was invalid + * or exceeds limits) the break remains unchanged. + */ +void *ucm_brk_syscall(void *addr); + + #endif diff --git a/src/ucs/config/ucm_opts.c b/src/ucs/config/ucm_opts.c index d96ef3846df..18abe3228e4 100644 --- a/src/ucs/config/ucm_opts.c +++ b/src/ucs/config/ucm_opts.c @@ -101,6 +101,9 @@ UCS_CONFIG_REGISTER_TABLE(ucm_global_config_table, "UCM", UCM_CONFIG_PREFIX, ucm_global_config_t, &ucs_config_global_list) UCS_STATIC_INIT { - (void)ucs_config_parser_fill_opts(&ucm_global_opts, ucm_global_config_table, - UCS_DEFAULT_ENV_PREFIX, UCM_CONFIG_PREFIX, 0); + ucm_global_config_t ucm_opts; + (void)ucs_config_parser_fill_opts(&ucm_opts, ucm_global_config_table, + UCS_DEFAULT_ENV_PREFIX, UCM_CONFIG_PREFIX, + 0); + ucm_library_init(&ucm_opts); } From 60e2cdf89ee0d5d7886e3fa00d844f701e26e258 Mon Sep 17 00:00:00 2001 From: Yossi Itigin Date: Sun, 13 Dec 2020 17:18:14 +0200 Subject: [PATCH 043/107] UCM: Remove UCS dependencies - Define own ucm_assert() macro. - Define own ucm_get_tid() function. - Remove ucs_init_once_mutex_unlock function, use pthread_mutex_unlock() directly. --- src/ucm/bistro/bistro.c | 4 ++-- src/ucm/bistro/bistro_int.h | 2 +- src/ucm/bistro/bistro_ppc64.c | 8 ++++---- src/ucm/malloc/malloc_hook.c | 4 ++-- src/ucm/mmap/install.c | 6 +++--- src/ucm/util/log.c | 5 ----- src/ucm/util/log.h | 18 ++++++++++++++++++ src/ucm/util/sys.c | 5 +++++ src/ucm/util/sys.h | 6 ++++++ src/ucs/Makefile.am | 1 - src/ucs/type/init_once.c | 20 -------------------- src/ucs/type/init_once.h | 2 +- 12 files changed, 42 insertions(+), 39 deletions(-) delete mode 100644 src/ucs/type/init_once.c diff --git a/src/ucm/bistro/bistro.c b/src/ucm/bistro/bistro.c index 51a807e91d5..1eec37ee9ad 100644 --- a/src/ucm/bistro/bistro.c +++ b/src/ucm/bistro/bistro.c @@ -16,7 +16,7 @@ ucs_status_t ucm_bistro_remove_restore_point(ucm_bistro_restore_point_t *rp) { - ucs_assert(rp != NULL); + ucm_assert(rp != NULL); free(rp); return UCS_OK; } @@ -103,7 +103,7 @@ ucs_status_t ucm_bistro_restore(ucm_bistro_restore_point_t *rp) void *ucm_bistro_restore_addr(ucm_bistro_restore_point_t *rp) { - ucs_assert(rp != NULL); + ucm_assert(rp != NULL); return rp->addr; } diff --git a/src/ucm/bistro/bistro_int.h b/src/ucm/bistro/bistro_int.h index e6c08a4994a..021dbc10760 100644 --- a/src/ucm/bistro/bistro_int.h +++ b/src/ucm/bistro/bistro_int.h @@ -37,7 +37,7 @@ static inline void *ucm_bistro_lookup(const char *symbol) { void *addr; - ucs_assert(symbol != NULL); + ucm_assert(symbol != NULL); addr = dlsym(RTLD_NEXT, symbol); if (!addr) { diff --git a/src/ucm/bistro/bistro_ppc64.c b/src/ucm/bistro/bistro_ppc64.c index 4b14250cd97..8a08655b365 100644 --- a/src/ucm/bistro/bistro_ppc64.c +++ b/src/ucm/bistro/bistro_ppc64.c @@ -76,7 +76,7 @@ struct ucm_bistro_restore_point { static void ucm_bistro_fill_base_patch(ucm_bistro_base_patch_t *patch, uint32_t reg, uintptr_t value) { - ucs_assert(patch != NULL); + ucm_assert(patch != NULL); patch->addis = ADDIS ( reg, 0, (value >> 48)); patch->ori1 = ORI ( reg, reg, (value >> 32)); @@ -88,7 +88,7 @@ static void ucm_bistro_fill_base_patch(ucm_bistro_base_patch_t *patch, static void ucm_bistro_fill_patch(ucm_bistro_patch_t *patch, uint32_t reg, uintptr_t value) { - ucs_assert(patch != NULL); + ucm_assert(patch != NULL); ucm_bistro_fill_base_patch(&patch->super, reg, value); @@ -185,7 +185,7 @@ ucs_status_t ucm_bistro_restore(ucm_bistro_restore_point_t *rp) { ucs_status_t status; - ucs_assert(rp != NULL); + ucm_assert(rp != NULL); status = ucm_bistro_apply_patch(rp->func, &rp->func_patch, sizeof(rp->func_patch)); if (UCS_STATUS_IS_ERR(status)) { @@ -202,7 +202,7 @@ ucs_status_t ucm_bistro_restore(ucm_bistro_restore_point_t *rp) void *ucm_bistro_restore_addr(ucm_bistro_restore_point_t *rp) { - ucs_assert(rp != NULL); + ucm_assert(rp != NULL); return rp->entry; } diff --git a/src/ucm/malloc/malloc_hook.c b/src/ucm/malloc/malloc_hook.c index 7446791174e..9bfbdb3bba7 100644 --- a/src/ucm/malloc/malloc_hook.c +++ b/src/ucm/malloc/malloc_hook.c @@ -141,8 +141,8 @@ static void ucm_malloc_mmaped_ptr_add(void *ptr) hash_it = kh_put(mmap_ptrs, &ucm_malloc_hook_state.ptrs, ptr, &hash_extra_status); - ucs_assert_always(hash_extra_status >= 0); - ucs_assert_always(hash_it != kh_end(&ucm_malloc_hook_state.ptrs)); + ucm_assert_always(hash_extra_status >= 0); + ucm_assert_always(hash_it != kh_end(&ucm_malloc_hook_state.ptrs)); ucs_recursive_spin_unlock(&ucm_malloc_hook_state.lock); } diff --git a/src/ucm/mmap/install.c b/src/ucm/mmap/install.c index 368552c2a07..c1b284eab62 100644 --- a/src/ucm/mmap/install.c +++ b/src/ucm/mmap/install.c @@ -115,7 +115,7 @@ static void ucm_mmap_event_test_callback(ucm_event_type_t event_type, * So ignore calls from other threads to ensure the only requested events * are proceeded. */ - if (data->tid == ucs_get_tid()) { + if (data->tid == ucm_get_tid()) { data->fired_events |= event_type; } } @@ -272,7 +272,7 @@ ucm_mmap_test_events_nolock(int events, int exclusive, const char *event_type) handler.cb = ucm_mmap_event_test_callback; handler.arg = &data; data.out_events = events; - data.tid = ucs_get_tid(); + data.tid = ucm_get_tid(); ucm_event_handler_add(&handler); ucm_fire_mmap_events_internal(events, &data, exclusive); @@ -362,7 +362,7 @@ static ucs_status_t ucs_mmap_install_reloc(int events) if (ucm_mmap_hook_mode() == UCM_MMAP_HOOK_RELOC) { status = ucm_reloc_modify(&entry->patch); } else { - ucs_assert(ucm_mmap_hook_mode() == UCM_MMAP_HOOK_BISTRO); + ucm_assert(ucm_mmap_hook_mode() == UCM_MMAP_HOOK_BISTRO); status = ucm_bistro_patch(entry->patch.symbol, entry->patch.value, NULL); } diff --git a/src/ucm/util/log.c b/src/ucm/util/log.c index 66fe58042d1..7325a341952 100644 --- a/src/ucm/util/log.c +++ b/src/ucm/util/log.c @@ -246,11 +246,6 @@ static void ucm_log_vsnprintf(char *buf, size_t max, const char *fmt, va_list ap *pb = '\0'; } -static pid_t ucm_get_tid(void) -{ - return syscall(SYS_gettid); -} - static void ucm_log_snprintf(char *buf, size_t max, const char *fmt, ...) { va_list ap; diff --git a/src/ucm/util/log.h b/src/ucm/util/log.h index 9dcfd317406..ac0e32ab418 100644 --- a/src/ucm/util/log.h +++ b/src/ucm/util/log.h @@ -22,6 +22,7 @@ ## __VA_ARGS__); \ } + #define ucm_fatal(_message, ...) ucm_log(UCS_LOG_LEVEL_FATAL, _message, ## __VA_ARGS__) #define ucm_error(_message, ...) ucm_log(UCS_LOG_LEVEL_ERROR, _message, ## __VA_ARGS__) #define ucm_warn(_message, ...) ucm_log(UCS_LOG_LEVEL_WARN, _message, ## __VA_ARGS__) @@ -30,8 +31,25 @@ #define ucm_debug(_message, ...) ucm_log(UCS_LOG_LEVEL_DEBUG, _message, ## __VA_ARGS__) #define ucm_trace(_message, ...) ucm_log(UCS_LOG_LEVEL_TRACE, _message, ## __VA_ARGS__) + +#define ucm_assert_always(_expression) \ + do { \ + if (!(_expression)) { \ + ucm_fatal("Assertion `%s' failed", #_expression); \ + } \ + } while (0) + + +#if ENABLE_ASSERT +# define ucm_assert(...) ucm_assert_always(__VA_ARGS__) +#else +# define ucm_assert(...) {} +#endif + + extern const char *ucm_log_level_names[]; + void __ucm_log(const char *file, unsigned line, const char *function, ucs_log_level_t level, const char *message, ...) UCS_F_PRINTF(5, 6); diff --git a/src/ucm/util/sys.c b/src/ucm/util/sys.c index d588f8d157e..2129ba203fb 100644 --- a/src/ucm/util/sys.c +++ b/src/ucm/util/sys.c @@ -371,3 +371,8 @@ void *ucm_brk_syscall(void *addr) #endif return result; } + +pid_t ucm_get_tid() +{ + return syscall(SYS_gettid); +} diff --git a/src/ucm/util/sys.h b/src/ucm/util/sys.h index 1384ef6664f..215de109ef7 100644 --- a/src/ucm/util/sys.h +++ b/src/ucm/util/sys.h @@ -8,6 +8,7 @@ #ifndef UCM_UTIL_SYS_H_ #define UCM_UTIL_SYS_H_ +#include #include @@ -101,4 +102,9 @@ char *ucm_concat_path(char *buffer, size_t max, const char *dir, const char *fil void *ucm_brk_syscall(void *addr); +/** + * @return System thread id of the current thread. + */ +pid_t ucm_get_tid(); + #endif diff --git a/src/ucs/Makefile.am b/src/ucs/Makefile.am index 688ffce0f5e..577453ec363 100644 --- a/src/ucs/Makefile.am +++ b/src/ucs/Makefile.am @@ -165,7 +165,6 @@ libucs_la_SOURCES = \ time/timerq.c \ type/class.c \ type/status.c \ - type/init_once.c \ type/spinlock.c if HAVE_AARCH64_THUNDERX2 diff --git a/src/ucs/type/init_once.c b/src/ucs/type/init_once.c deleted file mode 100644 index cfb05c9cb8c..00000000000 --- a/src/ucs/type/init_once.c +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Copyright (C) Mellanox Technologies Ltd. 2001-2019. ALL RIGHTS RESERVED. - * - * See file LICENSE for terms. - */ - -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif - -#include -#include - - -unsigned ucs_init_once_mutex_unlock(pthread_mutex_t *lock) -{ - int ret = pthread_mutex_unlock(lock); - ucs_assert_always(ret == 0); - return 0; -} diff --git a/src/ucs/type/init_once.h b/src/ucs/type/init_once.h index 4b7e967ccbc..fc75104f19f 100644 --- a/src/ucs/type/init_once.h +++ b/src/ucs/type/init_once.h @@ -51,7 +51,7 @@ unsigned ucs_init_once_mutex_unlock(pthread_mutex_t *lock); */ #define UCS_INIT_ONCE(_once) \ for (pthread_mutex_lock(&(_once)->lock); \ - !(_once)->initialized || ucs_init_once_mutex_unlock(&(_once)->lock); \ + !(_once)->initialized || pthread_mutex_unlock(&(_once)->lock); \ (_once)->initialized = 1) #endif From 67a25c9141d02ae59b155580f93c9bd9561c2511 Mon Sep 17 00:00:00 2001 From: Yossi Itigin Date: Sun, 13 Dec 2020 19:23:31 +0200 Subject: [PATCH 044/107] TEST/MEMHOOKS: Set UCM configuration after loading --- contrib/test_jenkins.sh | 21 +++++---- test/mpi/test_memhooks.c | 94 ++++++++++++++++++++++++++++++---------- 2 files changed, 85 insertions(+), 30 deletions(-) diff --git a/contrib/test_jenkins.sh b/contrib/test_jenkins.sh index 5f82d47b9e7..e907e628ce3 100755 --- a/contrib/test_jenkins.sh +++ b/contrib/test_jenkins.sh @@ -1134,16 +1134,21 @@ run_ucx_perftest() { # Test malloc hooks with mpi # test_malloc_hooks_mpi() { - for tname in malloc_hooks malloc_hooks_unmapped external_events flag_no_install + for mode in reloc bistro do - echo "==== Running memory hook (${tname}) on MPI ====" - $MPIRUN -np 1 $AFFINITY ./test/mpi/test_memhooks -t $tname - done + for tname in malloc_hooks malloc_hooks_unmapped external_events flag_no_install + do + echo "==== Running memory hook (${tname} mode ${mode}) on MPI ====" + $MPIRUN -np 1 $AFFINITY \ + ./test/mpi/test_memhooks -t $tname -m ${mode} + done - echo "==== Running memory hook (malloc_hooks) on MPI with LD_PRELOAD ====" - ucm_lib=$PWD/src/ucm/.libs/libucm.so - ls -l $ucm_lib - $MPIRUN -np 1 -x LD_PRELOAD=$ucm_lib $AFFINITY ./test/mpi/test_memhooks -t malloc_hooks + echo "==== Running memory hook (malloc_hooks mode ${mode}) on MPI with LD_PRELOAD ====" + ucm_lib=$PWD/src/ucm/.libs/libucm.so + ls -l $ucm_lib + $MPIRUN -np 1 -x LD_PRELOAD=$ucm_lib $AFFINITY \ + ./test/mpi/test_memhooks -t malloc_hooks -m ${mode} + done } # diff --git a/test/mpi/test_memhooks.c b/test/mpi/test_memhooks.c index c60ad01ec5e..25c90105b77 100644 --- a/test/mpi/test_memhooks.c +++ b/test/mpi/test_memhooks.c @@ -45,22 +45,22 @@ #define SHMAT_FAILED ((void*)-1) -void* open_dyn_lib(const char *lib_path); -void* flag_no_install_init(const char *path); +void *event_init(const char *path, ucm_mmap_hook_mode_t mmap_mode); +void *ext_event_init(const char *path, ucm_mmap_hook_mode_t mmap_mode); +void* flag_no_install_init(const char *path, ucm_mmap_hook_mode_t mmap_mode); int malloc_hooks_run_all(void *dl); int malloc_hooks_run_unmapped(void *dl); int ext_event_run(void *dl); -void *ext_event_init(const char *path); typedef struct memtest_type { const char *name; - void* (*init)(const char *path); + void* (*init)(const char *path, ucm_mmap_hook_mode_t mmap_mode); int (*run) (void *arg); } memtest_type_t; memtest_type_t tests[] = { - {"malloc_hooks", open_dyn_lib, malloc_hooks_run_all}, - {"malloc_hooks_unmapped", open_dyn_lib, malloc_hooks_run_unmapped}, + {"malloc_hooks", event_init, malloc_hooks_run_all}, + {"malloc_hooks_unmapped", event_init, malloc_hooks_run_unmapped}, {"external_events", ext_event_init, ext_event_run}, {"flag_no_install", flag_no_install_init, ext_event_run}, {NULL} @@ -78,6 +78,9 @@ static void usage() { printf(" malloc_hooks_unmapped : Test VM_UNMAPPED event only\n"); printf(" external_events : Test of ucm_set_external_event() API\n"); printf(" flag_no_install : Test of UCM_EVENT_FLAG_NO_INSTALL flag\n"); + printf(" -m Memory hooks mode (bistro)\n"); + printf(" reloc : Change .plt/.got tables\n"); + printf(" bistro : Binary code patching\n"); printf("\n"); } @@ -102,10 +105,27 @@ static ucs_status_t set_event_handler(void *dl, int events) return set_handler(events, 0, event_callback, NULL); } -static ucs_status_t disable_memory_hooks(void *dl) +static ucs_status_t init_ucm_config(void *dl_ucm, int enable_hooks, + ucm_mmap_hook_mode_t mmap_mode) { - setenv("UCX_MEM_MALLOC_HOOKS", "n", 1); - setenv("UCX_MEM_MMAP_RELOC", "n", 1); + void (*library_init)(const ucm_global_config_t *ucm_opts); + ucm_global_config_t *ucm_opts; + + DL_FIND_FUNC(dl_ucm, "ucm_library_init", library_init, + return UCS_ERR_NO_ELEM); + DL_FIND_FUNC(dl_ucm, "ucm_global_opts", ucm_opts, + return UCS_ERR_NO_ELEM); + + if (enable_hooks) { + ucm_opts->mmap_hook_mode = mmap_mode; + } else { + ucm_opts->enable_malloc_hooks = 0; + ucm_opts->enable_malloc_reloc = 0; + ucm_opts->mmap_hook_mode = UCM_MMAP_HOOK_NONE; + } + + library_init(NULL); + return UCS_OK; } @@ -122,8 +142,27 @@ void* open_dyn_lib(const char *lib_path) return dl; } +void *event_init(const char *path, ucm_mmap_hook_mode_t mmap_mode) +{ + ucs_status_t status; + void *dl_ucm; + + dl_ucm = open_dyn_lib(path); + if (dl_ucm == NULL) { + return NULL; + } + + status = init_ucm_config(dl_ucm, 1, mmap_mode); + CHKERR_JUMP(status != UCS_OK, "Failed to initialize UCM", fail); + + return dl_ucm; -void *ext_event_init(const char *path) +fail: + dlclose(dl_ucm); + return NULL; +} + +void *ext_event_init(const char *path, ucm_mmap_hook_mode_t mmap_mode) { void (*set_ext_event)(int events); ucs_status_t status; @@ -134,8 +173,8 @@ void *ext_event_init(const char *path) return NULL; } - status = disable_memory_hooks(dl_ucm); - CHKERR_JUMP(status != UCS_OK, "Failed to disable memory hooks", fail); + status = init_ucm_config(dl_ucm, 0, mmap_mode); + CHKERR_JUMP(status != UCS_OK, "Failed to initialize UCM", fail); DL_FIND_FUNC(dl_ucm, "ucm_set_external_event", set_ext_event, goto fail); set_ext_event(UCM_EVENT_VM_MAPPED | UCM_EVENT_VM_UNMAPPED); @@ -151,7 +190,7 @@ void *ext_event_init(const char *path) return NULL; } -void* flag_no_install_init(const char *path) +void* flag_no_install_init(const char *path, ucm_mmap_hook_mode_t mmap_mode) { void *dl_ucm; ucs_status_t status; @@ -161,8 +200,8 @@ void* flag_no_install_init(const char *path) return NULL; } - status = disable_memory_hooks(dl_ucm); - CHKERR_JUMP(status != UCS_OK, "Failed to disable memory hooks", fail); + status = init_ucm_config(dl_ucm, 0, mmap_mode); + CHKERR_JUMP(status != UCS_OK, "Failed to initialize UCM", fail); status = set_event_handler(dl_ucm, UCM_EVENT_VM_MAPPED | UCM_EVENT_VM_UNMAPPED | @@ -387,14 +426,14 @@ int ext_event_run(void *dl) total_mapped = 0; ptr_direct_mmap = mmap(NULL, size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, 0); - printf("totmapped %lu\n", total_mapped); + printf("total_mapped=%lu\n", total_mapped); /* No callback should be called as we registered events to be external */ CHKERR_JUMP(total_mapped != 0, "Callback for mmap invoked, while hooks were not set", fail); DL_FIND_FUNC(dl, "ucm_vm_mmap", ucm_event, goto fail); ucm_event(ptr_direct_mmap, size); CHKERR_JUMP(total_mapped == 0, "Callback for mmap is not called", fail); - printf("After ucm_vm_mmap called: mapped=%zu\n", total_mapped); + printf("After ucm_vm_mmap called: total_mapped=%zu\n", total_mapped); /* Call munmap directly */ total_unmapped = 0; @@ -405,7 +444,7 @@ int ext_event_run(void *dl) DL_FIND_FUNC(dl, "ucm_vm_munmap", ucm_event, goto fail); ucm_event(ptr_direct_mmap, size); CHKERR_JUMP(total_unmapped == 0, "Callback for mmap is not called", fail); - printf("After ucm_vm_munmap: unmapped=%zu\n", total_unmapped); + printf("After ucm_vm_munmap: total_unmapped=%zu\n", total_unmapped); ret = 0; @@ -416,13 +455,14 @@ int ext_event_run(void *dl) int main(int argc, char **argv) { - const char *ucm_path = UCS_PP_MAKE_STRING(UCM_LIB_DIR) "/" "libucm.so"; - memtest_type_t *test = tests; + const char *ucm_path = UCS_PP_MAKE_STRING(UCM_LIB_DIR)"/libucm.so"; + memtest_type_t *test = tests; + ucm_mmap_hook_mode_t mmap_mode = UCM_MMAP_HOOK_BISTRO; void *dl; int ret; int c; - while ((c = getopt(argc, argv, "t:h")) != -1) { + while ((c = getopt(argc, argv, "t:m:h")) != -1) { switch (c) { case 't': for (test = tests; test->name != NULL; ++test) { @@ -435,6 +475,16 @@ int main(int argc, char **argv) return -1; } break; + case 'm': + if (!strcasecmp(optarg, "bistro")) { + mmap_mode = UCM_MMAP_HOOK_BISTRO; + } else if (!strcasecmp(optarg, "reloc")) { + mmap_mode = UCM_MMAP_HOOK_RELOC; + } else { + fprintf(stderr, "Wrong mmap mode %s\n", optarg); + return -1; + } + break; case 'h': default: usage(); @@ -444,7 +494,7 @@ int main(int argc, char **argv) /* Some tests need to modify UCM config before to call ucp_init, * which may be called by MPI_Init */ - dl = test->init(ucm_path); + dl = test->init(ucm_path, mmap_mode); if (dl == NULL) { return -1; } From 796c105a84f1c2d2403f92ac916a28d3abbc5c5b Mon Sep 17 00:00:00 2001 From: Sergey Oblomov Date: Tue, 15 Dec 2020 13:03:36 +0200 Subject: [PATCH 045/107] UCP/TCP/KEEPALIVE: added processing of auto time - added processing of "auto" timeouts for keepalive - added parser type UNITS to allow set number with "auto" & "inf" - updated jenkins script --- contrib/test_jenkins.sh | 4 ++-- src/ucp/core/ucp_context.c | 8 +++++--- src/ucp/core/ucp_context.h | 6 ++++-- src/ucp/core/ucp_worker.c | 4 ++-- src/uct/tcp/tcp.h | 9 +++++++++ src/uct/tcp/tcp_ep.c | 19 ++++++++++++++----- src/uct/tcp/tcp_iface.c | 19 +++++++++++++++++-- test/gtest/ucp/test_ucp_peer_failure.cc | 2 +- 8 files changed, 54 insertions(+), 17 deletions(-) diff --git a/contrib/test_jenkins.sh b/contrib/test_jenkins.sh index da1ddd51248..967a371aeb3 100755 --- a/contrib/test_jenkins.sh +++ b/contrib/test_jenkins.sh @@ -880,7 +880,7 @@ run_ucp_hello() { mem_types_list+="cuda cuda-managed " fi - export UCX_KEEPALIVE_TIMEOUT=1s + export UCX_KEEPALIVE_INTERVAL=1s export UCX_KEEPALIVE_NUM_EPS=10 for test_mode in -w -f -b -erecv -esend -ekeepalive @@ -893,7 +893,7 @@ run_ucp_hello() { done rm -f ./ucp_hello_world - unset UCX_KEEPALIVE_TIMEOUT + unset UCX_KEEPALIVE_INTERVAL unset UCX_KEEPALIVE_NUM_EPS } diff --git a/src/ucp/core/ucp_context.c b/src/ucp/core/ucp_context.c index f5dfd8052df..3ac14633a21 100644 --- a/src/ucp/core/ucp_context.c +++ b/src/ucp/core/ucp_context.c @@ -311,9 +311,10 @@ static ucs_config_field_t ucp_config_table[] = { "Experimental: enable new protocol selection logic", ucs_offsetof(ucp_config_t, ctx.proto_enable), UCS_CONFIG_TYPE_BOOL}, - {"KEEPALIVE_TIMEOUT", "inf", - "Time period between keepalive rounds (\"inf\" - disabled).", - ucs_offsetof(ucp_config_t, ctx.keepalive_timeout), UCS_CONFIG_TYPE_TIME_UNITS}, + /* TODO: set for keepalive more reasonable values */ + {"KEEPALIVE_INTERVAL", "0", + "Time interval between keepalive rounds (0 - disabled).", + ucs_offsetof(ucp_config_t, ctx.keepalive_interval), UCS_CONFIG_TYPE_TIME}, {"KEEPALIVE_NUM_EPS", "0", "Maximal number of endpoints to check on every keepalive round\n" @@ -1542,6 +1543,7 @@ static ucs_status_t ucp_fill_config(ucp_context_h context, } } + context->config.keepalive_interval = ucs_time_from_sec(context->config.ext.keepalive_interval); return UCS_OK; err_free_alloc_methods: diff --git a/src/ucp/core/ucp_context.h b/src/ucp/core/ucp_context.h index 5bd0ebc9aca..e961fb1e5ba 100644 --- a/src/ucp/core/ucp_context.h +++ b/src/ucp/core/ucp_context.h @@ -108,7 +108,7 @@ typedef struct ucp_context_config { /** Enable new protocol selection logic */ int proto_enable; /** Time period between keepalive rounds (0 - disabled) */ - ucs_time_t keepalive_timeout; + double keepalive_interval; /** Maximal number of endpoints to check on every keepalive round * (0 - disabled, inf - check all endpoints on every round) */ unsigned keepalive_num_eps; @@ -258,9 +258,11 @@ typedef struct ucp_context { /* Config environment prefix used to create the context */ char *env_prefix; + /* Time period between keepalive rounds */ + ucs_time_t keepalive_interval; + /* MD to compare for transport selection scores */ char *selection_cmp; - } config; /* All configurations about multithreading support */ diff --git a/src/ucp/core/ucp_worker.c b/src/ucp/core/ucp_worker.c index 25c6ffc72bc..81c502a59fb 100644 --- a/src/ucp/core/ucp_worker.c +++ b/src/ucp/core/ucp_worker.c @@ -2660,7 +2660,7 @@ void ucp_worker_print_info(ucp_worker_h worker, FILE *stream) static int ucp_worker_keepalive_is_enabled(ucp_worker_h worker) { - return (worker->context->config.ext.keepalive_timeout != UCS_TIME_INFINITY) && + return (worker->context->config.keepalive_interval != 0) && (worker->context->config.ext.keepalive_num_eps != 0); } @@ -2698,7 +2698,7 @@ static unsigned ucp_worker_keepalive_progress(void *arg) ucp_ep_h ep; if (ucs_likely((now - worker->keepalive.last_round) < - worker->context->config.ext.keepalive_timeout)) { + worker->context->config.keepalive_interval)) { return 0; } diff --git a/src/uct/tcp/tcp.h b/src/uct/tcp/tcp.h index 6fc553f3e68..852e12fc1f9 100644 --- a/src/uct/tcp/tcp.h +++ b/src/uct/tcp/tcp.h @@ -60,6 +60,13 @@ /* Maximal value for connection sequence number */ #define UCT_TCP_CM_CONN_SN_MAX UINT64_MAX +/* The seconds the connection needs to remain idle before TCP starts sending + * keepalive probes */ +#define UCT_TCP_EP_DEFAULT_KEEPALIVE_IDLE 10 + +/* The seconds between individual keepalive probes */ +#define UCT_TCP_EP_DEFAULT_KEEPALIVE_INTVL 1 + /** * TCP EP connection manager ID @@ -555,6 +562,8 @@ ucs_status_t uct_tcp_cm_handle_incoming_conn(uct_tcp_iface_t *iface, ucs_status_t uct_tcp_cm_conn_start(uct_tcp_ep_t *ep); +int uct_tcp_keepalive_is_enabled(uct_tcp_iface_t *iface); + static inline void uct_tcp_iface_outstanding_inc(uct_tcp_iface_t *iface) { iface->outstanding++; diff --git a/src/uct/tcp/tcp_ep.c b/src/uct/tcp/tcp_ep.c index 8e9973b492f..011fc8bd588 100644 --- a/src/uct/tcp/tcp_ep.c +++ b/src/uct/tcp/tcp_ep.c @@ -415,6 +415,15 @@ static inline void uct_tcp_ep_ctx_move(uct_tcp_ep_ctx_t *to_ctx, memset(from_ctx, 0, sizeof(*from_ctx)); } +static int uct_tcp_ep_time_seconds(ucs_time_t time_val, int auto_val) +{ + if (time_val == UCS_TIME_AUTO) { + return auto_val; + } + + return ucs_max(1, (int)ucs_time_to_sec(time_val)); +} + static ucs_status_t uct_tcp_ep_keepalive_enable(uct_tcp_ep_t *ep) { #ifdef UCT_TCP_EP_KEEPALIVE @@ -425,14 +434,14 @@ static ucs_status_t uct_tcp_ep_keepalive_enable(uct_tcp_ep_t *ep) int intvl_sec; ucs_status_t status; - if ((iface->config.keepalive.idle == 0) || - (iface->config.keepalive.cnt == 0) || - (iface->config.keepalive.intvl == 0)) { + if (!uct_tcp_keepalive_is_enabled(iface)) { return UCS_OK; } - idle_sec = ucs_max(1, (int)ucs_time_to_sec(iface->config.keepalive.idle)); - intvl_sec = ucs_max(1, (int)ucs_time_to_sec(iface->config.keepalive.intvl)); + idle_sec = uct_tcp_ep_time_seconds(iface->config.keepalive.idle, + UCT_TCP_EP_DEFAULT_KEEPALIVE_IDLE); + intvl_sec = uct_tcp_ep_time_seconds(iface->config.keepalive.intvl, + UCT_TCP_EP_DEFAULT_KEEPALIVE_INTVL); status = ucs_socket_setopt(ep->fd, IPPROTO_TCP, TCP_KEEPINTVL, &intvl_sec, sizeof(intvl_sec)); diff --git a/src/uct/tcp/tcp_iface.c b/src/uct/tcp/tcp_iface.c index 89bd1c6655f..1e5e66c5e92 100644 --- a/src/uct/tcp/tcp_iface.c +++ b/src/uct/tcp/tcp_iface.c @@ -86,7 +86,7 @@ static ucs_config_field_t uct_tcp_iface_config_table[] = { ucs_offsetof(uct_tcp_iface_config_t, port_range), UCS_CONFIG_TYPE_RANGE_SPEC}, #ifdef UCT_TCP_EP_KEEPALIVE - {"KEEPIDLE", "10s", + {"KEEPIDLE", UCS_PP_MAKE_STRING(UCT_TCP_EP_DEFAULT_KEEPALIVE_IDLE) "s", "The time the connection needs to remain idle before TCP starts sending " "keepalive probes.", ucs_offsetof(uct_tcp_iface_config_t, keepalive.idle), @@ -98,7 +98,7 @@ static ucs_config_field_t uct_tcp_iface_config_table[] = { ucs_offsetof(uct_tcp_iface_config_t, keepalive.cnt), UCS_CONFIG_TYPE_UINT}, - {"KEEPINTVL", "10s", + {"KEEPINTVL", UCS_PP_MAKE_STRING(UCT_TCP_EP_DEFAULT_KEEPALIVE_INTVL) "s", "The time between individual keepalive probes.", ucs_offsetof(uct_tcp_iface_config_t, keepalive.intvl), UCS_CONFIG_TYPE_TIME_UNITS}, @@ -169,6 +169,10 @@ static ucs_status_t uct_tcp_iface_query(uct_iface_h tl_iface, uct_iface_attr_t * attr->cap.am.max_short = am_buf_size; attr->cap.am.max_bcopy = am_buf_size; + if (uct_tcp_keepalive_is_enabled(iface)) { + attr->cap.flags |= UCT_IFACE_FLAG_EP_KEEPALIVE; + } + if (iface->config.zcopy.max_iov > UCT_TCP_EP_ZCOPY_SERVICE_IOV_COUNT) { /* AM */ attr->cap.am.max_iov = iface->config.zcopy.max_iov - @@ -793,6 +797,17 @@ ucs_status_t uct_tcp_query_devices(uct_md_h md, return status; } +int uct_tcp_keepalive_is_enabled(uct_tcp_iface_t *iface) +{ +#ifdef UCT_TCP_EP_KEEPALIVE + return (iface->config.keepalive.idle != UCS_TIME_INFINITY) && + (iface->config.keepalive.cnt != 0) && + (iface->config.keepalive.intvl != UCS_TIME_INFINITY); +#else /* UCT_TCP_EP_KEEPALIVE */ + return 0; +#endif /* UCT_TCP_EP_KEEPALIVE */ +} + UCT_TL_DEFINE(&uct_tcp_component, tcp, uct_tcp_query_devices, uct_tcp_iface_t, UCT_TCP_CONFIG_PREFIX, uct_tcp_iface_config_table, uct_tcp_iface_config_t); diff --git a/test/gtest/ucp/test_ucp_peer_failure.cc b/test/gtest/ucp/test_ucp_peer_failure.cc index 2796b15e8d1..9e3da96e7c7 100644 --- a/test/gtest/ucp/test_ucp_peer_failure.cc +++ b/test/gtest/ucp/test_ucp_peer_failure.cc @@ -481,7 +481,7 @@ class test_ucp_peer_failure_keepalive : public test_ucp_peer_failure }; UCS_TEST_P(test_ucp_peer_failure_keepalive, kill_receiver, - "KEEPALIVE_TIMEOUT=0.3", "KEEPALIVE_NUM_EPS=inf") { + "KEEPALIVE_INTERVAL=0.3", "KEEPALIVE_NUM_EPS=inf") { /* TODO: wireup is not tested yet */ scoped_log_handler err_handler(wrap_errors_logger); From 34f62cd4b490d336571e401c0d0c955a42160d81 Mon Sep 17 00:00:00 2001 From: Yossi Itigin Date: Thu, 29 Oct 2020 18:36:22 +0200 Subject: [PATCH 046/107] UCT/RC: Protect rc_iface->ep_list with a spinlock Since we may create RC QP from the progress thread in wireup_cm pack callback, it has to be thread safe wrt other iface operations --- src/uct/ib/rc/base/rc_ep.c | 4 ++++ src/uct/ib/rc/base/rc_iface.c | 7 +++++++ src/uct/ib/rc/base/rc_iface.h | 1 + 3 files changed, 12 insertions(+) diff --git a/src/uct/ib/rc/base/rc_ep.c b/src/uct/ib/rc/base/rc_ep.c index e93597500fb..daf56b8b7d3 100644 --- a/src/uct/ib/rc/base/rc_ep.c +++ b/src/uct/ib/rc/base/rc_ep.c @@ -158,7 +158,11 @@ UCS_CLASS_INIT_FUNC(uct_rc_ep_t, uct_rc_iface_t *iface, uint32_t qp_num, ucs_arbiter_group_init(&self->arb_group); + ucs_spin_lock(&iface->ep_list_lock); ucs_list_add_head(&iface->ep_list, &self->list); + ucs_spin_unlock(&iface->ep_list_lock); + + ucs_debug("created rc ep %p", self); return UCS_OK; err_txqp_cleanup: diff --git a/src/uct/ib/rc/base/rc_iface.c b/src/uct/ib/rc/base/rc_iface.c index b786e3fc3a0..cacbaa0cf46 100644 --- a/src/uct/ib/rc/base/rc_iface.c +++ b/src/uct/ib/rc/base/rc_iface.c @@ -289,14 +289,17 @@ ucs_status_t uct_rc_iface_flush(uct_iface_h tl_iface, unsigned flags, } count = 0; + ucs_spin_lock(&iface->ep_list_lock); ucs_list_for_each(ep, &iface->ep_list, list) { status = uct_ep_flush(&ep->super.super, 0, NULL); if ((status == UCS_ERR_NO_RESOURCE) || (status == UCS_INPROGRESS)) { ++count; } else if (status != UCS_OK) { + ucs_spin_unlock(&iface->ep_list_lock); return status; } } + ucs_spin_unlock(&iface->ep_list_lock); if (count != 0) { UCT_TL_IFACE_STAT_FLUSH_WAIT(&iface->super.super); @@ -573,6 +576,7 @@ UCS_CLASS_INIT_FUNC(uct_rc_iface_t, uct_rc_iface_ops_t *ops, uct_md_h md, ucs_arbiter_init(&self->tx.arbiter); ucs_list_head_init(&self->ep_list); ucs_list_head_init(&self->ep_gc_list); + ucs_spinlock_init(&self->ep_list_lock, 0); /* Check FC parameters correctness */ if ((config->fc.hard_thresh <= 0) || (config->fc.hard_thresh >= 1)) { @@ -694,10 +698,13 @@ static UCS_CLASS_CLEANUP_FUNC(uct_rc_iface_t) ucs_free(self->eps[i]); } + ucs_spin_lock(&self->ep_list_lock); if (!ucs_list_is_empty(&self->ep_list)) { ucs_warn("some eps were not destroyed"); } + ucs_spin_unlock(&self->ep_list_lock); + ucs_spinlock_destroy(&self->ep_list_lock); ucs_arbiter_cleanup(&self->tx.arbiter); UCS_STATS_NODE_FREE(self->stats); diff --git a/src/uct/ib/rc/base/rc_iface.h b/src/uct/ib/rc/base/rc_iface.h index 3f107f71da5..642341d9b77 100644 --- a/src/uct/ib/rc/base/rc_iface.h +++ b/src/uct/ib/rc/base/rc_iface.h @@ -274,6 +274,7 @@ struct uct_rc_iface { uct_rc_ep_t **eps[UCT_RC_QP_TABLE_SIZE]; ucs_list_link_t ep_list; ucs_list_link_t ep_gc_list; + ucs_spinlock_t ep_list_lock; /* Progress function (either regular or TM aware) */ ucs_callback_t progress; From 68dc78f63be79659ab72334318588b25fd47a883 Mon Sep 17 00:00:00 2001 From: Sergey Oblomov Date: Mon, 7 Dec 2020 14:26:44 +0200 Subject: [PATCH 047/107] RC/IFACE: added lock on rc_iface->eps access - added lock on access rc_iface->eps access --- src/uct/ib/rc/base/rc_iface.c | 14 +++++++++++++- src/uct/ib/rc/base/rc_iface.h | 1 + 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/uct/ib/rc/base/rc_iface.c b/src/uct/ib/rc/base/rc_iface.c index cacbaa0cf46..38814518de5 100644 --- a/src/uct/ib/rc/base/rc_iface.c +++ b/src/uct/ib/rc/base/rc_iface.c @@ -250,6 +250,7 @@ void uct_rc_iface_add_qp(uct_rc_iface_t *iface, uct_rc_ep_t *ep, { uct_rc_ep_t ***ptr, **memb; + ucs_spin_lock(&iface->eps_lock); ptr = &iface->eps[qp_num >> UCT_RC_QP_TABLE_ORDER]; if (*ptr == NULL) { *ptr = ucs_calloc(UCS_BIT(UCT_RC_QP_TABLE_MEMB_ORDER), sizeof(**ptr), @@ -259,16 +260,19 @@ void uct_rc_iface_add_qp(uct_rc_iface_t *iface, uct_rc_ep_t *ep, memb = &(*ptr)[qp_num & UCS_MASK(UCT_RC_QP_TABLE_MEMB_ORDER)]; ucs_assert(*memb == NULL); *memb = ep; + ucs_spin_unlock(&iface->eps_lock); } void uct_rc_iface_remove_qp(uct_rc_iface_t *iface, unsigned qp_num) { uct_rc_ep_t **memb; + ucs_spin_lock(&iface->eps_lock); memb = &iface->eps[qp_num >> UCT_RC_QP_TABLE_ORDER] [qp_num & UCS_MASK(UCT_RC_QP_TABLE_MEMB_ORDER)]; ucs_assert(*memb != NULL); *memb = NULL; + ucs_spin_unlock(&iface->eps_lock); } ucs_status_t uct_rc_iface_flush(uct_iface_h tl_iface, unsigned flags, @@ -586,11 +590,16 @@ UCS_CLASS_INIT_FUNC(uct_rc_iface_t, uct_rc_iface_ops_t *ops, uct_md_h md, goto err; } + status = ucs_spinlock_init(&self->eps_lock, 0); + if (status != UCS_OK) { + goto err; + } + /* Create RX buffers mempool */ status = uct_ib_iface_recv_mpool_init(&self->super, &config->super, "rc_recv_desc", &self->rx.mp); if (status != UCS_OK) { - goto err; + goto err_destroy_eps_lock; } /* Create TX buffers mempool */ @@ -672,6 +681,8 @@ UCS_CLASS_INIT_FUNC(uct_rc_iface_t, uct_rc_iface_ops_t *ops, uct_md_h md, ucs_mpool_cleanup(&self->tx.mp, 1); err_destroy_rx_mp: ucs_mpool_cleanup(&self->rx.mp, 1); +err_destroy_eps_lock: + ucs_spinlock_destroy(&self->eps_lock); err: return status; } @@ -709,6 +720,7 @@ static UCS_CLASS_CLEANUP_FUNC(uct_rc_iface_t) UCS_STATS_NODE_FREE(self->stats); + ucs_spinlock_destroy(&self->eps_lock); ops->cleanup_rx(self); uct_rc_iface_tx_ops_cleanup(self); ucs_mpool_cleanup(&self->tx.mp, 1); diff --git a/src/uct/ib/rc/base/rc_iface.h b/src/uct/ib/rc/base/rc_iface.h index 642341d9b77..c4cb59e9544 100644 --- a/src/uct/ib/rc/base/rc_iface.h +++ b/src/uct/ib/rc/base/rc_iface.h @@ -272,6 +272,7 @@ struct uct_rc_iface { UCS_STATS_NODE_DECLARE(stats) uct_rc_ep_t **eps[UCT_RC_QP_TABLE_SIZE]; + ucs_spinlock_t eps_lock; ucs_list_link_t ep_list; ucs_list_link_t ep_gc_list; ucs_spinlock_t ep_list_lock; From e8761c8d7a1893f1b252810e583befb960c1e4bf Mon Sep 17 00:00:00 2001 From: Evgeny Leksikov Date: Tue, 15 Dec 2020 14:46:58 +0200 Subject: [PATCH 048/107] UCT/RC: use common spin lock for iface eps and ep_list --- src/uct/ib/rc/base/rc_ep.c | 4 ++-- src/uct/ib/rc/base/rc_iface.c | 21 +++++++++------------ src/uct/ib/rc/base/rc_iface.h | 3 +-- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/src/uct/ib/rc/base/rc_ep.c b/src/uct/ib/rc/base/rc_ep.c index daf56b8b7d3..f3ec3c4a305 100644 --- a/src/uct/ib/rc/base/rc_ep.c +++ b/src/uct/ib/rc/base/rc_ep.c @@ -158,9 +158,9 @@ UCS_CLASS_INIT_FUNC(uct_rc_ep_t, uct_rc_iface_t *iface, uint32_t qp_num, ucs_arbiter_group_init(&self->arb_group); - ucs_spin_lock(&iface->ep_list_lock); + ucs_spin_lock(&iface->eps_lock); ucs_list_add_head(&iface->ep_list, &self->list); - ucs_spin_unlock(&iface->ep_list_lock); + ucs_spin_unlock(&iface->eps_lock); ucs_debug("created rc ep %p", self); return UCS_OK; diff --git a/src/uct/ib/rc/base/rc_iface.c b/src/uct/ib/rc/base/rc_iface.c index 38814518de5..3ddd3553811 100644 --- a/src/uct/ib/rc/base/rc_iface.c +++ b/src/uct/ib/rc/base/rc_iface.c @@ -293,17 +293,17 @@ ucs_status_t uct_rc_iface_flush(uct_iface_h tl_iface, unsigned flags, } count = 0; - ucs_spin_lock(&iface->ep_list_lock); + ucs_spin_lock(&iface->eps_lock); ucs_list_for_each(ep, &iface->ep_list, list) { status = uct_ep_flush(&ep->super.super, 0, NULL); if ((status == UCS_ERR_NO_RESOURCE) || (status == UCS_INPROGRESS)) { ++count; } else if (status != UCS_OK) { - ucs_spin_unlock(&iface->ep_list_lock); + ucs_spin_unlock(&iface->eps_lock); return status; } } - ucs_spin_unlock(&iface->ep_list_lock); + ucs_spin_unlock(&iface->eps_lock); if (count != 0) { UCT_TL_IFACE_STAT_FLUSH_WAIT(&iface->super.super); @@ -576,11 +576,16 @@ UCS_CLASS_INIT_FUNC(uct_rc_iface_t, uct_rc_iface_ops_t *ops, uct_md_h md, self->tx.reads_completed = 0; uct_ib_fence_info_init(&self->tx.fi); + + status = ucs_spinlock_init(&self->eps_lock, 0); + if (status != UCS_OK) { + goto err; + } + memset(self->eps, 0, sizeof(self->eps)); ucs_arbiter_init(&self->tx.arbiter); ucs_list_head_init(&self->ep_list); ucs_list_head_init(&self->ep_gc_list); - ucs_spinlock_init(&self->ep_list_lock, 0); /* Check FC parameters correctness */ if ((config->fc.hard_thresh <= 0) || (config->fc.hard_thresh >= 1)) { @@ -590,11 +595,6 @@ UCS_CLASS_INIT_FUNC(uct_rc_iface_t, uct_rc_iface_ops_t *ops, uct_md_h md, goto err; } - status = ucs_spinlock_init(&self->eps_lock, 0); - if (status != UCS_OK) { - goto err; - } - /* Create RX buffers mempool */ status = uct_ib_iface_recv_mpool_init(&self->super, &config->super, "rc_recv_desc", &self->rx.mp); @@ -709,13 +709,10 @@ static UCS_CLASS_CLEANUP_FUNC(uct_rc_iface_t) ucs_free(self->eps[i]); } - ucs_spin_lock(&self->ep_list_lock); if (!ucs_list_is_empty(&self->ep_list)) { ucs_warn("some eps were not destroyed"); } - ucs_spin_unlock(&self->ep_list_lock); - ucs_spinlock_destroy(&self->ep_list_lock); ucs_arbiter_cleanup(&self->tx.arbiter); UCS_STATS_NODE_FREE(self->stats); diff --git a/src/uct/ib/rc/base/rc_iface.h b/src/uct/ib/rc/base/rc_iface.h index c4cb59e9544..adac6711648 100644 --- a/src/uct/ib/rc/base/rc_iface.h +++ b/src/uct/ib/rc/base/rc_iface.h @@ -271,11 +271,10 @@ struct uct_rc_iface { UCS_STATS_NODE_DECLARE(stats) + ucs_spinlock_t eps_lock; /* common lock for eps and ep_list */ uct_rc_ep_t **eps[UCT_RC_QP_TABLE_SIZE]; - ucs_spinlock_t eps_lock; ucs_list_link_t ep_list; ucs_list_link_t ep_gc_list; - ucs_spinlock_t ep_list_lock; /* Progress function (either regular or TM aware) */ ucs_callback_t progress; From c5896de849bd2c07aa8978752afd720a5604f474 Mon Sep 17 00:00:00 2001 From: Yossi Itigin Date: Wed, 16 Dec 2020 01:44:34 +0200 Subject: [PATCH 049/107] CONFIG/SPEC: Bump version to 1.11.0 --- configure.ac | 2 +- ucx.spec.in | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 269abb01376..2e6236fa35f 100644 --- a/configure.ac +++ b/configure.ac @@ -9,7 +9,7 @@ AC_PREREQ([2.63]) define([ucx_ver_major], 1) -define([ucx_ver_minor], 10) +define([ucx_ver_minor], 11) define([ucx_ver_patch], 0) define([ts], esyscmd([sh -c "date +%Y%m%d%H%M%S"])) diff --git a/ucx.spec.in b/ucx.spec.in index 16ce412fff8..869bf69ffb7 100644 --- a/ucx.spec.in +++ b/ucx.spec.in @@ -302,6 +302,8 @@ process to map the memory of another process into its virtual address space. %changelog +* Wed Dec 16 2020 Yossi Itigin 1.11.0-1 +- Bump version to 1.11.0 * Wed Nov 11 2020 Yossi Itigin 1.10.0-1 - Make the RPM relocatable * Tue Jul 07 2020 Yossi Itigin 1.10.0-1 From b609ad1a8e9abcf80978d36efe154d6e0149eee0 Mon Sep 17 00:00:00 2001 From: Artemy Kovalyov Date: Sun, 15 Nov 2020 17:09:24 +0200 Subject: [PATCH 050/107] UCT/DC: Flush TX --- src/uct/ib/Makefile.am | 1 + src/uct/ib/dc/dc_mlx5.c | 41 ++++++----- src/uct/ib/dc/dc_mlx5.h | 4 +- src/uct/ib/dc/dc_mlx5.inl | 22 ++++++ src/uct/ib/dc/dc_mlx5_ep.c | 101 +++++++++++++++++---------- src/uct/ib/dc/dc_mlx5_ep.h | 23 +++--- src/uct/ib/rc/accel/rc_mlx5_iface.c | 4 +- src/uct/ib/rc/base/rc_ep.c | 87 ++++++++++++----------- src/uct/ib/rc/base/rc_ep.h | 24 ++++--- src/uct/ib/rc/base/rc_iface.h | 17 ++--- src/uct/ib/rc/verbs/rc_verbs_iface.c | 4 +- test/gtest/uct/test_flush.cc | 31 +++++--- 12 files changed, 224 insertions(+), 135 deletions(-) create mode 100644 src/uct/ib/dc/dc_mlx5.inl diff --git a/src/uct/ib/Makefile.am b/src/uct/ib/Makefile.am index 61f2cdd08e6..c7a93fdef19 100644 --- a/src/uct/ib/Makefile.am +++ b/src/uct/ib/Makefile.am @@ -101,6 +101,7 @@ endif # HAVE_TL_RC if HAVE_TL_DC noinst_HEADERS += \ dc/dc_mlx5_ep.h \ + dc/dc_mlx5.inl \ dc/dc_mlx5.h libuct_ib_la_SOURCES += \ diff --git a/src/uct/ib/dc/dc_mlx5.c b/src/uct/ib/dc/dc_mlx5.c index eed7997c1ca..ed39605ca32 100644 --- a/src/uct/ib/dc/dc_mlx5.c +++ b/src/uct/ib/dc/dc_mlx5.c @@ -8,11 +8,11 @@ # include "config.h" #endif +#include "dc_mlx5.inl" #include "dc_mlx5.h" #include "dc_mlx5_ep.h" #include -#include #include #include #include @@ -226,11 +226,7 @@ uct_dc_mlx5_poll_tx(uct_dc_mlx5_iface_t *iface) iface, dci, qp_num, txqp, hw_ci); uct_rc_mlx5_txqp_process_tx_cqe(txqp, cqe, hw_ci); - - uct_rc_txqp_available_set(txqp, uct_ib_mlx5_txwq_update_bb(txwq, hw_ci)); - ucs_assert(uct_rc_txqp_available(txqp) <= txwq->bb_max); - - uct_rc_iface_update_reads(&iface->super.super); + uct_dc_mlx5_update_tx_res(iface, txwq, txqp, hw_ci); /** * Note: DCI is released after handling completion callbacks, @@ -1327,7 +1323,10 @@ uct_dc_mlx5_dci_keepalive_handle_failure(uct_dc_mlx5_iface_t *iface, ucs_mpool_put(op); reset_dci: - uct_dc_mlx5_iface_reset_dci(iface, dci, ep_status); + uct_rc_txqp_available_set(txqp, iface->super.super.config.tx_qp_len); + uct_rc_txqp_purge_outstanding(&iface->super.super, txqp, ep_status, + txwq->sw_pi, 0); + uct_dc_mlx5_iface_reset_dci(iface, dci); } ucs_status_t uct_dc_mlx5_iface_keepalive_init(uct_dc_mlx5_iface_t *iface) @@ -1349,21 +1348,15 @@ ucs_status_t uct_dc_mlx5_iface_keepalive_init(uct_dc_mlx5_iface_t *iface) return UCS_OK; } -void uct_dc_mlx5_iface_reset_dci(uct_dc_mlx5_iface_t *iface, uint8_t dci, - ucs_status_t ep_status) +void uct_dc_mlx5_iface_reset_dci(uct_dc_mlx5_iface_t *iface, uint8_t dci) { - uct_ib_mlx5_md_t *md = ucs_derived_of(iface->super.super.super.super.md, - uct_ib_mlx5_md_t); - uct_ib_mlx5_txwq_t *txwq; - uct_rc_txqp_t *txqp; + uct_ib_mlx5_md_t *md = ucs_derived_of(iface->super.super.super.super.md, + uct_ib_mlx5_md_t); + uct_ib_mlx5_txwq_t *txwq = &iface->tx.dcis[dci].txwq; ucs_status_t status; ucs_debug("iface %p reset dci[%d]", iface, dci); - UCT_DC_MLX5_IFACE_TXQP_DCI_GET(iface, dci, txqp, txwq); - uct_rc_txqp_available_set(txqp, (int16_t)iface->super.super.config.tx_qp_len); - uct_rc_txqp_purge_outstanding(&iface->super.super, txqp, ep_status, - txwq->sw_pi, 0); ucs_assert(txwq->super.type == UCT_IB_MLX5_OBJ_TYPE_VERBS); /* Synchronize CQ index with the driver, since it would remove pending @@ -1402,14 +1395,28 @@ void uct_dc_mlx5_iface_set_ep_failed(uct_dc_mlx5_iface_t *iface, ucs_status_t status; ucs_log_level_t log_lvl; + if (ep->flags & (UCT_DC_MLX5_EP_FLAG_ERR_HANDLER_INVOKED | + UCT_DC_MLX5_EP_FLAG_FLUSH_CANCEL)) { + return; + } + if (ep_status == UCS_ERR_CANCELED) { return; } + if (ep == iface->tx.fc_ep) { + /* Do not report errors on flow control endpoint */ + ucs_debug("got error on DC flow-control endpoint, iface %p: %s", iface, + ucs_status_string(ep_status)); + return; + } + status = uct_iface_handle_ep_err(&ib_iface->super.super, &ep->super.super, ep_status); log_lvl = uct_ib_iface_failure_log_level(ib_iface, status, ep_status); uct_ib_mlx5_completion_with_err(ib_iface, (uct_ib_mlx5_err_cqe_t*)cqe, txwq, log_lvl); + + ep->flags |= UCT_DC_MLX5_EP_FLAG_ERR_HANDLER_INVOKED; } diff --git a/src/uct/ib/dc/dc_mlx5.h b/src/uct/ib/dc/dc_mlx5.h index a0d1d23793b..8ba435a038e 100644 --- a/src/uct/ib/dc/dc_mlx5.h +++ b/src/uct/ib/dc/dc_mlx5.h @@ -257,9 +257,7 @@ void uct_dc_mlx5_iface_set_ep_failed(uct_dc_mlx5_iface_t *iface, uct_ib_mlx5_txwq_t *txwq, ucs_status_t ep_status); -void uct_dc_mlx5_iface_reset_dci(uct_dc_mlx5_iface_t *iface, - uint8_t dci, - ucs_status_t ep_status); +void uct_dc_mlx5_iface_reset_dci(uct_dc_mlx5_iface_t *iface, uint8_t dci); #if HAVE_DEVX diff --git a/src/uct/ib/dc/dc_mlx5.inl b/src/uct/ib/dc/dc_mlx5.inl new file mode 100644 index 00000000000..3d670e9f8c6 --- /dev/null +++ b/src/uct/ib/dc/dc_mlx5.inl @@ -0,0 +1,22 @@ +/** +* Copyright (C) Mellanox Technologies Ltd. 2020. ALL RIGHTS RESERVED. +* +* See file LICENSE for terms. +*/ + +#include "dc_mlx5.h" + +#include +#include "uct/ib/rc/base/rc_iface.h" +#include "uct/ib/rc/base/rc_ep.h" + + +static UCS_F_ALWAYS_INLINE void +uct_dc_mlx5_update_tx_res(uct_dc_mlx5_iface_t *iface, uct_ib_mlx5_txwq_t *txwq, + uct_rc_txqp_t *txqp, uint16_t hw_ci) +{ + uct_rc_txqp_available_set(txqp, uct_ib_mlx5_txwq_update_bb(txwq, hw_ci)); + ucs_assert(uct_rc_txqp_available(txqp) <= txwq->bb_max); + + uct_rc_iface_update_reads(&iface->super.super); +} diff --git a/src/uct/ib/dc/dc_mlx5_ep.c b/src/uct/ib/dc/dc_mlx5_ep.c index 622aa65e948..3e9b747aeeb 100644 --- a/src/uct/ib/dc/dc_mlx5_ep.c +++ b/src/uct/ib/dc/dc_mlx5_ep.c @@ -8,10 +8,10 @@ # include "config.h" #endif +#include "dc_mlx5.inl" #include "dc_mlx5_ep.h" #include "dc_mlx5.h" -#include #include #define UCT_DC_MLX5_IFACE_TXQP_GET(_iface, _ep, _txqp, _txwq) \ @@ -554,24 +554,11 @@ ucs_status_t uct_dc_mlx5_ep_flush(uct_ep_h tl_ep, unsigned flags, { uct_dc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_dc_mlx5_iface_t); uct_dc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_dc_mlx5_ep_t); + uct_ib_mlx5_md_t *md = ucs_derived_of(iface->super.super.super.super.md, + uct_ib_mlx5_md_t); ucs_status_t status; UCT_DC_MLX5_TXQP_DECL(txqp, txwq); - if (ucs_unlikely(flags & UCT_FLUSH_FLAG_CANCEL)) { - if (uct_dc_mlx5_iface_is_dci_rand(iface)) { - return UCS_ERR_UNSUPPORTED; - } - - uct_ep_pending_purge(tl_ep, NULL, 0); - if (ep->dci == UCT_DC_MLX5_EP_NO_DCI) { - /* No dci -> no WQEs -> HW is clean, nothing to cancel */ - return UCS_OK; - } - - uct_dc_mlx5_ep_handle_failure(ep, NULL, UCS_ERR_CANCELED); - return UCS_OK; - } - if (!uct_dc_mlx5_iface_has_tx_resources(iface)) { return UCS_ERR_NO_RESOURCE; } @@ -600,6 +587,21 @@ ucs_status_t uct_dc_mlx5_ep_flush(uct_ep_h tl_ep, unsigned flags, UCT_DC_MLX5_IFACE_TXQP_GET(iface, ep, txqp, txwq); + if (ucs_unlikely(flags & UCT_FLUSH_FLAG_CANCEL)) { + ucs_assert(ucs_arbiter_group_is_empty(&ep->arb_group)); + ucs_assert(!(ep->flags & UCT_DC_MLX5_EP_FLAG_FLUSH_CANCEL)); + if (uct_dc_mlx5_iface_is_dci_rand(iface)) { + return UCS_ERR_UNSUPPORTED; + } + + status = uct_ib_mlx5_modify_qp_state(md, &txwq->super, IBV_QPS_ERR); + if (status != UCS_OK) { + return status; + } + + ep->flags |= UCT_DC_MLX5_EP_FLAG_FLUSH_CANCEL; + } + return uct_rc_txqp_add_flush_comp(&iface->super.super, &ep->super, txqp, comp, txwq->sig_pi); } @@ -826,6 +828,7 @@ ucs_status_t uct_dc_mlx5_ep_fc_ctrl(uct_ep_t *tl_ep, unsigned op, uct_dc_mlx5_iface_t); uct_ib_iface_t *ib_iface = &iface->super.super.super; struct ibv_ah_attr ah_attr = {.is_global = 0}; + uct_rc_iface_send_op_t *grant_ep; uct_dc_fc_sender_data_t sender; uct_dc_fc_request_t *dc_req; struct mlx5_wqe_av mlx5_av; @@ -892,9 +895,18 @@ ucs_status_t uct_dc_mlx5_ep_fc_ctrl(uct_ep_t *tl_ep, unsigned op, sender.global.gid = ib_iface->gid_info.gid; sender.global.is_global = dc_ep->flags & UCT_DC_MLX5_EP_FLAG_GRH; + grant_ep = ucs_mpool_get(&iface->super.super.tx.send_op_mp); + if (ucs_unlikely(grant_ep == NULL)) { + ucs_error("failed to allocate fc hard req grant op"); + return UCS_ERR_NO_MEMORY; + } + UCS_STATS_UPDATE_COUNTER(dc_ep->fc.stats, UCT_RC_FC_STAT_TX_HARD_REQ, 1); + uct_rc_ep_init_send_op(grant_ep, UCT_RC_IFACE_SEND_OP_FLAG_FC_GRANT, + NULL, (uct_rc_send_handler_t)ucs_mpool_put); + grant_ep->ep = tl_ep; uct_rc_mlx5_txqp_inline_post(&iface->super, UCT_IB_QPT_DCI, txqp, txwq, MLX5_OPCODE_SEND_IMM, &sender.global, sizeof(sender.global), op, sender.ep, @@ -904,6 +916,7 @@ ucs_status_t uct_dc_mlx5_ep_fc_ctrl(uct_ep_t *tl_ep, unsigned op, uct_dc_mlx5_ep_get_grh(dc_ep), uct_ib_mlx5_wqe_av_size(&dc_ep->av), MLX5_WQE_CTRL_SOLICITED, INT_MAX); + uct_rc_txqp_add_send_op_sn(txqp, grant_ep, txwq->prev_sw_pi); } return UCS_OK; @@ -932,6 +945,28 @@ static void uct_dc_mlx5_ep_keepalive_cleanup(uct_dc_mlx5_ep_t *ep) } } +static void uct_dc_mlx5_txqp_purge_outstanding(uct_dc_mlx5_iface_t *iface, + uct_rc_txqp_t *txqp, + ucs_status_t status, + uint16_t sn, int warn) +{ + uct_rc_iface_send_op_t *op; + uct_dc_mlx5_ep_t *ep; + + ucs_queue_for_each_extract(op, &txqp->outstanding, queue, + UCS_CIRCULAR_COMPARE16(op->sn, <=, sn)) { + if (op->flags & UCT_RC_IFACE_SEND_OP_FLAG_FC_GRANT) { + /* grant operation aborted so need to clear "wait-for-grant" flag */ + ep = ucs_derived_of(op->ep, uct_dc_mlx5_ep_t); + uct_dc_mlx5_ep_clear_fc_grant_flag(iface, ep); + ucs_mpool_put(op); + } else { + uct_rc_txqp_purge_outstanding_op(&iface->super.super, op, status, + warn); + } + } +} + UCS_CLASS_INIT_FUNC(uct_dc_mlx5_ep_t, uct_dc_mlx5_iface_t *iface, const uct_dc_mlx5_iface_addr_t *if_addr, uct_ib_mlx5_base_av_t *av) @@ -975,10 +1010,9 @@ static UCS_CLASS_CLEANUP_FUNC(uct_dc_mlx5_ep_t) "iface (%p) ep (%p) dci leak detected: dci=%d", iface, self, self->dci); - /* TODO should be removed by flush */ - uct_rc_txqp_purge_outstanding(&iface->super.super, - &iface->tx.dcis[self->dci].txqp, UCS_ERR_CANCELED, - iface->tx.dcis[self->dci].txwq.sw_pi, 1); + uct_dc_mlx5_txqp_purge_outstanding(iface, &iface->tx.dcis[self->dci].txqp, + UCS_ERR_CANCELED, + iface->tx.dcis[self->dci].txwq.sw_pi, 1); ucs_assert(ucs_queue_is_empty(&iface->tx.dcis[self->dci].txqp.outstanding)); iface->tx.dcis[self->dci].ep = NULL; } @@ -1310,34 +1344,25 @@ ucs_status_t uct_dc_mlx5_ep_check_fc(uct_dc_mlx5_iface_t *iface, uct_dc_mlx5_ep_ void uct_dc_mlx5_ep_handle_failure(uct_dc_mlx5_ep_t *ep, void *arg, ucs_status_t ep_status) { + struct mlx5_cqe64 *cqe = arg; uct_iface_h tl_iface = ep->super.super.iface; uint8_t dci = ep->dci; uct_dc_mlx5_iface_t *iface = ucs_derived_of(tl_iface, uct_dc_mlx5_iface_t); + uct_rc_txqp_t *txqp = &iface->tx.dcis[dci].txqp; uct_ib_mlx5_txwq_t *txwq = &iface->tx.dcis[dci].txwq; + uint16_t pi = ntohs(cqe->wqe_counter); ucs_assert(!uct_dc_mlx5_iface_is_dci_rand(iface)); - uct_dc_mlx5_iface_reset_dci(iface, dci, ep_status); - /* since we removed all outstanding ops on the dci, it should be released */ + uct_dc_mlx5_update_tx_res(iface, txwq, txqp, pi); + uct_dc_mlx5_txqp_purge_outstanding(iface, txqp, ep_status, pi, 0); + ucs_assert(ep->dci != UCT_DC_MLX5_EP_NO_DCI); uct_dc_mlx5_iface_dci_put(iface, dci); - ucs_assert_always(ep->dci == UCT_DC_MLX5_EP_NO_DCI); - - if (uct_dc_mlx5_ep_fc_wait_for_grant(ep)) { - /* No need to wait for grant on this ep anymore */ - uct_dc_mlx5_ep_clear_fc_grant_flag(iface, ep); - } + uct_dc_mlx5_iface_set_ep_failed(iface, ep, cqe, txwq, ep_status); - if (ep == iface->tx.fc_ep) { - ucs_assert(ep_status != UCS_ERR_CANCELED); - /* Cannot handle errors on flow-control endpoint. - * Or shall we ignore them? - */ - ucs_debug("got error on DC flow-control endpoint, iface %p: %s", iface, - ucs_status_string(ep_status)); - } else { - uct_dc_mlx5_iface_set_ep_failed(iface, ep, (struct mlx5_cqe64*)arg, - txwq, ep_status); + if (ep->dci == UCT_DC_MLX5_EP_NO_DCI) { + uct_dc_mlx5_iface_reset_dci(iface, dci); } } diff --git a/src/uct/ib/dc/dc_mlx5_ep.h b/src/uct/ib/dc/dc_mlx5_ep.h index 4650df9df7e..4e17fdf51e1 100644 --- a/src/uct/ib/dc/dc_mlx5_ep.h +++ b/src/uct/ib/dc/dc_mlx5_ep.h @@ -17,22 +17,29 @@ enum uct_dc_mlx5_ep_flags { - UCT_DC_MLX5_EP_FLAG_TX_WAIT = UCS_BIT(0), /* ep is in the tx_wait state. See - description of the dcs+quota dci - selection policy above */ - UCT_DC_MLX5_EP_FLAG_GRH = UCS_BIT(1), /* ep has GRH address. Used by - dc_mlx5 endpoint */ - UCT_DC_MLX5_EP_FLAG_VALID = UCS_BIT(2), /* ep is a valid endpoint */ + UCT_DC_MLX5_EP_FLAG_TX_WAIT = UCS_BIT(0), /* ep is in the tx_wait state. See + description of the dcs+quota dci + selection policy above */ + UCT_DC_MLX5_EP_FLAG_GRH = UCS_BIT(1), /* ep has GRH address. Used by + dc_mlx5 endpoint */ + UCT_DC_MLX5_EP_FLAG_VALID = UCS_BIT(2), /* ep is a valid endpoint */ /* Indicates that FC grant has been requested, but is not received yet. * Flush will not complete until an outgoing grant request is acked. * It is needed to avoid the following cases: * 1) Grant arrives for the recently deleted ep. * 2) QP resources are available, but there are some pending requests. */ - UCT_DC_MLX5_EP_FLAG_FC_WAIT_FOR_GRANT = UCS_BIT(3), + UCT_DC_MLX5_EP_FLAG_FC_WAIT_FOR_GRANT = UCS_BIT(3), + /* Keepalive Request scheduled: indicates that keepalive request * is scheduled in outstanding queue and no more keepalive actions * are needed */ - UCT_DC_MLX5_EP_FLAG_KEEPALIVE_POSTED = UCS_BIT(4) + UCT_DC_MLX5_EP_FLAG_KEEPALIVE_POSTED = UCS_BIT(4), + + /* Flush cancel was executed on EP */ + UCT_DC_MLX5_EP_FLAG_FLUSH_CANCEL = UCS_BIT(5), + + /* Error handler already called or flush(CANCEL) disabled it */ + UCT_DC_MLX5_EP_FLAG_ERR_HANDLER_INVOKED = UCS_BIT(6), }; diff --git a/src/uct/ib/rc/accel/rc_mlx5_iface.c b/src/uct/ib/rc/accel/rc_mlx5_iface.c index ee3fccf0642..6081da04030 100644 --- a/src/uct/ib/rc/accel/rc_mlx5_iface.c +++ b/src/uct/ib/rc/accel/rc_mlx5_iface.c @@ -198,12 +198,12 @@ uct_rc_mlx5_iface_handle_failure(uct_ib_iface_t *ib_iface, void *arg, uct_rc_mlx5_common_update_tx_res(iface, &ep->tx.wq, &ep->super.txqp, pi); uct_rc_txqp_purge_outstanding(iface, &ep->super.txqp, ep_status, pi, 0); - if (ep->super.flags & (UCT_RC_EP_FLAG_NO_ERR_HANDLER | + if (ep->super.flags & (UCT_RC_EP_FLAG_ERR_HANDLER_INVOKED | UCT_RC_EP_FLAG_FLUSH_CANCEL)) { return; } - ep->super.flags |= UCT_RC_EP_FLAG_NO_ERR_HANDLER; + ep->super.flags |= UCT_RC_EP_FLAG_ERR_HANDLER_INVOKED; status = uct_iface_handle_ep_err(&iface->super.super.super, &ep->super.super.super, ep_status); diff --git a/src/uct/ib/rc/base/rc_ep.c b/src/uct/ib/rc/base/rc_ep.c index e93597500fb..267d67369ea 100644 --- a/src/uct/ib/rc/base/rc_ep.c +++ b/src/uct/ib/rc/base/rc_ep.c @@ -426,52 +426,61 @@ ucs_status_t uct_rc_ep_fc_grant(uct_pending_req_t *self) return status; } +void uct_rc_txqp_purge_outstanding_op(uct_rc_iface_t *iface, + uct_rc_iface_send_op_t *op, + ucs_status_t status, int warn) +{ + uct_rc_iface_send_desc_t *desc; + + if (op->handler != (uct_rc_send_handler_t)ucs_mpool_put) { + /* Allow clean flush cancel op from destroy flow */ + if (warn && (op->handler != uct_rc_ep_flush_op_completion_handler)) { + ucs_warn("destroying uncompleted operation %p handler %s", + op, ucs_debug_get_symbol_name(op->handler)); + } + + if (op->user_comp != NULL) { + /* This must be uct_rc_ep_get_bcopy_handler, + * uct_rc_ep_get_bcopy_handler_no_completion, + * uct_rc_ep_get_zcopy_completion_handler, + * uct_rc_ep_flush_op_completion_handler or + * one of the atomic handlers, + * so invoke user completion */ + uct_invoke_completion(op->user_comp, status); + } + + /* Need to release rdma_read resources taken by get operations */ + if ((op->handler == uct_rc_ep_get_bcopy_handler) || + (op->handler == uct_rc_ep_get_bcopy_handler_no_completion)) { + uct_rc_op_release_get_bcopy(op); + uct_rc_iface_update_reads(iface); + } else if (op->handler == uct_rc_ep_get_zcopy_completion_handler) { + uct_rc_op_release_get_zcopy(op); + uct_rc_iface_update_reads(iface); + } + } + + op->flags &= ~(UCT_RC_IFACE_SEND_OP_FLAG_INUSE | + UCT_RC_IFACE_SEND_OP_FLAG_ZCOPY); + if ((op->handler == uct_rc_ep_send_op_completion_handler) || + (op->handler == uct_rc_ep_get_zcopy_completion_handler)) { + uct_rc_iface_put_send_op(op); + } else if (op->handler == uct_rc_ep_flush_op_completion_handler) { + ucs_mpool_put(op); + } else { + desc = ucs_derived_of(op, uct_rc_iface_send_desc_t); + ucs_mpool_put(desc); + } +} + void uct_rc_txqp_purge_outstanding(uct_rc_iface_t *iface, uct_rc_txqp_t *txqp, ucs_status_t status, uint16_t sn, int warn) { uct_rc_iface_send_op_t *op; - uct_rc_iface_send_desc_t *desc; ucs_queue_for_each_extract(op, &txqp->outstanding, queue, UCS_CIRCULAR_COMPARE16(op->sn, <=, sn)) { - if (op->handler != (uct_rc_send_handler_t)ucs_mpool_put) { - /* Allow clean flush cancel op from destroy flow */ - if (warn && (op->handler != uct_rc_ep_flush_op_completion_handler)) { - ucs_warn("destroying txqp %p with uncompleted operation %p handler %s", - txqp, op, ucs_debug_get_symbol_name(op->handler)); - } - - if (op->user_comp != NULL) { - /* This must be uct_rc_ep_get_bcopy_handler, - * uct_rc_ep_get_bcopy_handler_no_completion, - * uct_rc_ep_get_zcopy_completion_handler, - * uct_rc_ep_flush_op_completion_handler or - * one of the atomic handlers, - * so invoke user completion */ - uct_invoke_completion(op->user_comp, status); - } - - /* Need to release rdma_read resources taken by get operations */ - if ((op->handler == uct_rc_ep_get_bcopy_handler) || - (op->handler == uct_rc_ep_get_bcopy_handler_no_completion)) { - uct_rc_op_release_get_bcopy(op); - uct_rc_iface_update_reads(iface); - } else if (op->handler == uct_rc_ep_get_zcopy_completion_handler) { - uct_rc_op_release_get_zcopy(op); - uct_rc_iface_update_reads(iface); - } - } - op->flags &= ~(UCT_RC_IFACE_SEND_OP_FLAG_INUSE | - UCT_RC_IFACE_SEND_OP_FLAG_ZCOPY); - if ((op->handler == uct_rc_ep_send_op_completion_handler) || - (op->handler == uct_rc_ep_get_zcopy_completion_handler)) { - uct_rc_iface_put_send_op(op); - } else if (op->handler == uct_rc_ep_flush_op_completion_handler) { - ucs_mpool_put(op); - } else { - desc = ucs_derived_of(op, uct_rc_iface_send_desc_t); - ucs_mpool_put(desc); - } + uct_rc_txqp_purge_outstanding_op(iface, op, status, warn); } } diff --git a/src/uct/ib/rc/base/rc_ep.h b/src/uct/ib/rc/base/rc_ep.h index fa3f031c820..fe52ffd8899 100644 --- a/src/uct/ib/rc/base/rc_ep.h +++ b/src/uct/ib/rc/base/rc_ep.h @@ -42,38 +42,38 @@ enum { /* Keepalive Request scheduled: indicates that keepalive request * is scheduled in pending queue and no more keepalive actions * are needed */ - UCT_RC_EP_FLAG_KEEPALIVE_PENDING = UCS_BIT(0), + UCT_RC_EP_FLAG_KEEPALIVE_PENDING = UCS_BIT(0), /* EP is connected to peer */ - UCT_RC_EP_FLAG_CONNECTED = UCS_BIT(1), + UCT_RC_EP_FLAG_CONNECTED = UCS_BIT(1), /* Flush cancel was executed on EP */ - UCT_RC_EP_FLAG_FLUSH_CANCEL = UCS_BIT(2), + UCT_RC_EP_FLAG_FLUSH_CANCEL = UCS_BIT(2), /* Error handler already called or flush(CANCEL) disabled it */ - UCT_RC_EP_FLAG_NO_ERR_HANDLER = UCS_BIT(3), + UCT_RC_EP_FLAG_ERR_HANDLER_INVOKED = UCS_BIT(3), /* Soft Credit Request: indicates that peer needs to piggy-back credits * grant to counter AM (if any). Can be bundled with * UCT_RC_EP_FLAG_FC_GRANT */ - UCT_RC_EP_FLAG_FC_SOFT_REQ = UCS_BIT(UCT_AM_ID_BITS), + UCT_RC_EP_FLAG_FC_SOFT_REQ = UCS_BIT(UCT_AM_ID_BITS), /* Hard Credit Request: indicates that wnd is close to be exhausted. * The peer must send separate AM with credit grant as soon as it * receives AM with this bit set. Can be bundled with * UCT_RC_EP_FLAG_FC_GRANT */ - UCT_RC_EP_FLAG_FC_HARD_REQ = UCS_BIT((UCT_AM_ID_BITS) + 1), + UCT_RC_EP_FLAG_FC_HARD_REQ = UCS_BIT((UCT_AM_ID_BITS) + 1), /* Credit Grant: ep should update its FC wnd as soon as it receives AM with * this bit set. Can be bundled with either soft or hard request bits */ - UCT_RC_EP_FLAG_FC_GRANT = UCS_BIT((UCT_AM_ID_BITS) + 2), + UCT_RC_EP_FLAG_FC_GRANT = UCS_BIT((UCT_AM_ID_BITS) + 2), /* Special FC AM with Credit Grant: Just an empty message indicating * credit grant. Can't be bundled with any other FC flag (as it consumes * all 3 FC bits). */ - UCT_RC_EP_FC_PURE_GRANT = (UCT_RC_EP_FLAG_FC_HARD_REQ | - UCT_RC_EP_FLAG_FC_SOFT_REQ | - UCT_RC_EP_FLAG_FC_GRANT) + UCT_RC_EP_FC_PURE_GRANT = (UCT_RC_EP_FLAG_FC_HARD_REQ | + UCT_RC_EP_FLAG_FC_SOFT_REQ | + UCT_RC_EP_FLAG_FC_GRANT) }; /* @@ -277,6 +277,10 @@ void uct_rc_fc_cleanup(uct_rc_fc_t *fc); ucs_status_t uct_rc_ep_fc_grant(uct_pending_req_t *self); +void uct_rc_txqp_purge_outstanding_op(uct_rc_iface_t *iface, + uct_rc_iface_send_op_t *op, + ucs_status_t status, int warn); + void uct_rc_txqp_purge_outstanding(uct_rc_iface_t *iface, uct_rc_txqp_t *txqp, ucs_status_t status, uint16_t sn, int warn); diff --git a/src/uct/ib/rc/base/rc_iface.h b/src/uct/ib/rc/base/rc_iface.h index 3f107f71da5..261456f6b33 100644 --- a/src/uct/ib/rc/base/rc_iface.h +++ b/src/uct/ib/rc/base/rc_iface.h @@ -98,19 +98,20 @@ enum { /* flags for uct_rc_iface_send_op_t */ enum { + UCT_RC_IFACE_SEND_OP_FLAG_FC_GRANT = UCS_BIT(11), #ifdef NVALGRIND - UCT_RC_IFACE_SEND_OP_FLAG_IOV = 0, + UCT_RC_IFACE_SEND_OP_FLAG_IOV = 0, #else - UCT_RC_IFACE_SEND_OP_FLAG_IOV = UCS_BIT(12), /* save iovec to make mem defined */ + UCT_RC_IFACE_SEND_OP_FLAG_IOV = UCS_BIT(12), /* save iovec to make mem defined */ #endif #if UCS_ENABLE_ASSERT - UCT_RC_IFACE_SEND_OP_FLAG_ZCOPY = UCS_BIT(13), /* zcopy */ - UCT_RC_IFACE_SEND_OP_FLAG_IFACE = UCS_BIT(14), /* belongs to iface ops buffer */ - UCT_RC_IFACE_SEND_OP_FLAG_INUSE = UCS_BIT(15) /* queued on a txqp */ + UCT_RC_IFACE_SEND_OP_FLAG_ZCOPY = UCS_BIT(13), /* zcopy */ + UCT_RC_IFACE_SEND_OP_FLAG_IFACE = UCS_BIT(14), /* belongs to iface ops buffer */ + UCT_RC_IFACE_SEND_OP_FLAG_INUSE = UCS_BIT(15) /* queued on a txqp */ #else - UCT_RC_IFACE_SEND_OP_FLAG_ZCOPY = 0, - UCT_RC_IFACE_SEND_OP_FLAG_IFACE = 0, - UCT_RC_IFACE_SEND_OP_FLAG_INUSE = 0 + UCT_RC_IFACE_SEND_OP_FLAG_ZCOPY = 0, + UCT_RC_IFACE_SEND_OP_FLAG_IFACE = 0, + UCT_RC_IFACE_SEND_OP_FLAG_INUSE = 0 #endif }; diff --git a/src/uct/ib/rc/verbs/rc_verbs_iface.c b/src/uct/ib/rc/verbs/rc_verbs_iface.c index 7b155956f89..d1ddf9f9f32 100644 --- a/src/uct/ib/rc/verbs/rc_verbs_iface.c +++ b/src/uct/ib/rc/verbs/rc_verbs_iface.c @@ -78,12 +78,12 @@ static void uct_rc_verbs_handle_failure(uct_ib_iface_t *ib_iface, void *arg, ep->txcnt.ci + count, 0); uct_rc_verbs_update_tx_res(iface, ep, count); - if (ep->super.flags & (UCT_RC_EP_FLAG_NO_ERR_HANDLER | + if (ep->super.flags & (UCT_RC_EP_FLAG_ERR_HANDLER_INVOKED | UCT_RC_EP_FLAG_FLUSH_CANCEL)) { return; } - ep->super.flags |= UCT_RC_EP_FLAG_NO_ERR_HANDLER; + ep->super.flags |= UCT_RC_EP_FLAG_ERR_HANDLER_INVOKED; status = uct_iface_handle_ep_err(&iface->super.super.super, &ep->super.super.super, ep_status); diff --git a/test/gtest/uct/test_flush.cc b/test/gtest/uct/test_flush.cc index b91733d7a43..1a5dc4162e0 100644 --- a/test/gtest/uct/test_flush.cc +++ b/test/gtest/uct/test_flush.cc @@ -552,7 +552,8 @@ UCT_INSTANTIATE_TEST_CASE(uct_flush_test) class uct_cancel_test : public uct_test { public: - static const size_t BUF_SIZE = 8 * 1024; + static const size_t BUF_SIZE = 8 * 1024; + static const size_t BUF_SIZE_DC = 1 * 1024; class peer { public: @@ -569,6 +570,8 @@ class uct_cancel_test : public uct_test { } void connect() { + m_e->destroy_eps(); + m_peer->m_e->destroy_eps(); m_e->connect(0, *m_peer->m_e, 0); m_peer->m_e->connect(0, *m_e, 0); } @@ -607,6 +610,10 @@ class uct_cancel_test : public uct_test { size_t header_length = 0; uct_iov_t iov; + if (has_transport("dc_mlx5")) { + size = ucs_min(BUF_SIZE_DC, size); + } + iov.buffer = (char*)sendbuf.ptr() + header_length; iov.count = 1; iov.length = size - header_length; @@ -620,6 +627,10 @@ class uct_cancel_test : public uct_test { mapped_buffer &sendbuf = *s->m_buf; mapped_buffer &recvbuf = *s->m_peer->m_buf; + if (has_transport("dc_mlx5")) { + size = ucs_min(BUF_SIZE_DC, size); + } + UCS_TEST_GET_BUFFER_IOV(iov, iovcnt, sendbuf.ptr(), size, sendbuf.memh(), s->m_e->iface_attr().cap.get.max_iov); @@ -647,7 +658,7 @@ class uct_cancel_test : public uct_test { done.count = flushing.size() + 1; done.status = UCS_OK; done.func = NULL; - ucs_time_t loop_end_limit = ucs_get_time() + ucs_time_from_sec(50.0); + ucs_time_t loop_end_limit = ucs_get_time() + ucs_time_from_sec(200.0); while (!flushing.empty() && (ucs_get_time() < loop_end_limit)) { std::list::iterator iter = flushing.begin(); while (iter != flushing.end()) { @@ -666,14 +677,17 @@ class uct_cancel_test : public uct_test { short_progress_loop(); } ASSERT_UCS_OK_OR_INPROGRESS(status); + double holdup = 200.0 - ucs_time_to_sec(loop_end_limit - ucs_get_time()); + if (holdup > 10.0) { + UCS_TEST_MESSAGE << "flush took " << holdup << " sec"; + } /* coverity[loop_condition] */ while (done.count != 1) { progress(); } - m_s1->m_e->destroy_eps(); - m_s1->m_e->connect(0, *m_s0->m_e, 0); + m_s0->connect(); /* there is a chance that one side getting disconect error before * calling flush(CANCEL) */ @@ -707,7 +721,8 @@ class uct_cancel_test : public uct_test { } void do_test(send_func_t send) { - for (int i = 0; i < count(); ++i) { + ucs_time_t loop_end_limit = ucs_get_time() + ucs_time_from_sec(300.0); + for (int i = 0; (i < count()) && (ucs_get_time() < loop_end_limit); ++i) { fill(send); flush_and_reconnect(); } @@ -764,9 +779,9 @@ class uct_cancel_test : public uct_test { } void check_skip_test_tl() { - const resource *r = dynamic_cast(GetParam()); - - if ((r->tl_name != "rc_mlx5") && (r->tl_name != "rc_verbs")) { + if ((GetParam()->tl_name != "dc_mlx5") && + (GetParam()->tl_name != "rc_verbs") && + (GetParam()->tl_name != "rc_mlx5")) { UCS_TEST_SKIP_R("not supported yet"); } From ac17a6f8359156dc4825385c3c4fecdf5dac4d6a Mon Sep 17 00:00:00 2001 From: Evgeny Leksikov Date: Wed, 9 Dec 2020 17:36:02 +0200 Subject: [PATCH 051/107] UCT/CM/RDMACM: add UCX_RDMA_CM_SOURCE_ADDRESS config --- src/ucs/sys/sock.c | 27 ++++++++ src/ucs/sys/sock.h | 14 ++++ src/uct/ib/rdmacm/rdmacm_cm.c | 48 +++++++++++++ src/uct/ib/rdmacm/rdmacm_cm.h | 10 +++ src/uct/ib/rdmacm/rdmacm_cm_ep.c | 8 ++- src/uct/ib/rdmacm/rdmacm_md.c | 18 ++++- test/gtest/common/test_helpers.cc | 12 ++++ test/gtest/common/test_helpers.h | 2 + test/gtest/ucs/test_sock.cc | 18 +++-- test/gtest/uct/ib/test_sockaddr.cc | 34 +++++++-- test/gtest/uct/test_mm.cc | 4 +- test/gtest/uct/uct_p2p_test.h | 7 +- test/gtest/uct/uct_test.cc | 107 +++++++++++++++++------------ test/gtest/uct/uct_test.h | 26 ++++--- 14 files changed, 262 insertions(+), 73 deletions(-) diff --git a/src/ucs/sys/sock.c b/src/ucs/sys/sock.c index 2aa937eb0b9..52088165396 100644 --- a/src/ucs/sys/sock.c +++ b/src/ucs/sys/sock.c @@ -672,6 +672,33 @@ const char* ucs_sockaddr_str(const struct sockaddr *sock_addr, return str; } +ucs_status_t ucs_sock_ipstr_to_sockaddr(const char *ip_str, + struct sockaddr_storage *sa_storage) +{ + struct sockaddr_in* sa_in; + struct sockaddr_in6* sa_in6; + int ret; + + /* try IPv4 */ + sa_in = (struct sockaddr_in*)sa_storage; + sa_in->sin_family = AF_INET; + ret = inet_pton(AF_INET, ip_str, &sa_in->sin_addr); + if (ret == 1) { + return UCS_OK; + } + + /* try IPv6 */ + sa_in6 = (struct sockaddr_in6*)sa_storage; + sa_in6->sin6_family = AF_INET6; + ret = inet_pton(AF_INET6, ip_str, &sa_in6->sin6_addr); + if (ret == 1) { + return UCS_OK; + } + + ucs_error("invalid address %s", ip_str); + return UCS_ERR_INVALID_ADDR; +} + int ucs_sockaddr_cmp(const struct sockaddr *sa1, const struct sockaddr *sa2, ucs_status_t *status_p) diff --git a/src/ucs/sys/sock.h b/src/ucs/sys/sock.h index a9fdab6e39a..2fbbfe0c9c5 100644 --- a/src/ucs/sys/sock.h +++ b/src/ucs/sys/sock.h @@ -377,6 +377,20 @@ const char* ucs_sockaddr_str(const struct sockaddr *sock_addr, char *str, size_t max_size); +/** + * Extract the IP address from a given string and return it as a sockaddr storage. + * + * @param [in] ip_str A string to take IP address from. + * @param [out] sa_storage sockaddr storage filled with the IP address and + * address family. + * + * @return UCS_OK if @a ip_str has a valid IP address or UCS_ERR_INVALID_ADDR + * otherwise. + */ +ucs_status_t ucs_sock_ipstr_to_sockaddr(const char *ip_str, + struct sockaddr_storage *sa_storage); + + /** * Check if the address family of the given sockaddr is IPv4 or IPv6 * diff --git a/src/uct/ib/rdmacm/rdmacm_cm.c b/src/uct/ib/rdmacm/rdmacm_cm.c index 558208f9772..0f6f22bc04e 100644 --- a/src/uct/ib/rdmacm/rdmacm_cm.c +++ b/src/uct/ib/rdmacm/rdmacm_cm.c @@ -621,9 +621,46 @@ static uct_iface_ops_t uct_rdmacm_cm_iface_ops = { .iface_is_reachable = (uct_iface_is_reachable_func_t)ucs_empty_function_return_zero }; +static ucs_status_t +uct_rdmacm_cm_ipstr_to_sockaddr(const char *ip_str, struct sockaddr **saddr_p, + const char *debug_name) +{ + struct sockaddr_storage *sa_storage; + ucs_status_t status; + + /* NULL-pointer for empty parameter */ + if (ip_str[0] == '\0') { + sa_storage = NULL; + goto out; + } + + sa_storage = ucs_calloc(1, sizeof(struct sockaddr_storage), debug_name); + if (sa_storage == NULL) { + status = UCS_ERR_NO_MEMORY; + ucs_error("cannot allocate memory for rdmacm source address"); + goto err; + } + + status = ucs_sock_ipstr_to_sockaddr(ip_str, sa_storage); + if (status != UCS_OK) { + goto err_free; + } + +out: + *saddr_p = (struct sockaddr*)sa_storage; + return UCS_OK; + +err_free: + ucs_free(sa_storage); +err: + return status; +} + UCS_CLASS_INIT_FUNC(uct_rdmacm_cm_t, uct_component_h component, uct_worker_h worker, const uct_cm_config_t *config) { + const uct_rdmacm_cm_config_t *rdmacm_config = ucs_derived_of(config, + uct_rdmacm_cm_config_t); uct_priv_worker_t *worker_priv; ucs_status_t status; @@ -657,11 +694,20 @@ UCS_CLASS_INIT_FUNC(uct_rdmacm_cm_t, uct_component_h component, goto err_destroy_ev_ch; } + status = uct_rdmacm_cm_ipstr_to_sockaddr(rdmacm_config->src_addr, + &self->config.src_addr, + "rdmacm_src_addr"); + if (status != UCS_OK) { + goto ucs_async_remove_handler; + } + ucs_debug("created rdmacm_cm %p with event_channel %p (fd=%d)", self, self->ev_ch, self->ev_ch->fd); return UCS_OK; +ucs_async_remove_handler: + ucs_async_remove_handler(self->ev_ch->fd, 1); err_destroy_ev_ch: rdma_destroy_event_channel(self->ev_ch); err: @@ -672,6 +718,8 @@ UCS_CLASS_CLEANUP_FUNC(uct_rdmacm_cm_t) { ucs_status_t status; + ucs_free(self->config.src_addr); + status = ucs_async_remove_handler(self->ev_ch->fd, 1); if (status != UCS_OK) { ucs_warn("failed to remove event handler for fd %d: %s", diff --git a/src/uct/ib/rdmacm/rdmacm_cm.h b/src/uct/ib/rdmacm/rdmacm_cm.h index 35ab34bb1c7..32c699a8401 100644 --- a/src/uct/ib/rdmacm/rdmacm_cm.h +++ b/src/uct/ib/rdmacm/rdmacm_cm.h @@ -23,9 +23,19 @@ typedef struct uct_rdmacm_cm { uct_cm_t super; struct rdma_event_channel *ev_ch; khash_t(uct_rdmacm_cm_cqs) cqs; + + struct { + struct sockaddr *src_addr; + } config; } uct_rdmacm_cm_t; +typedef struct uct_rdmacm_cm_config { + uct_cm_config_t super; + char *src_addr; +} uct_rdmacm_cm_config_t; + + UCS_CLASS_DECLARE_NEW_FUNC(uct_rdmacm_cm_t, uct_cm_t, uct_component_h, uct_worker_h, const uct_cm_config_t *); UCS_CLASS_DECLARE_DELETE_FUNC(uct_rdmacm_cm_t, uct_cm_t); diff --git a/src/uct/ib/rdmacm/rdmacm_cm_ep.c b/src/uct/ib/rdmacm/rdmacm_cm_ep.c index e33f10257e2..4aad63ab10b 100644 --- a/src/uct/ib/rdmacm/rdmacm_cm_ep.c +++ b/src/uct/ib/rdmacm/rdmacm_cm_ep.c @@ -252,8 +252,8 @@ ucs_status_t uct_rdmacm_cm_ep_pack_cb(uct_rdmacm_cm_ep_t *cep, static ucs_status_t uct_rdamcm_cm_ep_client_init(uct_rdmacm_cm_ep_t *cep, const uct_ep_params_t *params) { - uct_rdmacm_cm_t *rdmacm_cm = uct_rdmacm_cm_ep_get_cm(cep); uct_cm_base_ep_t *cm_ep = &cep->super; + uct_rdmacm_cm_t *rdmacm_cm = uct_rdmacm_cm_ep_get_cm(cep); char ip_port_str[UCS_SOCKADDR_STRING_LEN]; char ep_str[UCT_RDMACM_EP_STRING_LEN]; ucs_status_t status; @@ -284,8 +284,10 @@ static ucs_status_t uct_rdamcm_cm_ep_client_init(uct_rdmacm_cm_ep_t *cep, * thread. Therefore, all ep fields have to be initialized before this * function is called. */ ucs_trace("%s: rdma_resolve_addr on cm_id %p", - uct_rdmacm_cm_ep_str(cep, ep_str, UCT_RDMACM_EP_STRING_LEN), cep->id); - if (rdma_resolve_addr(cep->id, NULL, (struct sockaddr*)params->sockaddr->addr, + uct_rdmacm_cm_ep_str(cep, ep_str, UCT_RDMACM_EP_STRING_LEN), + cep->id); + if (rdma_resolve_addr(cep->id, rdmacm_cm->config.src_addr, + (struct sockaddr*)params->sockaddr->addr, 1000/* TODO */)) { ucs_error("rdma_resolve_addr() to dst addr %s failed: %m", ucs_sockaddr_str((struct sockaddr*)params->sockaddr->addr, diff --git a/src/uct/ib/rdmacm/rdmacm_md.c b/src/uct/ib/rdmacm/rdmacm_md.c index 22e301e3016..28f9a5f61b8 100644 --- a/src/uct/ib/rdmacm/rdmacm_md.c +++ b/src/uct/ib/rdmacm/rdmacm_md.c @@ -18,11 +18,23 @@ static ucs_config_field_t uct_rdmacm_md_config_table[] = { {"ADDR_RESOLVE_TIMEOUT", "500ms", "Time to wait for address resolution to complete", - ucs_offsetof(uct_rdmacm_md_config_t, addr_resolve_timeout), UCS_CONFIG_TYPE_TIME}, + ucs_offsetof(uct_rdmacm_md_config_t, addr_resolve_timeout), UCS_CONFIG_TYPE_TIME}, {NULL} }; +static ucs_config_field_t uct_rdmacm_cm_config_table[] = { + {"", "", NULL, + ucs_offsetof(uct_rdmacm_cm_config_t, super), UCS_CONFIG_TYPE_TABLE(uct_cm_config_table)}, + + {"SOURCE_ADDRESS", "", + "If non-empty, specify the local source address (IPv4 or IPv6) to use \n" + "when creating a client connection", + ucs_offsetof(uct_rdmacm_cm_config_t, src_addr), UCS_CONFIG_TYPE_STRING}, + + {NULL} +}; + static void uct_rdmacm_md_close(uct_md_h md); static uct_md_ops_t uct_rdmacm_md_ops = { @@ -253,8 +265,8 @@ uct_component_t uct_rdmacm_component = { .cm_config = { .name = "RDMA-CM connection manager", .prefix = "RDMA_CM_", - .table = uct_cm_config_table, - .size = sizeof(uct_cm_config_t), + .table = uct_rdmacm_cm_config_table, + .size = sizeof(uct_rdmacm_cm_config_t), }, .tl_list = UCT_COMPONENT_TL_LIST_INITIALIZER(&uct_rdmacm_component), #if HAVE_RDMACM_QP_LESS diff --git a/test/gtest/common/test_helpers.cc b/test/gtest/common/test_helpers.cc index 69c2a8aa69b..d32b9843886 100644 --- a/test/gtest/common/test_helpers.cc +++ b/test/gtest/common/test_helpers.cc @@ -685,6 +685,18 @@ const struct sockaddr* sock_addr_storage::get_sock_addr_ptr() const { return m_is_valid ? (struct sockaddr *)(&m_storage) : NULL; } +const void* sock_addr_storage::get_sock_addr_in_buf() const { + const struct sockaddr* saddr = get_sock_addr_ptr(); + + ucs_assert_always(saddr != NULL); + ucs_assert_always((saddr->sa_family == AF_INET) || + (saddr->sa_family == AF_INET6)); + + return (saddr->sa_family == AF_INET) ? + (const void*)&((struct sockaddr_in*)saddr)->sin_addr : + (const void*)&((struct sockaddr_in6*)saddr)->sin6_addr; +} + std::ostream& operator<<(std::ostream& os, const sock_addr_storage& sa_storage) { return os << ucs::sockaddr_to_str(sa_storage.get_sock_addr_ptr()); diff --git a/test/gtest/common/test_helpers.h b/test/gtest/common/test_helpers.h index 4e157d1cc31..51452dcfc7f 100644 --- a/test/gtest/common/test_helpers.h +++ b/test/gtest/common/test_helpers.h @@ -347,6 +347,8 @@ class sock_addr_storage { const struct sockaddr* get_sock_addr_ptr() const; + const void* get_sock_addr_in_buf() const; + private: struct sockaddr_storage m_storage; size_t m_size; diff --git a/test/gtest/ucs/test_sock.cc b/test/gtest/ucs/test_sock.cc index 3a0c5f70126..9f2330b73e7 100644 --- a/test/gtest/ucs/test_sock.cc +++ b/test/gtest/ucs/test_sock.cc @@ -154,13 +154,14 @@ UCS_TEST_F(test_socket, sockaddr_get_inet_addr) { } } -UCS_TEST_F(test_socket, sockaddr_str) { - const uint16_t port = 65534; - const char *ipv4_addr = "192.168.122.157"; - const char *ipv6_addr = "fe80::218:e7ff:fe16:fb97"; +UCS_TEST_F(test_socket, str_sockaddr_str) { + const uint16_t port = 65534; + const char *ipv4_addr = "192.168.122.157"; + const char *ipv6_addr = "fe80::218:e7ff:fe16:fb97"; struct sockaddr_in sa_in; struct sockaddr_in6 sa_in6; char ipv4_addr_out[128], ipv6_addr_out[128], *str, test_str[1024]; + ucs_status_t status; sa_in.sin_family = AF_INET; sa_in.sin_port = htons(port); @@ -170,8 +171,13 @@ UCS_TEST_F(test_socket, sockaddr_str) { sprintf(ipv4_addr_out, "%s:%d", ipv4_addr, port); sprintf(ipv6_addr_out, "%s:%d", ipv6_addr, port); - inet_pton(AF_INET, ipv4_addr, &(sa_in.sin_addr)); - inet_pton(AF_INET6, ipv6_addr, &(sa_in6.sin6_addr)); + status = ucs_sock_ipstr_to_sockaddr(ipv4_addr, + (struct sockaddr_storage *)&sa_in); + ASSERT_EQ(UCS_OK, status); + + status = ucs_sock_ipstr_to_sockaddr(ipv6_addr, + (struct sockaddr_storage *)&sa_in6); + ASSERT_EQ(UCS_OK, status); /* Check with short `str_len` to fit IP address only */ { diff --git a/test/gtest/uct/ib/test_sockaddr.cc b/test/gtest/uct/ib/test_sockaddr.cc index 30ee1c7d760..98f1be51b92 100644 --- a/test/gtest/uct/ib/test_sockaddr.cc +++ b/test/gtest/uct/ib/test_sockaddr.cc @@ -108,8 +108,9 @@ class test_uct_sockaddr : public uct_test { /* initiate the client's private data callback argument */ client->max_conn_priv = server->iface_attr().max_conn_priv; - UCS_TEST_MESSAGE << "Testing " << m_listen_addr - << " Interface: " << GetParam()->dev_name; + UCS_TEST_MESSAGE << "Testing " << GetParam()->component_name + << " on " << m_listen_addr + << " interface: " << GetParam()->dev_name; } size_t iface_priv_data_do_pack(void *priv_data) @@ -391,12 +392,33 @@ class test_uct_cm_sockaddr : public uct_test { } void init() { + struct { + bool is_set; + char cstr[UCS_SOCKADDR_STRING_LEN]; + } src_addr = { + .is_set = false, + .cstr = {0} + }; + uct_test::init(); /* This address is accessible, as it was tested at the resource creation */ m_listen_addr = GetParam()->listen_sock_addr; m_connect_addr = GetParam()->connect_sock_addr; + const ucs::sock_addr_storage &src_sock_addr = + GetParam()->source_sock_addr; + if (src_sock_addr.get_sock_addr_ptr() != NULL) { + int sa_family = src_sock_addr.get_sock_addr_ptr()->sa_family; + const char *ret = inet_ntop(sa_family, + src_sock_addr.get_sock_addr_in_buf(), + src_addr.cstr, UCS_SOCKADDR_STRING_LEN); + EXPECT_EQ(src_addr.cstr, ret); + set_config((std::string("RDMA_CM_SOURCE_ADDRESS?=") + + src_addr.cstr).c_str()); + src_addr.is_set = true; + } + uint16_t port = ucs::get_port(); m_listen_addr.set_port(port); m_connect_addr.set_port(port); @@ -418,8 +440,12 @@ class test_uct_cm_sockaddr : public uct_test { m_long_priv_data.resize(m_long_priv_data_len); ucs::fill_random(m_long_priv_data); - UCS_TEST_MESSAGE << "Testing " << m_listen_addr - << " Interface: " << GetParam()->dev_name; + UCS_TEST_MESSAGE << "Testing " << GetParam()->component_name << " on " + << m_listen_addr << " interface " + << GetParam()->dev_name + << (src_addr.is_set ? + (std::string(" with RDMA_CM_SOURCE_ADDRESS=") + + src_addr.cstr) : ""); } protected: diff --git a/test/gtest/uct/test_mm.cc b/test/gtest/uct/test_mm.cc index c664f030b1b..06a6ecd7cf7 100644 --- a/test/gtest/uct/test_mm.cc +++ b/test/gtest/uct/test_mm.cc @@ -21,8 +21,8 @@ class test_uct_mm : public uct_test { std::string shm_dir; mm_resource(const resource& res, const std::string& shm_dir = "") : - resource(res.component, res.md_name, res.local_cpus, res.tl_name, - res.dev_name, res.dev_type), + resource(res.component, res.component_name, res.md_name, + res.local_cpus, res.tl_name, res.dev_name, res.dev_type), shm_dir(shm_dir) { } diff --git a/test/gtest/uct/uct_p2p_test.h b/test/gtest/uct/uct_p2p_test.h index 700ce6d83af..6912e0e0933 100644 --- a/test/gtest/uct/uct_p2p_test.h +++ b/test/gtest/uct/uct_p2p_test.h @@ -43,9 +43,10 @@ class uct_p2p_test : public uct_test { bool loopback; p2p_resource(const resource& res) : - resource(res.component, res.md_name, res.local_cpus, - res.tl_name, res.dev_name, res.dev_type), - loopback(false) { } + resource(res.component, res.component_name, res.md_name, + res.local_cpus, res.tl_name, res.dev_name, + res.dev_type), loopback(false) { + } }; virtual void test_xfer(send_func_t send, size_t length, unsigned flags, diff --git a/test/gtest/uct/uct_test.cc b/test/gtest/uct/uct_test.cc index 28890bc026d..484aa40caaa 100644 --- a/test/gtest/uct/uct_test.cc +++ b/test/gtest/uct/uct_test.cc @@ -27,42 +27,47 @@ std::string resource::name() const { return ss.str(); } -resource::resource() : component(NULL), md_name(""), tl_name(""), dev_name(""), - variant_name(""), dev_type(UCT_DEVICE_TYPE_LAST), +resource::resource() : component(NULL), dev_type(UCT_DEVICE_TYPE_LAST), variant(DEFAULT_VARIANT) { CPU_ZERO(&local_cpus); } -resource::resource(uct_component_h component, const std::string& md_name, - const ucs_cpu_set_t& local_cpus, const std::string& tl_name, - const std::string& dev_name, uct_device_type_t dev_type) : - component(component), md_name(md_name), local_cpus(local_cpus), - tl_name(tl_name), dev_name(dev_name), variant_name(""), - dev_type(dev_type), variant(DEFAULT_VARIANT) +resource::resource(uct_component_h component, const std::string& component_name, + const std::string& md_name, const ucs_cpu_set_t& local_cpus, + const std::string& tl_name, const std::string& dev_name, + uct_device_type_t dev_type) : + component(component), component_name(component_name), + md_name(md_name), local_cpus(local_cpus), tl_name(tl_name), + dev_name(dev_name), dev_type(dev_type), + variant(DEFAULT_VARIANT) { } -resource::resource(uct_component_h component, const uct_md_attr_t& md_attr, +resource::resource(uct_component_h component, + const uct_component_attr& cmpt_attr, + const uct_md_attr_t& md_attr, const uct_md_resource_desc_t& md_resource, const uct_tl_resource_desc_t& tl_resource) : component(component), + component_name(cmpt_attr.name), md_name(md_resource.md_name), local_cpus(md_attr.local_cpus), tl_name(tl_resource.tl_name), dev_name(tl_resource.dev_name), - variant_name(""), dev_type(tl_resource.dev_type), variant(DEFAULT_VARIANT) { } -resource_speed::resource_speed(uct_component_h component, const uct_worker_h& worker, - const uct_md_h& md, const uct_md_attr_t& md_attr, +resource_speed::resource_speed(uct_component_h component, + const uct_component_attr& cmpt_attr, + const uct_worker_h& worker, const uct_md_h& md, + const uct_md_attr_t& md_attr, const uct_md_resource_desc_t& md_resource, const uct_tl_resource_desc_t& tl_resource) : - resource(component, md_attr, md_resource, - tl_resource) { + resource(component, cmpt_attr, md_attr, + md_resource, tl_resource) { ucs_status_t status; uct_iface_params_t iface_params = { 0 }; uct_iface_config_t *iface_config; @@ -176,8 +181,6 @@ uct_test::uct_test() { status = uct_component_query(GetParam()->component, &component_attr); ASSERT_UCS_OK(status); - UCS_TEST_MESSAGE << "Testing component: " << component_attr.name; - if (component_attr.flags & UCT_COMPONENT_FLAG_CM) { status = uct_cm_config_read(GetParam()->component, NULL, NULL, &m_cm_config); ASSERT_UCS_OK(status); @@ -197,52 +200,67 @@ uct_test::~uct_test() { } void uct_test::init_sockaddr_rsc(resource *rsc, struct sockaddr *listen_addr, - struct sockaddr *connect_addr, size_t size) + struct sockaddr *connect_addr, size_t size, + bool init_src) { rsc->listen_sock_addr.set_sock_addr(*listen_addr, size); rsc->connect_sock_addr.set_sock_addr(*connect_addr, size); + if (init_src) { + /* src_addr == dst_addr to be sure they are reachable */ + rsc->source_sock_addr.set_sock_addr(*connect_addr, size); + } } -void uct_test::set_interface_rscs(uct_component_h cmpt, const char *name, - ucs_cpu_set_t local_cpus, struct ifaddrs *ifa, +void uct_test::set_interface_rscs(uct_component_h cmpt, const char *cmpt_name, + const char *md_name, ucs_cpu_set_t local_cpus, + struct ifaddrs *ifa, std::vector& all_resources) { int i; - /* Create two resources on the same interface. the first one will have the - * ip of the interface and the second one will have INADDR_ANY */ - for (i = 0; i < 2; i++) { - resource rsc(cmpt, std::string(name), local_cpus, "sockaddr", - std::string(ifa->ifa_name), UCT_DEVICE_TYPE_NET); - - if (i == 0) { - /* first rsc */ + /* Create three resources on the same interface: + * 0 - has the ip of the dst interface + * 1 - has the ip of the dst interface and IP of src interface + * 2 - has INADDR_ANY + */ + for (i = 0; i < 3; i++) { + resource rsc(cmpt, std::string(cmpt_name), std::string(md_name), + local_cpus, "sockaddr", std::string(ifa->ifa_name), + UCT_DEVICE_TYPE_NET); + bool init_src_addr = (i == 1); + + if (i < 2) { if (ifa->ifa_addr->sa_family == AF_INET) { uct_test::init_sockaddr_rsc(&rsc, ifa->ifa_addr, ifa->ifa_addr, - sizeof(struct sockaddr_in)); + sizeof(struct sockaddr_in), + init_src_addr); } else if (ifa->ifa_addr->sa_family == AF_INET6) { uct_test::init_sockaddr_rsc(&rsc, ifa->ifa_addr, ifa->ifa_addr, - sizeof(struct sockaddr_in6)); + sizeof(struct sockaddr_in6), + init_src_addr); } else { UCS_TEST_ABORT("Unknown sa_family " << ifa->ifa_addr->sa_family); } all_resources.push_back(rsc); } else { - /* second rsc */ if (ifa->ifa_addr->sa_family == AF_INET) { struct sockaddr_in sin; memset(&sin, 0, sizeof(struct sockaddr_in)); sin.sin_family = AF_INET; sin.sin_addr.s_addr = INADDR_ANY; uct_test::init_sockaddr_rsc(&rsc, (struct sockaddr*)&sin, - ifa->ifa_addr, sizeof(struct sockaddr_in)); + ifa->ifa_addr, + sizeof(struct sockaddr_in), + init_src_addr); } else if (ifa->ifa_addr->sa_family == AF_INET6) { struct sockaddr_in6 sin; memset(&sin, 0, sizeof(struct sockaddr_in6)); sin.sin6_family = AF_INET6; sin.sin6_addr = in6addr_any; uct_test::init_sockaddr_rsc(&rsc, (struct sockaddr*)&sin, - ifa->ifa_addr, sizeof(struct sockaddr_in6)); + ifa->ifa_addr, + sizeof(struct sockaddr_in6), + init_src_addr); } else { UCS_TEST_ABORT("Unknown sa_family " << ifa->ifa_addr->sa_family); } @@ -284,8 +302,9 @@ void uct_test::set_md_sockaddr_resources(const md_resource& md_rsc, uct_md_h md, if (uct_md_is_sockaddr_accessible(md, &sock_addr, UCT_SOCKADDR_ACC_LOCAL) && uct_md_is_sockaddr_accessible(md, &sock_addr, UCT_SOCKADDR_ACC_REMOTE)) { - uct_test::set_interface_rscs(md_rsc.cmpt, md_rsc.rsc_desc.md_name, - local_cpus, ifa, all_resources); + uct_test::set_interface_rscs(md_rsc.cmpt, md_rsc.cmpt_attr.name, + md_rsc.rsc_desc.md_name, local_cpus, + ifa, all_resources); } } @@ -305,7 +324,8 @@ void uct_test::set_cm_sockaddr_resources(uct_component_h cmpt, const char *cmpt_ continue; } - uct_test::set_interface_rscs(cmpt, cmpt_name, local_cpus, ifa, all_resources); + uct_test::set_interface_rscs(cmpt, cmpt_name, "", local_cpus, ifa, + all_resources); } freeifaddrs(ifaddr); @@ -388,15 +408,16 @@ std::vector uct_test::enum_resources(const std::string& tl_name resource_speed tcp_fastest_rsc; for (unsigned j = 0; j < num_tl_resources; ++j) { - if (tcp_fastest_dev && (std::string("tcp") == tl_resources[j].tl_name)) { - resource_speed rsc(iter->cmpt, worker, md, md_attr, - iter->rsc_desc, tl_resources[j]); + if (tcp_fastest_dev && (std::string("tcp") == + tl_resources[j].tl_name)) { + resource_speed rsc(iter->cmpt, iter->cmpt_attr, worker, md, + md_attr, iter->rsc_desc, tl_resources[j]); if (!tcp_fastest_rsc.bw || (rsc.bw > tcp_fastest_rsc.bw)) { tcp_fastest_rsc = rsc; } } else { - resource rsc(iter->cmpt, md_attr, iter->rsc_desc, - tl_resources[j]); + resource rsc(iter->cmpt, iter->cmpt_attr, md_attr, + iter->rsc_desc, tl_resources[j]); all_resources.push_back(rsc); } } @@ -434,9 +455,9 @@ void uct_test::generate_test_variant(int variant, for (std::vector::iterator iter = r.begin(); iter != r.end(); ++iter) { if (tl_name.empty() || ((*iter)->tl_name == tl_name)) { - resource rsc((*iter)->component, (*iter)->md_name, - (*iter)->local_cpus, (*iter)->tl_name, - (*iter)->dev_name, (*iter)->dev_type); + resource rsc((*iter)->component, (*iter)->component_name, + (*iter)->md_name, (*iter)->local_cpus, + (*iter)->tl_name, (*iter)->dev_name, (*iter)->dev_type); rsc.variant = variant; rsc.variant_name = variant_name; test_res.push_back(rsc); diff --git a/test/gtest/uct/uct_test.h b/test/gtest/uct/uct_test.h index 4c4817672a2..18e7c21a630 100644 --- a/test/gtest/uct/uct_test.h +++ b/test/gtest/uct/uct_test.h @@ -46,6 +46,7 @@ struct resource { virtual ~resource() {}; virtual std::string name() const; uct_component_h component; + std::string component_name; std::string md_name; ucs_cpu_set_t local_cpus; std::string tl_name; @@ -54,13 +55,16 @@ struct resource { uct_device_type_t dev_type; ucs::sock_addr_storage listen_sock_addr; /* sockaddr to listen on */ ucs::sock_addr_storage connect_sock_addr; /* sockaddr to connect to */ + ucs::sock_addr_storage source_sock_addr; /* sockaddr to connect from */ int variant; resource(); - resource(uct_component_h component, const std::string& md_name, - const ucs_cpu_set_t& local_cpus, const std::string& tl_name, - const std::string& dev_name, uct_device_type_t dev_type); - resource(uct_component_h component, const uct_md_attr_t& md_attr, + resource(uct_component_h component, const std::string& component_name, + const std::string& md_name, const ucs_cpu_set_t& local_cpus, + const std::string& tl_name, const std::string& dev_name, + uct_device_type_t dev_type); + resource(uct_component_h component, const uct_component_attr& cmpnt_attr, + const uct_md_attr_t& md_attr, const uct_md_resource_desc_t& md_resource, const uct_tl_resource_desc_t& tl_resource); }; @@ -69,8 +73,10 @@ struct resource_speed : public resource { double bw; resource_speed() : resource(), bw(0) { } - resource_speed(uct_component_h component, const uct_worker_h& worker, - const uct_md_h& md, const uct_md_attr_t& md_attr, + resource_speed(uct_component_h component, + const uct_component_attr& cmpnt_attr, + const uct_worker_h& worker, const uct_md_h& md, + const uct_md_attr_t& md_attr, const uct_md_resource_desc_t& md_resource, const uct_tl_resource_desc_t& tl_resource); }; @@ -389,11 +395,13 @@ class uct_test : public testing::TestWithParam, static void set_cm_sockaddr_resources(uct_component_h cmpt, const char *cmpt_name, ucs_cpu_set_t local_cpus, std::vector& all_resources); - static void set_interface_rscs(uct_component_h comt, const char * name, - ucs_cpu_set_t local_cpus, struct ifaddrs *ifa, + static void set_interface_rscs(uct_component_h cmpt, const char *cmpt_name, + const char *md_name, ucs_cpu_set_t local_cpus, + struct ifaddrs *ifa, std::vector& all_resources); static void init_sockaddr_rsc(resource *rsc, struct sockaddr *listen_addr, - struct sockaddr *connect_addr, size_t size); + struct sockaddr *connect_addr, size_t size, + bool init_src); uct_test::entity* create_entity(size_t rx_headroom, uct_error_handler_t err_handler = NULL, uct_tag_unexp_eager_cb_t eager_cb = NULL, From bc170a07add4de084c5d82395c7d003bc017a190 Mon Sep 17 00:00:00 2001 From: Mikhail Brinskii Date: Wed, 9 Dec 2020 18:30:01 +0200 Subject: [PATCH 052/107] UCP/AM: Define persistent data flag for AM handlers --- src/ucp/api/ucp.h | 18 +++++--- src/ucp/core/ucp_am.c | 82 +++++++++++++++++++++++++---------- test/gtest/ucp/test_ucp_am.cc | 46 +++++++++++++++++++- 3 files changed, 116 insertions(+), 30 deletions(-) diff --git a/src/ucp/api/ucp.h b/src/ucp/api/ucp.h index 598c46a6c03..eee23515750 100644 --- a/src/ucp/api/ucp.h +++ b/src/ucp/api/ucp.h @@ -463,13 +463,21 @@ enum { * @ingroup UCP_WORKER * @brief Flags for a UCP Active Message callback. * - * Flags that indicate how to handle UCP Active Messages - * Currently only UCP_AM_FLAG_WHOLE_MSG is supported, - * which indicates the entire message is handled in one - * callback. + * Flags that indicate how to handle UCP Active Messages. */ enum ucp_am_cb_flags { - UCP_AM_FLAG_WHOLE_MSG = UCS_BIT(0) + /** + * Indicates that the entire message will be handled in one callback. + */ + UCP_AM_FLAG_WHOLE_MSG = UCS_BIT(0), + + /** + * Guarantees that the specified @ref ucp_am_recv_callback_t callback, + * will always be called with @ref UCP_AM_RECV_ATTR_FLAG_DATA flag set, + * so the data will be accessible outside the callback, until + * @ref ucp_am_data_release is called. + */ + UCP_AM_FLAG_PERSISTENT_DATA = UCS_BIT(1) }; diff --git a/src/ucp/core/ucp_am.c b/src/ucp/core/ucp_am.c index 326b766e74a..13ea8fe3f24 100644 --- a/src/ucp/core/ucp_am.c +++ b/src/ucp/core/ucp_am.c @@ -1086,17 +1086,23 @@ UCS_PROFILE_FUNC(ucs_status_ptr_t, ucp_am_recv_data_nbx, return ret; } +static UCS_F_ALWAYS_INLINE void* +ucp_am_data_ptr(ucp_am_hdr_t *am_hdr, size_t hdr_size) +{ + return UCS_PTR_BYTE_OFFSET(am_hdr, hdr_size + am_hdr->header_length); +} + static UCS_F_ALWAYS_INLINE ucs_status_t ucp_am_invoke_cb(ucp_worker_h worker, ucp_am_hdr_t *am_hdr, - size_t am_hdr_length, size_t data_length, + size_t am_hdr_length, void *data, size_t data_length, ucp_ep_h reply_ep, uint64_t recv_flags) { uint16_t am_id = am_hdr->am_id; uint32_t user_hdr_length = am_hdr->header_length; - void *am_data = UCS_PTR_BYTE_OFFSET(am_hdr, am_hdr_length); ucp_am_entry_t *am_cb = &ucs_array_elem(&worker->am, am_id); ucp_am_recv_param_t param; unsigned flags; + void *user_hdr; if (ucs_unlikely(!ucp_am_recv_check_id(worker, am_id))) { return UCS_OK; @@ -1105,10 +1111,10 @@ ucp_am_invoke_cb(ucp_worker_h worker, ucp_am_hdr_t *am_hdr, if (ucs_likely(am_cb->flags & UCP_AM_CB_PRIV_FLAG_NBX)) { param.recv_attr = recv_flags; param.reply_ep = reply_ep; + user_hdr = user_hdr_length ? + UCS_PTR_BYTE_OFFSET(am_hdr, am_hdr_length) : NULL; - return am_cb->cb(am_cb->context, user_hdr_length ? am_data : NULL, - user_hdr_length, - UCS_PTR_BYTE_OFFSET(am_data, user_hdr_length), + return am_cb->cb(am_cb->context, user_hdr, user_hdr_length, data, data_length - user_hdr_length, ¶m); } @@ -1122,8 +1128,7 @@ ucp_am_invoke_cb(ucp_worker_h worker, ucp_am_hdr_t *am_hdr, flags = (recv_flags & UCP_AM_RECV_ATTR_FLAG_DATA) ? UCP_CB_PARAM_FLAG_DATA : 0; - return am_cb->cb_old(am_cb->context, am_data, data_length, reply_ep, - flags); + return am_cb->cb_old(am_cb->context, data, data_length, reply_ep, flags); } static UCS_F_ALWAYS_INLINE ucs_status_t @@ -1132,17 +1137,48 @@ ucp_am_handler_common(ucp_worker_h worker, ucp_am_hdr_t *am_hdr, size_t hdr_size uint64_t recv_flags) { ucp_recv_desc_t *desc = NULL; - void *data; + ucp_am_entry_t *am_cb = &ucs_array_elem(&worker->am, am_hdr->am_id); + void *data = ucp_am_data_ptr(am_hdr, hdr_size); ucs_status_t status; - recv_flags |= (am_flags & UCT_CB_PARAM_FLAG_DESC) ? - UCP_AM_RECV_ATTR_FLAG_DATA : 0; + ucs_assert(total_length >= am_hdr->header_length + hdr_size); + + if (am_flags & UCT_CB_PARAM_FLAG_DESC) { + recv_flags |= UCP_AM_RECV_ATTR_FLAG_DATA; + } else if (am_cb->flags & UCP_AM_FLAG_PERSISTENT_DATA) { + /* If user requested persistent data pointer and UCT_CB_PARAM_FLAG_DESC + * is not set by UCT, need to initialize rdesc before calling the callback. + * Note: for UCT_CB_PARAM_FLAG_DESC case, can't initialize desc in advance, + * because it will overwrite user header (ucp_recv_desc_init uses headroom + * right before the data, where user header resides). + */ + status = ucp_recv_desc_init(worker, data, + total_length - hdr_size - am_hdr->header_length, + 0, + 0, /* pass as a const */ + 0, 0, -hdr_size, &desc); + if (ucs_unlikely(UCS_STATUS_IS_ERR(status))) { + goto err; + } + data = desc + 1; + recv_flags |= UCP_AM_RECV_ATTR_FLAG_DATA; + } + + status = ucp_am_invoke_cb(worker, am_hdr, hdr_size, data, + total_length - hdr_size, reply_ep, recv_flags); + if (desc != NULL) { + if (status != UCS_INPROGRESS) { + /* data is not needed - release previously initialized desc */ + ucp_recv_desc_release(desc); + } + /* initialized desc means that UCT_CB_PARAM_FLAG_DESC is not set in + * am_flags, thus UCS_OK is the only valid return code + */ + return UCS_OK; + } - status = ucp_am_invoke_cb(worker, am_hdr, hdr_size, - total_length - hdr_size, reply_ep, - recv_flags); if (status != UCS_INPROGRESS) { - return UCS_OK; /* we do not need UCT desc, just return UCS_OK */ + return UCS_OK; /* user does not want to keep the data, just return UCS_OK */ } if (ucs_unlikely(!(am_flags & UCT_CB_PARAM_FLAG_DESC))) { @@ -1150,21 +1186,21 @@ ucp_am_handler_common(ucp_worker_h worker, ucp_am_hdr_t *am_hdr, size_t hdr_size return UCS_OK; } - ucs_assert(total_length >= am_hdr->header_length + hdr_size); - data = UCS_PTR_BYTE_OFFSET(am_hdr, hdr_size + am_hdr->header_length); status = ucp_recv_desc_init(worker, data, total_length - hdr_size - am_hdr->header_length, 0, UCT_CB_PARAM_FLAG_DESC, /* pass as a const */ 0, 0, -hdr_size, &desc); if (ucs_unlikely(UCS_STATUS_IS_ERR(status))) { - ucs_error("worker %p could not allocate descriptor for active" - " message on callback : %u", worker, am_hdr->am_id); - return UCS_OK; + goto err; } - ucs_assert(desc != NULL); return UCS_INPROGRESS; + +err: + ucs_error("worker %p could not allocate descriptor for active" + " message on callback : %u", worker, am_hdr->am_id); + return UCS_OK; } UCS_PROFILE_FUNC(ucs_status_t, ucp_am_handler_reply, @@ -1261,11 +1297,11 @@ ucp_am_handle_unfinished(ucp_worker_h worker, ucp_recv_desc_t *first_rdesc, first_hdr = (ucp_am_first_hdr_t*)(first_rdesc + 1); recv_flags = ucp_am_hdr_reply_ep(worker, first_hdr->super.super.flags, reply_ep, &reply_ep); + msg = ucp_am_data_ptr(&first_hdr->super.super, sizeof(*first_hdr)); status = ucp_am_invoke_cb(worker, &first_hdr->super.super, - sizeof(*first_hdr), first_hdr->total_size, - reply_ep, - recv_flags | UCP_AM_RECV_ATTR_FLAG_DATA); + sizeof(*first_hdr), msg, first_hdr->total_size, + reply_ep, recv_flags | UCP_AM_RECV_ATTR_FLAG_DATA); if (status != UCS_INPROGRESS) { ucs_free(first_rdesc); /* user does not need to hold this data */ return; diff --git a/test/gtest/ucp/test_ucp_am.cc b/test/gtest/ucp/test_ucp_am.cc index 136c5735f2a..90f857a1417 100644 --- a/test/gtest/ucp/test_ucp_am.cc +++ b/test/gtest/ucp/test_ucp_am.cc @@ -337,8 +337,8 @@ class test_ucp_am_nbx : public test_ucp_am_base { } } - void set_am_data_handler(entity &e, uint16_t am_id, - ucp_am_recv_callback_t cb, void *arg) + void set_am_data_handler(entity &e, uint16_t am_id, ucp_am_recv_callback_t cb, + void *arg, unsigned flags = 0) { ucp_am_handler_param_t param; @@ -349,6 +349,12 @@ class test_ucp_am_nbx : public test_ucp_am_base { param.id = am_id; param.cb = cb; param.arg = arg; + + if (flags != 0) { + param.field_mask |= UCP_AM_HANDLER_PARAM_FIELD_FLAGS; + param.flags = flags; + } + ASSERT_UCS_OK(ucp_worker_set_am_recv_handler(e.worker(), ¶m)); } @@ -496,6 +502,21 @@ class test_ucp_am_nbx : public test_ucp_am_base { return UCS_OK; } + static ucs_status_t am_data_hold_cb(void *arg, const void *header, + size_t header_length, void *data, + size_t length, + const ucp_am_recv_param_t *param) + { + void **rx_data_p = reinterpret_cast(arg); + + EXPECT_TRUE(param->recv_attr & UCP_AM_RECV_ATTR_FLAG_DATA); + EXPECT_EQ(NULL, *rx_data_p); + + *rx_data_p = data; + + return UCS_INPROGRESS; + } + static const uint16_t TEST_AM_NBX_ID = 0; ucp_datatype_t m_dt; volatile bool m_am_received; @@ -599,6 +620,27 @@ UCS_TEST_P(test_ucp_am_nbx, rx_rts_reply_am_on_closed_ep, "RNDV_THRESH=32K") test_recv_on_closed_ep(64 * UCS_KBYTE, UCP_AM_SEND_REPLY); } +UCS_TEST_P(test_ucp_am_nbx, rx_persistent_data) +{ + void *rx_data = NULL; + char data = 'd'; + + set_am_data_handler(receiver(), TEST_AM_NBX_ID, am_data_hold_cb, &rx_data, + UCP_AM_FLAG_PERSISTENT_DATA); + + ucp_request_param_t param; + + param.op_attr_mask = 0ul; + ucs_status_ptr_t sptr = ucp_am_send_nbx(sender().ep(), TEST_AM_NBX_ID, NULL, + 0ul, &data, sizeof(data), ¶m); + wait_for_flag(&rx_data); + EXPECT_TRUE(rx_data != NULL); + EXPECT_EQ(data, *reinterpret_cast(rx_data)); + + ucp_am_data_release(receiver().worker(), rx_data); + EXPECT_EQ(UCS_OK, request_wait(sptr)); +} + UCP_INSTANTIATE_TEST_CASE(test_ucp_am_nbx) From 957be76f0b98ef03888c3ca20402c8abc3a158a1 Mon Sep 17 00:00:00 2001 From: Raul Akhmetshin Date: Mon, 23 Nov 2020 10:46:40 +0200 Subject: [PATCH 053/107] UCT/API: Added new UCT EP AM short IOV API. --- src/ucp/core/ucp_ep.c | 1 + src/ucp/wireup/wireup_ep.c | 1 + src/uct/api/tl.h | 5 ++ src/uct/api/uct.h | 42 ++++++++++- src/uct/base/uct_iface.c | 40 ++++++++++ src/uct/base/uct_iface.h | 3 + src/uct/ib/dc/dc_mlx5.c | 1 + src/uct/ib/rc/accel/rc_mlx5_iface.c | 1 + src/uct/ib/rc/verbs/rc_verbs_iface.c | 1 + src/uct/ib/rdmacm/rdmacm_cm.c | 1 + src/uct/ib/ud/accel/ud_mlx5.c | 1 + src/uct/ib/ud/verbs/ud_verbs.c | 1 + src/uct/sm/mm/base/mm_iface.c | 1 + src/uct/sm/self/self.c | 1 + src/uct/tcp/tcp_iface.c | 1 + src/uct/tcp/tcp_sockcm.c | 1 + src/uct/ugni/rdma/ugni_rdma_iface.c | 2 + src/uct/ugni/smsg/ugni_smsg_iface.c | 1 + src/uct/ugni/udt/ugni_udt_iface.c | 1 + test/gtest/uct/test_p2p_am.cc | 106 ++++++++++++++++++--------- test/gtest/uct/test_p2p_err.cc | 32 ++++++++ test/gtest/uct/test_p2p_mix.cc | 23 ++++++ test/gtest/uct/test_p2p_mix.h | 3 + test/gtest/uct/test_stats.cc | 23 ++++++ 24 files changed, 257 insertions(+), 36 deletions(-) diff --git a/src/ucp/core/ucp_ep.c b/src/ucp/core/ucp_ep.c index 344b3bee642..c0e36c9d76c 100644 --- a/src/ucp/core/ucp_ep.c +++ b/src/ucp/core/ucp_ep.c @@ -970,6 +970,7 @@ static uct_iface_t ucp_failed_tl_iface = { .ep_get_bcopy = (uct_ep_get_bcopy_func_t)ucs_empty_function_return_ep_timeout, .ep_get_zcopy = (uct_ep_get_zcopy_func_t)ucs_empty_function_return_ep_timeout, .ep_am_short = (uct_ep_am_short_func_t)ucs_empty_function_return_ep_timeout, + .ep_am_short_iov = (uct_ep_am_short_iov_func_t)ucs_empty_function_return_ep_timeout, .ep_am_bcopy = (uct_ep_am_bcopy_func_t)ucs_empty_function_return_bc_ep_timeout, .ep_am_zcopy = (uct_ep_am_zcopy_func_t)ucs_empty_function_return_ep_timeout, .ep_atomic_cswap64 = (uct_ep_atomic_cswap64_func_t)ucs_empty_function_return_ep_timeout, diff --git a/src/ucp/wireup/wireup_ep.c b/src/ucp/wireup/wireup_ep.c index 614de433247..b15f24cc990 100644 --- a/src/ucp/wireup/wireup_ep.c +++ b/src/ucp/wireup/wireup_ep.c @@ -347,6 +347,7 @@ UCS_CLASS_INIT_FUNC(ucp_wireup_ep_t, ucp_ep_h ucp_ep) .ep_get_bcopy = (uct_ep_get_bcopy_func_t)ucs_empty_function_return_no_resource, .ep_get_zcopy = (uct_ep_get_zcopy_func_t)ucs_empty_function_return_no_resource, .ep_am_short = (uct_ep_am_short_func_t)ucs_empty_function_return_no_resource, + .ep_am_short_iov = (uct_ep_am_short_iov_func_t)ucs_empty_function_return_no_resource, .ep_am_bcopy = ucp_wireup_ep_am_bcopy, .ep_am_zcopy = (uct_ep_am_zcopy_func_t)ucs_empty_function_return_no_resource, .ep_tag_eager_short = (uct_ep_tag_eager_short_func_t)ucs_empty_function_return_no_resource, diff --git a/src/uct/api/tl.h b/src/uct/api/tl.h index e6f7667e21a..b3a3cabaf35 100644 --- a/src/uct/api/tl.h +++ b/src/uct/api/tl.h @@ -75,6 +75,10 @@ typedef ucs_status_t (*uct_ep_am_short_func_t)(uct_ep_h ep, const void *payload, unsigned length); +typedef ucs_status_t (*uct_ep_am_short_iov_func_t)(uct_ep_h ep, uint8_t id, + const uct_iov_t *iov, + size_t iovcnt); + typedef ssize_t (*uct_ep_am_bcopy_func_t)(uct_ep_h ep, uint8_t id, uct_pack_callback_t pack_cb, @@ -299,6 +303,7 @@ typedef struct uct_iface_ops { /* endpoint - active message */ uct_ep_am_short_func_t ep_am_short; + uct_ep_am_short_iov_func_t ep_am_short_iov; uct_ep_am_bcopy_func_t ep_am_bcopy; uct_ep_am_zcopy_func_t ep_am_zcopy; diff --git a/src/uct/api/uct.h b/src/uct/api/uct.h index 5b6a673b7c0..ece5f190ae6 100644 --- a/src/uct/api/uct.h +++ b/src/uct/api/uct.h @@ -889,8 +889,9 @@ struct uct_iface_attr { } get; /**< Attributes for GET operations */ struct { - size_t max_short; /**< Total max. size (incl. the header) */ - size_t max_bcopy; /**< Total max. size (incl. the header) */ + size_t max_short; /**< Total maximum size (incl. the header) + @anchor uct_iface_attr_cap_am_max_short */ + size_t max_bcopy; /**< Total maximum size (incl. the header) */ size_t min_zcopy; /**< Minimal size for am_zcopy (incl. the header and total of @ref uct_iov_t::length of the @a iov parameter) */ @@ -2639,6 +2640,43 @@ UCT_INLINE_API ucs_status_t uct_ep_am_short(uct_ep_h ep, uint8_t id, uint64_t he } +/** + * @ingroup UCT_AM + * @brief Short io-vector send operation. + * + * This routine sends a message using @ref uct_short_protocol_desc "short" protocol. + * The input data in @a iov array of @ref ::uct_iov_t structures is sent to remote + * side to contiguous buffer keeping the order of the data in the array. + * + * @param [in] ep Destination endpoint handle. + * @param [in] id Active message id. Must be in range 0..UCT_AM_ID_MAX-1. + * @param [in] iov Points to an array of @ref ::uct_iov_t structures. + * The @a iov pointer must be a valid address of an array + * of @ref ::uct_iov_t structures. A particular structure + * pointer must be a valid address. A NULL terminated + * array is not required. @a stride and @a count fields in + * @ref ::uct_iov_t structure are ignored in current + * implementation. The total size of the data buffers in + * the array is limited by + * @ref uct_iface_attr_cap_am_max_short + * "uct_iface_attr::cap::am::max_short". + * @param [in] iovcnt Size of the @a iov data @ref ::uct_iov_t structures + * array. If @a iovcnt is zero, the data is considered empty. + * @a iovcnt is limited by @ref uct_iface_attr_cap_am_max_iov + * "uct_iface_attr::cap::am::max_iov". + * + * @return UCS_OK Operation completed successfully. + * @return UCS_ERR_NO_RESOURCE Could not start the operation due to lack of + * send resources. + * @return otherwise Error code. + */ +UCT_INLINE_API ucs_status_t uct_ep_am_short_iov(uct_ep_h ep, uint8_t id, + const uct_iov_t *iov, size_t iovcnt) +{ + return ep->iface->ops.ep_am_short_iov(ep, id, iov, iovcnt); +} + + /** * @ingroup UCT_AM * @brief diff --git a/src/uct/base/uct_iface.c b/src/uct/base/uct_iface.c index abe1a826a93..c9ae04c25dc 100644 --- a/src/uct/base/uct_iface.c +++ b/src/uct/base/uct_iface.c @@ -11,6 +11,7 @@ #include "uct_iface.h" #include "uct_cm.h" +#include "uct_iov.inl" #include #include @@ -549,3 +550,42 @@ ucs_config_field_t uct_iface_config_table[] = { {NULL} }; + +ucs_status_t uct_base_ep_am_short_iov(uct_ep_h ep, uint8_t id, const uct_iov_t *iov, + size_t iovcnt) +{ + uint64_t header = 0; + size_t length; + void *buffer; + size_t iov_it; + size_t offset; + ucs_status_t status; + + length = uct_iov_total_length(iov, iovcnt); + if (length > UCS_ALLOCA_MAX_SIZE) { + buffer = ucs_malloc(length, "uct_base_ep_am_short_iov buffer"); + } else { + buffer = ucs_alloca(length); + } + + for (iov_it = 0, offset = 0; iov_it < iovcnt; ++iov_it) { + memcpy(UCS_PTR_BYTE_OFFSET(buffer, offset), iov[iov_it].buffer, + iov[iov_it].length); + offset += iov[iov_it].length; + } + + /* There is uint64_t header in the parameter list of uct_ep_am_short. Thus the + * minmum message size is 8b. If the total length of iov is less than 8b, the + * remainder of the header is filled with zeros. */ + offset = ucs_min(sizeof(header), length); + memcpy(&header, buffer, offset); + + status = uct_ep_am_short(ep, id, header, UCS_PTR_BYTE_OFFSET(buffer, offset), + length - offset); + + if (length > UCS_ALLOCA_MAX_SIZE) { + ucs_free(buffer); + } + + return status; +} diff --git a/src/uct/base/uct_iface.h b/src/uct/base/uct_iface.h index 6c2753c96b7..d18c3e5a709 100644 --- a/src/uct/base/uct_iface.h +++ b/src/uct/base/uct_iface.h @@ -708,4 +708,7 @@ void uct_am_short_fill_data(void *buffer, uint64_t header, const void *payload, memcpy(packet->payload, payload, length); } +ucs_status_t uct_base_ep_am_short_iov(uct_ep_h ep, uint8_t id, const uct_iov_t *iov, + size_t iovcnt); + #endif diff --git a/src/uct/ib/dc/dc_mlx5.c b/src/uct/ib/dc/dc_mlx5.c index eed7997c1ca..4d2ccfb5ee1 100644 --- a/src/uct/ib/dc/dc_mlx5.c +++ b/src/uct/ib/dc/dc_mlx5.c @@ -1084,6 +1084,7 @@ static uct_rc_iface_ops_t uct_dc_mlx5_iface_ops = { .ep_get_bcopy = uct_dc_mlx5_ep_get_bcopy, .ep_get_zcopy = uct_dc_mlx5_ep_get_zcopy, .ep_am_short = uct_dc_mlx5_ep_am_short, + .ep_am_short_iov = uct_base_ep_am_short_iov, .ep_am_bcopy = uct_dc_mlx5_ep_am_bcopy, .ep_am_zcopy = uct_dc_mlx5_ep_am_zcopy, .ep_atomic_cswap64 = uct_dc_mlx5_ep_atomic_cswap64, diff --git a/src/uct/ib/rc/accel/rc_mlx5_iface.c b/src/uct/ib/rc/accel/rc_mlx5_iface.c index ee3fccf0642..8d2f91885a2 100644 --- a/src/uct/ib/rc/accel/rc_mlx5_iface.c +++ b/src/uct/ib/rc/accel/rc_mlx5_iface.c @@ -764,6 +764,7 @@ static uct_rc_iface_ops_t uct_rc_mlx5_iface_ops = { .ep_get_bcopy = uct_rc_mlx5_ep_get_bcopy, .ep_get_zcopy = uct_rc_mlx5_ep_get_zcopy, .ep_am_short = uct_rc_mlx5_ep_am_short, + .ep_am_short_iov = uct_base_ep_am_short_iov, .ep_am_bcopy = uct_rc_mlx5_ep_am_bcopy, .ep_am_zcopy = uct_rc_mlx5_ep_am_zcopy, .ep_atomic_cswap64 = uct_rc_mlx5_ep_atomic_cswap64, diff --git a/src/uct/ib/rc/verbs/rc_verbs_iface.c b/src/uct/ib/rc/verbs/rc_verbs_iface.c index 7b155956f89..c2e2ebaf3a7 100644 --- a/src/uct/ib/rc/verbs/rc_verbs_iface.c +++ b/src/uct/ib/rc/verbs/rc_verbs_iface.c @@ -456,6 +456,7 @@ static uct_rc_iface_ops_t uct_rc_verbs_iface_ops = { { { .ep_am_short = uct_rc_verbs_ep_am_short, + .ep_am_short_iov = uct_base_ep_am_short_iov, .ep_am_bcopy = uct_rc_verbs_ep_am_bcopy, .ep_am_zcopy = uct_rc_verbs_ep_am_zcopy, .ep_put_short = uct_rc_verbs_ep_put_short, diff --git a/src/uct/ib/rdmacm/rdmacm_cm.c b/src/uct/ib/rdmacm/rdmacm_cm.c index e0d284713a1..0318c73e2ec 100644 --- a/src/uct/ib/rdmacm/rdmacm_cm.c +++ b/src/uct/ib/rdmacm/rdmacm_cm.c @@ -548,6 +548,7 @@ static uct_iface_ops_t uct_rdmacm_cm_iface_ops = { .ep_put_bcopy = (uct_ep_put_bcopy_func_t)ucs_empty_function_return_unsupported, .ep_get_bcopy = (uct_ep_get_bcopy_func_t)ucs_empty_function_return_unsupported, .ep_am_short = (uct_ep_am_short_func_t)ucs_empty_function_return_unsupported, + .ep_am_short_iov = (uct_ep_am_short_iov_func_t)ucs_empty_function_return_unsupported, .ep_am_bcopy = (uct_ep_am_bcopy_func_t)ucs_empty_function_return_unsupported, .ep_atomic_cswap64 = (uct_ep_atomic_cswap64_func_t)ucs_empty_function_return_unsupported, .ep_atomic64_post = (uct_ep_atomic64_post_func_t)ucs_empty_function_return_unsupported, diff --git a/src/uct/ib/ud/accel/ud_mlx5.c b/src/uct/ib/ud/accel/ud_mlx5.c index b8190c09af1..0f2d85b2232 100644 --- a/src/uct/ib/ud/accel/ud_mlx5.c +++ b/src/uct/ib/ud/accel/ud_mlx5.c @@ -691,6 +691,7 @@ static uct_ud_iface_ops_t uct_ud_mlx5_iface_ops = { { .ep_put_short = uct_ud_mlx5_ep_put_short, .ep_am_short = uct_ud_mlx5_ep_am_short, + .ep_am_short_iov = uct_base_ep_am_short_iov, .ep_am_bcopy = uct_ud_mlx5_ep_am_bcopy, .ep_am_zcopy = uct_ud_mlx5_ep_am_zcopy, .ep_pending_add = uct_ud_ep_pending_add, diff --git a/src/uct/ib/ud/verbs/ud_verbs.c b/src/uct/ib/ud/verbs/ud_verbs.c index 3aeeee0f552..41234d35987 100644 --- a/src/uct/ib/ud/verbs/ud_verbs.c +++ b/src/uct/ib/ud/verbs/ud_verbs.c @@ -519,6 +519,7 @@ static uct_ud_iface_ops_t uct_ud_verbs_iface_ops = { { .ep_put_short = uct_ud_verbs_ep_put_short, .ep_am_short = uct_ud_verbs_ep_am_short, + .ep_am_short_iov = uct_base_ep_am_short_iov, .ep_am_bcopy = uct_ud_verbs_ep_am_bcopy, .ep_am_zcopy = uct_ud_verbs_ep_am_zcopy, .ep_pending_add = uct_ud_ep_pending_add, diff --git a/src/uct/sm/mm/base/mm_iface.c b/src/uct/sm/mm/base/mm_iface.c index 7654dea4afc..af7110f2375 100644 --- a/src/uct/sm/mm/base/mm_iface.c +++ b/src/uct/sm/mm/base/mm_iface.c @@ -400,6 +400,7 @@ static uct_iface_ops_t uct_mm_iface_ops = { .ep_put_bcopy = uct_sm_ep_put_bcopy, .ep_get_bcopy = uct_sm_ep_get_bcopy, .ep_am_short = uct_mm_ep_am_short, + .ep_am_short_iov = uct_base_ep_am_short_iov, .ep_am_bcopy = uct_mm_ep_am_bcopy, .ep_atomic_cswap64 = uct_sm_ep_atomic_cswap64, .ep_atomic64_post = uct_sm_ep_atomic64_post, diff --git a/src/uct/sm/self/self.c b/src/uct/sm/self/self.c index eae999f6196..c835c534317 100644 --- a/src/uct/sm/self/self.c +++ b/src/uct/sm/self/self.c @@ -290,6 +290,7 @@ static uct_iface_ops_t uct_self_iface_ops = { .ep_put_bcopy = uct_sm_ep_put_bcopy, .ep_get_bcopy = uct_sm_ep_get_bcopy, .ep_am_short = uct_self_ep_am_short, + .ep_am_short_iov = uct_base_ep_am_short_iov, .ep_am_bcopy = uct_self_ep_am_bcopy, .ep_atomic_cswap64 = uct_sm_ep_atomic_cswap64, .ep_atomic64_post = uct_sm_ep_atomic64_post, diff --git a/src/uct/tcp/tcp_iface.c b/src/uct/tcp/tcp_iface.c index 89bd1c6655f..0447873da49 100644 --- a/src/uct/tcp/tcp_iface.c +++ b/src/uct/tcp/tcp_iface.c @@ -336,6 +336,7 @@ ucs_status_t uct_tcp_iface_set_sockopt(uct_tcp_iface_t *iface, int fd, static uct_iface_ops_t uct_tcp_iface_ops = { .ep_am_short = uct_tcp_ep_am_short, + .ep_am_short_iov = uct_base_ep_am_short_iov, .ep_am_bcopy = uct_tcp_ep_am_bcopy, .ep_am_zcopy = uct_tcp_ep_am_zcopy, .ep_put_zcopy = uct_tcp_ep_put_zcopy, diff --git a/src/uct/tcp/tcp_sockcm.c b/src/uct/tcp/tcp_sockcm.c index 70a9a00bfe6..cc5026b50e3 100644 --- a/src/uct/tcp/tcp_sockcm.c +++ b/src/uct/tcp/tcp_sockcm.c @@ -141,6 +141,7 @@ static uct_iface_ops_t uct_tcp_sockcm_iface_ops = { .ep_put_bcopy = (uct_ep_put_bcopy_func_t)ucs_empty_function_return_unsupported, .ep_get_bcopy = (uct_ep_get_bcopy_func_t)ucs_empty_function_return_unsupported, .ep_am_short = (uct_ep_am_short_func_t)ucs_empty_function_return_unsupported, + .ep_am_short_iov = (uct_ep_am_short_iov_func_t)ucs_empty_function_return_unsupported, .ep_am_bcopy = (uct_ep_am_bcopy_func_t)ucs_empty_function_return_unsupported, .ep_atomic_cswap64 = (uct_ep_atomic_cswap64_func_t)ucs_empty_function_return_unsupported, .ep_atomic64_post = (uct_ep_atomic64_post_func_t)ucs_empty_function_return_unsupported, diff --git a/src/uct/ugni/rdma/ugni_rdma_iface.c b/src/uct/ugni/rdma/ugni_rdma_iface.c index 69e1ebac557..57775aef812 100644 --- a/src/uct/ugni/rdma/ugni_rdma_iface.c +++ b/src/uct/ugni/rdma/ugni_rdma_iface.c @@ -186,6 +186,7 @@ static uct_iface_ops_t uct_ugni_aries_rdma_iface_ops = { .ep_get_bcopy = uct_ugni_ep_get_bcopy, .ep_get_zcopy = uct_ugni_ep_get_zcopy, .ep_am_short = uct_ugni_ep_am_short, + .ep_am_short_iov = uct_base_ep_am_short_iov, .ep_atomic_cswap64 = uct_ugni_ep_atomic_cswap64, .ep_atomic_cswap32 = uct_ugni_ep_atomic_cswap32, .ep_atomic64_post = uct_ugni_ep_atomic64_post, @@ -217,6 +218,7 @@ static uct_iface_ops_t uct_ugni_gemini_rdma_iface_ops = { .ep_get_bcopy = uct_ugni_ep_get_bcopy, .ep_get_zcopy = uct_ugni_ep_get_zcopy, .ep_am_short = uct_ugni_ep_am_short, + .ep_am_short_iov = uct_base_ep_am_short_iov, .ep_atomic_cswap64 = uct_ugni_ep_atomic_cswap64, .ep_pending_add = uct_ugni_ep_pending_add, .ep_pending_purge = uct_ugni_ep_pending_purge, diff --git a/src/uct/ugni/smsg/ugni_smsg_iface.c b/src/uct/ugni/smsg/ugni_smsg_iface.c index 2e658e445b6..a9ed245e568 100644 --- a/src/uct/ugni/smsg/ugni_smsg_iface.c +++ b/src/uct/ugni/smsg/ugni_smsg_iface.c @@ -229,6 +229,7 @@ static UCS_CLASS_CLEANUP_FUNC(uct_ugni_smsg_iface_t) static uct_iface_ops_t uct_ugni_smsg_iface_ops = { .ep_am_short = uct_ugni_smsg_ep_am_short, + .ep_am_short_iov = uct_base_ep_am_short_iov, .ep_am_bcopy = uct_ugni_smsg_ep_am_bcopy, .ep_pending_add = uct_ugni_ep_pending_add, .ep_pending_purge = uct_ugni_ep_pending_purge, diff --git a/src/uct/ugni/udt/ugni_udt_iface.c b/src/uct/ugni/udt/ugni_udt_iface.c index 7fde0b38ee7..5a101eacda7 100644 --- a/src/uct/ugni/udt/ugni_udt_iface.c +++ b/src/uct/ugni/udt/ugni_udt_iface.c @@ -355,6 +355,7 @@ static UCS_CLASS_DEFINE_DELETE_FUNC(uct_ugni_udt_iface_t, uct_iface_t); static uct_iface_ops_t uct_ugni_udt_iface_ops = { .ep_am_short = uct_ugni_udt_ep_am_short, + .ep_am_short_iov = uct_base_ep_am_short_iov, .ep_am_bcopy = uct_ugni_udt_ep_am_bcopy, .ep_pending_add = uct_ugni_udt_ep_pending_add, .ep_pending_purge = uct_ugni_udt_ep_pending_purge, diff --git a/test/gtest/uct/test_p2p_am.cc b/test/gtest/uct/test_p2p_am.cc index 303b3b6a78a..4a963e095fe 100644 --- a/test/gtest/uct/test_p2p_am.cc +++ b/test/gtest/uct/test_p2p_am.cc @@ -182,6 +182,15 @@ class uct_p2p_am_test : public uct_p2p_test sendbuf.length() - sizeof(hdr)); } + ucs_status_t am_short_iov(uct_ep_h ep, const mapped_buffer &sendbuf, + const mapped_buffer &recvbuf) + { + UCS_TEST_GET_BUFFER_IOV(iov, iovcnt, (char *)sendbuf.ptr(), sendbuf.length(), + sendbuf.memh(), sender().iface_attr().cap.am.max_iov); + + return uct_ep_am_short_iov(ep, AM_ID, iov, iovcnt); + } + ucs_status_t am_bcopy(uct_ep_h ep, const mapped_buffer& sendbuf, const mapped_buffer& recvbuf) { @@ -337,6 +346,11 @@ UCS_TEST_SKIP_COND_P(uct_p2p_am_test, am_sync, blocking_send(static_cast(&uct_p2p_am_test::am_short), sender_ep(), sendbuf_short, recvbuf, false); am_sync_finish(am_count); + + am_count = m_am_count; + blocking_send(static_cast(&uct_p2p_am_test::am_short_iov), + sender_ep(), sendbuf_short, recvbuf, false); + am_sync_finish(am_count); } if (receiver().iface_attr().cap.flags & UCT_IFACE_FLAG_AM_BCOPY) { @@ -380,6 +394,11 @@ UCS_TEST_SKIP_COND_P(uct_p2p_am_test, am_async, blocking_send(static_cast(&uct_p2p_am_test::am_short), sender_ep(), sendbuf_short, recvbuf, false); am_async_finish(am_count); + + am_count = m_am_count; + blocking_send(static_cast(&uct_p2p_am_test::am_short_iov), + sender_ep(), sendbuf_short, recvbuf, false); + am_async_finish(am_count); } if (receiver().iface_attr().cap.flags & UCT_IFACE_FLAG_AM_BCOPY) { @@ -511,6 +530,42 @@ class uct_p2p_am_misc : public uct_p2p_am_test return UCS_LOG_FUNC_RC_CONTINUE; } + void am_max_multi(send_func_t send) + { + ucs_status_t status; + + mapped_buffer small_sendbuf(sizeof(SEED1), SEED1, sender()); + mapped_buffer sendbuf(ucs_min(sender().iface_attr().cap.am.max_short, 8192ul), + SEED1, sender()); + mapped_buffer recvbuf(0, 0, sender()); /* dummy */ + + m_am_count = 0; + set_keep_data(false); + + status = uct_iface_set_am_handler(receiver().iface(), AM_ID, am_handler, + this, UCT_CB_FLAG_ASYNC); + ASSERT_UCS_OK(status); + + /* exhaust all resources or time out 1sec */ + ucs_time_t loop_end_limit = ucs_get_time() + ucs_time_from_sec(1.0); + do { + status = (this->*send)(sender_ep(), sendbuf, recvbuf); + } while ((ucs_get_time() < loop_end_limit) && (status == UCS_OK)); + if (status != UCS_ERR_NO_RESOURCE) { + ASSERT_UCS_OK(status); + } + + /* should be able to send again after a while */ + ucs_time_t deadline = ucs_get_time() + + (ucs::test_time_multiplier() * + ucs_time_from_sec(DEFAULT_TIMEOUT_SEC)); + do { + progress(); + status = (this->*send)(sender_ep(), small_sendbuf, recvbuf); + } while ((status == UCS_ERR_NO_RESOURCE) && (ucs_get_time() < deadline)); + EXPECT_EQ(UCS_OK, status); + } + bool m_rx_buf_limit_failed; }; @@ -533,6 +588,15 @@ UCS_TEST_SKIP_COND_P(uct_p2p_am_test, am_short_keep_data, TEST_UCT_FLAG_DIR_SEND_TO_RECV); } +UCS_TEST_SKIP_COND_P(uct_p2p_am_test, am_short_iov_keep_data, + !check_caps(UCT_IFACE_FLAG_AM_SHORT, UCT_IFACE_FLAG_AM_DUP)) +{ + set_keep_data(true); + test_xfer_multi(static_cast(&uct_p2p_am_test::am_short_iov), + sizeof(uint64_t), sender().iface_attr().cap.am.max_short, + TEST_UCT_FLAG_DIR_SEND_TO_RECV); +} + UCS_TEST_SKIP_COND_P(uct_p2p_am_test, am_bcopy_keep_data, !check_caps(UCT_IFACE_FLAG_AM_BCOPY, UCT_IFACE_FLAG_AM_DUP)) { @@ -600,41 +664,15 @@ UCS_TEST_SKIP_COND_P(uct_p2p_am_misc, no_rx_buffs, } UCS_TEST_SKIP_COND_P(uct_p2p_am_misc, am_max_short_multi, - !check_caps(UCT_IFACE_FLAG_AM_SHORT)) { - ucs_status_t status; - - m_am_count = 0; - set_keep_data(false); - - status = uct_iface_set_am_handler(receiver().iface(), AM_ID, am_handler, - this, UCT_CB_FLAG_ASYNC); - ASSERT_UCS_OK(status); - - size_t size = ucs_min(sender().iface_attr().cap.am.max_short, 8192ul); - std::string sendbuf(size, 0); - mem_buffer::pattern_fill(&sendbuf[0], sendbuf.size(), SEED1); - ucs_assert(SEED1 == *(uint64_t*)&sendbuf[0]); - - /* exhaust all resources or time out 1sec */ - ucs_time_t loop_end_limit = ucs_get_time() + ucs_time_from_sec(1.0); - do { - status = uct_ep_am_short(sender_ep(), AM_ID, SEED1, - ((uint64_t*)&sendbuf[0]) + 1, - sendbuf.size() - sizeof(uint64_t)); - } while ((ucs_get_time() < loop_end_limit) && (status == UCS_OK)); - if (status != UCS_ERR_NO_RESOURCE) { - ASSERT_UCS_OK(status); - } + !check_caps(UCT_IFACE_FLAG_AM_SHORT)) +{ + am_max_multi(static_cast(&uct_p2p_am_test::am_short)); +} - /* should be able to send again after a while */ - ucs_time_t deadline = ucs_get_time() + - (ucs::test_time_multiplier() * - ucs_time_from_sec(DEFAULT_TIMEOUT_SEC)); - do { - progress(); - status = uct_ep_am_short(sender_ep(), AM_ID, SEED1, NULL, 0); - } while ((status == UCS_ERR_NO_RESOURCE) && (ucs_get_time() < deadline)); - EXPECT_EQ(UCS_OK, status); +UCS_TEST_SKIP_COND_P(uct_p2p_am_misc, am_max_short_iov_multi, + !check_caps(UCT_IFACE_FLAG_AM_SHORT)) +{ + am_max_multi(static_cast(&uct_p2p_am_test::am_short_iov)); } UCT_INSTANTIATE_TEST_CASE(uct_p2p_am_misc) diff --git a/test/gtest/uct/test_p2p_err.cc b/test/gtest/uct/test_p2p_err.cc index e1d9e7f8ad6..362422ea7d3 100644 --- a/test/gtest/uct/test_p2p_err.cc +++ b/test/gtest/uct/test_p2p_err.cc @@ -16,6 +16,7 @@ class uct_p2p_err_test : public uct_p2p_test { OP_PUT_BCOPY, OP_PUT_ZCOPY, OP_AM_SHORT, + OP_AM_SHORT_IOV, OP_AM_BCOPY, OP_AM_ZCOPY }; @@ -71,6 +72,11 @@ class uct_p2p_err_test : public uct_p2p_test { case OP_AM_SHORT: status = uct_ep_am_short(sender_ep(), am_id, 0, buffer, length); break; + case OP_AM_SHORT_IOV: { + UCS_TEST_GET_BUFFER_IOV(iov, iovcnt, buffer, length, memh, 1); + status = uct_ep_am_short_iov(sender_ep(), am_id, iov, iovcnt); + break; + } case OP_AM_BCOPY: arg.buffer = buffer; arg.length = length; @@ -231,6 +237,23 @@ UCS_TEST_SKIP_COND_P(uct_p2p_err_test, invalid_am_short_length, recvbuf.pattern_check(2); } +UCS_TEST_SKIP_COND_P(uct_p2p_err_test, invalid_am_short_iov_length, + !check_caps(UCT_IFACE_FLAG_AM_SHORT)) +{ + size_t max_short = sender().iface_attr().cap.am.max_short; + if (max_short > (2 * UCS_MBYTE)) { + UCS_TEST_SKIP_R("max_short too large"); + } + + mapped_buffer sendbuf(max_short + 1, 1, sender()); + mapped_buffer recvbuf(max_short + 1, 2, receiver()); + + test_error_run(OP_AM_SHORT_IOV, 0, sendbuf.ptr(), sendbuf.length(), + UCT_MEM_HANDLE_NULL, recvbuf.addr(), recvbuf.rkey(), "length"); + + recvbuf.pattern_check(2); +} + UCS_TEST_SKIP_COND_P(uct_p2p_err_test, invalid_am_bcopy_length, !check_caps(UCT_IFACE_FLAG_AM_BCOPY | UCT_IFACE_FLAG_ERRHANDLE_BCOPY_LEN)) { @@ -279,6 +302,15 @@ UCS_TEST_SKIP_COND_P(uct_p2p_err_test, short_invalid_am_id, "active message id"); } +UCS_TEST_SKIP_COND_P(uct_p2p_err_test, short_iov_invalid_am_id, + !check_caps(UCT_IFACE_FLAG_AM_SHORT)) +{ + mapped_buffer sendbuf(4, 2, sender()); + + test_error_run(OP_AM_SHORT_IOV, UCT_AM_ID_MAX, sendbuf.ptr(), sendbuf.length(), + UCT_MEM_HANDLE_NULL, 0, UCT_INVALID_RKEY, "active message id"); +} + UCS_TEST_SKIP_COND_P(uct_p2p_err_test, bcopy_invalid_am_id, !check_caps(UCT_IFACE_FLAG_AM_BCOPY)) { mapped_buffer sendbuf(4, 2, sender()); diff --git a/test/gtest/uct/test_p2p_mix.cc b/test/gtest/uct/test_p2p_mix.cc index 4d6266e3fc3..2d1f84f02a0 100644 --- a/test/gtest/uct/test_p2p_mix.cc +++ b/test/gtest/uct/test_p2p_mix.cc @@ -88,6 +88,26 @@ ucs_status_t uct_p2p_mix_test::am_short(const mapped_buffer &sendbuf, return status; } +ucs_status_t uct_p2p_mix_test::am_short_iov(const mapped_buffer &sendbuf, + const mapped_buffer &recvbuf, + uct_completion_t *comp) +{ + ucs_status_t status; + uct_iov_t iov; + + iov.buffer = sendbuf.ptr(); + iov.length = sendbuf.length(); + iov.count = 1; + iov.stride = 0; + iov.memh = sendbuf.memh(); + + status = uct_ep_am_short_iov(sender().ep(0), AM_ID, &iov, 1); + if (status == UCS_OK) { + ucs_atomic_add32(&am_pending, +1); + } + return status; +} + ucs_status_t uct_p2p_mix_test::am_zcopy(const mapped_buffer &sendbuf, const mapped_buffer &recvbuf, uct_completion_t *comp) @@ -170,6 +190,9 @@ void uct_p2p_mix_test::init() { if (sender().iface_attr().cap.flags & UCT_IFACE_FLAG_AM_SHORT) { m_avail_send_funcs.push_back(&uct_p2p_mix_test::am_short); m_send_size = ucs_min(m_send_size, sender().iface_attr().cap.am.max_short); + + m_avail_send_funcs.push_back(&uct_p2p_mix_test::am_short_iov); + m_send_size = ucs_min(m_send_size, sender().iface_attr().cap.am.max_short); } if (sender().iface_attr().cap.flags & UCT_IFACE_FLAG_AM_ZCOPY) { m_avail_send_funcs.push_back(&uct_p2p_mix_test::am_zcopy); diff --git a/test/gtest/uct/test_p2p_mix.h b/test/gtest/uct/test_p2p_mix.h index 51d9bdc834e..24be23ae9a4 100644 --- a/test/gtest/uct/test_p2p_mix.h +++ b/test/gtest/uct/test_p2p_mix.h @@ -49,6 +49,9 @@ class uct_p2p_mix_test : public uct_p2p_test { const mapped_buffer &recvbuf, uct_completion_t *comp); + ucs_status_t am_short_iov(const mapped_buffer &sendbuf, + const mapped_buffer &recvbuf, uct_completion_t *comp); + ucs_status_t am_zcopy(const mapped_buffer &sendbuf, const mapped_buffer &recvbuf, uct_completion_t *comp); diff --git a/test/gtest/uct/test_stats.cc b/test/gtest/uct/test_stats.cc index eda38295296..e7f431801d1 100644 --- a/test/gtest/uct/test_stats.cc +++ b/test/gtest/uct/test_stats.cc @@ -220,6 +220,29 @@ UCS_TEST_SKIP_COND_P(test_uct_stats, am_short, check_am_rx_counters(sizeof(hdr) + sizeof(send_data)); } +UCS_TEST_SKIP_COND_P(test_uct_stats, am_short_iov, + !check_caps(UCT_IFACE_FLAG_AM_SHORT)) +{ + ucs_status_t status; + + init_bufs(0, sender().iface_attr().cap.am.max_short); + + status = uct_iface_set_am_handler(receiver().iface(), 0, am_handler, 0, + UCT_CB_FLAG_ASYNC); + EXPECT_UCS_OK(status); + + UCS_TEST_GET_BUFFER_IOV(iov, iovcnt, lbuf->ptr(), lbuf->length(), lbuf->memh(), + sender().iface_attr().cap.am.max_iov); + + UCT_TEST_CALL_AND_TRY_AGAIN(uct_ep_am_short_iov(sender_ep(), 0, iov, iovcnt), + status); + EXPECT_UCS_OK(status); + + EXPECT_STAT(sender, uct_ep, UCT_EP_STAT_AM, 1UL); + EXPECT_STAT(sender, uct_ep, UCT_EP_STAT_BYTES_SHORT, lbuf->length()); + check_am_rx_counters(lbuf->length()); +} + UCS_TEST_SKIP_COND_P(test_uct_stats, am_bcopy, !check_caps(UCT_IFACE_FLAG_AM_BCOPY)) { From 92d6b2a986469729abd21e0dbe43e871ac8419a0 Mon Sep 17 00:00:00 2001 From: dmitrygx Date: Wed, 16 Dec 2020 11:02:25 +0200 Subject: [PATCH 054/107] UCT/IB: Fixes for SL selection --- src/uct/ib/base/ib_iface.c | 14 +-- src/uct/ib/base/ib_iface.h | 3 +- src/uct/ib/mlx5/ib_mlx5.c | 13 ++- test/gtest/uct/ib/test_ib.cc | 161 ++++++++++++++++++++++++++++------- 4 files changed, 150 insertions(+), 41 deletions(-) diff --git a/src/uct/ib/base/ib_iface.c b/src/uct/ib/base/ib_iface.c index f998a70184a..cd56f23934f 100644 --- a/src/uct/ib/base/ib_iface.c +++ b/src/uct/ib/base/ib_iface.c @@ -647,7 +647,7 @@ void uct_ib_iface_fill_ah_attr_from_gid_lid(uct_ib_iface_t *iface, uint16_t lid, memset(ah_attr, 0, sizeof(*ah_attr)); - ucs_assert(iface->config.sl != UCT_IB_SL_INVALID); + ucs_assert(iface->config.sl < UCT_IB_SL_NUM); ah_attr->sl = iface->config.sl; ah_attr->port_num = iface->config.port_num; @@ -1179,9 +1179,12 @@ static void uct_ib_iface_set_path_mtu(uct_ib_iface_t *iface, uint8_t uct_ib_iface_config_select_sl(const uct_ib_iface_config_t *ib_config) { - ucs_assert((ib_config->sl <= UCT_IB_SL_MAX) || - (ib_config->sl == UCS_ULUNITS_AUTO)); - return (ib_config->sl == UCS_ULUNITS_AUTO) ? 0 : (uint8_t)ib_config->sl; + if (ib_config->sl == UCS_ULUNITS_AUTO) { + return 0; + } + + ucs_assert(ib_config->sl < UCT_IB_SL_NUM); + return (uint8_t)ib_config->sl; } UCS_CLASS_INIT_FUNC(uct_ib_iface_t, uct_ib_iface_ops_t *ops, uct_md_h md, @@ -1244,7 +1247,8 @@ UCS_CLASS_INIT_FUNC(uct_ib_iface_t, uct_ib_iface_ops_t *ops, uct_md_h md, self->config.rx_max_batch = ucs_min(config->rx.max_batch, config->rx.queue_len / 4); self->config.port_num = port_num; - self->config.sl = UCT_IB_SL_INVALID; + /* initialize to invalid value */ + self->config.sl = UCT_IB_SL_NUM; self->config.hop_limit = config->hop_limit; self->release_desc.cb = uct_ib_iface_release_desc; self->config.enable_res_domain = config->enable_res_domain; diff --git a/src/uct/ib/base/ib_iface.h b/src/uct/ib/base/ib_iface.h index c778c090d24..22dbe10adbc 100644 --- a/src/uct/ib/base/ib_iface.h +++ b/src/uct/ib/base/ib_iface.h @@ -26,8 +26,7 @@ #define UCT_IB_ADDRESS_INVALID_PATH_MTU ((enum ibv_mtu)0) #define UCT_IB_ADDRESS_INVALID_PKEY 0 #define UCT_IB_ADDRESS_DEFAULT_PKEY 0xffff -#define UCT_IB_SL_MAX 15 -#define UCT_IB_SL_INVALID (UCT_IB_SL_MAX + 1) +#define UCT_IB_SL_NUM 16 /* Forward declarations */ typedef struct uct_ib_iface_config uct_ib_iface_config_t; diff --git a/src/uct/ib/mlx5/ib_mlx5.c b/src/uct/ib/mlx5/ib_mlx5.c index 5081aaf68e9..bf4a9cce14a 100644 --- a/src/uct/ib/mlx5/ib_mlx5.c +++ b/src/uct/ib/mlx5/ib_mlx5.c @@ -736,7 +736,7 @@ uct_ib_mlx5_select_sl(const uct_ib_iface_config_t *ib_config, /* which SLs are allowed by user config */ sl_allow_mask = (ib_config->sl == UCS_ULUNITS_AUTO) ? - UCS_MASK(UCT_IB_SL_MAX) : UCS_BIT(ib_config->sl); + UCS_MASK(UCT_IB_SL_NUM) : UCS_BIT(ib_config->sl); if (have_sl_mask_cap) { sls_with_ar = sl_allow_mask & hw_sl_mask; @@ -815,7 +815,16 @@ uct_ib_mlx5_iface_select_sl(uct_ib_iface_t *iface, uint16_t ooo_sl_mask = 0; ucs_status_t status; - ucs_assert(iface->config.sl == UCT_IB_SL_INVALID); + ucs_assert(iface->config.sl == UCT_IB_SL_NUM); + + if (uct_ib_device_is_port_roce(uct_ib_iface_device(iface), + iface->config.port_num)) { + /* Ethernet priority for RoCE devices can't be selected regardless + * AR support requested by user, pass empty ooo_sl_mask */ + return uct_ib_mlx5_select_sl(ib_config, UCS_NO, 0, 1, + UCT_IB_IFACE_ARG(iface), + &iface->config.sl); + } #if HAVE_DEVX status = uct_ib_mlx5_devx_query_ooo_sl_mask(md, iface->config.port_num, diff --git a/test/gtest/uct/ib/test_ib.cc b/test/gtest/uct/ib/test_ib.cc index e83fb55a09f..801b8ebf563 100644 --- a/test/gtest/uct/ib/test_ib.cc +++ b/test/gtest/uct/ib/test_ib.cc @@ -376,6 +376,56 @@ UCS_TEST_P(test_uct_ib_gid_idx, non_default_gid_idx, "GID_INDEX=1") { UCT_INSTANTIATE_IB_TEST_CASE(test_uct_ib_gid_idx); +#if HAVE_DEVX +class test_uct_ib_sl : public test_uct_ib_with_specific_port { +public: + void check_port_attr() { + ucs_status_t status; + ucs::handle uct_md; + + UCS_TEST_CREATE_HANDLE(uct_md_h, uct_md, uct_ib_md_close, uct_ib_md_open, + &uct_ib_component, + ibv_get_device_name(m_ibctx->device), + m_md_config); + + uct_ib_mlx5_md_t *ib_md = ucs_derived_of(uct_md, uct_ib_mlx5_md_t); + + /* check if OOO SL mask can be detected for the port */ + status = uct_ib_mlx5_devx_query_ooo_sl_mask(ib_md, m_port, + &m_ooo_sl_mask); + EXPECT_TRUE((status == UCS_OK) || (status == UCS_ERR_UNSUPPORTED)); + if (status != UCS_OK) { + UCS_TEST_SKIP_R("OOO SL mask couldn't be detected for " + + m_dev_name + ":" + ucs::to_string(m_port)); + } + } + +protected: + uint16_t m_ooo_sl_mask; +}; + +UCS_TEST_P(test_uct_ib_sl, check_ib_sl_config) { + // go over all SLs, check UCTs could be initialized on a specific SL + // and able to send/recv traffic + for (uint8_t sl = 0; sl < UCT_IB_SL_NUM; ++sl) { + if (!has_transport("rc_verbs") && !has_transport("ud_verbs")) { + // if AR is configured on the given SL, set AR_ENABLE to "y", + // otherwise - to "n" in order to test that AR_ENABLE parameter + // works as expected w/o errors and warnings + modify_config("IB_AR_ENABLE", + (m_ooo_sl_mask & UCS_BIT(sl)) ? "y" : "n"); + } + modify_config("IB_SL", ucs::to_string(static_cast(sl))); + + test_uct_ib::init(); + send_recv_short(); + test_uct_ib::cleanup(); + } +} + +UCT_INSTANTIATE_IB_TEST_CASE(test_uct_ib_sl); +#endif + class test_uct_ib_utils : public ucs::test { }; @@ -519,10 +569,39 @@ class test_uct_ib_sl_utils : public test_uct_ib_utils { return status; } + static ucs_log_func_rc_t + wrap_errors_check_sl_masks_logger(const char *file, unsigned line, + const char *function, + ucs_log_level_t level, + const ucs_log_component_config_t * + comp_conf, + const char *message, va_list ap) + { + if (level == UCS_LOG_LEVEL_ERROR) { + std::string err_str = format_message(message, ap); + + for (uint8_t sl = 0; sl < UCT_IB_SL_NUM; ++sl) { + std::string sl_val = ucs::to_string(static_cast(sl)); + + if ((err_str.find(sl_val + ", ") == std::string::npos) && + (err_str.find(sl_val + " }") == std::string::npos)) { + return UCS_LOG_FUNC_RC_CONTINUE; + } + } + + return UCS_LOG_FUNC_RC_STOP; + } + + return UCS_LOG_FUNC_RC_CONTINUE; + } + ucs_status_t select_sl_nok(ucs_ternary_auto_value_t ar_enable, - uint64_t ooo_sl_mask, + unsigned long config_sl, uint64_t ooo_sl_mask, const uct_ib_iface_config_t &config) const { - scoped_log_handler slh(wrap_errors_logger); + scoped_log_handler slh(((ooo_sl_mask != m_ooo_sl_mask_not_detected) && + (config_sl == UCS_ULUNITS_AUTO)) ? + wrap_errors_check_sl_masks_logger : + wrap_errors_logger); uint8_t sl; EXPECT_NE(UCS_AUTO, ar_enable); @@ -540,7 +619,7 @@ class test_uct_ib_sl_utils : public test_uct_ib_utils { if (exp_status == UCS_OK) { status = select_sl_ok(ar_enable, config_sl, ooo_sl_mask, config); } else { - status = select_sl_nok(ar_enable, ooo_sl_mask, config); + status = select_sl_nok(ar_enable, config_sl, ooo_sl_mask, config); } EXPECT_EQ(exp_status, status); } @@ -562,42 +641,60 @@ UCS_TEST_F(test_uct_ib_sl_utils, sl_selection) { ucs_ternary_auto_value_t ar_enable = static_cast(i); - /* select the default SL, with empty OOO SL mask */ + // select the default SL, with empty OOO SL mask select_sl(ar_enable, UCS_ULUNITS_AUTO, 0, (ar_enable == UCS_YES) ? err_status : UCS_OK); - /* select the default SL, without OOO SL mask (not detected) */ + // select the default SL, without OOO SL mask (not detected) select_sl(ar_enable, UCS_ULUNITS_AUTO, m_ooo_sl_mask_not_detected, ((ar_enable != UCS_TRY) && (ar_enable != UCS_AUTO)) ? err_status : UCS_OK); - /* select the default SL, with OOO SL mask { 4 } */ - select_sl(ar_enable, UCS_ULUNITS_AUTO, UCS_BIT(4), UCS_OK); - - /* select SL=8, with empty OOO SL mask */ - select_sl(ar_enable, 8, 0, - (ar_enable == UCS_YES) ? err_status : UCS_OK); - - /* select SL=8, without OOO SL mask (not detected) */ - select_sl(ar_enable, 8, m_ooo_sl_mask_not_detected, - ((ar_enable != UCS_TRY) && (ar_enable != UCS_AUTO)) ? - err_status : UCS_OK); - - /* select SL=8, with OOO SL mask { 8 } */ - select_sl(ar_enable, 8, UCS_BIT(8), - (ar_enable == UCS_NO) ? err_status : UCS_OK); - - /* select SL=8, with OOO SL mask { 0 } */ - select_sl(ar_enable, 8, UCS_BIT(0), - (ar_enable == UCS_YES) ? err_status : UCS_OK); - - /* select SL=8, with OOO SL mask { 0, 4, 8 } */ - select_sl(ar_enable, 8, UCS_BIT(0) | UCS_BIT(4) | UCS_BIT(8), - (ar_enable == UCS_NO) ? err_status : UCS_OK); - - /* select SL=8, with OOO SL mask { 0, 8, 11 } */ - select_sl(ar_enable, 8, UCS_BIT(0) | UCS_BIT(4) | UCS_BIT(11), - (ar_enable == UCS_YES) ? err_status : UCS_OK); + for (uint8_t sl = 0; sl < UCT_IB_SL_NUM; ++sl) { + // select the default SL, with OOO SL mask which contains only + select_sl(ar_enable, UCS_ULUNITS_AUTO, UCS_BIT(sl), UCS_OK); + + // select SL=, with empty OOO SL mask + select_sl(ar_enable, sl, 0, + (ar_enable == UCS_YES) ? err_status : UCS_OK); + + // select SL=, without OOO SL mask (not detected) + select_sl(ar_enable, sl, m_ooo_sl_mask_not_detected, + ((ar_enable != UCS_TRY) && (ar_enable != UCS_AUTO)) ? + err_status : UCS_OK); + + /* select SL=, with OOO SL mask which contains only */ + select_sl(ar_enable, sl, UCS_BIT(sl), + (ar_enable == UCS_NO) ? err_status : UCS_OK); + + // select SL=, with OOO SL mask which doesn't contain , but + // it contains one another element + select_sl(ar_enable, sl, UCS_BIT(UCT_IB_SL_NUM - 1 - sl), + (ar_enable == UCS_YES) ? err_status : UCS_OK); + + // select SL=, with OOO SL mask which contains and two + // other elements + select_sl(ar_enable, sl, + UCS_BIT(sl) | + UCS_BIT((sl + 1) % UCT_IB_SL_NUM) | + UCS_BIT((sl + 2) % UCT_IB_SL_NUM), + (ar_enable == UCS_NO) ? err_status : UCS_OK); + + // select SL=, with OOO SL mask which doesn't contain + select_sl(ar_enable, sl, + UCS_BIT((sl + 1) % UCT_IB_SL_NUM) | + UCS_BIT((sl + 2) % UCT_IB_SL_NUM) | + UCS_BIT((sl + 3) % UCT_IB_SL_NUM), + (ar_enable == UCS_YES) ? err_status : UCS_OK); + + // select SL=, with full OOO SL mask + select_sl(ar_enable, sl, UCS_MASK(UCT_IB_SL_NUM), + (ar_enable == UCS_NO) ? err_status : UCS_OK); + + // select SL=, with full OOO SL mask, except + select_sl(ar_enable, sl, UCS_MASK(UCT_IB_SL_NUM) & ~UCS_BIT(sl), + (ar_enable == UCS_YES) ? err_status : UCS_OK); + } } } From bd987885cdc6f0bcfa8f63580312474a5c876bdc Mon Sep 17 00:00:00 2001 From: Yossi Itigin Date: Thu, 17 Dec 2020 19:00:50 +0200 Subject: [PATCH 055/107] TEST/UCM/JENKINS: Extend memtype hook tests --- contrib/test_jenkins.sh | 22 ++++++ test/apps/Makefile.am | 20 ++++++ test/apps/test_cuda_hook.c | 71 +++++++++++++++++++ test/gtest/common/mem_buffer.cc | 2 +- test/gtest/ucm/malloc_hook.cc | 118 ++++++++++++++++++++++++++------ 5 files changed, 210 insertions(+), 23 deletions(-) create mode 100644 test/apps/test_cuda_hook.c diff --git a/contrib/test_jenkins.sh b/contrib/test_jenkins.sh index a424aa64563..d8f0dcaee51 100755 --- a/contrib/test_jenkins.sh +++ b/contrib/test_jenkins.sh @@ -1338,6 +1338,28 @@ test_malloc_hook() { then ./test/apps/test_tcmalloc fi + + if [ "X$have_cuda" == "Xyes" ] + then + for mode in reloc bistro + do + export UCX_MEM_MMAP_HOOK_MODE=${mode} + + # Run cuda memory hooks with dynamic link + ./test/apps/test_cuda_hook_dynamic + + # Run cuda memory hooks with static link + if ./test/apps/test_cuda_hook_static + then + echo "Static link with cuda is expected to fail" + exit 1 + else + echo "Test failed - as expected" + fi + + unset UCX_MEM_MMAP_HOOK_MODE + done + fi } test_jucx() { diff --git a/test/apps/Makefile.am b/test/apps/Makefile.am index dd2e7d7d75b..48e2c75d7ab 100644 --- a/test/apps/Makefile.am +++ b/test/apps/Makefile.am @@ -54,6 +54,26 @@ test_init_mt_CPPFLAGS = $(BASE_CPPFLAGS) test_init_mt_CFLAGS = $(BASE_CFLAGS) $(OPENMP_CFLAGS) test_init_mt_LDADD = $(top_builddir)/src/ucp/libucp.la +if HAVE_CUDA +noinst_PROGRAMS += test_cuda_hook_dynamic \ + test_cuda_hook_static + +test_cuda_hook_dynamic_SOURCES = test_cuda_hook.c +test_cuda_hook_dynamic_CPPFLAGS = $(BASE_CPPFLAGS) $(CUDA_CPPFLAGS) +test_cuda_hook_dynamic_CFLAGS = $(BASE_CFLAGS) +test_cuda_hook_dynamic_LDFLAGS = $(CUDA_LDFLAGS) +test_cuda_hook_dynamic_LDADD = $(top_builddir)/src/ucp/libucp.la \ + $(top_builddir)/src/ucm/libucm.la + +noinst_PROGRAMS += test_cuda_hook_static +test_cuda_hook_static_SOURCES = test_cuda_hook.c +test_cuda_hook_static_CPPFLAGS = $(BASE_CPPFLAGS) $(CUDA_CPPFLAGS) +test_cuda_hook_static_CFLAGS = $(BASE_CFLAGS) +test_cuda_hook_static_LDFLAGS = -lcudart_static -lrt -ldl -lpthread +test_cuda_hook_static_LDADD = $(top_builddir)/src/ucp/libucp.la \ + $(top_builddir)/src/ucm/libucm.la +endif + if HAVE_TCMALLOC noinst_PROGRAMS += test_tcmalloc test_tcmalloc_SOURCES = test_tcmalloc.c diff --git a/test/apps/test_cuda_hook.c b/test/apps/test_cuda_hook.c new file mode 100644 index 00000000000..37391ff06ec --- /dev/null +++ b/test/apps/test_cuda_hook.c @@ -0,0 +1,71 @@ +/** + * Copyright (C) Mellanox Technologies Ltd. 2020. ALL RIGHTS RESERVED. + * + * See file LICENSE for terms. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include + + +static void event_cb(ucm_event_type_t event_type, ucm_event_t *event, void *arg) +{ + int *count_p = arg; + const char *title; + + if (event_type == UCM_EVENT_MEM_TYPE_ALLOC) { + title = "allocate"; + } else if (event_type == UCM_EVENT_MEM_TYPE_FREE) { + title = "free"; + } else { + printf("unexpected memory event type %d\n", event_type); + return; + } + + printf("%s %s address %p size %zu\n", title, + ucs_memory_type_names[event->mem_type.mem_type], + event->mem_type.address, event->mem_type.size); + ++(*count_p); +} + +int main(int argc, char **argv) +{ + static const ucm_event_type_t memtype_events = UCM_EVENT_MEM_TYPE_ALLOC | + UCM_EVENT_MEM_TYPE_FREE; + static const int num_expected_events = 2; + ucs_status_t status; + ucp_context_h context; + ucp_params_t params; + int num_events; + void *dptr; + int res; + + params.field_mask = UCP_PARAM_FIELD_FEATURES; + params.features = UCP_FEATURE_TAG | UCP_FEATURE_STREAM; + status = ucp_init(¶ms, NULL, &context); + if (status != UCS_OK) { + printf("failed to create context\n"); + return -1; + } + + num_events = 0; + ucm_set_event_handler(memtype_events, 1000, event_cb, &num_events); + + dptr = NULL; + res = cudaMalloc(&dptr, 4096); + printf("cudaMalloc() returned %p result %d\n", dptr, res); + + cudaFree(dptr); + + ucm_unset_event_handler(memtype_events, event_cb, &num_events); + printf("got %d/%d memory events\n", num_events, num_expected_events); + + ucp_cleanup(context); + + return (num_events >= num_expected_events) ? 0 : -1; +} diff --git a/test/gtest/common/mem_buffer.cc b/test/gtest/common/mem_buffer.cc index 00a8a1df2d0..99e37c66b0d 100644 --- a/test/gtest/common/mem_buffer.cc +++ b/test/gtest/common/mem_buffer.cc @@ -23,7 +23,7 @@ do { \ cudaError_t cerr = _code; \ if (cerr != cudaSuccess) { \ - UCS_TEST_ABORT(# _code << " failed" << _details); \ + UCS_TEST_ABORT(#_code << " failed with code " << cerr << _details); \ } \ } while (0) diff --git a/test/gtest/ucm/malloc_hook.cc b/test/gtest/ucm/malloc_hook.cc index 7e0e56a69b2..624fda474da 100644 --- a/test/gtest/ucm/malloc_hook.cc +++ b/test/gtest/ucm/malloc_hook.cc @@ -218,6 +218,11 @@ class malloc_hook : public ucs::test { ucm_bistro_restore(m_rp); } + ucm_bistro_restore_point_t* rp() + { + return m_rp; + } + protected: ucm_bistro_restore_point_t *m_rp; }; @@ -1034,8 +1039,6 @@ typedef int (munmap_f_t)(void *addr, size_t len); UCS_TEST_SKIP_COND_F(malloc_hook, bistro_patch, RUNNING_ON_VALGRIND) { const char *symbol = "munmap"; - ucm_bistro_restore_point_t *rp = NULL; - ucs_status_t status; munmap_f_t *munmap_f; void *ptr; int res; @@ -1043,30 +1046,27 @@ UCS_TEST_SKIP_COND_F(malloc_hook, bistro_patch, RUNNING_ON_VALGRIND) { uint64_t UCS_V_UNUSED origin; /* set hook to mmap call */ - status = ucm_bistro_patch(symbol, (void*)bistro_hook<0>::munmap, &rp); - ASSERT_UCS_OK(status); - EXPECT_NE((intptr_t)rp, 0); - - munmap_f = (munmap_f_t*)ucm_bistro_restore_addr(rp); - EXPECT_NE((intptr_t)munmap_f, 0); + { + bistro_patch patch(symbol, (void*)bistro_hook<0>::munmap); - /* save partial body of patched function */ - patched = *(uint64_t*)munmap_f; + munmap_f = (munmap_f_t*)ucm_bistro_restore_addr(patch.rp()); + EXPECT_NE((intptr_t)munmap_f, 0); - bistro_call_counter = 0; - ptr = mmap(NULL, 4096, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0); - EXPECT_NE(ptr, MAP_FAILED); + /* save partial body of patched function */ + patched = *(uint64_t*)munmap_f; - /* try to call munmap, we should jump into munmap_hook instead */ - res = munmap_f(ptr, 4096); - EXPECT_EQ(res, 0); - /* due to cache coherency issues on ARM systems could be executed - * original function body, so, skip counter evaluation */ - EXPECT_GT(bistro_call_counter, 0); + bistro_call_counter = 0; + ptr = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, 0, 0); + EXPECT_NE(ptr, MAP_FAILED); - /* restore original mmap body */ - status = ucm_bistro_restore(rp); - ASSERT_UCS_OK(status); + /* try to call munmap, we should jump into munmap_hook instead */ + res = munmap_f(ptr, 4096); + EXPECT_EQ(res, 0); + /* due to cache coherency issues on ARM systems could be executed + * original function body, so, skip counter evaluation */ + EXPECT_GT(bistro_call_counter, 0); + } bistro_call_counter = 0; /* now try to call mmap, we should NOT jump into mmap_hook */ @@ -1201,6 +1201,80 @@ UCS_TEST_SKIP_COND_F(malloc_hook, test_event_unmap, EXPECT_TRUE(status == UCS_OK); } +class memtype_hooks : public ucs::test_with_param { +public: + void mem_event(ucm_event_type_t event_type, ucm_event_t *event) + { + m_events.push_back(event_t(event_type, *event)); + } + +protected: + typedef std::pair event_t; + + bool is_event_fired(ucm_event_type_t event_type, void *address, size_t size) + { + for (size_t i = 0; i < m_events.size(); ++i) { + if (event_type != m_events[i].first) { + continue; + } + + if ((event_type == UCM_EVENT_MEM_TYPE_ALLOC) && + (m_events[i].second.mem_type.address == address) && + (m_events[i].second.mem_type.mem_type == mem_type()) && + (m_events[i].second.mem_type.size == size)) { + return true; + } + + if ((event_type == UCM_EVENT_MEM_TYPE_FREE) && + (m_events[i].second.mem_type.address == address) && + (m_events[i].second.mem_type.size == size)) { + return true; + } + } + + return false; + } + +protected: + ucs_memory_type_t mem_type() const + { + return GetParam(); + } + + std::vector m_events; +}; + +UCS_TEST_SKIP_COND_P(memtype_hooks, alloc_free, + RUNNING_ON_VALGRIND || (GetParam() == UCS_MEMORY_TYPE_HOST)) +{ + + /* vector operations should not generate events */ + m_events.reserve(16); + + mmap_event event_handler(this); + + UCS_TEST_MESSAGE << ucs_memory_type_names[mem_type()]; + + ucs_status_t status; + status = event_handler.set(UCM_EVENT_MEM_TYPE_ALLOC | + UCM_EVENT_MEM_TYPE_FREE); + ASSERT_UCS_OK(status); + + const size_t size = 64 * UCS_KBYTE; + ucs::auto_ptr buffer(new mem_buffer(size, mem_type())); + void *ptr = buffer->ptr(); + + EXPECT_TRUE(is_event_fired(UCM_EVENT_MEM_TYPE_ALLOC, ptr, size)); + m_events.clear(); + + buffer.reset(); + + EXPECT_TRUE(is_event_fired(UCM_EVENT_MEM_TYPE_FREE, ptr, size)); +} + +INSTANTIATE_TEST_CASE_P(mem_types, memtype_hooks, + ::testing::ValuesIn(mem_buffer::supported_mem_types())); + class malloc_hook_dlopen : public malloc_hook { protected: class library { From b20bf4cb93be53391e8131f8985ee8dbbe097a1b Mon Sep 17 00:00:00 2001 From: Yossi Itigin Date: Fri, 13 Nov 2020 18:27:38 +0200 Subject: [PATCH 056/107] UCS/TOOLS/BUILD: Infrastructure for FUSE-based monitoring file-system Add build scripts and initial files for virtual monitoring file-system, which can be used to inspect UCX objects (protocols, transports, data structures), their relations (expressed by fs hierarchy and symlinks) and modify settings during runtime whenever possible. - ucx_vfs is a daemon which tracks the mountpoint and process status for every user, and takes care of cleanup when the process dies. - ucs/vfs/base/vfs_obj.{c,h} is the data structure ("model") which keeps the object heirarchy - ucs/vfs/fuse is serving the filesystem operations ("view") - ucs/vfs/sock adds convenience functions to communicate between the daemon and UCS. --- Makefile.am | 1 + config/m4/fuse3.m4 | 51 +++++++++++++++++++++++++++++++++++ configure.ac | 7 ++++- contrib/buildrpm.sh | 1 + src/tools/vfs/Makefile.am | 17 ++++++++++++ src/tools/vfs/vfs_daemon.h | 14 ++++++++++ src/tools/vfs/vfs_main.c | 18 +++++++++++++ src/ucs/Makefile.am | 6 ++++- src/ucs/configure.m4 | 7 +++++ src/ucs/sys/compiler.h | 12 +++++++++ src/ucs/sys/compiler_def.h | 12 --------- src/ucs/sys/init.c | 8 ++++++ src/ucs/vfs/base/vfs_obj.c | 14 ++++++++++ src/ucs/vfs/base/vfs_obj.h | 12 +++++++++ src/ucs/vfs/fuse/Makefile.am | 19 +++++++++++++ src/ucs/vfs/fuse/configure.m4 | 12 +++++++++ src/ucs/vfs/fuse/vfs_fuse.c | 44 ++++++++++++++++++++++++++++++ src/ucs/vfs/sock/Makefile.am | 11 ++++++++ src/ucs/vfs/sock/configure.m4 | 6 +++++ src/ucs/vfs/sock/vfs_sock.c | 11 ++++++++ src/ucs/vfs/sock/vfs_sock.h | 14 ++++++++++ ucx.spec.in | 25 ++++++++++++++++- 22 files changed, 307 insertions(+), 15 deletions(-) create mode 100644 config/m4/fuse3.m4 create mode 100644 src/tools/vfs/Makefile.am create mode 100644 src/tools/vfs/vfs_daemon.h create mode 100644 src/tools/vfs/vfs_main.c create mode 100644 src/ucs/vfs/base/vfs_obj.c create mode 100644 src/ucs/vfs/base/vfs_obj.h create mode 100644 src/ucs/vfs/fuse/Makefile.am create mode 100644 src/ucs/vfs/fuse/configure.m4 create mode 100644 src/ucs/vfs/fuse/vfs_fuse.c create mode 100644 src/ucs/vfs/sock/Makefile.am create mode 100644 src/ucs/vfs/sock/configure.m4 create mode 100644 src/ucs/vfs/sock/vfs_sock.c create mode 100644 src/ucs/vfs/sock/vfs_sock.h diff --git a/Makefile.am b/Makefile.am index 64364aa4fbf..321d64bada1 100644 --- a/Makefile.am +++ b/Makefile.am @@ -37,6 +37,7 @@ SUBDIRS += $(UCG_SUBDIR) endif SUBDIRS += \ + src/tools/vfs \ src/tools/info \ src/tools/perf \ src/tools/profile \ diff --git a/config/m4/fuse3.m4 b/config/m4/fuse3.m4 new file mode 100644 index 00000000000..c79b31a5019 --- /dev/null +++ b/config/m4/fuse3.m4 @@ -0,0 +1,51 @@ +# +# Copyright (C) Mellanox Technologies Ltd. 2020. ALL RIGHTS RESERVED. +# See file LICENSE for terms. +# + +fuse3_happy="no" + +AC_ARG_WITH([fuse3], + [AS_HELP_STRING([--with-fuse3=(DIR)], + [Enable the use of FUSEv3 (default is guess).])], + [], [with_fuse3=guess]) + +AS_IF([test "x$with_fuse3" != xno], + [ + AS_IF([test "x$with_fuse3" = "xguess" \ + -o "x$with_fuse3" = "xyes" \ + -o "x$with_fuse3" = "x"], + [FUSE3_CPPFLAGS=$(pkg-config --cflags fuse3) + FUSE3_LIBS=$(pkg-config --libs fuse3)], + [FUSE3_CPPFLAGS="-I${with_fuse3}/include/fuse3" + FUSE3_LIBS="-L${with_fuse3}/lib -L${with_fuse3}/lib64"]) + + save_CPPFLAGS="$CPPFLAGS" + save_LDFLAGS="$LDFLAGS" + + CPPFLAGS="$FUSE3_CPPFLAGS $CPPFLAGS" + LDFLAGS="$FUSE3_LIBS $LDFLAGS" + + fuse3_happy="yes" + AC_CHECK_DECLS([fuse_open_channel, fuse_mount, fuse_unmount], + [AC_SUBST([FUSE3_CPPFLAGS], [$FUSE3_CPPFLAGS]) + AC_DEFINE([FUSE_USE_VERSION], 30, [Fuse API version])], + [fuse3_happy="no"], + [[#define FUSE_USE_VERSION 30 + #include ]]) + + AC_CHECK_FUNCS([fuse_open_channel fuse_mount fuse_unmount], + [AC_SUBST([FUSE3_LIBS], [$FUSE3_LIBS])], + [fuse3_happy="no"]) + + AS_IF([test "x$fuse3_happy" != "xyes" -a "x$with_fuse3" != "xguess"], + [AC_MSG_ERROR([FUSEv3 requested but could not be found])]) + + CPPFLAGS="$save_CPPFLAGS" + LDFLAGS="$save_LDFLAGS" + ], + [AC_MSG_WARN([FUSEv3 was explicitly disabled])] +) + +AM_CONDITIONAL([HAVE_FUSE3], [test "x$fuse3_happy" != xno]) +vfs_enable=$fuse3_happy diff --git a/configure.ac b/configure.ac index 2e6236fa35f..b6f1db0cb4d 100644 --- a/configure.ac +++ b/configure.ac @@ -209,6 +209,7 @@ AS_IF([test "x$with_docs_only" = xyes], AM_CONDITIONAL([HAVE_AARCH64_THUNDERX2], [false]) AM_CONDITIONAL([HAVE_AARCH64_THUNDERX1], [false]) AM_CONDITIONAL([HAVE_AARCH64_HI1620], [false]) + AM_CONDITIONAL([HAVE_FUSE3], [false]) ], [ AM_CONDITIONAL([DOCS_ONLY], [false]) @@ -217,6 +218,7 @@ AS_IF([test "x$with_docs_only" = xyes], m4_include([config/m4/ucm.m4]) m4_include([config/m4/mpi.m4]) m4_include([config/m4/rte.m4]) + m4_include([config/m4/fuse3.m4]) m4_include([config/m4/java.m4]) m4_include([config/m4/cuda.m4]) m4_include([config/m4/rocm.m4]) @@ -328,6 +330,7 @@ AS_IF([test "x$with_docs_only" = xyes], # Print which transports are built # build_modules="${uct_modules}" +build_modules="${build_modules}${ucs_modules}" build_modules="${build_modules}${uct_ib_modules}" build_modules="${build_modules}${uct_cuda_modules}" build_modules="${build_modules}${ucm_modules}" @@ -357,10 +360,10 @@ AC_CONFIG_FILES([ debian/rules debian/control debian/changelog - src/ucs/Makefile src/ucp/Makefile src/ucp/api/ucp_version.h src/ucp/core/ucp_version.c + src/tools/vfs/Makefile src/tools/info/Makefile src/tools/profile/Makefile test/apps/Makefile @@ -397,8 +400,10 @@ AC_MSG_NOTICE([ C compiler: ${CC} ${BASE_CFLAGS}]) AC_MSG_NOTICE([ C++ compiler: ${CXX} ${BASE_CXXFLAGS}]) AC_MSG_NOTICE([ Multi-thread: ${mt_enable}]) AC_MSG_NOTICE([ MPI tests: ${mpi_enable}]) +AC_MSG_NOTICE([ VFS support: ${vfs_enable}]) AC_MSG_NOTICE([ Devel headers: ${enable_devel_headers}]) AC_MSG_NOTICE([ Bindings: <$(echo ${build_bindings}|tr ':' ' ') >]) +AC_MSG_NOTICE([ UCS modules: <$(echo ${ucs_modules}|tr ':' ' ') >]) AC_MSG_NOTICE([ UCT modules: <$(echo ${uct_modules}|tr ':' ' ') >]) AC_MSG_NOTICE([ CUDA modules: <$(echo ${uct_cuda_modules}|tr ':' ' ') >]) AC_MSG_NOTICE([ ROCM modules: <$(echo ${uct_rocm_modules}|tr ':' ' ') >]) diff --git a/contrib/buildrpm.sh b/contrib/buildrpm.sh index 46d4187b8fb..7a4da8d63ff 100755 --- a/contrib/buildrpm.sh +++ b/contrib/buildrpm.sh @@ -106,6 +106,7 @@ if [ $opt_binrpm -eq 1 ]; then with_args+=" $(with_arg rocm)" with_args+=" $(with_arg ugni)" with_args+=" $(with_arg xpmem)" + with_args+=" $(with_arg vfs)" with_args+=" $(with_arg java)" echo rpmbuild -bb $rpmmacros $rpmopts $rpmspec $defines $with_args | bash -eEx diff --git a/src/tools/vfs/Makefile.am b/src/tools/vfs/Makefile.am new file mode 100644 index 00000000000..a560c0a4de9 --- /dev/null +++ b/src/tools/vfs/Makefile.am @@ -0,0 +1,17 @@ +# +# Copyright (C) Mellanox Technologies Ltd. 2020. ALL RIGHTS RESERVED. +# +# See file LICENSE for terms. +# + +if HAVE_FUSE3 + +bin_PROGRAMS = ucx_vfs +ucx_vfs_CPPFLAGS = $(BASE_CPPFLAGS) $(FUSE3_CPPFLAGS) +ucx_vfs_CFLAGS = $(BASE_CFLAGS) +ucx_vfs_SOURCES = vfs_main.c +noinst_HEADERS = vfs_daemon.h +ucx_vfs_LDADD = $(FUSE3_LIBS) \ + $(top_builddir)/src/ucs/vfs/sock/libucs_vfs_sock.la + +endif diff --git a/src/tools/vfs/vfs_daemon.h b/src/tools/vfs/vfs_daemon.h new file mode 100644 index 00000000000..955ee6b19e5 --- /dev/null +++ b/src/tools/vfs/vfs_daemon.h @@ -0,0 +1,14 @@ +/** + * Copyright (C) Mellanox Technologies Ltd. 2020. ALL RIGHTS RESERVED. + * + * See file LICENSE for terms. + */ + +#ifndef VFS_DAEMON_H_ +#define VFS_DAEMON_H_ + +#include +#include + + +#endif diff --git a/src/tools/vfs/vfs_main.c b/src/tools/vfs/vfs_main.c new file mode 100644 index 00000000000..4239d46f4fb --- /dev/null +++ b/src/tools/vfs/vfs_main.c @@ -0,0 +1,18 @@ +/** + * Copyright (C) Mellanox Technologies Ltd. 2020. ALL RIGHTS RESERVED. + * + * See file LICENSE for terms. + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "vfs_daemon.h" + + +int main(int argc, char **argv) +{ + /* TODO implement VFS daemon logic */ + return 0; +} diff --git a/src/ucs/Makefile.am b/src/ucs/Makefile.am index aeb5c228065..e15a564a479 100644 --- a/src/ucs/Makefile.am +++ b/src/ucs/Makefile.am @@ -5,6 +5,8 @@ # See file LICENSE for terms. # +SUBDIRS = vfs/sock . vfs/fuse + AUTOMAKE_OPTIONS = nostdinc # avoid collision with built-in debug.h lib_LTLIBRARIES = libucs.la bin_PROGRAMS = @@ -64,6 +66,7 @@ nobase_dist_libucs_la_HEADERS = \ type/status.h \ type/thread_mode.h \ type/cpu_set.h \ + vfs/base/vfs_obj.h \ arch/atomic.h \ arch/x86_64/global_opts.h \ arch/x86_64/atomic.h \ @@ -166,7 +169,8 @@ libucs_la_SOURCES = \ time/timerq.c \ type/class.c \ type/status.c \ - type/spinlock.c + type/spinlock.c \ + vfs/base/vfs_obj.c if HAVE_AARCH64_THUNDERX2 libucs_la_SOURCES += \ diff --git a/src/ucs/configure.m4 b/src/ucs/configure.m4 index 153eb0d78e4..d48d4a6206b 100644 --- a/src/ucs/configure.m4 +++ b/src/ucs/configure.m4 @@ -5,6 +5,10 @@ # See file LICENSE for terms. # +ucs_modules="" +m4_include([src/ucs/vfs/sock/configure.m4]) +m4_include([src/ucs/vfs/fuse/configure.m4]) +AC_DEFINE_UNQUOTED([ucs_MODULES], ["${ucs_modules}"], [UCS loadable modules]) # # Internal profiling support. @@ -240,3 +244,6 @@ AS_IF([test "x$enable_builtin_memcpy" != xno], AC_CHECK_FUNCS([__clear_cache], [], []) AC_CHECK_FUNCS([__aarch64_sync_cache_range], [], []) + + +AC_CONFIG_FILES([src/ucs/Makefile]) diff --git a/src/ucs/sys/compiler.h b/src/ucs/sys/compiler.h index 9be16ff7f3a..3089fb3f9c2 100644 --- a/src/ucs/sys/compiler.h +++ b/src/ucs/sys/compiler.h @@ -84,4 +84,16 @@ #define UCS_CACHELINE_PADDING_MISALIGN(...) \ ((UCS_PP_FOREACH(UCS_CACHELINE_PADDING_SIZEOF, _, __VA_ARGS__)) % UCS_SYS_CACHE_LINE_SIZE) +/* + * Define code which runs at global constructor phase + */ +#define UCS_STATIC_INIT \ + static void UCS_F_CTOR UCS_PP_APPEND_UNIQUE_ID(ucs_initializer_ctor)() + +/* + * Define code which runs at global destructor phase + */ +#define UCS_STATIC_CLEANUP \ + static void UCS_F_DTOR UCS_PP_APPEND_UNIQUE_ID(ucs_initializer_dtor)() + #endif diff --git a/src/ucs/sys/compiler_def.h b/src/ucs/sys/compiler_def.h index 9e9b29e12a6..98db780d3c9 100644 --- a/src/ucs/sys/compiler_def.h +++ b/src/ucs/sys/compiler_def.h @@ -179,16 +179,4 @@ /* Check if an expression is a compile-time constant */ #define ucs_is_constant(expr) __builtin_constant_p(expr) -/* - * Define code which runs at global constructor phase - */ -#define UCS_STATIC_INIT \ - static void UCS_F_CTOR UCS_PP_APPEND_UNIQUE_ID(ucs_initializer_ctor)() - -/* - * Define code which runs at global destructor phase - */ -#define UCS_STATIC_CLEANUP \ - static void UCS_F_DTOR UCS_PP_APPEND_UNIQUE_ID(ucs_initializer_dtor)() - #endif /* UCS_COMPILER_DEF_H */ diff --git a/src/ucs/sys/init.c b/src/ucs/sys/init.c index 1d228af4b63..9b6a33895cf 100644 --- a/src/ucs/sys/init.c +++ b/src/ucs/sys/init.c @@ -9,6 +9,7 @@ #endif #include +#include #include #include #include @@ -78,6 +79,12 @@ static UCS_F_NOOPTIMIZE void ucs_check_cpu_flags(void) } } +static void ucs_modules_load() +{ + UCS_MODULE_FRAMEWORK_DECLARE(ucs); + UCS_MODULE_FRAMEWORK_LOAD(ucs, UCS_MODULE_LOAD_FLAG_GLOBAL); +} + static void UCS_F_CTOR ucs_init() { ucs_check_cpu_flags(); @@ -97,6 +104,7 @@ static void UCS_F_CTOR ucs_init() ucs_debug("%s loaded at 0x%lx", ucs_debug_get_lib_path(), ucs_debug_get_lib_base_addr()); ucs_debug("cmd line: %s", ucs_get_process_cmdline()); + ucs_modules_load(); } static void UCS_F_DTOR ucs_cleanup(void) diff --git a/src/ucs/vfs/base/vfs_obj.c b/src/ucs/vfs/base/vfs_obj.c new file mode 100644 index 00000000000..3c65ccf6468 --- /dev/null +++ b/src/ucs/vfs/base/vfs_obj.c @@ -0,0 +1,14 @@ +/** + * Copyright (C) Mellanox Technologies Ltd. 2020. ALL RIGHTS RESERVED. + * + * See file LICENSE for terms. + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "vfs_obj.h" + + +/* TODO implement VFS object tree structure */ diff --git a/src/ucs/vfs/base/vfs_obj.h b/src/ucs/vfs/base/vfs_obj.h new file mode 100644 index 00000000000..ca22b0afe6b --- /dev/null +++ b/src/ucs/vfs/base/vfs_obj.h @@ -0,0 +1,12 @@ +/** + * Copyright (C) Mellanox Technologies Ltd. 2020. ALL RIGHTS RESERVED. + * + * See file LICENSE for terms. + */ + +#ifndef UCS_VFS_H_ +#define UCS_VFS_H_ + +/* This header file defines API for manipulating VFS object tree structure */ + +#endif diff --git a/src/ucs/vfs/fuse/Makefile.am b/src/ucs/vfs/fuse/Makefile.am new file mode 100644 index 00000000000..8ad3511277b --- /dev/null +++ b/src/ucs/vfs/fuse/Makefile.am @@ -0,0 +1,19 @@ +# +# Copyright (C) Mellanox Technologies Ltd. 2020. ALL RIGHTS RESERVED. +# +# See file LICENSE for terms. +# + +if HAVE_FUSE3 + +module_LTLIBRARIES = libucs_fuse.la +libucs_fuse_la_CPPFLAGS = $(BASE_CPPFLAGS) $(FUSE3_CPPFLAGS) +libucs_fuse_la_CFLAGS = $(BASE_CFLAGS) $(CUDA_CFLAGS) +libucs_fuse_la_LIBADD = $(FUSE3_LIBS) \ + $(top_builddir)/src/ucs/vfs/sock/libucs_vfs_sock.la \ + $(top_builddir)/src/ucs/libucs.la +libucs_fuse_la_LDFLAGS = -version-info $(SOVERSION) +libucs_fuse_la_SOURCES = vfs_fuse.c +include $(top_srcdir)/config/module.am + +endif diff --git a/src/ucs/vfs/fuse/configure.m4 b/src/ucs/vfs/fuse/configure.m4 new file mode 100644 index 00000000000..bf45ea11dcb --- /dev/null +++ b/src/ucs/vfs/fuse/configure.m4 @@ -0,0 +1,12 @@ +# +# Copyright (C) Mellanox Technologies Ltd. 2020. ALL RIGHTS RESERVED. +# + + +AC_CHECK_DECLS([inotify_init, inotify_add_watch, IN_ATTRIB], + [AC_DEFINE([HAVE_INOTIFY], 1, [Enable inotify support])], + [], + [[#include ]]) + +AS_IF([test "x$fuse3_happy" = "xyes"], [ucs_modules="${ucs_modules}:fuse"]) +AC_CONFIG_FILES([src/ucs/vfs/fuse/Makefile]) diff --git a/src/ucs/vfs/fuse/vfs_fuse.c b/src/ucs/vfs/fuse/vfs_fuse.c new file mode 100644 index 00000000000..4552039e238 --- /dev/null +++ b/src/ucs/vfs/fuse/vfs_fuse.c @@ -0,0 +1,44 @@ +/** + * Copyright (C) Mellanox Technologies Ltd. 2020. ALL RIGHTS RESERVED. + * + * See file LICENSE for terms. + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include +#include +#include +#include + + +static struct { + pthread_t thread_id; +} ucs_vfs_fuse_context = { + .thread_id = -1, +}; + +static void *ucs_vfs_fuse_thread_func(void *arg) +{ + return NULL; +} + +static void ucs_fuse_thread_stop() +{ + pthread_join(ucs_vfs_fuse_context.thread_id, NULL); +} + +UCS_STATIC_INIT +{ + pthread_create(&ucs_vfs_fuse_context.thread_id, NULL, + ucs_vfs_fuse_thread_func, NULL); +} + +UCS_STATIC_CLEANUP +{ + if (ucs_vfs_fuse_context.thread_id != -1) { + ucs_fuse_thread_stop(); + } +} diff --git a/src/ucs/vfs/sock/Makefile.am b/src/ucs/vfs/sock/Makefile.am new file mode 100644 index 00000000000..8e9b372ab8c --- /dev/null +++ b/src/ucs/vfs/sock/Makefile.am @@ -0,0 +1,11 @@ +# +# Copyright (C) Mellanox Technologies Ltd. 2020. ALL RIGHTS RESERVED. +# +# See file LICENSE for terms. +# + +noinst_LTLIBRARIES = libucs_vfs_sock.la +libucs_vfs_sock_la_SOURCES = vfs_sock.c +noinst_HEADERS = vfs_sock.h +libucs_vfs_sock_la_CPPFLAGS = $(BASE_CPPFLAGS) +libucs_vfs_sock_la_LDFLAGS = $(BASE_LDFLAGS) diff --git a/src/ucs/vfs/sock/configure.m4 b/src/ucs/vfs/sock/configure.m4 new file mode 100644 index 00000000000..6abed32fc8e --- /dev/null +++ b/src/ucs/vfs/sock/configure.m4 @@ -0,0 +1,6 @@ +# +# Copyright (C) Mellanox Technologies Ltd. 2020. ALL RIGHTS RESERVED. +# + + +AC_CONFIG_FILES([src/ucs/vfs/sock/Makefile]) diff --git a/src/ucs/vfs/sock/vfs_sock.c b/src/ucs/vfs/sock/vfs_sock.c new file mode 100644 index 00000000000..98a2d894f1a --- /dev/null +++ b/src/ucs/vfs/sock/vfs_sock.c @@ -0,0 +1,11 @@ +/** + * Copyright (C) Mellanox Technologies Ltd. 2020. ALL RIGHTS RESERVED. + * + * See file LICENSE for terms. + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "vfs_sock.h" diff --git a/src/ucs/vfs/sock/vfs_sock.h b/src/ucs/vfs/sock/vfs_sock.h new file mode 100644 index 00000000000..6f1848e99c9 --- /dev/null +++ b/src/ucs/vfs/sock/vfs_sock.h @@ -0,0 +1,14 @@ +/** + * Copyright (C) Mellanox Technologies Ltd. 2020. ALL RIGHTS RESERVED. + * + * See file LICENSE for terms. + */ + +#ifndef UCS_VFS_SOCK_H_ +#define UCS_VFS_SOCK_H_ + +/* This header file defines socket operations for communicating between UCS + * library and VFS daemon */ + + +#endif diff --git a/ucx.spec.in b/ucx.spec.in index 869bf69ffb7..7a53cff978a 100644 --- a/ucx.spec.in +++ b/ucx.spec.in @@ -13,6 +13,7 @@ %bcond_with rocm %bcond_with ugni %bcond_with xpmem +%bcond_with vfs Name: ucx Version: @VERSION@ @@ -64,6 +65,9 @@ BuildRequires: hsa-rocr-dev %if %{with xpmem} BuildRequires: xpmem-devel %endif +%if %{with vfs} +BuildRequires: fuse3-devel +%endif %description UCX is an optimized communication framework for high-performance distributed @@ -110,6 +114,7 @@ Provides header files and examples for developing with UCX. %_with_arg rdmacm rdmacm \ %_with_arg rocm rocm \ %_with_arg xpmem xpmem \ + %_with_arg vfs fuse3 \ %_with_arg ugni ugni \ %{?configure_options} make %{?_smp_mflags} V=1 @@ -124,7 +129,9 @@ rm -f %{buildroot}%{_libdir}/ucx/lib*.a %files %{_libdir}/lib*.so.* -%{_bindir}/uc* +%{_bindir}/ucx_info +%{_bindir}/ucx_perftest +%{_bindir}/ucx_read_profile @HAVE_GLIBCXX_NOTHROW_TRUE@%{_bindir}/io_demo %{_datadir}/ucx %exclude %{_datadir}/ucx/examples @@ -300,9 +307,25 @@ process to map the memory of another process into its virtual address space. %{_libdir}/ucx/libuct_xpmem.so.* %endif +%if %{with vfs} +%package vfs +Requires: %{name}%{?_isa} = %{version}-%{release} +Summary: UCX Virtual Filesystem support. +Group: System Environment/Libraries + +%description vfs +Provides a virtual filesystem over FUSE which allows real-time monitoring of UCX +library internals, protocol objects, transports status, and more. + +%files vfs +%{_libdir}/ucx/libucs_fuse.so.* +%{_bindir}/ucx_vfs +%endif %changelog * Wed Dec 16 2020 Yossi Itigin 1.11.0-1 +- Add VFS sub-package +* Wed Dec 16 2020 Yossi Itigin 1.11.0-1 - Bump version to 1.11.0 * Wed Nov 11 2020 Yossi Itigin 1.10.0-1 - Make the RPM relocatable From daa69c5196a611b1b4cf825a953676f5bd298d4b Mon Sep 17 00:00:00 2001 From: Sergey Oblomov Date: Fri, 18 Dec 2020 10:47:11 +0200 Subject: [PATCH 057/107] RC_MLX5/IFACE: fixed assert - fixed issue in iface recv prepost when adaptive routing enabled - fixed assert: rc_mlx5_common.c:128 Assertion `rc_iface->rx.srq.available >= count' failed - don't try prepost recv twice --- src/uct/ib/rc/accel/rc_mlx5_common.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/uct/ib/rc/accel/rc_mlx5_common.c b/src/uct/ib/rc/accel/rc_mlx5_common.c index 9bdbc6005a2..76caf22a564 100644 --- a/src/uct/ib/rc/accel/rc_mlx5_common.c +++ b/src/uct/ib/rc/accel/rc_mlx5_common.c @@ -91,6 +91,7 @@ unsigned uct_rc_mlx5_iface_srq_post_recv(uct_rc_mlx5_iface_common_t *iface) sizeof(struct mlx5_wqe_srq_next_seg)); ucs_assert(UCS_CIRCULAR_COMPARE16(srq->ready_idx, <=, srq->free_idx)); + ucs_assert(rc_iface->rx.srq.available > 0); wqe_index = srq->ready_idx; for (;;) { @@ -140,6 +141,12 @@ unsigned uct_rc_mlx5_iface_srq_post_recv(uct_rc_mlx5_iface_common_t *iface) void uct_rc_mlx5_iface_common_prepost_recvs(uct_rc_mlx5_iface_common_t *iface) { + /* prepost recvs only if quota available (recvs were not preposted + * before) */ + if (iface->super.rx.srq.quota == 0) { + return; + } + iface->super.rx.srq.available = iface->super.rx.srq.quota; iface->super.rx.srq.quota = 0; uct_rc_mlx5_iface_srq_post_recv(iface); From c39e054d0b6212d14cd8f33beac6b43c2178e6e0 Mon Sep 17 00:00:00 2001 From: Raul Akhmetshin Date: Mon, 14 Dec 2020 22:05:26 +0200 Subject: [PATCH 058/107] TOOLS/PERF: Added performance tests for am short iov. --- contrib/ucx_perftest_config/test_types_uct | 14 ++++++++------ src/tools/perf/api/libperf.h | 1 + src/tools/perf/lib/libperf.c | 9 ++++++--- src/tools/perf/lib/uct_tests.cc | 13 ++++++++++--- src/tools/perf/perftest.c | 14 ++++++++++---- test/gtest/uct/test_uct_perf.cc | 18 ++++++++++++++++++ 6 files changed, 53 insertions(+), 16 deletions(-) diff --git a/contrib/ucx_perftest_config/test_types_uct b/contrib/ucx_perftest_config/test_types_uct index 2769ee481de..40edda3cb20 100644 --- a/contrib/ucx_perftest_config/test_types_uct +++ b/contrib/ucx_perftest_config/test_types_uct @@ -6,12 +6,14 @@ put_short_lat -t put_lat -D short put_bcopy_lat -t put_lat -D bcopy put_zcopy_lat -t put_lat -D zcopy # AM -am_short_lat -t am_lat -D short -am_bcopy_lat -t am_lat -D bcopy -am_zcopy_lat -t am_lat -D zcopy -am_short_bw -t am_bw -D short -am_bcopy_bw -t am_bw -D bcopy -am_zcopy_bw -t am_bw -D zcopy +am_short_lat -t am_lat -D short +am_short_iov_lat -t am_lat -D shortiov +am_bcopy_lat -t am_lat -D bcopy +am_zcopy_lat -t am_lat -D zcopy +am_short_bw -t am_bw -D short +am_short_iov_bw -t am_bw -D shortiov +am_bcopy_bw -t am_bw -D bcopy +am_zcopy_bw -t am_bw -D zcopy # GET get_bcopy -t get -D bcopy get_zcopy -t get -D zcopy diff --git a/src/tools/perf/api/libperf.h b/src/tools/perf/api/libperf.h index 9b1da56314b..7f3080ae10e 100644 --- a/src/tools/perf/api/libperf.h +++ b/src/tools/perf/api/libperf.h @@ -64,6 +64,7 @@ typedef enum { typedef enum { UCT_PERF_DATA_LAYOUT_SHORT, + UCT_PERF_DATA_LAYOUT_SHORT_IOV, UCT_PERF_DATA_LAYOUT_BCOPY, UCT_PERF_DATA_LAYOUT_ZCOPY, UCT_PERF_DATA_LAYOUT_LAST diff --git a/src/tools/perf/lib/libperf.c b/src/tools/perf/lib/libperf.c index 71196f0b54d..17538f366c2 100644 --- a/src/tools/perf/lib/libperf.c +++ b/src/tools/perf/lib/libperf.c @@ -480,7 +480,8 @@ void uct_perf_iface_flush_b(ucx_perf_context_t *perf) static inline uint64_t __get_flag(uct_perf_data_layout_t layout, uint64_t short_f, uint64_t bcopy_f, uint64_t zcopy_f) { - return (layout == UCT_PERF_DATA_LAYOUT_SHORT) ? short_f : + return ((layout == UCT_PERF_DATA_LAYOUT_SHORT) || + (layout == UCT_PERF_DATA_LAYOUT_SHORT_IOV)) ? short_f : (layout == UCT_PERF_DATA_LAYOUT_BCOPY) ? bcopy_f : (layout == UCT_PERF_DATA_LAYOUT_ZCOPY) ? zcopy_f : 0; @@ -502,7 +503,8 @@ static inline ucs_status_t __get_atomic_flag(size_t size, uint64_t *op32, static inline size_t __get_max_size(uct_perf_data_layout_t layout, size_t short_m, size_t bcopy_m, uint64_t zcopy_m) { - return (layout == UCT_PERF_DATA_LAYOUT_SHORT) ? short_m : + return ((layout == UCT_PERF_DATA_LAYOUT_SHORT) || + (layout == UCT_PERF_DATA_LAYOUT_SHORT_IOV)) ? short_m : (layout == UCT_PERF_DATA_LAYOUT_BCOPY) ? bcopy_m : (layout == UCT_PERF_DATA_LAYOUT_ZCOPY) ? zcopy_m : 0; @@ -692,7 +694,8 @@ static ucs_status_t uct_perf_test_check_capabilities(ucx_perf_params_t *params, } } - if (UCT_PERF_DATA_LAYOUT_ZCOPY == params->uct.data_layout) { + if ((UCT_PERF_DATA_LAYOUT_ZCOPY == params->uct.data_layout) || + (UCT_PERF_DATA_LAYOUT_SHORT_IOV == params->uct.data_layout)) { if (params->msg_size_cnt > max_iov) { if ((params->flags & UCX_PERF_TEST_FLAG_VERBOSE) || !params->msg_size_cnt) { diff --git a/src/tools/perf/lib/uct_tests.cc b/src/tools/perf/lib/uct_tests.cc index 96afbb5a53e..88964ce1d48 100644 --- a/src/tools/perf/lib/uct_tests.cc +++ b/src/tools/perf/lib/uct_tests.cc @@ -73,7 +73,8 @@ class uct_perf_test_runner { const size_t iovcnt = perf->params.msg_size_cnt; size_t iov_length_it, iov_it; - ucs_assert(UCT_PERF_DATA_LAYOUT_ZCOPY == DATA); + ucs_assert((UCT_PERF_DATA_LAYOUT_ZCOPY == DATA) || + (UCT_PERF_DATA_LAYOUT_SHORT_IOV == DATA)); ucs_assert(NULL != perf->params.msg_size_list); ucs_assert(iovcnt > 0); ucs_assert(perf->params.msg_size_list[0] >= header_size); @@ -100,9 +101,10 @@ class uct_perf_test_runner { } void uct_perf_test_prepare_iov_buffer() { - if (UCT_PERF_DATA_LAYOUT_ZCOPY == DATA) { + if ((UCT_PERF_DATA_LAYOUT_ZCOPY == DATA) || + (UCT_PERF_DATA_LAYOUT_SHORT_IOV == DATA)) { size_t start_iov_buffer_size = 0; - if (UCX_PERF_CMD_AM == CMD) { + if ((UCX_PERF_CMD_AM == CMD) && (UCT_PERF_DATA_LAYOUT_ZCOPY == DATA)) { start_iov_buffer_size = m_perf.params.am_hdr_size; } uct_perf_get_buffer_iov(m_perf.uct.iov, m_perf.send_buffer, @@ -249,6 +251,10 @@ class uct_perf_test_runner { return uct_ep_am_short(ep, UCT_PERF_TEST_AM_ID, am_short_hdr, (char*)buffer + sizeof(am_short_hdr), length - sizeof(am_short_hdr)); + case UCT_PERF_DATA_LAYOUT_SHORT_IOV: + set_sn(buffer, m_perf.uct.send_mem.mem_type, &sn); + return uct_ep_am_short_iov(ep, UCT_PERF_TEST_AM_ID, m_perf.uct.iov, + m_perf.params.msg_size_cnt); case UCT_PERF_DATA_LAYOUT_BCOPY: set_sn(buffer, m_perf.uct.send_mem.mem_type, &sn); packed_len = uct_ep_am_bcopy(ep, UCT_PERF_TEST_AM_ID, pack_cb, @@ -690,6 +696,7 @@ class uct_perf_test_runner { TEST_CASE(_perf, UCS_PP_TUPLE_0 _case, UCS_PP_TUPLE_1 _case, _data, false) #define TEST_CASE_ALL_DATA(_perf, _case) \ TEST_CASE_ALL_OSD(_perf, _case, UCT_PERF_DATA_LAYOUT_SHORT) \ + TEST_CASE_ALL_OSD(_perf, _case, UCT_PERF_DATA_LAYOUT_SHORT_IOV) \ TEST_CASE_ALL_OSD(_perf, _case, UCT_PERF_DATA_LAYOUT_BCOPY) \ TEST_CASE_ALL_OSD(_perf, _case, UCT_PERF_DATA_LAYOUT_ZCOPY) diff --git a/src/tools/perf/perftest.c b/src/tools/perf/perftest.c index ddc66defaff..1fb91cdfff3 100644 --- a/src/tools/perf/perftest.c +++ b/src/tools/perf/perftest.c @@ -296,6 +296,9 @@ static void print_header(struct perftest_context *ctx) case UCT_PERF_DATA_LAYOUT_SHORT: test_data_str = "short"; break; + case UCT_PERF_DATA_LAYOUT_SHORT_IOV: + test_data_str = "short iov"; + break; case UCT_PERF_DATA_LAYOUT_BCOPY: test_data_str = "bcopy"; break; @@ -447,10 +450,11 @@ static void usage(const struct perftest_context *ctx, const char *program) printf(" -d device to use for testing\n"); printf(" -x transport to use for testing\n"); printf(" -D data layout for sender side:\n"); - printf(" short - short messages (default, cannot be used for get)\n"); - printf(" bcopy - copy-out (cannot be used for atomics)\n"); - printf(" zcopy - zero-copy (cannot be used for atomics)\n"); - printf(" iov - scatter-gather list (iovec)\n"); + printf(" short - short messages (default, cannot be used for get)\n"); + printf(" shortiov - short io-vector messages (only for active messages)\n"); + printf(" bcopy - copy-out (cannot be used for atomics)\n"); + printf(" zcopy - zero-copy (cannot be used for atomics)\n"); + printf(" iov - scatter-gather list (iovec)\n"); printf(" -W flow control window size, for active messages (%u)\n", ctx->params.super.uct.fc_window); printf(" -H active message header size (%zu)\n", @@ -654,6 +658,8 @@ static ucs_status_t parse_test_params(perftest_params_t *params, char opt, case 'D': if (!strcmp(opt_arg, "short")) { params->super.uct.data_layout = UCT_PERF_DATA_LAYOUT_SHORT; + } else if (!strcmp(opt_arg, "shortiov")) { + params->super.uct.data_layout = UCT_PERF_DATA_LAYOUT_SHORT_IOV; } else if (!strcmp(opt_arg, "bcopy")) { params->super.uct.data_layout = UCT_PERF_DATA_LAYOUT_BCOPY; } else if (!strcmp(opt_arg, "zcopy")) { diff --git a/test/gtest/uct/test_uct_perf.cc b/test/gtest/uct/test_uct_perf.cc index 81e8dde05e6..3bd7c398b0d 100644 --- a/test/gtest/uct/test_uct_perf.cc +++ b/test/gtest/uct/test_uct_perf.cc @@ -42,6 +42,24 @@ const test_perf::test_spec test_uct_perf::tests[] = ucs_offsetof(ucx_perf_result_t, msgrate.total_average), 1e-6, 0.8, 80.0, 0 }, + { "am short iov latency", "usec", + UCX_PERF_API_UCT, UCX_PERF_CMD_AM, UCX_PERF_TEST_TYPE_PINGPONG, + UCT_PERF_DATA_LAYOUT_SHORT_IOV, 0, 1, { 8, 8 }, 1, 100000lu, + ucs_offsetof(ucx_perf_result_t, latency.total_average), 1e6, 0.01, 2.5, + 0 }, + + { "am short iov rate", "Mpps", + UCX_PERF_API_UCT, UCX_PERF_CMD_AM, UCX_PERF_TEST_TYPE_STREAM_UNI, + UCT_PERF_DATA_LAYOUT_SHORT_IOV, 0, 1, { 8, 8 }, 1, 2000000lu, + ucs_offsetof(ucx_perf_result_t, msgrate.total_average), 1e-6, 0.8, 80.0, + 0 }, + + { "am short iov rate64", "Mpps", + UCX_PERF_API_UCT, UCX_PERF_CMD_AM, UCX_PERF_TEST_TYPE_STREAM_UNI, + UCT_PERF_DATA_LAYOUT_SHORT_IOV, 0, 1, { 32, 32 }, 1, 2000000lu, + ucs_offsetof(ucx_perf_result_t, msgrate.total_average), 1e-6, 0.8, 80.0, + 0 }, + { "am bcopy latency", "usec", UCX_PERF_API_UCT, UCX_PERF_CMD_AM, UCX_PERF_TEST_TYPE_PINGPONG, UCT_PERF_DATA_LAYOUT_BCOPY, 0, 1, { 8 }, 1, 100000lu, From cb5de58c7a96ba3fe7c7b53b5ecaa9598fbcd004 Mon Sep 17 00:00:00 2001 From: Peter Rudenko Date: Wed, 16 Dec 2020 15:11:10 +0200 Subject: [PATCH 059/107] UCP: call user calbback on ucp_ep_close_nbx. --- src/ucp/api/ucp.h | 18 +++++++++------ src/ucp/core/ucp_ep.c | 2 +- test/gtest/ucp/test_ucp_wireup.cc | 37 +++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 8 deletions(-) diff --git a/src/ucp/api/ucp.h b/src/ucp/api/ucp.h index 70620ab3f88..3a14ca8935e 100644 --- a/src/ucp/api/ucp.h +++ b/src/ucp/api/ucp.h @@ -283,7 +283,11 @@ typedef enum { flag is not set then @ref ucp_ep_close_nbx schedules flushes on all outstanding - operations. */ + operations. + @note this flag is incompatible + with @ref UCP_OP_ATTR_FLAG_NO_IMM_CMPL, + since it forces immediate completion. + */ } ucp_ep_close_flags_t; @@ -599,7 +603,7 @@ typedef enum { UCP_OP_ATTR_FLAG_NO_IMM_CMPL = UCS_BIT(16), /**< deny immediate completion */ UCP_OP_ATTR_FLAG_FAST_CMPL = UCS_BIT(17), /**< expedite local completion, - even if it delays remote + even if it delays remote data delivery. Note for implementer: this option can disable zero copy @@ -2662,11 +2666,11 @@ void ucp_rkey_destroy(ucp_rkey_h rkey); * @brief Add user defined callback for Active Message. * * This routine installs a user defined callback to handle incoming Active - * Messages with a specific id. This callback is called whenever an Active - * Message that was sent from the remote peer by @ref ucp_am_send_nb is + * Messages with a specific id. This callback is called whenever an Active + * Message that was sent from the remote peer by @ref ucp_am_send_nb is * received on this worker. * - * @param [in] worker UCP worker on which to set the Active Message + * @param [in] worker UCP worker on which to set the Active Message * handler. * @param [in] id Active Message id. * @param [in] cb Active Message callback. NULL to clear. @@ -3649,7 +3653,7 @@ ucs_status_ptr_t ucp_put_nb(ucp_ep_h ep, const void *buffer, size_t length, * progress of the operation. The application is * responsible for releasing the handle using * @ref ucp_request_free "ucp_request_free()" routine. - * + * * @note Only the datatype ucp_dt_make_contig(1) is supported * for @a param->datatype, see @ref ucp_dt_make_contig. */ @@ -3775,7 +3779,7 @@ ucs_status_ptr_t ucp_get_nb(ucp_ep_h ep, void *buffer, size_t length, * progress of the operation. The application is * responsible for releasing the handle using * @ref ucp_request_free "ucp_request_free()" routine. - * + * * @note Only the datatype ucp_dt_make_contig(1) is supported * for @a param->datatype, see @ref ucp_dt_make_contig. */ diff --git a/src/ucp/core/ucp_ep.c b/src/ucp/core/ucp_ep.c index 344b3bee642..72f5c790c09 100644 --- a/src/ucp/core/ucp_ep.c +++ b/src/ucp/core/ucp_ep.c @@ -1056,7 +1056,7 @@ ucs_status_ptr_t ucp_ep_close_nbx(ucp_ep_h ep, const ucp_request_param_t *param) ucp_ep_disconnected(ep, 1); } else { - request = ucp_ep_flush_internal(ep, 0, &ucp_request_null_param, NULL, + request = ucp_ep_flush_internal(ep, 0, param, NULL, ucp_ep_close_flushed_callback, "close"); if (!UCS_PTR_IS_PTR(request)) { if (ucp_ep_is_cm_local_connected(ep)) { diff --git a/test/gtest/ucp/test_ucp_wireup.cc b/test/gtest/ucp/test_ucp_wireup.cc index b90fcd876de..09781883c12 100644 --- a/test/gtest/ucp/test_ucp_wireup.cc +++ b/test/gtest/ucp/test_ucp_wireup.cc @@ -62,6 +62,9 @@ class test_ucp_wireup : public ucp_test { void disconnect(ucp_test::entity &e); + static void close_completion(void *request, ucs_status_t status, + void *user_data); + static void send_completion(void *request, ucs_status_t status); static void tag_recv_completion(void *request, ucs_status_t status, @@ -315,6 +318,14 @@ void test_ucp_wireup::send_completion(void *request, ucs_status_t status) { } +void test_ucp_wireup::close_completion(void *request, ucs_status_t status, + void *user_data) +{ + ASSERT_UCS_OK(status); + ASSERT_NE((test_ucp_wireup *)NULL, (test_ucp_wireup *)user_data); +} + + void test_ucp_wireup::tag_recv_completion(void *request, ucs_status_t status, ucp_tag_recv_info_t *info) { @@ -806,6 +817,32 @@ UCS_TEST_P(test_ucp_wireup_2sided, connect_disconnect) { } } +UCS_TEST_P(test_ucp_wireup_2sided, close_nbx_callback) { + sender().connect(&receiver(), get_ep_params()); + if (!is_loopback()) { + receiver().connect(&sender(), get_ep_params()); + } + + std::vector reqs; + ucp_request_param_t param; + + param.op_attr_mask = UCP_OP_ATTR_FIELD_CALLBACK | + UCP_OP_ATTR_FIELD_USER_DATA | + UCP_OP_ATTR_FLAG_NO_IMM_CMPL; + param.cb.send = close_completion; + param.user_data = this; + + reqs.push_back(ucp_ep_close_nbx(sender().revoke_ep(), ¶m)); + EXPECT_FALSE(UCS_PTR_IS_ERR(reqs.back())); + + if (!is_loopback()) { + reqs.push_back(ucp_ep_close_nbx(receiver().revoke_ep(), ¶m)); + EXPECT_FALSE(UCS_PTR_IS_ERR(reqs.back())); + } + + waitall(reqs); +} + UCS_TEST_P(test_ucp_wireup_2sided, multi_ep_2sided) { const unsigned count = 10; From ea23a9d60aab8dea07f0e844bb62c702965c4ed3 Mon Sep 17 00:00:00 2001 From: Sergey Oblomov Date: Thu, 17 Dec 2020 09:36:18 +0200 Subject: [PATCH 060/107] WIREUP: exclude ep_check lanes where KA supported from map - exclude from ep_check map lanes where built-in keepalive feature is supported - updated ep-check gtests to disable built-in keepalive --- src/ucp/wireup/select.c | 16 ++++++++++++++++ test/gtest/ucp/test_ucp_peer_failure.cc | 2 ++ test/gtest/ucp/test_ucp_wireup.cc | 7 +++++++ test/gtest/uct/test_peer_failure.cc | 7 +++++++ 4 files changed, 32 insertions(+) diff --git a/src/ucp/wireup/select.c b/src/ucp/wireup/select.c index ed6a0a52e41..077dc337a22 100644 --- a/src/ucp/wireup/select.c +++ b/src/ucp/wireup/select.c @@ -1520,6 +1520,22 @@ static void ucp_wireup_init_keepalive_map(ucp_worker_h worker, dev_map_used = 0; + /* find all devices with built-in keepalive support */ + for (lane = 0; lane < key->num_lanes; ++lane) { + rsc_index = key->lanes[lane].rsc_index; + if (rsc_index == UCP_NULL_RESOURCE) { + continue; + } + + dev_index = context->tl_rscs[rsc_index].dev_index; + ucs_assert(dev_index < (sizeof(dev_map_used) * 8)); + iface_attr = ucp_worker_iface_get_attr(worker, rsc_index); + if (iface_attr->cap.flags & UCT_IFACE_FLAG_EP_KEEPALIVE) { + dev_map_used |= UCS_BIT(dev_index); + } + } + + /* send ep_check on devices without built-in keepalive */ for (lane = 0; lane < key->num_lanes; ++lane) { /* add lanes to ep_check map */ rsc_index = key->lanes[lane].rsc_index; diff --git a/test/gtest/ucp/test_ucp_peer_failure.cc b/test/gtest/ucp/test_ucp_peer_failure.cc index 9e3da96e7c7..e17ddad6dca 100644 --- a/test/gtest/ucp/test_ucp_peer_failure.cc +++ b/test/gtest/ucp/test_ucp_peer_failure.cc @@ -464,6 +464,8 @@ class test_ucp_peer_failure_keepalive : public test_ucp_peer_failure test_ucp_peer_failure_keepalive() { m_sbuf.resize(1 * UCS_MBYTE); m_rbuf.resize(1 * UCS_MBYTE); + + m_env.push_back(new ucs::scoped_setenv("UCX_TCP_KEEPIDLE", "inf")); } void init() { diff --git a/test/gtest/ucp/test_ucp_wireup.cc b/test/gtest/ucp/test_ucp_wireup.cc index b90fcd876de..87ce12f5d91 100644 --- a/test/gtest/ucp/test_ucp_wireup.cc +++ b/test/gtest/ucp/test_ucp_wireup.cc @@ -1421,6 +1421,10 @@ UCP_INSTANTIATE_TEST_CASE_TLS(test_ucp_wireup_asymmetric, ib, "ib") class test_ucp_wireup_keepalive : public test_ucp_wireup { public: + test_ucp_wireup_keepalive() { + m_env.push_back(new ucs::scoped_setenv("UCX_TCP_KEEPIDLE", "inf")); + } + static void get_test_variants(std::vector& variants) { test_ucp_wireup::get_test_variants(variants, @@ -1445,6 +1449,9 @@ class test_ucp_wireup_keepalive : public test_ucp_wireup { sender().connect(&receiver(), get_ep_params()); receiver().connect(&sender(), get_ep_params()); } + +protected: + ucs::ptr_vector m_env; }; /* test if EP has non-empty keepalive lanes mask */ diff --git a/test/gtest/uct/test_peer_failure.cc b/test/gtest/uct/test_peer_failure.cc index 93e922296b0..87d39a3250e 100644 --- a/test/gtest/uct/test_peer_failure.cc +++ b/test/gtest/uct/test_peer_failure.cc @@ -468,6 +468,10 @@ UCT_INSTANTIATE_TEST_CASE(test_uct_peer_failure_multiple) class test_uct_peer_failure_keepalive : public test_uct_peer_failure { public: + test_uct_peer_failure_keepalive() { + m_env.push_back(new ucs::scoped_setenv("UCX_TCP_KEEPIDLE", "inf")); + } + void kill_receiver() { /* Hack: for SHM-based transports we can't really terminate @@ -483,6 +487,9 @@ class test_uct_peer_failure_keepalive : public test_uct_peer_failure test_uct_peer_failure::kill_receiver(); } + +protected: + ucs::ptr_vector m_env; }; UCS_TEST_SKIP_COND_P(test_uct_peer_failure_keepalive, killed, From c818f5251ce32a77b6ec3e3a67732f722a692494 Mon Sep 17 00:00:00 2001 From: Raul Akhmetshin Date: Fri, 18 Dec 2020 19:19:13 +0200 Subject: [PATCH 061/107] TOOLS/PERF: Addressed review comments. --- src/tools/perf/lib/libperf.c | 3 ++- test/gtest/uct/test_uct_perf.cc | 12 ++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/tools/perf/lib/libperf.c b/src/tools/perf/lib/libperf.c index 17538f366c2..6d1562f83a0 100644 --- a/src/tools/perf/lib/libperf.c +++ b/src/tools/perf/lib/libperf.c @@ -706,7 +706,8 @@ static ucs_status_t uct_perf_test_check_capabilities(ucx_perf_params_t *params, return UCS_ERR_UNSUPPORTED; } /* if msg_size_cnt == 1 the message size checked above */ - if ((UCX_PERF_CMD_AM == params->command) && (params->msg_size_cnt > 1)) { + if ((UCT_PERF_DATA_LAYOUT_ZCOPY == params->uct.data_layout) && + (UCX_PERF_CMD_AM == params->command) && (params->msg_size_cnt > 1)) { if (params->am_hdr_size > params->msg_size_list[0]) { if (params->flags & UCX_PERF_TEST_FLAG_VERBOSE) { ucs_error("AM header size (%lu) larger than the first IOV " diff --git a/test/gtest/uct/test_uct_perf.cc b/test/gtest/uct/test_uct_perf.cc index 3bd7c398b0d..df5e303d21b 100644 --- a/test/gtest/uct/test_uct_perf.cc +++ b/test/gtest/uct/test_uct_perf.cc @@ -24,19 +24,19 @@ class test_uct_perf : public uct_test, public test_perf { const test_perf::test_spec test_uct_perf::tests[] = { - { "am latency", "usec", + { "am short latency", "usec", UCX_PERF_API_UCT, UCX_PERF_CMD_AM, UCX_PERF_TEST_TYPE_PINGPONG, UCT_PERF_DATA_LAYOUT_SHORT, 0, 1, { 8 }, 1, 100000lu, ucs_offsetof(ucx_perf_result_t, latency.total_average), 1e6, 0.01, 2.5, 0 }, - { "am rate", "Mpps", + { "am short rate", "Mpps", UCX_PERF_API_UCT, UCX_PERF_CMD_AM, UCX_PERF_TEST_TYPE_STREAM_UNI, UCT_PERF_DATA_LAYOUT_SHORT, 0, 1, { 8 }, 1, 2000000lu, ucs_offsetof(ucx_perf_result_t, msgrate.total_average), 1e-6, 0.8, 80.0, 0 }, - { "am rate64", "Mpps", + { "am short rate64", "Mpps", UCX_PERF_API_UCT, UCX_PERF_CMD_AM, UCX_PERF_TEST_TYPE_STREAM_UNI, UCT_PERF_DATA_LAYOUT_SHORT, 0, 1, { 64 }, 1, 2000000lu, ucs_offsetof(ucx_perf_result_t, msgrate.total_average), 1e-6, 0.8, 80.0, @@ -44,19 +44,19 @@ const test_perf::test_spec test_uct_perf::tests[] = { "am short iov latency", "usec", UCX_PERF_API_UCT, UCX_PERF_CMD_AM, UCX_PERF_TEST_TYPE_PINGPONG, - UCT_PERF_DATA_LAYOUT_SHORT_IOV, 0, 1, { 8, 8 }, 1, 100000lu, + UCT_PERF_DATA_LAYOUT_SHORT_IOV, 0, 2, { 4, 4 }, 1, 100000lu, ucs_offsetof(ucx_perf_result_t, latency.total_average), 1e6, 0.01, 2.5, 0 }, { "am short iov rate", "Mpps", UCX_PERF_API_UCT, UCX_PERF_CMD_AM, UCX_PERF_TEST_TYPE_STREAM_UNI, - UCT_PERF_DATA_LAYOUT_SHORT_IOV, 0, 1, { 8, 8 }, 1, 2000000lu, + UCT_PERF_DATA_LAYOUT_SHORT_IOV, 0, 2, { 4, 4 }, 1, 2000000lu, ucs_offsetof(ucx_perf_result_t, msgrate.total_average), 1e-6, 0.8, 80.0, 0 }, { "am short iov rate64", "Mpps", UCX_PERF_API_UCT, UCX_PERF_CMD_AM, UCX_PERF_TEST_TYPE_STREAM_UNI, - UCT_PERF_DATA_LAYOUT_SHORT_IOV, 0, 1, { 32, 32 }, 1, 2000000lu, + UCT_PERF_DATA_LAYOUT_SHORT_IOV, 0, 2, { 32, 32 }, 1, 2000000lu, ucs_offsetof(ucx_perf_result_t, msgrate.total_average), 1e-6, 0.8, 80.0, 0 }, From 3897b8b11ba022677c38eeef37d287a79e433dc3 Mon Sep 17 00:00:00 2001 From: Raul Akhmetshin Date: Fri, 4 Dec 2020 12:16:38 +0200 Subject: [PATCH 062/107] UCT/IB/UD: Implemented AM short IOV in UD verbs. --- src/uct/ib/ud/verbs/ud_verbs.c | 37 +++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/src/uct/ib/ud/verbs/ud_verbs.c b/src/uct/ib/ud/verbs/ud_verbs.c index 41234d35987..25c78982c42 100644 --- a/src/uct/ib/ud/verbs/ud_verbs.c +++ b/src/uct/ib/ud/verbs/ud_verbs.c @@ -196,6 +196,41 @@ ucs_status_t uct_ud_verbs_ep_am_short(uct_ep_h tl_ep, uint8_t id, uint64_t hdr, return UCS_OK; } +static ucs_status_t uct_ud_verbs_ep_am_short_iov(uct_ep_h tl_ep, uint8_t id, + const uct_iov_t *iov, size_t iovcnt) +{ + uct_ud_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_ud_verbs_ep_t); + uct_ud_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_ud_verbs_iface_t); + uint32_t length = uct_iov_total_length(iov, iovcnt); + uct_ud_send_skb_t *skb; + ucs_status_t status; + + UCT_CHECK_LENGTH(sizeof(uct_ud_neth_t) + length, 0, + iface->super.config.max_inline, "am_short"); + + uct_ud_enter(&iface->super); + + status = uct_ud_am_skb_common(&iface->super, &ep->super, id, &skb); + if (status != UCS_OK) { + uct_ud_leave(&iface->super); + return status; + } + + skb->len = iface->tx.sge[0].length = sizeof(uct_ud_neth_t); + iface->tx.sge[0].addr = (uintptr_t)skb->neth; + iface->tx.wr_skb.num_sge = uct_ib_verbs_sge_fill_iov(iface->tx.sge + 1, + iov, iovcnt) + 1; + + uct_ud_verbs_post_send(iface, ep, &iface->tx.wr_skb, IBV_SEND_INLINE, + UCT_IB_MAX_ZCOPY_LOG_SGE(&iface->super.super)); + + iface->tx.wr_skb.num_sge = 1; + uct_ud_iface_complete_tx_skb(&iface->super, &ep->super, skb); + UCT_TL_EP_STAT_OP(&ep->super.super, AM, SHORT, length); + uct_ud_leave(&iface->super); + return UCS_OK; +} + static ssize_t uct_ud_verbs_ep_am_bcopy(uct_ep_h tl_ep, uint8_t id, uct_pack_callback_t pack_cb, void *arg, unsigned flags) @@ -519,7 +554,7 @@ static uct_ud_iface_ops_t uct_ud_verbs_iface_ops = { { .ep_put_short = uct_ud_verbs_ep_put_short, .ep_am_short = uct_ud_verbs_ep_am_short, - .ep_am_short_iov = uct_base_ep_am_short_iov, + .ep_am_short_iov = uct_ud_verbs_ep_am_short_iov, .ep_am_bcopy = uct_ud_verbs_ep_am_bcopy, .ep_am_zcopy = uct_ud_verbs_ep_am_zcopy, .ep_pending_add = uct_ud_ep_pending_add, From f8051f2230054d318917d55ee9a008b72522943b Mon Sep 17 00:00:00 2001 From: Devendar Bureddy Date: Fri, 18 Dec 2020 09:16:45 +0200 Subject: [PATCH 063/107] UCP/MEM_TYPE: Adjust mem type zcopy thresh if user sets UCX_ZCOPY_THRESH --- src/ucp/core/ucp_ep.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/ucp/core/ucp_ep.c b/src/ucp/core/ucp_ep.c index 344b3bee642..6b7ae1bde42 100644 --- a/src/ucp/core/ucp_ep.c +++ b/src/ucp/core/ucp_ep.c @@ -1499,6 +1499,7 @@ static void ucp_ep_config_init_attrs(ucp_worker_t *worker, ucp_rsc_index_t rsc_i uct_iface_attr_t *iface_attr; size_t it; size_t zcopy_thresh; + size_t mem_type_zcopy_thresh; int mem_type; iface_attr = ucp_worker_iface_get_attr(worker, rsc_index); @@ -1536,11 +1537,13 @@ static void ucp_ep_config_init_attrs(ucp_worker_t *worker, ucp_rsc_index_t rsc_i zcopy_thresh = ucs_min(zcopy_thresh, adjust_min_val); config->sync_zcopy_thresh[it] = zcopy_thresh; config->zcopy_thresh[it] = zcopy_thresh; + mem_type_zcopy_thresh = 1; } } else { config->zcopy_auto_thresh = 0; config->sync_zcopy_thresh[0] = config->zcopy_thresh[0] = ucs_min(context->config.ext.zcopy_thresh, adjust_min_val); + mem_type_zcopy_thresh = config->zcopy_thresh[0]; /* adjust max_short if zcopy_thresh is set externally */ ucp_ep_config_adjust_max_short(&config->max_short, @@ -1551,7 +1554,7 @@ static void ucp_ep_config_init_attrs(ucp_worker_t *worker, ucp_rsc_index_t rsc_i if (UCP_MEM_IS_HOST(mem_type)) { config->mem_type_zcopy_thresh[mem_type] = config->zcopy_thresh[0]; } else if (md_attr->cap.reg_mem_types & UCS_BIT(mem_type)) { - config->mem_type_zcopy_thresh[mem_type] = 1; + config->mem_type_zcopy_thresh[mem_type] = mem_type_zcopy_thresh; } } } From 2dfb8cdf8a482794dce624f5b3dcd1cf3acf0f95 Mon Sep 17 00:00:00 2001 From: dmitrygx Date: Sat, 19 Dec 2020 02:02:48 +0200 Subject: [PATCH 064/107] UCT/IB: Handle multiple flush cancel w/o completion --- src/uct/ib/dc/dc_mlx5_ep.c | 6 +++++- src/uct/ib/rc/base/rc_ep.c | 1 - 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/uct/ib/dc/dc_mlx5_ep.c b/src/uct/ib/dc/dc_mlx5_ep.c index 3e9b747aeeb..57a03f33fac 100644 --- a/src/uct/ib/dc/dc_mlx5_ep.c +++ b/src/uct/ib/dc/dc_mlx5_ep.c @@ -589,11 +589,14 @@ ucs_status_t uct_dc_mlx5_ep_flush(uct_ep_h tl_ep, unsigned flags, if (ucs_unlikely(flags & UCT_FLUSH_FLAG_CANCEL)) { ucs_assert(ucs_arbiter_group_is_empty(&ep->arb_group)); - ucs_assert(!(ep->flags & UCT_DC_MLX5_EP_FLAG_FLUSH_CANCEL)); if (uct_dc_mlx5_iface_is_dci_rand(iface)) { return UCS_ERR_UNSUPPORTED; } + if (ep->flags & UCT_DC_MLX5_EP_FLAG_FLUSH_CANCEL) { + goto out; + } + status = uct_ib_mlx5_modify_qp_state(md, &txwq->super, IBV_QPS_ERR); if (status != UCS_OK) { return status; @@ -602,6 +605,7 @@ ucs_status_t uct_dc_mlx5_ep_flush(uct_ep_h tl_ep, unsigned flags, ep->flags |= UCT_DC_MLX5_EP_FLAG_FLUSH_CANCEL; } +out: return uct_rc_txqp_add_flush_comp(&iface->super.super, &ep->super, txqp, comp, txwq->sig_pi); } diff --git a/src/uct/ib/rc/base/rc_ep.c b/src/uct/ib/rc/base/rc_ep.c index 34df2c17f25..923866fac6a 100644 --- a/src/uct/ib/rc/base/rc_ep.c +++ b/src/uct/ib/rc/base/rc_ep.c @@ -512,7 +512,6 @@ ucs_status_t uct_rc_ep_flush(uct_rc_ep_t *ep, int16_t max_available, if (ucs_unlikely(flags & UCT_FLUSH_FLAG_CANCEL)) { ucs_assert(ucs_arbiter_group_is_empty(&ep->arb_group)); - ucs_assert(!(ep->flags & UCT_RC_EP_FLAG_FLUSH_CANCEL)); ep->flags |= UCT_RC_EP_FLAG_FLUSH_CANCEL; } From be02557bfadbd0ae56c35078d1b50f93f3d1c83f Mon Sep 17 00:00:00 2001 From: dmitrygx Date: Sat, 19 Dec 2020 15:28:15 +0200 Subject: [PATCH 065/107] UCT/IB/RC: Don't modify QP state to error multiple times during FLUSH_CANCEL --- src/uct/ib/rc/accel/rc_mlx5_ep.c | 3 ++- src/uct/ib/rc/verbs/rc_verbs_ep.c | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/uct/ib/rc/accel/rc_mlx5_ep.c b/src/uct/ib/rc/accel/rc_mlx5_ep.c index 0be0a694004..089abf23e02 100644 --- a/src/uct/ib/rc/accel/rc_mlx5_ep.c +++ b/src/uct/ib/rc/accel/rc_mlx5_ep.c @@ -563,6 +563,7 @@ ucs_status_t uct_rc_mlx5_ep_flush(uct_ep_h tl_ep, unsigned flags, UCT_RC_MLX5_EP_DECL(tl_ep, iface, ep); uct_ib_mlx5_md_t *md = ucs_derived_of(iface->super.super.super.md, uct_ib_mlx5_md_t); + int already_canceled = ep->super.flags & UCT_RC_EP_FLAG_FLUSH_CANCEL; ucs_status_t status; uint16_t sn; @@ -585,7 +586,7 @@ ucs_status_t uct_rc_mlx5_ep_flush(uct_ep_h tl_ep, unsigned flags, sn = ep->tx.wq.sig_pi; } - if (ucs_unlikely(flags & UCT_FLUSH_FLAG_CANCEL)) { + if (ucs_unlikely((flags & UCT_FLUSH_FLAG_CANCEL) && !already_canceled)) { status = uct_ib_mlx5_modify_qp_state(md, &ep->tx.wq.super, IBV_QPS_ERR); if (status != UCS_OK) { return status; diff --git a/src/uct/ib/rc/verbs/rc_verbs_ep.c b/src/uct/ib/rc/verbs/rc_verbs_ep.c index ce5e717d148..ee8e2a2acf7 100644 --- a/src/uct/ib/rc/verbs/rc_verbs_ep.c +++ b/src/uct/ib/rc/verbs/rc_verbs_ep.c @@ -414,7 +414,8 @@ ucs_status_t uct_rc_verbs_ep_flush(uct_ep_h tl_ep, unsigned flags, uct_completion_t *comp) { uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t); - uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t); + uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t); + int already_canceled = ep->super.flags & UCT_RC_EP_FLAG_FLUSH_CANCEL; ucs_status_t status; status = uct_rc_ep_flush(&ep->super, iface->config.tx_max_wr, flags); @@ -427,7 +428,7 @@ ucs_status_t uct_rc_verbs_ep_flush(uct_ep_h tl_ep, unsigned flags, uct_rc_verbs_ep_post_flush(ep, IBV_SEND_SIGNALED); } - if (ucs_unlikely(flags & UCT_FLUSH_FLAG_CANCEL)) { + if (ucs_unlikely((flags & UCT_FLUSH_FLAG_CANCEL) && !already_canceled)) { status = uct_ib_modify_qp(ep->qp, IBV_QPS_ERR); if (status != UCS_OK) { return status; From 1990b322838e57690edec046099b27b1c0eaeb27 Mon Sep 17 00:00:00 2001 From: Devendar Bureddy Date: Mon, 21 Dec 2020 06:03:54 +0200 Subject: [PATCH 066/107] UCP/MEM_TYPE: Fix review comments --- src/ucp/core/ucp_ep.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ucp/core/ucp_ep.c b/src/ucp/core/ucp_ep.c index 6b7ae1bde42..f82882aa598 100644 --- a/src/ucp/core/ucp_ep.c +++ b/src/ucp/core/ucp_ep.c @@ -1529,6 +1529,7 @@ static void ucp_ep_config_init_attrs(ucp_worker_t *worker, ucp_rsc_index_t rsc_i if (context->config.ext.zcopy_thresh == UCS_MEMUNITS_AUTO) { config->zcopy_auto_thresh = 1; + mem_type_zcopy_thresh = 1; for (it = 0; it < UCP_MAX_IOV; ++it) { zcopy_thresh = ucp_ep_config_get_zcopy_auto_thresh( it + 1, &md_attr->reg_cost, context, @@ -1537,7 +1538,6 @@ static void ucp_ep_config_init_attrs(ucp_worker_t *worker, ucp_rsc_index_t rsc_i zcopy_thresh = ucs_min(zcopy_thresh, adjust_min_val); config->sync_zcopy_thresh[it] = zcopy_thresh; config->zcopy_thresh[it] = zcopy_thresh; - mem_type_zcopy_thresh = 1; } } else { config->zcopy_auto_thresh = 0; From 48c05474181020be7dab6e471dd0053c5534a89f Mon Sep 17 00:00:00 2001 From: Raul Akhmetshin Date: Tue, 22 Dec 2020 14:08:05 +0200 Subject: [PATCH 067/107] UCT/IB/UD: Addressed review comments. --- src/uct/ib/ud/verbs/ud_verbs.c | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/src/uct/ib/ud/verbs/ud_verbs.c b/src/uct/ib/ud/verbs/ud_verbs.c index 25c78982c42..aa970039172 100644 --- a/src/uct/ib/ud/verbs/ud_verbs.c +++ b/src/uct/ib/ud/verbs/ud_verbs.c @@ -102,9 +102,9 @@ static inline void uct_ud_verbs_ep_tx_inlv(uct_ud_verbs_iface_t *iface, uct_ud_verbs_ep_t *ep, const void *buffer, unsigned length) { - iface->tx.sge[1].addr = (uintptr_t)buffer; - iface->tx.sge[1].length = length; - ucs_assert(iface->tx.wr_inl.num_sge == 2); + iface->tx.sge[1].addr = (uintptr_t)buffer; + iface->tx.sge[1].length = length; + iface->tx.wr_inl.num_sge = 2; uct_ud_verbs_post_send(iface, ep, &iface->tx.wr_inl, IBV_SEND_INLINE, 2); } @@ -201,11 +201,12 @@ static ucs_status_t uct_ud_verbs_ep_am_short_iov(uct_ep_h tl_ep, uint8_t id, { uct_ud_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_ud_verbs_ep_t); uct_ud_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_ud_verbs_iface_t); - uint32_t length = uct_iov_total_length(iov, iovcnt); uct_ud_send_skb_t *skb; ucs_status_t status; - UCT_CHECK_LENGTH(sizeof(uct_ud_neth_t) + length, 0, + UCT_CHECK_IOV_SIZE(iovcnt, (size_t)iface->config.max_send_sge, + "uct_ud_verbs_ep_am_short_iov"); + UCT_CHECK_LENGTH(sizeof(uct_ud_neth_t) + uct_iov_total_length(iov, iovcnt), 0, iface->super.config.max_inline, "am_short"); uct_ud_enter(&iface->super); @@ -218,16 +219,15 @@ static ucs_status_t uct_ud_verbs_ep_am_short_iov(uct_ep_h tl_ep, uint8_t id, skb->len = iface->tx.sge[0].length = sizeof(uct_ud_neth_t); iface->tx.sge[0].addr = (uintptr_t)skb->neth; - iface->tx.wr_skb.num_sge = uct_ib_verbs_sge_fill_iov(iface->tx.sge + 1, - iov, iovcnt) + 1; - - uct_ud_verbs_post_send(iface, ep, &iface->tx.wr_skb, IBV_SEND_INLINE, - UCT_IB_MAX_ZCOPY_LOG_SGE(&iface->super.super)); + iface->tx.wr_inl.num_sge = uct_ib_verbs_sge_fill_iov(iface->tx.sge + 1, + iov, iovcnt) + 1; + uct_ud_verbs_post_send(iface, ep, &iface->tx.wr_inl, IBV_SEND_INLINE, + iface->tx.wr_inl.num_sge); - iface->tx.wr_skb.num_sge = 1; uct_ud_iface_complete_tx_skb(&iface->super, &ep->super, skb); - UCT_TL_EP_STAT_OP(&ep->super.super, AM, SHORT, length); + UCT_TL_EP_STAT_OP(&ep->super.super, AM, SHORT, uct_iov_total_length(iov, iovcnt)); uct_ud_leave(&iface->super); + return UCS_OK; } @@ -674,7 +674,6 @@ static UCS_CLASS_INIT_FUNC(uct_ud_verbs_iface_t, uct_md_h md, uct_worker_h worke self->tx.wr_inl.imm_data = 0; self->tx.wr_inl.next = 0; self->tx.wr_inl.sg_list = self->tx.sge; - self->tx.wr_inl.num_sge = 2; memset(&self->tx.wr_skb, 0, sizeof(self->tx.wr_skb)); self->tx.wr_skb.opcode = IBV_WR_SEND; From 0bfdac5fa2d6b3bc5af080d52fb6d7779aaa70cd Mon Sep 17 00:00:00 2001 From: Leonid Genkin Date: Tue, 8 Dec 2020 14:12:11 +0200 Subject: [PATCH 068/107] CODE: Use custom LLVM build, improve clang-format configuration --- .clang-format | 51 ++++++++++++++++++++++----------- buildlib/az-helpers.sh | 8 ++++++ buildlib/azure-pipelines-pr.yml | 17 +++++++++-- buildlib/fedora.Dockerfile | 9 +++++- 4 files changed, 65 insertions(+), 20 deletions(-) diff --git a/.clang-format b/.clang-format index e490ba75153..1d927867611 100644 --- a/.clang-format +++ b/.clang-format @@ -2,7 +2,8 @@ BasedOnStyle: LLVM AlignEscapedNewlines: DontAlign AlignConsecutiveAssignments: true -AlignConsecutiveDeclarations: false +AlignConsecutiveDeclarations: true +AlignConsecutiveStructMembers: true AlignConsecutiveMacros: true AlignAfterOpenBracket: true AlignOperands: true @@ -20,13 +21,13 @@ AlwaysBreakAfterReturnType: None PenaltyReturnTypeOnItsOwnLine: 100 PenaltyBreakAssignment: 100 PenaltyExcessCharacter: 10 -ColumnLimit: 85 +ColumnLimit: 80 AlwaysBreakBeforeMultilineStrings: false BinPackArguments: true BinPackParameters: true BreakBeforeBraces: Custom BraceWrapping: - AfterClass: true + AfterClass: false AfterControlStatement: false AfterEnum: false AfterFunction: true @@ -58,7 +59,36 @@ SpaceInEmptyParentheses: false SpaceBeforeParens: ControlStatementsExceptForEachMacros SpaceBeforeAssignmentOperators: true SpaceAfterCStyleCast: false -ForEachMacros: ["ucs_for_each_submask"] +SortIncludes: false +ForEachMacros: ['FOR_EACH_ENTITY', + 'kh_foreach', + 'kh_foreach_key', + 'kh_foreach_value', + 'ucp_unpacked_address_for_each', + 'ucs_array_for_each', + 'UCS_BITMAP_FOR_EACH_BIT', + 'ucs_for_each_bit', + 'ucs_for_each_submask', + 'ucs_hlist_for_each', + 'ucs_hlist_for_each_extract', + 'ucs_hlist_for_each_extract_if', + 'ucs_list_for_each', + 'ucs_list_for_each_safe', + 'ucs_memory_type_for_each', + 'UCS_PP_FOREACH', + 'UCS_PP_FOREACH_SEP', + 'ucs_profile_for_each_location', + 'ucs_ptr_array_for_each', + 'ucs_ptr_array_locked_for_each', + 'ucs_queue_for_each', + 'ucs_queue_for_each_extract', + 'ucs_queue_for_each_safe', + 'ucs_timerq_for_each_expired', + 'UCT_IB_IFACE_VERBS_FOREACH_RXWQE', + 'UCT_RC_VERBS_IFACE_FOREACH_TXWQE', + 'UCS_INIT_ONCE', + 'UCS_TEST_F', + 'UCX_PERF_TEST_FOREACH'] # CPP Standard: Cpp11 @@ -73,7 +103,6 @@ FixNamespaceComments: true NamespaceIndentation: None UseTab: Never ReflowComments: true -SortIncludes: true IncludeCategories: - Regex: '^"' Priority: 1 @@ -85,17 +114,7 @@ SpacesInAngles: false SpacesBeforeTrailingComments: 1 SpaceAfterTemplateKeyword: false SpacesInContainerLiterals: false - +--- # Java Language: Java DisableFormat: true - -# Issues: -# 1. -# Pointer alignment + declaration alignment: -# long_type_name var; -# void * ptr; -# Instead of: -#void *ptr; - -... diff --git a/buildlib/az-helpers.sh b/buildlib/az-helpers.sh index bd89fb38242..0a45cacff09 100644 --- a/buildlib/az-helpers.sh +++ b/buildlib/az-helpers.sh @@ -54,6 +54,14 @@ function azure_log_warning() { echo "##vso[task.logissue type=warning]${msg}" } +# Complete the task as "succeeeded with issues" +function azure_complete_with_issues() { + test "x$RUNNING_IN_AZURE" = "xno" && return + msg=$1 + set +x + echo "##vso[task.complete result=SucceededWithIssues;]DONE${msg}" +} + # Get IPv4 address of an interface function get_ip() { iface=$1 diff --git a/buildlib/azure-pipelines-pr.yml b/buildlib/azure-pipelines-pr.yml index f95ae93001a..17b5cfafd04 100644 --- a/buildlib/azure-pipelines-pr.yml +++ b/buildlib/azure-pipelines-pr.yml @@ -12,8 +12,7 @@ resources: image: ucfconsort.azurecr.io/ucx/centos7:1 endpoint: ucfconsort_registry - container: fedora - image: ucfconsort.azurecr.io/ucx/fedora33:1 - endpoint: ucfconsort_registry + image: rdmz-harbor.rdmz.labs.mlnx/ucx/fedora33:1 stages: - stage: Codestyle @@ -51,6 +50,10 @@ stages: # Check that the code is formatted according to the code style guidelines - job: format displayName: format code + pool: + name: MLNX + demands: + - ucx_instinct -equals yes container: fedora steps: - checkout: self @@ -69,7 +72,7 @@ stages: if ! git diff --quiet --exit-code then url="https://github.com/openucx/ucx/wiki/Code-style-checking" - azure_log_warning "Code is not formatted according to the code style, see $url for more info." + azure_complete_with_issues "Code is not formatted according to the code style, see $url for more info." fi condition: eq(variables['Build.Reason'], 'PullRequest') @@ -78,6 +81,10 @@ stages: jobs: - job: static_checks displayName: Static checks + pool: + name: MLNX + demands: + - ucx_instinct -equals yes container: fedora steps: - checkout: self @@ -158,6 +165,10 @@ stages: # Test RPM build - job: build_rpm displayName: Build tarball and source rpm + pool: + name: MLNX + demands: + - ucx_instinct -equals yes container: fedora steps: - checkout: self diff --git a/buildlib/fedora.Dockerfile b/buildlib/fedora.Dockerfile index 973519f17dd..cb75a540f5b 100644 --- a/buildlib/fedora.Dockerfile +++ b/buildlib/fedora.Dockerfile @@ -4,7 +4,7 @@ FROM fedora:33 RUN dnf install -y \ autoconf \ automake \ - clang \ + cmake \ cppcheck \ csclng \ cscppc \ @@ -20,6 +20,13 @@ RUN dnf install -y \ make \ maven \ numactl-devel \ + python \ rdma-core-devel \ rpm-build \ && dnf clean dbcache packages +RUN export BUILD_ROOT=/tmp/llvm-project && \ + git clone https://github.com/openucx/llvm-project.git --depth=1 -b ucx-clang-format --single-branch ${BUILD_ROOT} && \ + mkdir -p ${BUILD_ROOT}/build && cd ${BUILD_ROOT}/build && \ + cmake -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS=clang -G "Unix Makefiles" \ + ../llvm && \ + make -j$(nproc) && make install && rm -rf ${BUILD_ROOT} From 79499115ed6fb6f9fb410ba88d90aa9b68fa973a Mon Sep 17 00:00:00 2001 From: Yossi Itigin Date: Mon, 28 Dec 2020 17:06:38 +0200 Subject: [PATCH 069/107] UCT/IB/DEVX: Set modify-QP global address parameters only for GRH case --- src/uct/ib/rc/accel/rc_mlx5_devx.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/uct/ib/rc/accel/rc_mlx5_devx.c b/src/uct/ib/rc/accel/rc_mlx5_devx.c index 172b43ddf34..10f7c271658 100644 --- a/src/uct/ib/rc/accel/rc_mlx5_devx.c +++ b/src/uct/ib/rc/accel/rc_mlx5_devx.c @@ -377,16 +377,19 @@ uct_rc_mlx5_iface_common_devx_connect_qp(uct_rc_mlx5_iface_common_t *iface, UCT_IB_MLX5DV_SET(qpc, qpc, primary_address_path.rlid, ah_attr->dlid); UCT_IB_MLX5DV_SET(qpc, qpc, primary_address_path.mlid, ah_attr->src_path_bits & 0x7f); - UCT_IB_MLX5DV_SET(qpc, qpc, primary_address_path.hop_limit, - ah_attr->grh.hop_limit); - memcpy(UCT_IB_MLX5DV_ADDR_OF(qpc, qpc, primary_address_path.rgid_rip), - &ah_attr->grh.dgid, - UCT_IB_MLX5DV_FLD_SZ_BYTES(qpc, primary_address_path.rgid_rip)); UCT_IB_MLX5DV_SET(qpc, qpc, primary_address_path.sl, iface->super.super.config.sl); - /* TODO add flow_label support */ - UCT_IB_MLX5DV_SET(qpc, qpc, primary_address_path.tclass, - iface->super.super.config.traffic_class); + + if (ah_attr->is_global) { + UCT_IB_MLX5DV_SET(qpc, qpc, primary_address_path.hop_limit, + ah_attr->grh.hop_limit); + memcpy(UCT_IB_MLX5DV_ADDR_OF(qpc, qpc, primary_address_path.rgid_rip), + &ah_attr->grh.dgid, + UCT_IB_MLX5DV_FLD_SZ_BYTES(qpc, primary_address_path.rgid_rip)); + /* TODO add flow_label support */ + UCT_IB_MLX5DV_SET(qpc, qpc, primary_address_path.tclass, + iface->super.super.config.traffic_class); + } } UCT_IB_MLX5DV_SET(qpc, qpc, primary_address_path.vhca_port_num, ah_attr->port_num); From 91cfaa2e4056a9509ac4509ce30aaaa99ccab55c Mon Sep 17 00:00:00 2001 From: Yossi Itigin Date: Mon, 28 Dec 2020 23:04:45 +0200 Subject: [PATCH 070/107] AZP: Remove HWI test node due to server issues --- buildlib/azure-pipelines-pr.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/buildlib/azure-pipelines-pr.yml b/buildlib/azure-pipelines-pr.yml index f95ae93001a..ebaa009806a 100644 --- a/buildlib/azure-pipelines-pr.yml +++ b/buildlib/azure-pipelines-pr.yml @@ -198,11 +198,6 @@ stages: name: new demands: ucx_new -equals yes test_perf: 1 - - template: tests.yml - parameters: - name: hwi - demands: ucx_hwi -equals yes - test_perf: 0 - stage: io_demo dependsOn: [Codestyle] From a914f977ba11c0bc18e13454cb38ee2069648578 Mon Sep 17 00:00:00 2001 From: Devendar Bureddy Date: Tue, 29 Dec 2020 06:29:33 +0200 Subject: [PATCH 071/107] UCS: Move UCS_STATIC_INIT macro to compiler_def.h --- src/ucs/sys/compiler.h | 12 ------------ src/ucs/sys/compiler_def.h | 12 ++++++++++++ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/ucs/sys/compiler.h b/src/ucs/sys/compiler.h index 3089fb3f9c2..9be16ff7f3a 100644 --- a/src/ucs/sys/compiler.h +++ b/src/ucs/sys/compiler.h @@ -84,16 +84,4 @@ #define UCS_CACHELINE_PADDING_MISALIGN(...) \ ((UCS_PP_FOREACH(UCS_CACHELINE_PADDING_SIZEOF, _, __VA_ARGS__)) % UCS_SYS_CACHE_LINE_SIZE) -/* - * Define code which runs at global constructor phase - */ -#define UCS_STATIC_INIT \ - static void UCS_F_CTOR UCS_PP_APPEND_UNIQUE_ID(ucs_initializer_ctor)() - -/* - * Define code which runs at global destructor phase - */ -#define UCS_STATIC_CLEANUP \ - static void UCS_F_DTOR UCS_PP_APPEND_UNIQUE_ID(ucs_initializer_dtor)() - #endif diff --git a/src/ucs/sys/compiler_def.h b/src/ucs/sys/compiler_def.h index 98db780d3c9..9e9b29e12a6 100644 --- a/src/ucs/sys/compiler_def.h +++ b/src/ucs/sys/compiler_def.h @@ -179,4 +179,16 @@ /* Check if an expression is a compile-time constant */ #define ucs_is_constant(expr) __builtin_constant_p(expr) +/* + * Define code which runs at global constructor phase + */ +#define UCS_STATIC_INIT \ + static void UCS_F_CTOR UCS_PP_APPEND_UNIQUE_ID(ucs_initializer_ctor)() + +/* + * Define code which runs at global destructor phase + */ +#define UCS_STATIC_CLEANUP \ + static void UCS_F_DTOR UCS_PP_APPEND_UNIQUE_ID(ucs_initializer_dtor)() + #endif /* UCS_COMPILER_DEF_H */ From dc29cfdd7b1163adce66029dd1151035af7ee536 Mon Sep 17 00:00:00 2001 From: Artemy Kovalyov Date: Fri, 18 Dec 2020 14:38:21 +0200 Subject: [PATCH 072/107] GTEST/UCP/SOCKADDR: Increase flush timeout --- test/gtest/ucp/test_ucp_sockaddr.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/gtest/ucp/test_ucp_sockaddr.cc b/test/gtest/ucp/test_ucp_sockaddr.cc index 73ef845c36b..50b69241c70 100644 --- a/test/gtest/ucp/test_ucp_sockaddr.cc +++ b/test/gtest/ucp/test_ucp_sockaddr.cc @@ -480,7 +480,7 @@ class test_ucp_sockaddr : public ucp_test { void one_sided_disconnect(entity &e, enum ucp_ep_close_mode mode) { void *req = e.disconnect_nb(0, 0, mode); - ucs_time_t deadline = ucs_time_from_sec(10.0) + ucs_get_time(); + ucs_time_t deadline = ucs::get_deadline(); while (!is_request_completed(req) && (ucs_get_time() < deadline)) { /* TODO: replace the progress() with e().progress() when async progress is implemented. */ From 8a3932c1fdc44a814207a9f22f2ff770d15e3ab6 Mon Sep 17 00:00:00 2001 From: Raul Akhmetshin Date: Thu, 3 Dec 2020 13:56:51 +0200 Subject: [PATCH 073/107] UCT/IB/RC: Implemented AM short IOV in RC verbs. --- src/uct/ib/rc/base/rc_iface.h | 4 ++-- src/uct/ib/rc/verbs/rc_verbs.h | 5 ++++- src/uct/ib/rc/verbs/rc_verbs_ep.c | 21 ++++++++++++++++++++- src/uct/ib/rc/verbs/rc_verbs_iface.c | 3 +-- src/uct/ib/rc/verbs/rc_verbs_impl.h | 19 +++++++++++++++++-- 5 files changed, 44 insertions(+), 8 deletions(-) diff --git a/src/uct/ib/rc/base/rc_iface.h b/src/uct/ib/rc/base/rc_iface.h index 843562093fc..016cbe9ea00 100644 --- a/src/uct/ib/rc/base/rc_iface.h +++ b/src/uct/ib/rc/base/rc_iface.h @@ -23,9 +23,9 @@ #define UCT_RC_QP_TABLE_MEMB_ORDER (UCT_IB_QPN_ORDER - UCT_RC_QP_TABLE_ORDER) #define UCT_RC_QP_MAX_RETRY_COUNT 7 -#define UCT_RC_CHECK_AM_SHORT(_am_id, _length, _max_inline) \ +#define UCT_RC_CHECK_AM_SHORT(_am_id, _length, _header_t, _max_inline) \ UCT_CHECK_AM_ID(_am_id); \ - UCT_CHECK_LENGTH(sizeof(uct_rc_am_short_hdr_t) + _length, 0, _max_inline, "am_short"); + UCT_CHECK_LENGTH(sizeof(_header_t) + _length, 0, _max_inline, "am_short"); #define UCT_RC_CHECK_ZCOPY_DATA(_header_length, _length, _seg_size) \ UCT_CHECK_LENGTH(_header_length + _length, 0, _seg_size, "am_zcopy payload"); \ diff --git a/src/uct/ib/rc/verbs/rc_verbs.h b/src/uct/ib/rc/verbs/rc_verbs.h index 7cb3d457f2d..628848f9536 100644 --- a/src/uct/ib/rc/verbs/rc_verbs.h +++ b/src/uct/ib/rc/verbs/rc_verbs.h @@ -74,7 +74,7 @@ typedef struct uct_rc_verbs_iface { struct ibv_srq *srq; struct ibv_send_wr inl_am_wr; struct ibv_send_wr inl_rwrite_wr; - struct ibv_sge inl_sge[2]; + struct ibv_sge inl_sge[UCT_IB_MAX_IOV]; uct_rc_am_short_hdr_t am_inl_hdr; ucs_mpool_t short_desc_mp; uct_rc_iface_send_desc_t *fc_desc; /* used when max_inline is zero */ @@ -122,6 +122,9 @@ ucs_status_t uct_rc_verbs_ep_get_zcopy(uct_ep_h tl_ep, ucs_status_t uct_rc_verbs_ep_am_short(uct_ep_h tl_ep, uint8_t id, uint64_t hdr, const void *buffer, unsigned length); +ucs_status_t uct_rc_verbs_ep_am_short_iov(uct_ep_h ep, uint8_t id, + const uct_iov_t *iov, size_t iovcnt); + ssize_t uct_rc_verbs_ep_am_bcopy(uct_ep_h tl_ep, uint8_t id, uct_pack_callback_t pack_cb, void *arg, unsigned flags); diff --git a/src/uct/ib/rc/verbs/rc_verbs_ep.c b/src/uct/ib/rc/verbs/rc_verbs_ep.c index ee8e2a2acf7..75ab439f598 100644 --- a/src/uct/ib/rc/verbs/rc_verbs_ep.c +++ b/src/uct/ib/rc/verbs/rc_verbs_ep.c @@ -264,7 +264,7 @@ ucs_status_t uct_rc_verbs_ep_am_short(uct_ep_h tl_ep, uint8_t id, uint64_t hdr, uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t); uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t); - UCT_RC_CHECK_AM_SHORT(id, length, iface->config.max_inline); + UCT_RC_CHECK_AM_SHORT(id, length, uct_rc_am_short_hdr_t, iface->config.max_inline); UCT_RC_CHECK_RES_AND_FC(&iface->super, &ep->super, id); uct_rc_verbs_iface_fill_inl_am_sge(iface, id, hdr, buffer, length); UCT_TL_EP_STAT_OP(&ep->super.super, AM, SHORT, sizeof(hdr) + length); @@ -275,6 +275,25 @@ ucs_status_t uct_rc_verbs_ep_am_short(uct_ep_h tl_ep, uint8_t id, uint64_t hdr, return UCS_OK; } +ucs_status_t uct_rc_verbs_ep_am_short_iov(uct_ep_h tl_ep, uint8_t id, + const uct_iov_t *iov, size_t iovcnt) +{ + uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t); + uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t); + + UCT_RC_CHECK_AM_SHORT(id, uct_iov_total_length(iov, iovcnt), uct_rc_hdr_t, + iface->config.max_inline); + UCT_RC_CHECK_RES_AND_FC(&iface->super, &ep->super, id); + UCT_CHECK_IOV_SIZE(iovcnt, UCT_IB_MAX_IOV - 1, "uct_rc_verbs_ep_am_short_iov"); + uct_rc_verbs_iface_fill_inl_am_sge_iov(iface, id, iov, iovcnt); + UCT_TL_EP_STAT_OP(&ep->super.super, AM, SHORT, uct_iov_total_length(iov, iovcnt)); + uct_rc_verbs_ep_post_send(iface, ep, &iface->inl_am_wr, + IBV_SEND_INLINE | IBV_SEND_SOLICITED, INT_MAX); + UCT_RC_UPDATE_FC(&iface->super, &ep->super, id); + + return UCS_OK; +} + ssize_t uct_rc_verbs_ep_am_bcopy(uct_ep_h tl_ep, uint8_t id, uct_pack_callback_t pack_cb, void *arg, unsigned flags) diff --git a/src/uct/ib/rc/verbs/rc_verbs_iface.c b/src/uct/ib/rc/verbs/rc_verbs_iface.c index 21a42f15f1e..cab4c359c62 100644 --- a/src/uct/ib/rc/verbs/rc_verbs_iface.c +++ b/src/uct/ib/rc/verbs/rc_verbs_iface.c @@ -163,7 +163,6 @@ static void uct_rc_verbs_iface_init_inl_wrs(uct_rc_verbs_iface_t *iface) { memset(&iface->inl_am_wr, 0, sizeof(iface->inl_am_wr)); iface->inl_am_wr.sg_list = iface->inl_sge; - iface->inl_am_wr.num_sge = 2; iface->inl_am_wr.opcode = IBV_WR_SEND; iface->inl_am_wr.send_flags = IBV_SEND_INLINE; @@ -456,7 +455,7 @@ static uct_rc_iface_ops_t uct_rc_verbs_iface_ops = { { { .ep_am_short = uct_rc_verbs_ep_am_short, - .ep_am_short_iov = uct_base_ep_am_short_iov, + .ep_am_short_iov = uct_rc_verbs_ep_am_short_iov, .ep_am_bcopy = uct_rc_verbs_ep_am_bcopy, .ep_am_zcopy = uct_rc_verbs_ep_am_zcopy, .ep_put_short = uct_rc_verbs_ep_put_short, diff --git a/src/uct/ib/rc/verbs/rc_verbs_impl.h b/src/uct/ib/rc/verbs/rc_verbs_impl.h index ccb4c6c4409..ed0c18233b4 100644 --- a/src/uct/ib/rc/verbs/rc_verbs_impl.h +++ b/src/uct/ib/rc/verbs/rc_verbs_impl.h @@ -134,11 +134,26 @@ uct_rc_verbs_iface_fill_inl_am_sge(uct_rc_verbs_iface_t *iface, const void *buffer, unsigned length) { uct_rc_am_short_hdr_t *am = &iface->am_inl_hdr; - am->rc_hdr.am_id = id; - am->am_hdr = hdr; + + am->rc_hdr.am_id = id; + am->am_hdr = hdr; + iface->inl_am_wr.num_sge = 2; uct_rc_verbs_iface_fill_inl_sge(iface, am, sizeof(*am), buffer, length); } +static inline void +uct_rc_verbs_iface_fill_inl_am_sge_iov(uct_rc_verbs_iface_t *iface, uint8_t id, + const uct_iov_t *iov, size_t iovcnt) +{ + uct_rc_hdr_t *rch = &iface->am_inl_hdr.rc_hdr; + + rch->am_id = id; + iface->inl_sge[0].addr = (uintptr_t)rch; + iface->inl_sge[0].length = sizeof(*rch); + iface->inl_am_wr.num_sge = uct_ib_verbs_sge_fill_iov(iface->inl_sge + 1, iov, + iovcnt) + 1; +} + #define UCT_RC_VERBS_FILL_SGE(_wr, _sge, _length) \ _wr.sg_list = &_sge; \ _wr.num_sge = 1; \ From 44623a5dfc5d2935afa8ff93bb19a4d21b67628a Mon Sep 17 00:00:00 2001 From: Anatoly Vildemanov Date: Thu, 17 Dec 2020 16:02:51 +0300 Subject: [PATCH 074/107] AZP/IODEMO: Fix data racing between pipelines --- buildlib/io_demo/az-stage-io-demo.yaml | 8 ++++---- buildlib/io_demo/io-demo.yml | 25 +++++++++++++++++++++---- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/buildlib/io_demo/az-stage-io-demo.yaml b/buildlib/io_demo/az-stage-io-demo.yaml index 8ddf59fcac7..a9ec912f192 100644 --- a/buildlib/io_demo/az-stage-io-demo.yaml +++ b/buildlib/io_demo/az-stage-io-demo.yaml @@ -7,8 +7,8 @@ parameters: steps: - bash: | set -eEx - source ./buildlib/az-helpers.sh - ./buildlib/az-network-corrupter.sh \ + source $(workspace)/buildlib/az-helpers.sh + $(workspace)/buildlib/az-network-corrupter.sh \ initial_delay=$(initial_delay) \ cycles=$(cycles) \ downtime=$(downtime) \ @@ -27,8 +27,8 @@ steps: mkdir -p $(workspace)/${{ parameters.name }} # set UCX environment variables export UCX_SOCKADDR_CM_ENABLE=y - #export UCX_NET_DEVICES=$(ibdev2netdev | sed -ne 's/\(\w*\) port \([0-9]\) ==> '${roce_iface}' .*/\1:\2/p') - $(workspace)/../test/apps/iodemo/run_io_demo.sh \ + export UCX_NET_DEVICES=$(ibdev2netdev | sed -ne 's/\(\w*\) port \([0-9]\) ==> '${roce_iface}' .*/\1:\2/p') + $(workspace)/test/apps/iodemo/run_io_demo.sh \ -H $(agent_hosts) \ --tasks-per-node 1 \ --duration 480 \ diff --git a/buildlib/io_demo/io-demo.yml b/buildlib/io_demo/io-demo.yml index 80d6a658f6b..4d478df535d 100644 --- a/buildlib/io_demo/io-demo.yml +++ b/buildlib/io_demo/io-demo.yml @@ -36,10 +36,21 @@ jobs: ./contrib/configure-release --prefix=$(Build.Repository.LocalPath)/install make -j`nproc` make install - echo "##vso[task.setvariable variable=workspace;isOutput=true]$(Build.Repository.LocalPath)/buildlib" - echo "##vso[task.setvariable variable=io_demo_exe;isOutput=true]$(Build.Repository.LocalPath)/install/bin/io_demo" displayName: Build name: build + - task: CopyFiles@2 + inputs: + sourceFolder: '$(Build.Repository.LocalPath)' + contents: | + buildlib/az-helpers.sh + buildlib/az-network-corrupter.sh + install/** + test/apps/iodemo/run_io_demo.sh + targetFolder: '$(Build.ArtifactStagingDirectory)' + - task: PublishBuildArtifacts@1 + inputs: + pathToPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: drop_$(Build.BuildId) - job: test dependsOn: io_build @@ -60,12 +71,18 @@ jobs: maxParallel: 1 variables: - workspace: $[ dependencies.io_build.outputs['build.workspace'] ] - io_demo_exe: $[ dependencies.io_build.outputs['build.io_demo_exe'] ] + workspace: drop_$(Build.BuildId) + io_demo_exe: drop_$(Build.BuildId)/install/bin/io_demo displayName: "io_demo: " steps: - checkout: none + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: drop_$(Build.BuildId) + downloadPath: $(System.DefaultWorkingDirectory) + - bash: chmod u+rwx $(workspace) -R - template: az-stage-io-demo.yaml parameters: name: tag From c5c57eaa1d08eff76c7a70a850ec89840677f4de Mon Sep 17 00:00:00 2001 From: Peter Rudenko Date: Tue, 29 Dec 2020 15:47:52 +0200 Subject: [PATCH 075/107] JUCX: Request AM feature support. --- .../src/main/java/org/openucx/jucx/ucp/UcpConstants.java | 1 + .../src/main/java/org/openucx/jucx/ucp/UcpParams.java | 9 +++++++++ bindings/java/src/main/native/ucp_constants.cc | 1 + 3 files changed, 11 insertions(+) diff --git a/bindings/java/src/main/java/org/openucx/jucx/ucp/UcpConstants.java b/bindings/java/src/main/java/org/openucx/jucx/ucp/UcpConstants.java index e47a25af70b..764f827f5e9 100644 --- a/bindings/java/src/main/java/org/openucx/jucx/ucp/UcpConstants.java +++ b/bindings/java/src/main/java/org/openucx/jucx/ucp/UcpConstants.java @@ -38,6 +38,7 @@ public class UcpConstants { static long UCP_FEATURE_AMO64; static long UCP_FEATURE_WAKEUP; static long UCP_FEATURE_STREAM; + static long UCP_FEATURE_AM; /** * UCP worker parameters field mask. diff --git a/bindings/java/src/main/java/org/openucx/jucx/ucp/UcpParams.java b/bindings/java/src/main/java/org/openucx/jucx/ucp/UcpParams.java index d4ace227ed7..8797279e8e0 100644 --- a/bindings/java/src/main/java/org/openucx/jucx/ucp/UcpParams.java +++ b/bindings/java/src/main/java/org/openucx/jucx/ucp/UcpParams.java @@ -102,6 +102,15 @@ public UcpParams requestTagFeature() { return this; } + /** + * Request Active Message support feature. + */ + public UcpParams requestAmFeature() { + this.fieldMask |= UcpConstants.UCP_PARAM_FIELD_FEATURES; + this.features |= UcpConstants.UCP_FEATURE_AM; + return this; + } + /** * Request remote memory access support. */ diff --git a/bindings/java/src/main/native/ucp_constants.cc b/bindings/java/src/main/native/ucp_constants.cc index c156aae4aea..5a5906ee061 100644 --- a/bindings/java/src/main/native/ucp_constants.cc +++ b/bindings/java/src/main/native/ucp_constants.cc @@ -30,6 +30,7 @@ Java_org_openucx_jucx_ucp_UcpConstants_loadConstants(JNIEnv *env, jclass cls) JUCX_DEFINE_LONG_CONSTANT(UCP_FEATURE_AMO64); JUCX_DEFINE_LONG_CONSTANT(UCP_FEATURE_WAKEUP); JUCX_DEFINE_LONG_CONSTANT(UCP_FEATURE_STREAM); + JUCX_DEFINE_LONG_CONSTANT(UCP_FEATURE_AM); // UCP worker parameters JUCX_DEFINE_LONG_CONSTANT(UCP_WORKER_PARAM_FIELD_THREAD_MODE); From 647019f5892d3e2549bc0905b1d833af6f05ffcb Mon Sep 17 00:00:00 2001 From: Peter Rudenko Date: Tue, 29 Dec 2020 17:05:08 +0200 Subject: [PATCH 076/107] JUCX: add memory type hint to ucp operations. --- .../org/openucx/jucx/ucp/UcpEndpoint.java | 102 ++++++++++++++---- .../java/org/openucx/jucx/ucp/UcpWorker.java | 45 ++++++-- bindings/java/src/main/native/endpoint.cc | 22 ++-- bindings/java/src/main/native/worker.cc | 7 +- 4 files changed, 136 insertions(+), 40 deletions(-) diff --git a/bindings/java/src/main/java/org/openucx/jucx/ucp/UcpEndpoint.java b/bindings/java/src/main/java/org/openucx/jucx/ucp/UcpEndpoint.java index cdb5c751d27..287c960500e 100644 --- a/bindings/java/src/main/java/org/openucx/jucx/ucp/UcpEndpoint.java +++ b/bindings/java/src/main/java/org/openucx/jucx/ucp/UcpEndpoint.java @@ -9,11 +9,13 @@ import java.io.Closeable; import java.nio.ByteBuffer; +import static org.openucx.jucx.ucs.UcsConstants.MEMORY_TYPE.UCS_MEMORY_TYPE_UNKNOWN; + public class UcpEndpoint extends UcxNativeStruct implements Closeable { - private final String paramsString; + private String paramsString; // Keep a reference to errorHandler to prevent it from GC and have valid ref // from JNI error handler. - private final UcpEndpointErrorHandler errorHandler; + private UcpEndpointErrorHandler errorHandler; @Override public String toString() { @@ -81,9 +83,15 @@ public UcpRequest putNonBlocking(ByteBuffer src, long remoteAddress, UcpRemoteKe public UcpRequest putNonBlocking(long localAddress, long size, long remoteAddress, UcpRemoteKey remoteKey, UcxCallback callback) { + return putNonBlocking(localAddress, size, remoteAddress, remoteKey, callback, + UCS_MEMORY_TYPE_UNKNOWN); + } + public UcpRequest putNonBlocking(long localAddress, long size, + long remoteAddress, UcpRemoteKey remoteKey, + UcxCallback callback, int memoryType) { return putNonBlockingNative(getNativeId(), localAddress, - size, remoteAddress, remoteKey.getNativeId(), callback); + size, remoteAddress, remoteKey.getNativeId(), callback, memoryType); } /** @@ -136,8 +144,16 @@ public UcpRequest getNonBlocking(long remoteAddress, UcpRemoteKey remoteKey, public UcpRequest getNonBlocking(long remoteAddress, UcpRemoteKey remoteKey, long localAddress, long size, UcxCallback callback) { + return getNonBlocking(remoteAddress, remoteKey, localAddress, size, callback, + UCS_MEMORY_TYPE_UNKNOWN); + } + + public UcpRequest getNonBlocking(long remoteAddress, UcpRemoteKey remoteKey, + long localAddress, long size, UcxCallback callback, + int memoryType) { + return getNonBlockingNative(getNativeId(), remoteAddress, remoteKey.getNativeId(), - localAddress, size, callback); + localAddress, size, callback, memoryType); } /** @@ -192,7 +208,13 @@ public UcpRequest sendTaggedNonBlocking(ByteBuffer sendBuffer, long tag, UcxCall public UcpRequest sendTaggedNonBlocking(long localAddress, long size, long tag, UcxCallback callback) { - return sendTaggedNonBlockingNative(getNativeId(), localAddress, size, tag, callback); + return sendTaggedNonBlocking(localAddress, size, tag, callback, UCS_MEMORY_TYPE_UNKNOWN); + } + + public UcpRequest sendTaggedNonBlocking(long localAddress, long size, + long tag, UcxCallback callback, int memoryType) { + return sendTaggedNonBlockingNative(getNativeId(), localAddress, size, tag, callback, + memoryType); } /** @@ -207,11 +229,17 @@ public UcpRequest sendTaggedNonBlocking(ByteBuffer sendBuffer, UcxCallback callb * Iov version of non blocking send operaation */ public UcpRequest sendTaggedNonBlocking(long[] localAddresses, long[] sizes, - long tag, UcxCallback callback) { + long tag, UcxCallback callback, int memoryType) { UcxParams.checkArraySizes(localAddresses, sizes); return sendTaggedIovNonBlockingNative(getNativeId(), localAddresses, sizes, - tag, callback); + tag, callback, memoryType); + } + + public UcpRequest sendTaggedNonBlocking(long[] localAddresses, long[] sizes, + long tag, UcxCallback callback) { + + return sendTaggedNonBlocking(localAddresses, sizes, tag, callback, UCS_MEMORY_TYPE_UNKNOWN); } /** @@ -222,14 +250,27 @@ public UcpRequest sendTaggedNonBlocking(long[] localAddresses, long[] sizes, * completion of the send operation. */ public UcpRequest sendStreamNonBlocking(long localAddress, long size, UcxCallback callback) { - return sendStreamNonBlockingNative(getNativeId(), localAddress, size, callback); + return sendStreamNonBlocking(localAddress, size, callback, UCS_MEMORY_TYPE_UNKNOWN); + } + + public UcpRequest sendStreamNonBlocking(long localAddress, long size, UcxCallback callback, + int memoryType) { + return sendStreamNonBlockingNative(getNativeId(), localAddress, size, callback, memoryType); } public UcpRequest sendStreamNonBlocking(long[] localAddresses, long[] sizes, UcxCallback callback) { UcxParams.checkArraySizes(localAddresses, sizes); - return sendStreamIovNonBlockingNative(getNativeId(), localAddresses, sizes, callback); + return sendStreamNonBlocking(localAddresses, sizes, callback, UCS_MEMORY_TYPE_UNKNOWN); + } + + public UcpRequest sendStreamNonBlocking(long[] localAddresses, long[] sizes, + UcxCallback callback, int memoryType) { + UcxParams.checkArraySizes(localAddresses, sizes); + + return sendStreamIovNonBlockingNative(getNativeId(), localAddresses, sizes, callback, + memoryType); } public UcpRequest sendStreamNonBlocking(ByteBuffer buffer, UcxCallback callback) { @@ -244,17 +285,30 @@ public UcpRequest sendStreamNonBlocking(ByteBuffer buffer, UcxCallback callback) * the UCP library will invoke the call-back when data is in the receive buffer * and ready for application access. */ + public UcpRequest recvStreamNonBlocking(long localAddress, long size, long flags, + UcxCallback callback, int memoryType) { + return recvStreamNonBlockingNative(getNativeId(), localAddress, size, flags, callback, + memoryType); + } + public UcpRequest recvStreamNonBlocking(long localAddress, long size, long flags, UcxCallback callback) { - return recvStreamNonBlockingNative(getNativeId(), localAddress, size, flags, callback); + return recvStreamNonBlocking(localAddress, size, flags, callback, UCS_MEMORY_TYPE_UNKNOWN); } public UcpRequest recvStreamNonBlocking(long[] localAddresses, long[] sizes, long flags, - UcxCallback callback) { + UcxCallback callback, int memoryType) { UcxParams.checkArraySizes(localAddresses, sizes); return recvStreamIovNonBlockingNative(getNativeId(), localAddresses, sizes, flags, - callback); + callback, memoryType); + } + + public UcpRequest recvStreamNonBlocking(long[] localAddresses, long[] sizes, long flags, + UcxCallback callback) { + + return recvStreamNonBlocking(localAddresses, sizes, flags, callback, + UCS_MEMORY_TYPE_UNKNOWN); } public UcpRequest recvStreamNonBlocking(ByteBuffer buffer, long flags, UcxCallback callback) { @@ -297,7 +351,8 @@ public UcpRequest closeNonBlockingFlush() { private static native UcpRequest putNonBlockingNative(long enpointId, long localAddress, long size, long remoteAddr, - long ucpRkeyId, UcxCallback callback); + long ucpRkeyId, UcxCallback callback, + int memoryType); private static native void putNonBlockingImplicitNative(long enpointId, long localAddress, long size, long remoteAddr, @@ -305,7 +360,8 @@ private static native void putNonBlockingImplicitNative(long enpointId, long loc private static native UcpRequest getNonBlockingNative(long enpointId, long remoteAddress, long ucpRkeyId, long localAddress, - long size, UcxCallback callback); + long size, UcxCallback callback, + int memoryType); private static native void getNonBlockingImplicitNative(long enpointId, long remoteAddress, long ucpRkeyId, long localAddress, @@ -313,29 +369,35 @@ private static native void getNonBlockingImplicitNative(long enpointId, long rem private static native UcpRequest sendTaggedNonBlockingNative(long enpointId, long localAddress, long size, long tag, - UcxCallback callback); + UcxCallback callback, + int memoryType); private static native UcpRequest sendTaggedIovNonBlockingNative(long enpointId, long[] localAddresses, long[] sizes, long tag, - UcxCallback callback); + UcxCallback callback, + int memoryType); private static native UcpRequest sendStreamNonBlockingNative(long enpointId, long localAddress, - long size, UcxCallback callback); + long size, UcxCallback callback, + int memoryType); private static native UcpRequest sendStreamIovNonBlockingNative(long enpointId, long[] localAddresses, long[] sizes, - UcxCallback callback); + UcxCallback callback, + int memoryType); private static native UcpRequest recvStreamNonBlockingNative(long enpointId, long localAddress, long size, long flags, - UcxCallback callback); + UcxCallback callback, + int memoryType); private static native UcpRequest recvStreamIovNonBlockingNative(long enpointId, long[] localAddresses, long[] sizes, long flags, - UcxCallback callback); + UcxCallback callback, + int memoryType); private static native UcpRequest flushNonBlockingNative(long enpointId, UcxCallback callback); diff --git a/bindings/java/src/main/java/org/openucx/jucx/ucp/UcpWorker.java b/bindings/java/src/main/java/org/openucx/jucx/ucp/UcpWorker.java index 7848246bda7..62ced23748d 100644 --- a/bindings/java/src/main/java/org/openucx/jucx/ucp/UcpWorker.java +++ b/bindings/java/src/main/java/org/openucx/jucx/ucp/UcpWorker.java @@ -10,6 +10,8 @@ import org.openucx.jucx.*; +import static org.openucx.jucx.ucs.UcsConstants.MEMORY_TYPE.UCS_MEMORY_TYPE_UNKNOWN; + /** * UCP worker is an opaque object representing the communication context. The * worker represents an instance of a local communication resource and the @@ -128,14 +130,20 @@ public UcpRequest recvTaggedNonBlocking(ByteBuffer recvBuffer, long tag, long ta if (!recvBuffer.isDirect()) { throw new UcxException("Recv buffer must be direct."); } - return recvTaggedNonBlockingNative(getNativeId(), UcxUtils.getAddress(recvBuffer), + return recvTaggedNonBlocking(UcxUtils.getAddress(recvBuffer), recvBuffer.remaining(), tag, tagMask, callback); } public UcpRequest recvTaggedNonBlocking(long localAddress, long size, long tag, long tagMask, UcxCallback callback) { + return recvTaggedNonBlocking(localAddress, size, tag, tagMask, callback, + UCS_MEMORY_TYPE_UNKNOWN); + } + + public UcpRequest recvTaggedNonBlocking(long localAddress, long size, long tag, long tagMask, + UcxCallback callback, int memoryType) { return recvTaggedNonBlockingNative(getNativeId(), localAddress, size, - tag, tagMask, callback); + tag, tagMask, callback, memoryType); } /** @@ -150,10 +158,18 @@ public UcpRequest recvTaggedNonBlocking(ByteBuffer recvBuffer, UcxCallback callb public UcpRequest recvTaggedNonBlocking(long[] localAddresses, long[] sizes, long tag, long tagMask, UcxCallback callback) { + + return recvTaggedNonBlocking(localAddresses, sizes, tag, tagMask, callback, + UCS_MEMORY_TYPE_UNKNOWN); + } + + public UcpRequest recvTaggedNonBlocking(long[] localAddresses, long[] sizes, + long tag, long tagMask, + UcxCallback callback, int memoryType) { UcxParams.checkArraySizes(localAddresses, sizes); return recvTaggedIovNonBlockingNative(getNativeId(), localAddresses, sizes, tag, - tagMask, callback); + tagMask, callback, memoryType); } /** @@ -201,9 +217,15 @@ public UcpTagMessage tagProbeNonBlocking(long tag, long tagMask, boolean remove) * If the receive operation cannot be stated the routine returns an error. */ public UcpRequest recvTaggedMessageNonBlocking(long address, long size, UcpTagMessage message, - UcxCallback callback) { + UcxCallback callback, int memoryType) { return recvTaggedMessageNonBlockingNative(getNativeId(), address, size, - message.getNativeId(), callback); + message.getNativeId(), callback, memoryType); + } + + public UcpRequest recvTaggedMessageNonBlocking(long address, long size, UcpTagMessage message, + UcxCallback callback) { + return recvTaggedMessageNonBlocking(address, size, message, callback, + UCS_MEMORY_TYPE_UNKNOWN); } public UcpRequest recvTaggedMessageNonBlocking(ByteBuffer buffer, UcpTagMessage message, @@ -212,6 +234,7 @@ public UcpRequest recvTaggedMessageNonBlocking(ByteBuffer buffer, UcpTagMessage message, callback); } + /** * This routine tries to cancels an outstanding communication request. After * calling this routine, the request will be in completed or canceled (but @@ -222,6 +245,9 @@ public UcpRequest recvTaggedMessageNonBlocking(ByteBuffer buffer, UcpTagMessage * case it is canceled the status argument is set to UCS_ERR_CANCELED. */ public void cancelRequest(UcpRequest request) { + if (request.getNativeId() == null) { + throw new UcxException("Request is not valid"); + } cancelRequestNative(getNativeId(), request.getNativeId()); } @@ -261,20 +287,23 @@ public ByteBuffer getAddress() { private static native UcpRequest recvTaggedNonBlockingNative(long workerId, long localAddress, long size, long tag, long tagMask, - UcxCallback callback); + UcxCallback callback, + int memoryType); private static native UcpRequest recvTaggedIovNonBlockingNative(long workerId, long[] localAddresses, long[] sizes, long tag, long tagMask, - UcxCallback callback); + UcxCallback callback, + int memoryType); private static native UcpTagMessage tagProbeNonBlockingNative(long workerId, long tag, long tagMask, boolean remove); private static native UcpRequest recvTaggedMessageNonBlockingNative(long workerId, long address, long size, long tagMsgId, - UcxCallback callback); + UcxCallback callback, + int memoryType); private static native void cancelRequestNative(long workerId, long requestId); } diff --git a/bindings/java/src/main/native/endpoint.cc b/bindings/java/src/main/native/endpoint.cc index 9f1151b398e..ac27c9c8a7c 100644 --- a/bindings/java/src/main/native/endpoint.cc +++ b/bindings/java/src/main/native/endpoint.cc @@ -138,7 +138,8 @@ JNIEXPORT jobject JNICALL Java_org_openucx_jucx_ucp_UcpEndpoint_putNonBlockingNative(JNIEnv *env, jclass cls, jlong ep_ptr, jlong laddr, jlong size, jlong raddr, - jlong rkey_ptr, jobject callback) + jlong rkey_ptr, jobject callback, + jint memory_type) { ucs_status_ptr_t request = ucp_put_nb((ucp_ep_h)ep_ptr, (void *)laddr, size, raddr, (ucp_rkey_h)rkey_ptr, jucx_request_callback); @@ -166,7 +167,8 @@ JNIEXPORT jobject JNICALL Java_org_openucx_jucx_ucp_UcpEndpoint_getNonBlockingNative(JNIEnv *env, jclass cls, jlong ep_ptr, jlong raddr, jlong rkey_ptr, jlong laddr, - jlong size, jobject callback) + jlong size, jobject callback, + jint memory_type) { ucs_status_ptr_t request = ucp_get_nb((ucp_ep_h)ep_ptr, (void *)laddr, size, raddr, (ucp_rkey_h)rkey_ptr, jucx_request_callback); @@ -194,7 +196,7 @@ JNIEXPORT jobject JNICALL Java_org_openucx_jucx_ucp_UcpEndpoint_sendTaggedNonBlockingNative(JNIEnv *env, jclass cls, jlong ep_ptr, jlong addr, jlong size, jlong tag, - jobject callback) + jobject callback, jint memory_type) { ucs_status_ptr_t request = ucp_tag_send_nb((ucp_ep_h)ep_ptr, (void *)addr, size, ucp_dt_make_contig(1), tag, jucx_request_callback); @@ -208,7 +210,7 @@ JNIEXPORT jobject JNICALL Java_org_openucx_jucx_ucp_UcpEndpoint_sendTaggedIovNonBlockingNative(JNIEnv *env, jclass cls, jlong ep_ptr, jlongArray addresses, jlongArray sizes, jlong tag, - jobject callback) + jobject callback, jint memory_type) { int iovcnt; @@ -234,7 +236,8 @@ Java_org_openucx_jucx_ucp_UcpEndpoint_sendTaggedIovNonBlockingNative(JNIEnv *env JNIEXPORT jobject JNICALL Java_org_openucx_jucx_ucp_UcpEndpoint_sendStreamNonBlockingNative(JNIEnv *env, jclass cls, jlong ep_ptr, jlong addr, - jlong size, jobject callback) + jlong size, jobject callback, + jint memory_type) { ucs_status_ptr_t request = ucp_stream_send_nb((ucp_ep_h)ep_ptr, (void *)addr, size, ucp_dt_make_contig(1), jucx_request_callback, 0); @@ -246,8 +249,8 @@ Java_org_openucx_jucx_ucp_UcpEndpoint_sendStreamNonBlockingNative(JNIEnv *env, j JNIEXPORT jobject JNICALL Java_org_openucx_jucx_ucp_UcpEndpoint_sendStreamIovNonBlockingNative(JNIEnv *env, jclass cls, jlong ep_ptr, jlongArray addresses, - jlongArray sizes, - jobject callback) + jlongArray sizes, jobject callback, + jint memory_type) { int iovcnt; @@ -274,7 +277,7 @@ JNIEXPORT jobject JNICALL Java_org_openucx_jucx_ucp_UcpEndpoint_recvStreamNonBlockingNative(JNIEnv *env, jclass cls, jlong ep_ptr, jlong addr, jlong size, jlong flags, - jobject callback) + jobject callback, jint memory_type) { size_t rlength; ucs_status_ptr_t request = ucp_stream_recv_nb((ucp_ep_h)ep_ptr, (void *)addr, size, @@ -295,7 +298,8 @@ JNIEXPORT jobject JNICALL Java_org_openucx_jucx_ucp_UcpEndpoint_recvStreamIovNonBlockingNative(JNIEnv *env, jclass cls, jlong ep_ptr, jlongArray addresses, jlongArray sizes, - jlong flags, jobject callback) + jlong flags, jobject callback, + jint memory_type) { size_t rlength; diff --git a/bindings/java/src/main/native/worker.cc b/bindings/java/src/main/native/worker.cc index cc1c4af45de..1a2df7a105c 100644 --- a/bindings/java/src/main/native/worker.cc +++ b/bindings/java/src/main/native/worker.cc @@ -145,7 +145,7 @@ Java_org_openucx_jucx_ucp_UcpWorker_recvTaggedNonBlockingNative(JNIEnv *env, jcl jlong ucp_worker_ptr, jlong laddr, jlong size, jlong tag, jlong tag_mask, - jobject callback) + jobject callback, jint memory_type) { ucs_status_ptr_t request = ucp_tag_recv_nb((ucp_worker_h)ucp_worker_ptr, (void *)laddr, size, @@ -162,7 +162,7 @@ Java_org_openucx_jucx_ucp_UcpWorker_recvTaggedIovNonBlockingNative(JNIEnv *env, jlong ucp_worker_ptr, jlongArray addresses, jlongArray sizes, jlong tag, jlong tag_mask, - jobject callback) + jobject callback, jint memory_type) { int iovcnt; ucp_dt_iov_t* iovec = get_ucp_iov(env, addresses, sizes, iovcnt); @@ -210,7 +210,8 @@ Java_org_openucx_jucx_ucp_UcpWorker_recvTaggedMessageNonBlockingNative(JNIEnv *e jlong ucp_worker_ptr, jlong laddr, jlong size, jlong msg_ptr, - jobject callback) + jobject callback, + jint memory_type) { ucs_status_ptr_t request = ucp_tag_msg_recv_nb((ucp_worker_h)ucp_worker_ptr, (void *)laddr, size, From b88f99a9fd12167f916c912c390612350f852c9c Mon Sep 17 00:00:00 2001 From: Peter Rudenko Date: Wed, 30 Dec 2020 10:38:34 +0200 Subject: [PATCH 077/107] JUCX: add requestAmFeature to test. --- .../java/src/test/java/org/openucx/jucx/UcpContextTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bindings/java/src/test/java/org/openucx/jucx/UcpContextTest.java b/bindings/java/src/test/java/org/openucx/jucx/UcpContextTest.java index 8450604083d..7db844601ec 100644 --- a/bindings/java/src/test/java/org/openucx/jucx/UcpContextTest.java +++ b/bindings/java/src/test/java/org/openucx/jucx/UcpContextTest.java @@ -28,7 +28,8 @@ public static void closeContext(UcpContext context) { @Test public void testCreateSimpleUcpContext() { - UcpParams contextParams = new UcpParams().requestTagFeature(); + UcpParams contextParams = new UcpParams().requestTagFeature() + .requestAmFeature(); UcpContext context = createContext(contextParams); closeContext(context); } From 1b362b698e3449834f8fb3b77b5703adf6bbf4e8 Mon Sep 17 00:00:00 2001 From: Mikhail Brinskii Date: Thu, 17 Dec 2020 13:32:13 +0200 Subject: [PATCH 078/107] UCP/AM: GPU support for UCP AM --- src/ucp/api/ucp.h | 7 +- src/ucp/core/ucp_am.c | 47 ++++++++----- src/ucp/core/ucp_ep.c | 17 +++-- src/ucp/core/ucp_ep.h | 4 +- src/ucp/proto/proto_am.inl | 11 ++- src/ucp/rndv/rndv.c | 4 +- src/ucp/tag/tag_send.c | 19 ++--- test/gtest/common/mem_buffer.cc | 7 +- test/gtest/common/mem_buffer.h | 5 +- test/gtest/ucp/test_ucp_am.cc | 121 ++++++++++++++++++++++++++------ 10 files changed, 172 insertions(+), 70 deletions(-) diff --git a/src/ucp/api/ucp.h b/src/ucp/api/ucp.h index 598c46a6c03..eaba270fdff 100644 --- a/src/ucp/api/ucp.h +++ b/src/ucp/api/ucp.h @@ -1,5 +1,5 @@ /* -* Copyright (C) Mellanox Technologies Ltd. 2001-2014. ALL RIGHTS RESERVED. +* Copyright (C) Mellanox Technologies Ltd. 2001-2020. ALL RIGHTS RESERVED. * Copyright (C) UT-Battelle, LLC. 2014-2017. ALL RIGHTS RESERVED. * Copyright (C) ARM Ltd. 2016-2017. ALL RIGHTS RESERVED. * Copyright (C) Los Alamos National Security, LLC. 2018 ALL RIGHTS RESERVED. @@ -2755,8 +2755,6 @@ ucs_status_ptr_t ucp_am_send_nb(ucp_ep_h ep, uint16_t id, * @note If UCP_OP_ATTR_FLAG_NO_IMM_CMPL flag is set in the op_attr_mask field * of @a param, then the operation will return a request handle, even if * it completes immediately. - * @note Currently Active Message API supports communication operations with - * host memory only. * @note This operation supports specific flags, which can be passed * in @a param by @ref ucp_request_param_t.flags. The exact set of flags * is defined by @ref ucp_send_am_flags. @@ -2811,9 +2809,6 @@ ucs_status_ptr_t ucp_am_send_nbx(ucp_ep_h ep, unsigned id, * The routine returns a request handle instead, which can further be used * for tracking operation progress. * - * @note Currently Active Message API supports communication operations with - * host memory only. - * * @param [in] worker Worker that is used for the receive operation. * @param [in] data_desc Data descriptor, provided in @ref ucp_am_recv_callback_t routine. diff --git a/src/ucp/core/ucp_am.c b/src/ucp/core/ucp_am.c index 0a194a321ab..ca90c2224a4 100644 --- a/src/ucp/core/ucp_am.c +++ b/src/ucp/core/ucp_am.c @@ -263,7 +263,7 @@ static UCS_F_ALWAYS_INLINE ssize_t ucp_am_get_short_max(const ucp_request_t *req, ssize_t max_short) { return (UCP_DT_IS_CONTIG(req->send.datatype) && - UCP_MEM_IS_ACCESSIBLE_FROM_CPU(req->send.mem_type)) ? + UCP_MEM_IS_HOST(req->send.mem_type)) ? max_short : -1; } @@ -382,7 +382,7 @@ ucp_am_bcopy_pack_data(void *buffer, ucp_request_t *req, size_t length) return user_header_length + ucp_dt_pack(req->send.ep->worker, req->send.datatype, - UCS_MEMORY_TYPE_HOST, + req->send.mem_type, UCS_PTR_BYTE_OFFSET(buffer, user_header_length), req->send.buffer, @@ -462,7 +462,7 @@ ucp_am_bcopy_pack_args_mid(void *dest, void *arg) ucs_assert(req->send.state.dt.offset > 0); return sizeof(*hdr) + ucp_dt_pack(req->send.ep->worker, req->send.datatype, - UCS_MEMORY_TYPE_HOST, hdr + 1, + req->send.mem_type, hdr + 1, req->send.buffer, &req->send.state.dt, length); } @@ -756,7 +756,8 @@ static ucs_status_t ucp_am_send_start_rndv(ucp_request_t *sreq) static void ucp_am_send_req_init(ucp_request_t *req, ucp_ep_h ep, const void *header, size_t header_length, const void *buffer, ucp_datatype_t datatype, - size_t count, uint16_t flags, uint16_t am_id) + size_t count, uint16_t flags, uint16_t am_id, + ucs_memory_type_t mem_type) { req->flags = UCP_REQUEST_FLAG_SEND_AM; req->send.ep = ep; @@ -766,13 +767,14 @@ static void ucp_am_send_req_init(ucp_request_t *req, ucp_ep_h ep, req->send.msg_proto.am.header_length = header_length; req->send.buffer = (void*)buffer; req->send.datatype = datatype; - req->send.mem_type = UCS_MEMORY_TYPE_HOST; req->send.lane = ep->am_lane; req->send.pending_lane = UCP_NULL_LANE; ucp_request_send_state_init(req, datatype, count); - req->send.length = ucp_dt_length(req->send.datatype, count, - req->send.buffer, &req->send.state.dt); + req->send.length = ucp_dt_length(req->send.datatype, count, + req->send.buffer, &req->send.state.dt); + req->send.mem_type = ucp_get_memory_type(ep->worker->context, req->send.buffer, + req->send.length, mem_type); } static UCS_F_ALWAYS_INLINE size_t @@ -892,9 +894,13 @@ ucp_am_try_send_short(ucp_ep_h ep, uint16_t id, uint32_t flags, const void *buffer, size_t length) { if (ucs_unlikely(((length != 0) && (header_length != 0)) || - ((ssize_t)(length + header_length) > - ucp_ep_config(ep)->am_u.max_eager_short)) || - (flags & UCP_AM_SEND_FLAG_RNDV)) { + (flags & UCP_AM_SEND_FLAG_RNDV))) { + goto out; + } + + if (ucs_unlikely(!ucp_proto_is_inline(ep, + &ucp_ep_config(ep)->am_u.max_eager_short, + length + header_length))) { goto out; } @@ -922,6 +928,7 @@ UCS_PROFILE_FUNC(ucs_status_ptr_t, ucp_am_send_nbx, ucp_request_t *req; uint32_t attr_mask; uint32_t flags; + ssize_t max_short; UCP_CONTEXT_CHECK_FEATURE_FLAGS(ep->worker->context, UCP_FEATURE_AM, return UCS_STATUS_PTR(UCS_ERR_INVALID_PARAM)); @@ -966,17 +973,21 @@ UCS_PROFILE_FUNC(ucs_status_ptr_t, ucp_am_send_nbx, goto out;}); ucp_am_send_req_init(req, ep, header, header_length, buffer, datatype, - count, flags, id); + count, flags, id, ucp_request_param_mem_type(param)); + + /* Note that max_eager_short.memtype_on is always initialized to real + * max_short value + */ + max_short = ucp_ep_config(ep)->am_u.max_eager_short.memtype_on; if (flags & UCP_AM_SEND_REPLY) { ret = ucp_am_send_req(req, count, &ucp_ep_config(ep)->am, param, ucp_ep_config(ep)->am_u.reply_proto, - ucs_min(ucp_ep_config(ep)->am_u.max_eager_short, - UCP_AM_SHORT_REPLY_MAX_SIZE), flags); + ucs_min(max_short, UCP_AM_SHORT_REPLY_MAX_SIZE), + flags); } else { ret = ucp_am_send_req(req, count, &ucp_ep_config(ep)->am, param, - ucp_ep_config(ep)->am_u.proto, - ucp_ep_config(ep)->am_u.max_eager_short, flags); + ucp_ep_config(ep)->am_u.proto, max_short, flags); } out: @@ -1007,11 +1018,12 @@ UCS_PROFILE_FUNC(ucs_status_ptr_t, ucp_am_recv_data_nbx, { ucp_am_rndv_rts_hdr_t *rts = data_desc; ucp_recv_desc_t *desc = (ucp_recv_desc_t*)data_desc - 1; + ucp_context_h context = worker->context; ucs_status_ptr_t ret; ucp_request_t *req; ucp_datatype_t datatype; - UCP_CONTEXT_CHECK_FEATURE_FLAGS(worker->context, UCP_FEATURE_AM, + UCP_CONTEXT_CHECK_FEATURE_FLAGS(context, UCP_FEATURE_AM, return UCS_STATUS_PTR(UCS_ERR_INVALID_PARAM)); UCP_WORKER_THREAD_CS_ENTER_CONDITIONAL(worker); @@ -1047,7 +1059,8 @@ UCS_PROFILE_FUNC(ucs_status_ptr_t, ucp_am_recv_data_nbx, ucp_dt_recv_state_init(&req->recv.state, buffer, datatype, count); req->recv.length = ucp_dt_length(datatype, count, buffer, &req->recv.state); - req->recv.mem_type = UCS_MEMORY_TYPE_HOST; + req->recv.mem_type = ucp_get_memory_type(context, buffer, req->recv.length, + ucp_request_param_mem_type(param)); req->recv.am.desc = (ucp_recv_desc_t*)rts - 1; ucp_request_set_callback_param(param, recv_am, req, recv.am); diff --git a/src/ucp/core/ucp_ep.c b/src/ucp/core/ucp_ep.c index e998f258d89..005c56a0880 100644 --- a/src/ucp/core/ucp_ep.c +++ b/src/ucp/core/ucp_ep.c @@ -1572,6 +1572,7 @@ ucs_status_t ucp_ep_config_init(ucp_worker_h worker, ucp_ep_config_t *config, size_t max_rndv_thresh, max_am_rndv_thresh; size_t min_rndv_thresh, min_am_rndv_thresh; size_t rma_zcopy_thresh; + size_t am_max_eager_short; double rndv_max_bw[UCS_MEMORY_TYPE_LAST], scale, bw; ucs_status_t status; size_t it; @@ -1629,6 +1630,8 @@ ucs_status_t ucp_ep_config_init(ucp_worker_h worker, ucp_ep_config_t *config, config->tag.offload.max_eager_short.memtype_off = -1; config->tag.max_eager_short.memtype_on = -1; config->tag.max_eager_short.memtype_off = -1; + config->am_u.max_eager_short.memtype_on = -1; + config->am_u.max_eager_short.memtype_off = -1; for (lane = 0; lane < config->key.num_lanes; ++lane) { rsc_index = config->key.lanes[lane].rsc_index; @@ -1816,7 +1819,7 @@ ucs_status_t ucp_ep_config_init(ucp_worker_h worker, ucp_ep_config_t *config, UCT_IFACE_FLAG_AM_BCOPY, UCT_IFACE_FLAG_AM_ZCOPY, sizeof(ucp_eager_hdr_t), SIZE_MAX); - config->am_u.max_eager_short = config->am.max_short; + am_max_eager_short = config->am.max_short; /* Calculate rendezvous thresholds which may be used by UCP AM * protocol. */ @@ -1825,19 +1828,23 @@ ucs_status_t ucp_ep_config_init(ucp_worker_h worker, ucp_ep_config_t *config, ucp_ep_config_set_rndv_thresh(worker, config, rkey_ptr_lanes, iface_attr->cap.get.min_zcopy, SIZE_MAX, &config->rndv.rma_thresh, - &config->am_u.max_eager_short); + &am_max_eager_short); } else { ucp_ep_config_set_rndv_thresh(worker, config, config->key.rma_bw_lanes, iface_attr->cap.get.min_zcopy, SIZE_MAX, &config->rndv.rma_thresh, - &config->am_u.max_eager_short); + &am_max_eager_short); } ucp_ep_config_set_am_rndv_thresh(worker, iface_attr, md_attr, config, iface_attr->cap.am.min_zcopy, SIZE_MAX, &config->rndv.am_thresh, - &config->am_u.max_eager_short); + &am_max_eager_short); + + ucp_ep_config_set_memtype_thresh(&config->am_u.max_eager_short, + am_max_eager_short, + context->num_mem_type_detect_mds); /* All keys must fit in RNDV packet. * TODO remove some MDs if they don't @@ -1850,7 +1857,7 @@ ucs_status_t ucp_ep_config_init(ucp_worker_h worker, ucp_ep_config_t *config, /* TODO: set threshold level based on all available lanes */ config->tag.eager = config->am; - config->tag.eager.max_short = config->am_u.max_eager_short; + config->tag.eager.max_short = am_max_eager_short; config->tag.lane = lane; config->tag.rndv.am_thresh = config->rndv.am_thresh; config->tag.rndv.rma_thresh = config->rndv.rma_thresh; diff --git a/src/ucp/core/ucp_ep.h b/src/ucp/core/ucp_ep.h index 4146643e5a5..8b4786fda01 100644 --- a/src/ucp/core/ucp_ep.h +++ b/src/ucp/core/ucp_ep.h @@ -1,5 +1,5 @@ /** - * Copyright (C) Mellanox Technologies Ltd. 2001-2015. ALL RIGHTS RESERVED. + * Copyright (C) Mellanox Technologies Ltd. 2001-2020. ALL RIGHTS RESERVED. * Copyright (C) Los Alamos National Security, LLC. 2019 ALL RIGHTS RESERVED. * * See file LICENSE for terms. @@ -336,7 +336,7 @@ struct ucp_ep_config { const ucp_request_send_proto_t *reply_proto; /* Maximal size for eager short */ - ssize_t max_eager_short; + ucp_memtype_thresh_t max_eager_short; } am_u; /* Protocol selection data */ diff --git a/src/ucp/proto/proto_am.inl b/src/ucp/proto/proto_am.inl index 6440c774003..3c2d18ea31b 100644 --- a/src/ucp/proto/proto_am.inl +++ b/src/ucp/proto/proto_am.inl @@ -1,5 +1,5 @@ /** - * Copyright (C) Mellanox Technologies Ltd. 2001-2019. ALL RIGHTS RESERVED. + * Copyright (C) Mellanox Technologies Ltd. 2001-2020. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ @@ -517,6 +517,15 @@ ucp_proto_get_short_max(const ucp_request_t *req, -1 : msg_config->max_short; } +static UCS_F_ALWAYS_INLINE int +ucp_proto_is_inline(ucp_ep_h ep, const ucp_memtype_thresh_t *max_eager_short, + ssize_t length) +{ + return (ucs_likely(length <= max_eager_short->memtype_off) || + (length <= max_eager_short->memtype_on && + ucp_memory_type_cache_is_empty(ep->worker->context))); +} + static UCS_F_ALWAYS_INLINE ucp_request_t* ucp_proto_ssend_ack_request_alloc(ucp_worker_h worker, ucs_ptr_map_key_t ep_id) { diff --git a/src/ucp/rndv/rndv.c b/src/ucp/rndv/rndv.c index a01f414769f..730c39d2890 100644 --- a/src/ucp/rndv/rndv.c +++ b/src/ucp/rndv/rndv.c @@ -751,7 +751,7 @@ UCS_PROFILE_FUNC_VOID(ucp_rndv_recv_frag_put_completion, (self), req->recv.remaining, freq->send.length); req->recv.remaining -= freq->send.length; if (req->recv.remaining == 0) { - ucp_request_complete_tag_recv(req, UCS_OK); + ucp_rndv_recv_req_complete(req, UCS_OK); if (!is_put_proto) { ucp_worker_del_request_id(worker, rreq_remote_id); } @@ -1122,7 +1122,7 @@ static void ucp_rndv_do_rkey_ptr(ucp_request_t *rndv_req, ucp_request_t *rreq, &rkey->tl_rkey[rkey_index].rkey, rndv_rts_hdr->address, &local_ptr); if (status != UCS_OK) { - ucp_request_complete_tag_recv(rreq, status); + ucp_rndv_recv_req_complete(rreq, status); ucp_rkey_destroy(rkey); ucp_rndv_req_send_ats(rndv_req, rreq, rndv_rts_hdr->sreq.req_id, status); return; diff --git a/src/ucp/tag/tag_send.c b/src/ucp/tag/tag_send.c index 31a8c5ee453..c075f854133 100644 --- a/src/ucp/tag/tag_send.c +++ b/src/ucp/tag/tag_send.c @@ -1,5 +1,5 @@ /** - * Copyright (C) Mellanox Technologies Ltd. 2001-2015. ALL RIGHTS RESERVED. + * Copyright (C) Mellanox Technologies Ltd. 2001-2020. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ @@ -143,28 +143,19 @@ ucp_tag_send_req_init(ucp_request_t* req, ucp_ep_h ep, const void* buffer, req->send.pending_lane = UCP_NULL_LANE; } -static UCS_F_ALWAYS_INLINE int -ucp_tag_eager_is_inline(ucp_ep_h ep, const ucp_memtype_thresh_t *max_eager_short, - ssize_t length) -{ - return (ucs_likely(length <= max_eager_short->memtype_off) || - (length <= max_eager_short->memtype_on && - ucp_memory_type_cache_is_empty(ep->worker->context))); -} - static UCS_F_ALWAYS_INLINE ucs_status_t ucp_tag_send_inline(ucp_ep_h ep, const void *buffer, size_t length, ucp_tag_t tag) { ucs_status_t status; - if (ucp_tag_eager_is_inline(ep, &ucp_ep_config(ep)->tag.max_eager_short, - length)) { + if (ucp_proto_is_inline(ep, &ucp_ep_config(ep)->tag.max_eager_short, length)) { UCS_STATIC_ASSERT(sizeof(ucp_tag_t) == sizeof(ucp_eager_hdr_t)); UCS_STATIC_ASSERT(sizeof(ucp_tag_t) == sizeof(uint64_t)); status = uct_ep_am_short(ucp_ep_get_am_uct_ep(ep), UCP_AM_ID_EAGER_ONLY, tag, buffer, length); - } else if (ucp_tag_eager_is_inline(ep, &ucp_ep_config(ep)->tag.offload.max_eager_short, - length)) { + } else if (ucp_proto_is_inline(ep, + &ucp_ep_config(ep)->tag.offload.max_eager_short, + length)) { UCS_STATIC_ASSERT(sizeof(ucp_tag_t) == sizeof(uct_tag_t)); status = uct_ep_tag_eager_short(ucp_ep_get_tag_uct_ep(ep), tag, buffer, length); diff --git a/test/gtest/common/mem_buffer.cc b/test/gtest/common/mem_buffer.cc index 00a8a1df2d0..7973b782e0a 100644 --- a/test/gtest/common/mem_buffer.cc +++ b/test/gtest/common/mem_buffer.cc @@ -1,5 +1,5 @@ /** - * Copyright (C) Mellanox Technologies Ltd. 2001-2019. ALL RIGHTS RESERVED. + * Copyright (C) Mellanox Technologies Ltd. 2001-2020. ALL RIGHTS RESERVED. * Copyright (C) Advanced Micro Devices, Inc. 2019. ALL RIGHTS RESERVED. * * See file LICENSE for terms. @@ -65,6 +65,11 @@ bool mem_buffer::is_rocm_supported() #endif } +bool mem_buffer::is_gpu_supported() +{ + return is_cuda_supported() || is_rocm_supported(); +} + const std::vector& mem_buffer::supported_mem_types() { static std::vector vec; diff --git a/test/gtest/common/mem_buffer.h b/test/gtest/common/mem_buffer.h index 5a55faa16f2..81fdd894161 100644 --- a/test/gtest/common/mem_buffer.h +++ b/test/gtest/common/mem_buffer.h @@ -1,5 +1,5 @@ /** - * Copyright (C) Mellanox Technologies Ltd. 2001-2019. ALL RIGHTS RESERVED. + * Copyright (C) Mellanox Technologies Ltd. 2001-2020. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ @@ -64,6 +64,9 @@ class mem_buffer { /* return the string name of a memory type */ static std::string mem_type_name(ucs_memory_type_t mem_type); + /* returns whether any other type of memory besides the CPU is supported */ + static bool is_gpu_supported(); + mem_buffer(size_t size, ucs_memory_type_t mem_type); virtual ~mem_buffer(); diff --git a/test/gtest/ucp/test_ucp_am.cc b/test/gtest/ucp/test_ucp_am.cc index d2f01047af1..1c2bd808405 100644 --- a/test/gtest/ucp/test_ucp_am.cc +++ b/test/gtest/ucp/test_ucp_am.cc @@ -1,5 +1,5 @@ /** - * Copyright (C) Mellanox Technologies Ltd. 2001-2015. ALL RIGHTS RESERVED. + * Copyright (C) Mellanox Technologies Ltd. 2001-2020. ALL RIGHTS RESERVED. * Copyright (c) UT-Battelle, LLC. 2015. ALL RIGHTS RESERVED. * Copyright (C) Los Alamos National Security, LLC. 2018. ALL RIGHTS RESERVED. * @@ -11,6 +11,7 @@ #include #include +#include #include "ucp_datatype.h" #include "ucp_test.h" @@ -295,6 +296,8 @@ UCP_INSTANTIATE_TEST_CASE(test_ucp_am) class test_ucp_am_nbx : public test_ucp_am_base { public: + static const uint64_t SEED = 0x1111111111111111lu; + test_ucp_am_nbx() { m_dt = ucp_dt_make_contig(1); @@ -352,6 +355,12 @@ class test_ucp_am_nbx : public test_ucp_am_base { ASSERT_UCS_OK(ucp_worker_set_am_recv_handler(e.worker(), ¶m)); } + void check_header(const void *header, size_t header_length) + { + std::string check_pattern((char*)header, header_length); + EXPECT_EQ(check_pattern, m_hdr); + } + ucs_status_ptr_t send_am(const ucp::data_type_desc_t& dt_desc, unsigned flags = 0, const void *hdr = NULL, unsigned hdr_length = 0) @@ -372,18 +381,21 @@ class test_ucp_am_nbx : public test_ucp_am_base { } void test_am_send_recv(size_t size, size_t header_size = 0ul, - unsigned flags = 0) + unsigned flags = 0, + ucs_memory_type_t mem_type = UCS_MEMORY_TYPE_HOST) { - std::string sbuf(size, 'd'); - std::string hbuf(header_size, 'h'); + mem_buffer sbuf(size, mem_type); + mem_buffer::pattern_fill(sbuf.ptr(), size, SEED, mem_type); + m_hdr.resize(header_size); + ucs::fill_random(m_hdr); m_am_received = false; set_am_data_handler(receiver(), TEST_AM_NBX_ID, am_data_cb, this); - ucp::data_type_desc_t sdt_desc(m_dt, &sbuf[0], size); + ucp::data_type_desc_t sdt_desc(m_dt, sbuf.ptr(), size); ucs_status_ptr_t sptr = send_am(sdt_desc, get_send_flag() | flags, - hbuf.c_str(), header_size); + m_hdr.data(), m_hdr.size()); wait_for_flag(&m_am_received); request_wait(sptr); @@ -408,14 +420,10 @@ class test_ucp_am_nbx : public test_ucp_am_base { const ucp_am_recv_param_t *rx_param) { EXPECT_FALSE(m_am_received); - EXPECT_EQ(std::string::npos, - std::string((const char*)data, length).find_first_not_of('d')); - if (header_length != 0) { - EXPECT_EQ(std::string::npos, - std::string((const char*)header, - header_length).find_first_not_of('h')); - } + mem_buffer::pattern_check(data, length, SEED); + + check_header(header, header_length); bool has_reply_ep = get_send_flag(); @@ -442,6 +450,7 @@ class test_ucp_am_nbx : public test_ucp_am_base { static const uint16_t TEST_AM_NBX_ID = 0; ucp_datatype_t m_dt; volatile bool m_am_received; + std::string m_hdr; }; UCS_TEST_P(test_ucp_am_nbx, set_invalid_handler) @@ -511,6 +520,37 @@ UCS_TEST_P(test_ucp_am_nbx, zero_send) UCP_INSTANTIATE_TEST_CASE(test_ucp_am_nbx) +class test_ucp_am_nbx_eager_memtype : public test_ucp_am_nbx { +public: + void init() { + modify_config("RNDV_THRESH", "inf"); + test_ucp_am_nbx::init(); + } + + static void get_test_variants(std::vector& variants) { + // 1. Do not instantiate test case if no GPU memtypes supported. + // 2. Do not exclude host memory type, because this generator is used by + // test_ucp_am_nbx_rndv_memtype class to generate combinations like + // host<->cuda, cuda-managed<->host, etc. + if (!mem_buffer::is_gpu_supported()) { + return; + } + + add_variant_memtypes(variants, test_ucp_am_base::get_test_variants, + std::numeric_limits::max()); + } +}; + +UCS_TEST_P(test_ucp_am_nbx_eager_memtype, basic) +{ + ucs_memory_type_t memtype = + mem_buffer::supported_mem_types()[get_variant_value()]; + test_am_send_recv(16 * UCS_KBYTE, 8, 0, memtype); +} + +UCP_INSTANTIATE_TEST_CASE_GPU_AWARE(test_ucp_am_nbx_eager_memtype) + + class test_ucp_am_nbx_dts : public test_ucp_am_nbx { public: static const uint64_t dts_bitmap = UCS_BIT(UCP_DATATYPE_CONTIG) | @@ -586,12 +626,14 @@ UCS_TEST_P(test_ucp_am_nbx_dts, send_eager_flag, "RNDV_THRESH=128") UCP_INSTANTIATE_TEST_CASE(test_ucp_am_nbx_dts) -class test_ucp_am_nbx_rndv: public test_ucp_am_nbx { +class test_ucp_am_nbx_rndv : public test_ucp_am_nbx { public: test_ucp_am_nbx_rndv() { - m_rx_dt = ucp_dt_make_contig(1); - m_status = UCS_OK; + m_rx_dt = ucp_dt_make_contig(1); + m_rx_memtype = UCS_MEMORY_TYPE_HOST; + m_status = UCS_OK; + m_rx_buf = NULL; modify_config("RNDV_THRESH", "128"); } @@ -603,9 +645,12 @@ class test_ucp_am_nbx_rndv: public test_ucp_am_nbx { EXPECT_TRUE(rx_param->recv_attr & UCP_AM_RECV_ATTR_FLAG_RNDV); EXPECT_FALSE(rx_param->recv_attr & UCP_AM_RECV_ATTR_FLAG_DATA); - m_rx_buf.resize(length, 'u'); + check_header(header, header_length); - m_rx_dt_desc.make(m_rx_dt, &m_rx_buf[0], length); + m_rx_buf = mem_buffer::allocate(length, m_rx_memtype); + mem_buffer::pattern_fill(m_rx_buf, length, 0ul, m_rx_memtype); + + m_rx_dt_desc.make(m_rx_dt, m_rx_buf, length); ucp_request_param_t params; params.op_attr_mask = UCP_OP_ATTR_FIELD_CALLBACK | @@ -633,7 +678,8 @@ class test_ucp_am_nbx_rndv: public test_ucp_am_nbx { ASSERT_FALSE(self->m_am_received); self->m_am_received = true; EXPECT_UCS_OK(status); - EXPECT_EQ(self->m_rx_buf, std::vector(length, 'd')); + mem_buffer::pattern_check(self->m_rx_buf, length, SEED, self->m_rx_memtype); + mem_buffer::release(self->m_rx_buf, self->m_rx_memtype); } static ucs_status_t am_data_reject_rndv_cb(void *arg, const void *header, @@ -664,9 +710,10 @@ class test_ucp_am_nbx_rndv: public test_ucp_am_nbx { } ucp_datatype_t m_rx_dt; + ucs_memory_type_t m_rx_memtype; ucs_status_t m_status; ucp::data_type_desc_t m_rx_dt_desc; - std::vector m_rx_buf; + void *m_rx_buf; }; UCS_TEST_P(test_ucp_am_nbx_rndv, rndv_auto, "RNDV_SCHEME=auto") @@ -763,7 +810,7 @@ UCS_TEST_P(test_ucp_am_nbx_rndv, deferred_reject_rndv) UCP_INSTANTIATE_TEST_CASE(test_ucp_am_nbx_rndv) -class test_ucp_am_nbx_rndv_dts: public test_ucp_am_nbx_rndv { +class test_ucp_am_nbx_rndv_dts : public test_ucp_am_nbx_rndv { public: static void get_test_variants(std::vector& variants) { @@ -797,3 +844,35 @@ UCS_TEST_P(test_ucp_am_nbx_rndv_dts, rndv, "RNDV_THRESH=256") } UCP_INSTANTIATE_TEST_CASE(test_ucp_am_nbx_rndv_dts); + + +class test_ucp_am_nbx_rndv_memtype : public test_ucp_am_nbx_rndv { +public: + static void get_test_variants(std::vector& variants) { + // Test will not be instantiated if no GPU memtypes supported, because + // of the check for supported memory types in + // test_ucp_am_nbx_eager_memtype::get_test_variants + add_variant_memtypes(variants, + test_ucp_am_nbx_eager_memtype::get_test_variants, + std::numeric_limits::max()); + } + + void init() + { + modify_config("RNDV_THRESH", "128"); + + test_ucp_am_nbx::init(); + m_rx_memtype = mem_type(1); + } + + ucs_memory_type_t mem_type(int idx) { + return ucs::supported_mem_type_pairs()[get_variant_value()][idx]; + } +}; + +UCS_TEST_P(test_ucp_am_nbx_rndv_memtype, rndv) +{ + test_am_send_recv(64 * UCS_KBYTE, 8, 0, mem_type(0)); +} + +UCP_INSTANTIATE_TEST_CASE_GPU_AWARE(test_ucp_am_nbx_rndv_memtype); From 9e8de6a11261ad1979f3c66139dadd9ef89463fc Mon Sep 17 00:00:00 2001 From: Yossi Itigin Date: Tue, 22 Dec 2020 18:45:18 +0200 Subject: [PATCH 079/107] UCS/VFS/TEST: Implement VFS socket operations --- src/ucs/vfs/sock/vfs_sock.c | 159 +++++++++++++++++++++++++++++++++++- src/ucs/vfs/sock/vfs_sock.h | 74 +++++++++++++++++ test/gtest/Makefile.am | 2 + test/gtest/ucs/test_vfs.cc | 101 +++++++++++++++++++++++ 4 files changed, 335 insertions(+), 1 deletion(-) create mode 100644 test/gtest/ucs/test_vfs.cc diff --git a/src/ucs/vfs/sock/vfs_sock.c b/src/ucs/vfs/sock/vfs_sock.c index 98a2d894f1a..97cc052bba3 100644 --- a/src/ucs/vfs/sock/vfs_sock.c +++ b/src/ucs/vfs/sock/vfs_sock.c @@ -5,7 +5,164 @@ */ #ifdef HAVE_CONFIG_H -# include "config.h" +#include "config.h" #endif #include "vfs_sock.h" + +#include +#include +#include +#include +#include +#include +#include +#include + + +typedef struct { + uint8_t action; +} UCS_S_PACKED ucs_vfs_msg_t; + + +int ucs_vfs_sock_get_address(struct sockaddr_un *un_addr) +{ + struct passwd *pw; + int ret; + + pw = getpwuid(geteuid()); + if (pw == NULL) { + return -errno; + } + + memset(un_addr, 0, sizeof(*un_addr)); + un_addr->sun_family = AF_UNIX; + snprintf(un_addr->sun_path, sizeof(un_addr->sun_path) - 1, + "/tmp/ucx-vfs-%s.sock", pw->pw_name); + return 0; +} + +int ucs_vfs_sock_setopt_passcred(int sockfd) +{ + int optval, ret; + + optval = 1; + ret = setsockopt(sockfd, SOL_SOCKET, SO_PASSCRED, &optval, sizeof(optval)); + if (ret < 0) { + return -errno; + } + + return 0; +} + +static int ucs_vfs_sock_retval(ssize_t ret, size_t expected) +{ + if (ret == expected) { + return 0; + } else if (ret < 0) { + return -errno; + } else { + return -EIO; + } +} + +int ucs_vfs_sock_send(int sockfd, const ucs_vfs_sock_message_t *vfs_msg) +{ + char cbuf[CMSG_SPACE(sizeof(*vfs_msg))] UCS_V_ALIGNED(sizeof(size_t)); + struct cmsghdr *cmsgp; + struct msghdr msgh; + ucs_vfs_msg_t msg; + struct iovec iov; + ssize_t nsent; + + memset(cbuf, 0, sizeof(cbuf)); + memset(&msgh, 0, sizeof(msgh)); + msg.action = vfs_msg->action; + iov.iov_base = &msg; + iov.iov_len = sizeof(msg); + msgh.msg_iov = &iov; + msgh.msg_iovlen = 1; + + if (vfs_msg->action == UCS_VFS_SOCK_ACTION_MOUNT_REPLY) { + /* send file descriptor */ + msgh.msg_control = cbuf; + msgh.msg_controllen = sizeof(cbuf); + cmsgp = CMSG_FIRSTHDR(&msgh); + cmsgp->cmsg_level = SOL_SOCKET; + cmsgp->cmsg_len = CMSG_LEN(sizeof(vfs_msg->fd)); + cmsgp->cmsg_type = SCM_RIGHTS; + memcpy(CMSG_DATA(cmsgp), &vfs_msg->fd, sizeof(vfs_msg->fd)); + } + + do { + nsent = sendmsg(sockfd, &msgh, 0); + } while ((nsent < 0) && (errno == EINTR)); + return ucs_vfs_sock_retval(nsent, iov.iov_len); +} + +int ucs_vfs_sock_recv(int sockfd, ucs_vfs_sock_message_t *vfs_msg) +{ + char cbuf[CMSG_SPACE(sizeof(*vfs_msg))] UCS_V_ALIGNED(sizeof(size_t)); + const struct ucred *cred; + struct cmsghdr *cmsgp; + struct msghdr msgh; + ucs_vfs_msg_t msg; + struct iovec iov; + ssize_t nrecvd; + + /* initialize to invalid values */ + vfs_msg->action = UCS_VFS_SOCK_ACTION_LAST; + vfs_msg->fd = -1; + vfs_msg->pid = -1; + + memset(cbuf, 0, sizeof(cbuf)); + memset(&msgh, 0, sizeof(msgh)); + iov.iov_base = &msg; + iov.iov_len = sizeof(msg); + msgh.msg_iov = &iov; + msgh.msg_iovlen = 1; + msgh.msg_control = cbuf; + msgh.msg_controllen = sizeof(cbuf); + + do { + nrecvd = recvmsg(sockfd, &msgh, MSG_WAITALL); + } while ((nrecvd < 0) && (errno == EINTR)); + if (nrecvd != iov.iov_len) { + assert(nrecvd < iov.iov_len); + return ucs_vfs_sock_retval(nrecvd, iov.iov_len); + } + + vfs_msg->action = msg.action; + + cmsgp = CMSG_FIRSTHDR(&msgh); + if ((cmsgp == NULL) || (cmsgp->cmsg_level != SOL_SOCKET)) { + return -EINVAL; + } + + if (msg.action == UCS_VFS_SOCK_ACTION_MOUNT_REPLY) { + /* expect file descriptor */ + if ((cmsgp->cmsg_type != SCM_RIGHTS) || + (cmsgp->cmsg_len != CMSG_LEN(sizeof(vfs_msg->fd)))) { + return -EINVAL; + } + + memcpy(&vfs_msg->fd, CMSG_DATA(cmsgp), sizeof(vfs_msg->fd)); + } else { + /* expect credentials */ + if ((cmsgp->cmsg_type != SCM_CREDENTIALS) || + (cmsgp->cmsg_len != CMSG_LEN(sizeof(*cred)))) { + return -EINVAL; + } + + cred = (const struct ucred*)CMSG_DATA(cmsgp); + if ((cred->uid != getuid()) || (cred->gid != getgid())) { + return -EPERM; + } + + if (msg.action == UCS_VFS_SOCK_ACTION_MOUNT) { + vfs_msg->pid = cred->pid; + } + } + + return 0; +} diff --git a/src/ucs/vfs/sock/vfs_sock.h b/src/ucs/vfs/sock/vfs_sock.h index 6f1848e99c9..bbb1da03eaf 100644 --- a/src/ucs/vfs/sock/vfs_sock.h +++ b/src/ucs/vfs/sock/vfs_sock.h @@ -7,8 +7,82 @@ #ifndef UCS_VFS_SOCK_H_ #define UCS_VFS_SOCK_H_ +#include +#include +#include + /* This header file defines socket operations for communicating between UCS * library and VFS daemon */ +/** + * VFS socket message type + */ +typedef enum { + UCS_VFS_SOCK_ACTION_STOP, /* daemon is asked to stop */ + UCS_VFS_SOCK_ACTION_MOUNT, /* daemon is asked to mount a file system */ + UCS_VFS_SOCK_ACTION_MOUNT_REPLY, /* daemon sends back FUSE file descriptor */ + UCS_VFS_SOCK_ACTION_LAST +} ucs_vfs_sock_action_t; + + +/** + * Parameters structure for sending/receiving a message over VFS socket + */ +typedef struct { + ucs_vfs_sock_action_t action; + + /* If action==MOUNT_REPLY: in/out parameter, holds FUSE file descriptor. + * Otherwise: unused + */ + int fd; + + /* If action==MOUNT: out parameter, holds the pid of sender process. + * Otherwise: unused + */ + pid_t pid; +} ucs_vfs_sock_message_t; + + +/** + * Return the Unix-domain socket address of the VFS daemon. + * + * @param [out] un_addr Filled with socket address. + * + * @return 0 on success, or the negative value of errno in case of failure. + */ +int ucs_vfs_sock_get_address(struct sockaddr_un *un_addr); + + +/** + * Enable receiving credentials of the remote process for every message. + * Typically used by the VFS daemon to verify sender identity. + * + * @param [in] fd Enable SO_PASSCRED on this socket. + * + * @return 0 on success, or the negative value of errno in case of failure. + */ +int ucs_vfs_sock_setopt_passcred(int sockfd); + + +/** + * Send a message on the VFS socket. + * + * @param [in] fd Socket file descriptor to send the message on. + * @param [in] vfs_msg Message to send. + * + * @return 0 on success, or the negative value of errno in case of failure. + */ +int ucs_vfs_sock_send(int sockfd, const ucs_vfs_sock_message_t *vfs_msg); + + +/** + * Receive a message on the VFS socket. + * + * @param [in] fd Socket file descriptor to receive the message on. + * @param [out] vfs_msg Filled with details of the received message. + * + * @return 0 on success, or the negative value of errno in case of failure. + */ +int ucs_vfs_sock_recv(int sockfd, ucs_vfs_sock_message_t *vfs_msg); #endif diff --git a/test/gtest/Makefile.am b/test/gtest/Makefile.am index aeae4abb8fd..0f3d24b30c8 100644 --- a/test/gtest/Makefile.am +++ b/test/gtest/Makefile.am @@ -46,6 +46,7 @@ noinst_PROGRAMS = gtest gtestdir = $(includedir) gtest_LDADD = \ $(top_builddir)/src/ucs/libucs.la \ + $(top_builddir)/src/ucs/vfs/sock/libucs_vfs_sock.la \ $(top_builddir)/src/uct/libuct.la \ $(top_builddir)/src/ucm/libucm.la \ $(top_builddir)/src/ucp/libucp.la \ @@ -174,6 +175,7 @@ gtest_SOURCES = \ ucs/test_type.cc \ ucs/test_log.cc \ ucs/test_iov.cc \ + ucs/test_vfs.cc \ ucs/arch/test_x86_64.cc if HAVE_IB diff --git a/test/gtest/ucs/test_vfs.cc b/test/gtest/ucs/test_vfs.cc new file mode 100644 index 00000000000..c4705a42756 --- /dev/null +++ b/test/gtest/ucs/test_vfs.cc @@ -0,0 +1,101 @@ +/** + * Copyright (C) Mellanox Technologies Ltd. 2020. ALL RIGHTS RESERVED. + * + * See file LICENSE for terms. + */ + +#include +extern "C" { +#include +} + +#include +#include + + +class test_vfs_sock : public ucs::test +{ +protected: + virtual void init() + { + int ret = socketpair(AF_UNIX, SOCK_STREAM, 0, m_sockets); + if (ret < 0) { + UCS_TEST_ABORT("failed to create socket pair: " << strerror(errno)); + } + + /* socket[1] will always receive credentials */ + ucs_vfs_sock_setopt_passcred(m_sockets[1]); + } + + virtual void cleanup() + { + close(m_sockets[1]); + close(m_sockets[0]); + } + +protected: + void + do_send_recv(ucs_vfs_sock_action_t action, int send_sockfd, int recv_sockfd, + int fd_in, ucs_vfs_sock_message_t *msg_out) + { + int ret; + + ucs_vfs_sock_message_t msg_in = {}; + msg_in.action = action; + msg_in.fd = fd_in; + ret = ucs_vfs_sock_send(send_sockfd, &msg_in); + ASSERT_EQ(0, ret) << strerror(-ret); + + ret = ucs_vfs_sock_recv(recv_sockfd, msg_out); + ASSERT_EQ(0, ret) << strerror(-ret); + EXPECT_EQ(action, msg_out->action); + } + + ino_t fd_inode(int fd) + { + struct stat st; + int ret = fstat(fd, &st); + if (ret < 0) { + UCS_TEST_ABORT("stat() failed: " << strerror(errno)); + } + return st.st_ino; + } + + int m_sockets[2]; +}; + +UCS_TEST_F(test_vfs_sock, send_recv_stop) { + /* send stop/start commands from socket[0] to socket[1] */ + ucs_vfs_sock_message_t msg_out = {}; + do_send_recv(UCS_VFS_SOCK_ACTION_STOP, m_sockets[0], m_sockets[1], -1, + &msg_out); +} + +UCS_TEST_F(test_vfs_sock, send_recv_mount) { + /* send mount request from socket[0] to socket[1] */ + ucs_vfs_sock_message_t msg_out = {}; + do_send_recv(UCS_VFS_SOCK_ACTION_MOUNT, m_sockets[0], m_sockets[1], -1, + &msg_out); + EXPECT_EQ(getpid(), msg_out.pid); +} + +UCS_TEST_F(test_vfs_sock, send_recv_mount_reply) { + /* open a file */ + int fd = open("/dev/null", O_WRONLY); + if (fd < 0) { + UCS_TEST_ABORT("failed to open /dev/null: " << strerror(errno)); + } + + /* send mount reply with fd from socket[1] to socket[0] */ + ucs_vfs_sock_message_t msg_out = {}; + do_send_recv(UCS_VFS_SOCK_ACTION_MOUNT_REPLY, m_sockets[1], m_sockets[0], + fd, &msg_out); + + UCS_TEST_MESSAGE << "send fd: " << fd << " recv fd: " << msg_out.fd; + /* expect to have different fd but same inode */ + ASSERT_NE(msg_out.fd, fd); + EXPECT_EQ(fd_inode(fd), fd_inode(msg_out.fd)); + + close(msg_out.fd); + close(fd); +} From 39891d47755618bfada1efcd47b8d651245e7b36 Mon Sep 17 00:00:00 2001 From: Yossi Itigin Date: Wed, 30 Dec 2020 14:40:01 +0200 Subject: [PATCH 080/107] GIT: Add .vscode, .classpath, and .externalToolBuilders to .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 0ffccf4851a..34663edab67 100644 --- a/.gitignore +++ b/.gitignore @@ -88,3 +88,6 @@ GTAGS *.swp compile_commands.json .idea/ +.externalToolBuilders +.classpath +.vscode From 9cc51e2731950fb4f7ab68a51fe802b4d5333545 Mon Sep 17 00:00:00 2001 From: Raul Akhmetshin Date: Fri, 4 Dec 2020 15:42:19 +0200 Subject: [PATCH 081/107] UCT/TCP: Implemented AM short IOV in TCP. --- src/uct/tcp/tcp.h | 7 ++- src/uct/tcp/tcp_ep.c | 111 +++++++++++++++++++++++++++++----------- src/uct/tcp/tcp_iface.c | 17 +++--- 3 files changed, 94 insertions(+), 41 deletions(-) diff --git a/src/uct/tcp/tcp.h b/src/uct/tcp/tcp.h index 852e12fc1f9..a6c14e49738 100644 --- a/src/uct/tcp/tcp.h +++ b/src/uct/tcp/tcp.h @@ -345,10 +345,10 @@ typedef struct uct_tcp_iface { size_t rx_seg_size; /* RX AM buffer size */ size_t sendv_thresh; /* Minimum size of user's payload from which * non-blocking vector send should be used */ - struct { - size_t max_iov; /* Maximum supported IOVs limited by + size_t max_iov; /* Maximum supported IOVs limited by * user configuration and service buffers * (TCP protocol and user's AM headers) */ + struct { size_t max_hdr; /* Maximum supported AM Zcopy header */ size_t hdr_offset; /* Offset in TX buffer to empty space that * can be used for AM Zcopy header */ @@ -500,6 +500,9 @@ void uct_tcp_ep_pending_queue_dispatch(uct_tcp_ep_t *ep); ucs_status_t uct_tcp_ep_am_short(uct_ep_h uct_ep, uint8_t am_id, uint64_t header, const void *payload, unsigned length); +ucs_status_t uct_tcp_ep_am_short_iov(uct_ep_h uct_ep, uint8_t am_id, + const uct_iov_t *iov, size_t iovcnt); + ssize_t uct_tcp_ep_am_bcopy(uct_ep_h uct_ep, uint8_t am_id, uct_pack_callback_t pack_cb, void *arg, unsigned flags); diff --git a/src/uct/tcp/tcp_ep.c b/src/uct/tcp/tcp_ep.c index 011fc8bd588..8d036d86e59 100644 --- a/src/uct/tcp/tcp_ep.c +++ b/src/uct/tcp/tcp_ep.c @@ -1494,17 +1494,17 @@ uct_tcp_ep_am_send(uct_tcp_ep_t *ep, const uct_tcp_am_hdr_t *hdr) static const void* uct_tcp_ep_am_sendv_get_trace_payload(uct_tcp_am_hdr_t *hdr, const void *header, - const struct iovec *payload_iov, + const struct iovec *iov, size_t iov_cnt, int short_sendv) { if (short_sendv == 0) { return header; } - /* If user requested trace data, we copy header and payload + /* If user requested trace data, we copy iov * to EP TX buffer in order to trace correct data */ - uct_am_short_fill_data(hdr + 1, *(const uint64_t*)header, - payload_iov->iov_base, payload_iov->iov_len); + ucs_iov_copy(iov + 1, iov_cnt - 1, 0, hdr + 1, SIZE_MAX, UCS_IOV_COPY_TO_BUF); + return (hdr + 1); } @@ -1533,8 +1533,8 @@ uct_tcp_ep_am_sendv(uct_tcp_ep_t *ep, int short_sendv, uct_tcp_am_hdr_t *hdr, uct_iface_trace_am(&iface->super, UCT_AM_TRACE_TYPE_SEND, hdr->am_id, /* the function will be invoked only in case of * data tracing is enabled */ - uct_tcp_ep_am_sendv_get_trace_payload(hdr, header, - &iov[2], short_sendv), + uct_tcp_ep_am_sendv_get_trace_payload(hdr, header, iov, + iov_cnt, short_sendv), hdr->length, "SEND: ep %p fd %d sent %zu/%zu bytes, " "moved by offset %zu, iov cnt %zu " "[addr %p len %zu] [addr %p len %zu]", @@ -1589,6 +1589,33 @@ static void uct_tcp_ep_post_put_ack(uct_tcp_ep_t *ep) ep->flags &= ~UCT_TCP_EP_FLAG_PUT_RX_SENDING_ACK; } +static inline ucs_status_t +uct_tcp_ep_am_short_sendv(uct_tcp_ep_t *ep, uct_tcp_iface_t *iface, + uct_tcp_am_hdr_t *hdr, uint64_t header, struct iovec *iov, + size_t iov_cnt) +{ + ucs_status_t status; + size_t offset; + + status = uct_tcp_ep_am_sendv(ep, 1, hdr, iface->config.tx_seg_size, &header, iov, + iov_cnt); + if (ucs_unlikely(status != UCS_OK)) { + return status; + } + + if (uct_tcp_ep_ctx_buf_need_progress(&ep->tx)) { + /* Copy only user's header and payload to the TX buffer, + * TCP AM header is placed at the beginning of the buffer */ + offset = (ep->tx.offset >= sizeof(*hdr)) ? ep->tx.offset - sizeof(*hdr) : 0; + + ucs_iov_copy(&iov[1], iov_cnt - 1, offset, + UCS_PTR_BYTE_OFFSET(hdr + 1, offset), + ep->tx.length - sizeof(*hdr) - offset, UCS_IOV_COPY_TO_BUF); + } + + return status; +} + ucs_status_t uct_tcp_ep_am_short(uct_ep_h uct_ep, uint8_t am_id, uint64_t header, const void *payload, unsigned length) { @@ -1597,7 +1624,6 @@ ucs_status_t uct_tcp_ep_am_short(uct_ep_h uct_ep, uint8_t am_id, uint64_t header uct_tcp_am_hdr_t *hdr = NULL; struct iovec iov[UCT_TCP_EP_AM_SHORTV_IOV_COUNT]; uint32_t UCS_V_UNUSED payload_length; - size_t offset; ucs_status_t status; UCT_CHECK_LENGTH(length + sizeof(header), 0, @@ -1619,11 +1645,6 @@ ucs_status_t uct_tcp_ep_am_short(uct_ep_h uct_ep, uint8_t am_id, uint64_t header if (length <= iface->config.sendv_thresh) { uct_am_short_fill_data(hdr + 1, header, payload, length); status = uct_tcp_ep_am_send(ep, hdr); - if (ucs_unlikely(status != UCS_OK)) { - return status; - } - - UCT_TL_EP_STAT_OP(&ep->super, AM, SHORT, payload_length); } else { iov[0].iov_base = hdr; iov[0].iov_len = sizeof(*hdr); @@ -1634,27 +1655,57 @@ ucs_status_t uct_tcp_ep_am_short(uct_ep_h uct_ep, uint8_t am_id, uint64_t header iov[2].iov_base = (void*)payload; iov[2].iov_len = length; - status = uct_tcp_ep_am_sendv(ep, 1, hdr, iface->config.tx_seg_size, - &header, iov, UCT_TCP_EP_AM_SHORTV_IOV_COUNT); - if (ucs_unlikely(status != UCS_OK)) { - return status; - } + status = uct_tcp_ep_am_short_sendv(ep, iface, hdr, header, iov, + UCT_TCP_EP_AM_SHORTV_IOV_COUNT); + } + + if (ucs_unlikely(status != UCS_OK)) { + return status; + } - UCT_TL_EP_STAT_OP(&ep->super, AM, SHORT, payload_length); + UCT_TL_EP_STAT_OP(&ep->super, AM, SHORT, payload_length); - if (uct_tcp_ep_ctx_buf_need_progress(&ep->tx)) { - /* Copy only user's header and payload to the TX buffer, - * TCP AM header is placed at the beginning of the buffer */ - offset = ((ep->tx.offset >= sizeof(*hdr)) ? - (ep->tx.offset - sizeof(*hdr)) : 0); + return status; +} - ucs_iov_copy(&iov[1], UCT_TCP_EP_AM_SHORTV_IOV_COUNT - 1, - offset, UCS_PTR_BYTE_OFFSET(hdr + 1, offset), - ep->tx.length - sizeof(*hdr) - offset, - UCS_IOV_COPY_TO_BUF); - } +ucs_status_t uct_tcp_ep_am_short_iov(uct_ep_h uct_ep, uint8_t am_id, + const uct_iov_t *uct_iov, size_t uct_iov_cnt) +{ + uct_tcp_ep_t *ep = ucs_derived_of(uct_ep, uct_tcp_ep_t); + uct_tcp_iface_t *iface = ucs_derived_of(uct_ep->iface, uct_tcp_iface_t); + uct_tcp_am_hdr_t *hdr = NULL; + struct iovec *iov = ucs_alloca((uct_iov_cnt + 1) * sizeof(*iov)); + ucs_iov_iter_t uct_iov_iter; + size_t UCS_V_UNUSED payload_length; + ucs_status_t status; + + UCT_CHECK_AM_ID(am_id); + UCT_CHECK_IOV_SIZE(uct_iov_cnt, iface->config.max_iov, "am_short_iov"); + UCT_CHECK_LENGTH(uct_iov_total_length(uct_iov, uct_iov_cnt), 0, + iface->config.tx_seg_size - sizeof(uct_tcp_am_hdr_t), + "am_short_iov"); + + status = uct_tcp_ep_am_prepare(iface, ep, am_id, &hdr); + if (status != UCS_OK) { + return status; } + ucs_assertv(hdr != NULL, "ep=%p", ep); + + ucs_iov_iter_init(&uct_iov_iter); + iov[0].iov_base = hdr; + iov[0].iov_len = sizeof(*hdr); + hdr->length = payload_length = uct_iov_to_iovec(&iov[1], &uct_iov_cnt, + uct_iov, uct_iov_cnt, + SIZE_MAX, &uct_iov_iter); + status = uct_tcp_ep_am_short_sendv(ep, iface, hdr, 0, iov, + uct_iov_cnt + 1); + if (ucs_unlikely(status != UCS_OK)) { + return status; + } + + UCT_TL_EP_STAT_OP(&ep->super, AM, SHORT, payload_length); + return status; } @@ -1703,7 +1754,7 @@ uct_tcp_ep_prepare_zcopy(uct_tcp_iface_t *iface, uct_tcp_ep_t *ep, uint8_t am_id uct_tcp_ep_zcopy_tx_t *ctx; ucs_status_t status; - UCT_CHECK_IOV_SIZE(iovcnt, iface->config.zcopy.max_iov, name); + UCT_CHECK_IOV_SIZE(iovcnt, iface->config.max_iov, name); UCT_CHECK_LENGTH(header_length, 0, iface->config.zcopy.max_hdr, name); status = uct_tcp_ep_am_prepare(iface, ep, am_id, &hdr); @@ -1731,7 +1782,7 @@ uct_tcp_ep_prepare_zcopy(uct_tcp_iface_t *iface, uct_tcp_ep_t *ep, uint8_t am_id /* User-defined payload */ ucs_iov_iter_init(&uct_iov_iter); - io_vec_cnt = iovcnt; + io_vec_cnt = iovcnt; *zcopy_payload_p = uct_iov_to_iovec(&ctx->iov[ctx->iov_cnt], &io_vec_cnt, iov, iovcnt, SIZE_MAX, &uct_iov_iter); *ctx_p = ctx; diff --git a/src/uct/tcp/tcp_iface.c b/src/uct/tcp/tcp_iface.c index 2d427d64032..d9955c95120 100644 --- a/src/uct/tcp/tcp_iface.c +++ b/src/uct/tcp/tcp_iface.c @@ -173,9 +173,9 @@ static ucs_status_t uct_tcp_iface_query(uct_iface_h tl_iface, uct_iface_attr_t * attr->cap.flags |= UCT_IFACE_FLAG_EP_KEEPALIVE; } - if (iface->config.zcopy.max_iov > UCT_TCP_EP_ZCOPY_SERVICE_IOV_COUNT) { + if (iface->config.max_iov > UCT_TCP_EP_ZCOPY_SERVICE_IOV_COUNT) { /* AM */ - attr->cap.am.max_iov = iface->config.zcopy.max_iov - + attr->cap.am.max_iov = iface->config.max_iov - UCT_TCP_EP_ZCOPY_SERVICE_IOV_COUNT; attr->cap.am.max_zcopy = iface->config.rx_seg_size - sizeof(uct_tcp_am_hdr_t); @@ -185,7 +185,7 @@ static ucs_status_t uct_tcp_iface_query(uct_iface_h tl_iface, uct_iface_attr_t * if (iface->config.put_enable) { /* PUT */ - attr->cap.put.max_iov = iface->config.zcopy.max_iov - + attr->cap.put.max_iov = iface->config.max_iov - UCT_TCP_EP_ZCOPY_SERVICE_IOV_COUNT; attr->cap.put.max_zcopy = UCT_TCP_EP_PUT_ZCOPY_MAX - UCT_TCP_EP_PUT_SERVICE_LENGTH; @@ -340,7 +340,7 @@ ucs_status_t uct_tcp_iface_set_sockopt(uct_tcp_iface_t *iface, int fd, static uct_iface_ops_t uct_tcp_iface_ops = { .ep_am_short = uct_tcp_ep_am_short, - .ep_am_short_iov = uct_base_ep_am_short_iov, + .ep_am_short_iov = uct_tcp_ep_am_short_iov, .ep_am_bcopy = uct_tcp_ep_am_bcopy, .ep_am_zcopy = uct_tcp_ep_am_zcopy, .ep_put_zcopy = uct_tcp_ep_put_zcopy, @@ -546,19 +546,18 @@ static UCS_CLASS_INIT_FUNC(uct_tcp_iface_t, uct_md_h md, uct_worker_h worker, /* Maximum IOV count allowed by user's configuration (considering TCP * protocol and user's AM headers that use 1st and 2nd IOVs * correspondingly) and system constraints */ - self->config.zcopy.max_iov = ucs_min(config->max_iov + + self->config.max_iov = ucs_min(config->max_iov + UCT_TCP_EP_ZCOPY_SERVICE_IOV_COUNT, ucs_iov_get_max()); /* Use a remaining part of TX segment for AM Zcopy header */ self->config.zcopy.hdr_offset = (sizeof(uct_tcp_ep_zcopy_tx_t) + - sizeof(struct iovec) * - self->config.zcopy.max_iov); + sizeof(struct iovec) * self->config.max_iov); if ((self->config.zcopy.hdr_offset > self->config.tx_seg_size) && - (self->config.zcopy.max_iov > UCT_TCP_EP_ZCOPY_SERVICE_IOV_COUNT)) { + (self->config.max_iov > UCT_TCP_EP_ZCOPY_SERVICE_IOV_COUNT)) { ucs_error("AM Zcopy context (%zu) must be <= TX segment size (%zu). " "It can be adjusted by decreasing maximum IOV count (%zu)", self->config.zcopy.hdr_offset, self->config.tx_seg_size, - self->config.zcopy.max_iov); + self->config.max_iov); return UCS_ERR_INVALID_PARAM; } From eedd443c42cb88810814cefd03b1991406792f78 Mon Sep 17 00:00:00 2001 From: Yossi Itigin Date: Tue, 22 Dec 2020 19:47:53 +0200 Subject: [PATCH 082/107] TOOLS/VFS: Add skeleton for VFS mount daemon --- src/tools/vfs/vfs_daemon.h | 41 ++++ src/tools/vfs/vfs_main.c | 365 +++++++++++++++++++++++++++++++++++- src/ucs/vfs/sock/vfs_sock.h | 1 + test/gtest/ucs/test_vfs.cc | 7 + 4 files changed, 412 insertions(+), 2 deletions(-) diff --git a/src/tools/vfs/vfs_daemon.h b/src/tools/vfs/vfs_daemon.h index 955ee6b19e5..523564d9d34 100644 --- a/src/tools/vfs/vfs_daemon.h +++ b/src/tools/vfs/vfs_daemon.h @@ -8,7 +8,48 @@ #define VFS_DAEMON_H_ #include +#include +#include +#include +#include +#include #include +#define VFS_DEFAULT_MOUNTPOINT_DIR "/tmp/ucx" +#define VFS_FUSE_MOUNT_PROG "fusermount3" + + +enum { + VFS_DAEMON_ACTION_START = UCS_VFS_SOCK_ACTION_NOP +}; + + +#define vfs_error(_fmt, ...) \ + { \ + fprintf(stderr, "Error: " _fmt "\n", ##__VA_ARGS__); \ + } + + +#define vfs_log(_fmt, ...) \ + { \ + if (g_opts.verbose) { \ + fprintf(stderr, "Debug: " _fmt "\n", ##__VA_ARGS__); \ + } \ + } + + +typedef struct { + int action; + int foreground; + int verbose; + const char *mountpoint_dir; + const char *mount_opts; + const char *sock_path; +} vfs_opts_t; + + +extern vfs_opts_t g_opts; +extern const char *vfs_action_names[]; + #endif diff --git a/src/tools/vfs/vfs_main.c b/src/tools/vfs/vfs_main.c index 4239d46f4fb..14f760369be 100644 --- a/src/tools/vfs/vfs_main.c +++ b/src/tools/vfs/vfs_main.c @@ -10,9 +10,370 @@ #include "vfs_daemon.h" +#include +#include +#include +#include -int main(int argc, char **argv) + +vfs_opts_t g_opts = { + .action = VFS_DAEMON_ACTION_START, + .foreground = 0, + .verbose = 0, + .mountpoint_dir = VFS_DEFAULT_MOUNTPOINT_DIR, + .mount_opts = "", + .sock_path = NULL +}; + +const char *vfs_action_names[] = { + [UCS_VFS_SOCK_ACTION_STOP] = "stop", + [UCS_VFS_SOCK_ACTION_MOUNT] = "mount", + [VFS_DAEMON_ACTION_START] = "start" +}; + +static struct sockaddr_un g_sockaddr; + + +static int vfs_run_fusermount(char **extra_argv) +{ + char command[128]; + pid_t child_pid; + int ret, status; + int devnull_fd; + char *p, *endp; + char *argv[16]; + int i, argc; + + argc = 0; + argv[argc++] = VFS_FUSE_MOUNT_PROG; + if (!g_opts.verbose) { + argv[argc++] = "-q"; + } + while (*extra_argv != NULL) { + argv[argc++] = *(extra_argv++); + } + argv[argc++] = NULL; + assert(argc <= ucs_static_array_size(argv)); + + /* save the whole command to log */ + p = command; + endp = command + sizeof(command); + for (i = 0; argv[i] != NULL; ++i) { + snprintf(p, endp - p, "%s ", argv[i]); + p += strlen(p); + } + *(p - 1) = '\0'; + + vfs_log("exec '%s'", command); + + child_pid = fork(); + if (child_pid == -1) { + vfs_error("fork() failed: %m"); + return -1; + } + + if (child_pid == 0) { + if (!g_opts.verbose) { + devnull_fd = open("/dev/null", O_WRONLY); + dup2(devnull_fd, 1); + dup2(devnull_fd, 2); + close(devnull_fd); + } + execvp(argv[0], argv); + vfs_error("failed to execute '%s': %m", command); + _exit(1); + } + + ret = waitpid(child_pid, &status, 0); + if (ret < 0) { + vfs_error("waitpid(%d) failed: %m", child_pid); + return -errno; + } else if (WIFEXITED(status) && (WEXITSTATUS(status) != 0)) { + vfs_error("'%s' exited with status %d", command, WEXITSTATUS(status)); + return -1; + } else if (!WIFEXITED(status)) { + vfs_error("'%s' did not exit properly (%d)", command, status); + return -1; + } + + return 0; +} + +static int vfs_unlink_socket(int silent_notexist) +{ + int ret; + + vfs_log("removing existing socket '%s'", g_sockaddr.sun_path); + + ret = unlink(g_sockaddr.sun_path); + if (ret < 0) { + ret = -errno; + if (silent_notexist && (errno == ENOENT)) { + vfs_log("could not unlink '%s': %m", g_sockaddr.sun_path); + } else { + vfs_error("could not unlink '%s': %m", g_sockaddr.sun_path); + } + return ret; + } + + return 0; +} + +/* return 0 or the (negative) value of errno in case of error */ +static int vfs_listen(int silent_addinuse_err) +{ + int listen_fd, ret; + + ret = umask(~S_IRWXU); + if (ret < 0) { + ret = -errno; + vfs_error("failed to set umask permissions: %m"); + goto out; + } + + listen_fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (listen_fd < 0) { + ret = -errno; + vfs_error("failed to create listening socket: %m"); + goto out; + } + + ret = bind(listen_fd, (const struct sockaddr*)&g_sockaddr, + sizeof(g_sockaddr)); + if (ret < 0) { + ret = -errno; + if ((errno != EADDRINUSE) || !silent_addinuse_err) { + vfs_error("bind(%s) failed: %m", g_sockaddr.sun_path); + } + goto out_close; + } + + ret = listen(listen_fd, 128); + if (ret < 0) { + ret = -errno; + vfs_error("listen() failed: %m"); + goto out_unlink; + } + + vfs_error("closing socket, TODO implement listening for connections"); + close(listen_fd); + +out_unlink: + vfs_unlink_socket(0); +out_close: + close(listen_fd); +out: + return ret; +} + +/* return 0 or the (negative) value of errno in case of error */ +static int vfs_connect_and_act() +{ + ucs_vfs_sock_message_t vfs_msg_out; + int connfd; + int ret; + + vfs_log("connecting to '%s'", g_sockaddr.sun_path); + + connfd = socket(AF_UNIX, SOCK_STREAM, 0); + if (connfd < 0) { + ret = -errno; + vfs_error("failed to create connection socket: %m"); + return ret; + } + + ret = connect(connfd, (const struct sockaddr*)&g_sockaddr, + sizeof(g_sockaddr)); + if (ret < 0) { + ret = -errno; + if (errno == ECONNREFUSED) { + vfs_log("connect(%s) failed: %m", g_sockaddr.sun_path); + } else { + vfs_error("connect(%s) failed: %m", g_sockaddr.sun_path); + } + goto err_close; + } + + if (g_opts.action < UCS_VFS_SOCK_ACTION_LAST) { + vfs_log("sending action '%s'", vfs_action_names[g_opts.action]); + + /* send action */ + vfs_msg_out.action = g_opts.action; + ret = ucs_vfs_sock_send(connfd, &vfs_msg_out); + if (ret < 0) { + vfs_error("failed to send: %d", ret); + goto err_close; + } + + ret = 0; + } + +err_close: + close(connfd); + return ret; +} + +/* return 0 or negative value in case of error */ +int vfs_start() +{ + int ret; + + ret = vfs_listen(1); + if (ret != -EADDRINUSE) { + return ret; + } + + /* Failed to listen because 'socket_name' path already exists - try to + * connect */ + ret = vfs_connect_and_act(); + if (ret != -ECONNREFUSED) { + return ret; + } + + /* Could not connect to the socket because no one is listening - remove the + * socket file and try listening again */ + ret = vfs_unlink_socket(0); + if (ret < 0) { + return ret; + } + + return vfs_listen(0); +} + +static void vfs_usage() +{ + struct sockaddr_un sock_addr = {}; + + ucs_vfs_sock_get_address(&sock_addr); + printf("Usage: ucx_vfs [options] [action]\n"); + printf("\n"); + printf("Options:\n"); + printf(" -d Set parent directory for mount points (default: %s)\n", + g_opts.mountpoint_dir); + printf(" -o Pass these mount options to mount.fuse\n"); + printf(" -f Do not daemonize; run in foreground\n"); + printf(" -v Enable verbose logging (requires -f)\n"); + printf(" -l Set listening unix socket path (default: %s)\n", + sock_addr.sun_path); + printf("\n"); + printf("Actions:\n"); + printf(" start Run the daemon and listen for connection from UCX\n"); + printf(" If a daemon is already running, do nothing\n"); + printf(" This is the default action.\n"); + printf(" stop Stop the running daemon\n"); + printf("\n"); +} + +static int vfs_parse_args(int argc, char **argv) { - /* TODO implement VFS daemon logic */ + const char *action_str; + int c, i; + + while ((c = getopt(argc, argv, "d:o:vfl:h")) != -1) { + switch (c) { + case 'd': + g_opts.mountpoint_dir = optarg; + break; + case 'o': + g_opts.mount_opts = optarg; + break; + case 'v': + ++g_opts.verbose; + break; + case 'f': + g_opts.foreground = 1; + break; + case 'l': + g_opts.sock_path = optarg; + break; + case 'h': + default: + vfs_usage(); + return -127; + } + } + + if (g_opts.verbose && !g_opts.foreground) { + vfs_error("Option -v requires -f"); + vfs_usage(); + return -1; + } + + if (optind < argc) { + action_str = argv[optind]; + g_opts.action = UCS_VFS_SOCK_ACTION_LAST; + for (i = 0; i < ucs_static_array_size(vfs_action_names); ++i) { + if ((vfs_action_names[i] != NULL) && + !strcmp(action_str, vfs_action_names[i])) { + g_opts.action = i; + } + } + if (g_opts.action == UCS_VFS_SOCK_ACTION_LAST) { + vfs_error("invalid action '%s'", action_str); + vfs_usage(); + return -1; + } + ++optind; + } + + if (optind < argc) { + vfs_error("only one action can be specified"); + vfs_usage(); + return -1; + } + return 0; } + +static int vfs_test_fuse() +{ + char *argv[] = {"-V", NULL}; + return vfs_run_fusermount(argv); +} + +int main(int argc, char **argv) +{ + int ret; + + ret = vfs_parse_args(argc, argv); + if (ret < 0) { + return -1; + } + + ret = vfs_test_fuse(); + if (ret < 0) { + return -1; + } + + ret = mkdir(g_opts.mountpoint_dir, S_IRWXU); + if ((ret < 0) && (errno != EEXIST)) { + vfs_error("could not create directory '%s': %m", g_opts.mountpoint_dir); + return -1; + } + + if (!g_opts.foreground) { + fuse_daemonize(0); + } + + if (g_opts.sock_path == NULL) { + ret = ucs_vfs_sock_get_address(&g_sockaddr); + if (ret < 0) { + vfs_error("failed to initialize socket address: %d", ret); + return -1; + } + } else { + g_sockaddr.sun_family = AF_UNIX; + strncpy(g_sockaddr.sun_path, g_opts.sock_path, + sizeof(g_sockaddr.sun_path)); + } + + switch (g_opts.action) { + case VFS_DAEMON_ACTION_START: + return vfs_start(); + case UCS_VFS_SOCK_ACTION_STOP: + return vfs_connect_and_act(); + default: + vfs_error("unexpected action %d", g_opts.action); + return -1; + } +} diff --git a/src/ucs/vfs/sock/vfs_sock.h b/src/ucs/vfs/sock/vfs_sock.h index bbb1da03eaf..b3f23732569 100644 --- a/src/ucs/vfs/sock/vfs_sock.h +++ b/src/ucs/vfs/sock/vfs_sock.h @@ -21,6 +21,7 @@ typedef enum { UCS_VFS_SOCK_ACTION_STOP, /* daemon is asked to stop */ UCS_VFS_SOCK_ACTION_MOUNT, /* daemon is asked to mount a file system */ UCS_VFS_SOCK_ACTION_MOUNT_REPLY, /* daemon sends back FUSE file descriptor */ + UCS_VFS_SOCK_ACTION_NOP, /* no-operation, used to test conneciton */ UCS_VFS_SOCK_ACTION_LAST } ucs_vfs_sock_action_t; diff --git a/test/gtest/ucs/test_vfs.cc b/test/gtest/ucs/test_vfs.cc index c4705a42756..01d30a93b55 100644 --- a/test/gtest/ucs/test_vfs.cc +++ b/test/gtest/ucs/test_vfs.cc @@ -99,3 +99,10 @@ UCS_TEST_F(test_vfs_sock, send_recv_mount_reply) { close(msg_out.fd); close(fd); } + +UCS_TEST_F(test_vfs_sock, send_recv_nop) { + /* send stop/start commands from socket[0] to socket[1] */ + ucs_vfs_sock_message_t msg_out = {}; + do_send_recv(UCS_VFS_SOCK_ACTION_NOP, m_sockets[0], m_sockets[1], -1, + &msg_out); +} \ No newline at end of file From 0c5526ec541b72e43c305a9b6b8f6357c22e99aa Mon Sep 17 00:00:00 2001 From: Anatoly Vildemanov Date: Wed, 30 Dec 2020 20:22:15 +0300 Subject: [PATCH 083/107] AZP/IODEMO: add LD_LIBRARY_PATH --- buildlib/io_demo/az-stage-io-demo.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildlib/io_demo/az-stage-io-demo.yaml b/buildlib/io_demo/az-stage-io-demo.yaml index a9ec912f192..f8a2050d721 100644 --- a/buildlib/io_demo/az-stage-io-demo.yaml +++ b/buildlib/io_demo/az-stage-io-demo.yaml @@ -28,6 +28,7 @@ steps: # set UCX environment variables export UCX_SOCKADDR_CM_ENABLE=y export UCX_NET_DEVICES=$(ibdev2netdev | sed -ne 's/\(\w*\) port \([0-9]\) ==> '${roce_iface}' .*/\1:\2/p') + export LD_LIBRARY_PATH=$(workspace)/install/lib:$LD_LIBRARY_PATH $(workspace)/test/apps/iodemo/run_io_demo.sh \ -H $(agent_hosts) \ --tasks-per-node 1 \ @@ -45,7 +46,6 @@ steps: -w 16 \ -t 10 \ ${{ parameters.iodemo_args }} - displayName: Launch with run_io_demo.sh ( ${{ parameters.name }} ) timeoutInMinutes: 10 From 496fb892b3dd9da8528aa595c96c4953423db762 Mon Sep 17 00:00:00 2001 From: Yossi Itigin Date: Thu, 31 Dec 2020 13:55:12 +0200 Subject: [PATCH 084/107] TOOLS/VFS: Fix review comments and statick check error --- src/tools/vfs/vfs_main.c | 14 ++++++++------ src/ucs/vfs/sock/vfs_sock.h | 2 +- test/gtest/ucs/test_vfs.cc | 2 +- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/tools/vfs/vfs_main.c b/src/tools/vfs/vfs_main.c index 14f760369be..59632087a0b 100644 --- a/src/tools/vfs/vfs_main.c +++ b/src/tools/vfs/vfs_main.c @@ -179,7 +179,7 @@ static int vfs_connect_and_act() if (connfd < 0) { ret = -errno; vfs_error("failed to create connection socket: %m"); - return ret; + goto out; } ret = connect(connfd, (const struct sockaddr*)&g_sockaddr, @@ -191,7 +191,7 @@ static int vfs_connect_and_act() } else { vfs_error("connect(%s) failed: %m", g_sockaddr.sun_path); } - goto err_close; + goto out_close; } if (g_opts.action < UCS_VFS_SOCK_ACTION_LAST) { @@ -202,14 +202,15 @@ static int vfs_connect_and_act() ret = ucs_vfs_sock_send(connfd, &vfs_msg_out); if (ret < 0) { vfs_error("failed to send: %d", ret); - goto err_close; + goto out_close; } ret = 0; } -err_close: +out_close: close(connfd); +out: return ret; } @@ -311,7 +312,7 @@ static int vfs_parse_args(int argc, char **argv) if (g_opts.action == UCS_VFS_SOCK_ACTION_LAST) { vfs_error("invalid action '%s'", action_str); vfs_usage(); - return -1; + return 0; } ++optind; } @@ -363,8 +364,9 @@ int main(int argc, char **argv) } } else { g_sockaddr.sun_family = AF_UNIX; + memset(g_sockaddr.sun_path, 0, sizeof(g_sockaddr.sun_path)); strncpy(g_sockaddr.sun_path, g_opts.sock_path, - sizeof(g_sockaddr.sun_path)); + sizeof(g_sockaddr.sun_path) - 1); } switch (g_opts.action) { diff --git a/src/ucs/vfs/sock/vfs_sock.h b/src/ucs/vfs/sock/vfs_sock.h index b3f23732569..1d33efb3a1f 100644 --- a/src/ucs/vfs/sock/vfs_sock.h +++ b/src/ucs/vfs/sock/vfs_sock.h @@ -21,7 +21,7 @@ typedef enum { UCS_VFS_SOCK_ACTION_STOP, /* daemon is asked to stop */ UCS_VFS_SOCK_ACTION_MOUNT, /* daemon is asked to mount a file system */ UCS_VFS_SOCK_ACTION_MOUNT_REPLY, /* daemon sends back FUSE file descriptor */ - UCS_VFS_SOCK_ACTION_NOP, /* no-operation, used to test conneciton */ + UCS_VFS_SOCK_ACTION_NOP, /* no-operation, used to test connection */ UCS_VFS_SOCK_ACTION_LAST } ucs_vfs_sock_action_t; diff --git a/test/gtest/ucs/test_vfs.cc b/test/gtest/ucs/test_vfs.cc index 01d30a93b55..9b58eecd2b0 100644 --- a/test/gtest/ucs/test_vfs.cc +++ b/test/gtest/ucs/test_vfs.cc @@ -105,4 +105,4 @@ UCS_TEST_F(test_vfs_sock, send_recv_nop) { ucs_vfs_sock_message_t msg_out = {}; do_send_recv(UCS_VFS_SOCK_ACTION_NOP, m_sockets[0], m_sockets[1], -1, &msg_out); -} \ No newline at end of file +} From 0ff146afd102eb3f1fe31ecb70311227c52b2a5f Mon Sep 17 00:00:00 2001 From: dmitrygx Date: Thu, 31 Dec 2020 15:47:18 +0000 Subject: [PATCH 085/107] UCT/IB/UD: Fix typedef for iface address --- src/uct/ib/base/ib_verbs.h | 4 ---- src/uct/ib/ud/base/ud_def.h | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/uct/ib/base/ib_verbs.h b/src/uct/ib/base/ib_verbs.h index 09f30c2b61e..5344833eb9b 100644 --- a/src/uct/ib/base/ib_verbs.h +++ b/src/uct/ib/base/ib_verbs.h @@ -323,8 +323,4 @@ static inline ucs_status_t uct_ib_qp_max_send_sge(struct ibv_qp *qp, return UCS_OK; } -typedef struct uct_ib_qpnum { - uct_ib_uint24_t qp_num; -} uct_ib_qpnum_t; - #endif /* UCT_IB_VERBS_H */ diff --git a/src/uct/ib/ud/base/ud_def.h b/src/uct/ib/ud/base/ud_def.h index e95fd0b612b..b69f45dab4b 100644 --- a/src/uct/ib/ud/base/ud_def.h +++ b/src/uct/ib/ud/base/ud_def.h @@ -40,7 +40,7 @@ typedef uint16_t uct_ud_psn_t; typedef struct uct_ud_iface uct_ud_iface_t; typedef struct uct_ud_ep uct_ud_ep_t; typedef struct uct_ud_ctl_hdr uct_ud_ctl_hdr_t; -typedef uct_ib_qpnum_t uct_ud_iface_addr_t; +typedef struct uct_ud_iface_addr uct_ud_iface_addr_t; typedef struct uct_ud_ep_addr uct_ud_ep_addr_t; typedef struct uct_ud_iface_peer uct_ud_iface_peer_t; From ad72dea6a3253b496254d1401aefc074973bdacc Mon Sep 17 00:00:00 2001 From: dmitrygx Date: Thu, 31 Dec 2020 10:30:08 +0000 Subject: [PATCH 086/107] UCP/WIREUP: Don't allow wireup EP over wireup EP --- src/ucp/wireup/wireup.c | 10 ++++++---- src/ucp/wireup/wireup_ep.c | 2 ++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/ucp/wireup/wireup.c b/src/ucp/wireup/wireup.c index 07e3a2cc4ed..1d1f2e2c83d 100644 --- a/src/ucp/wireup/wireup.c +++ b/src/ucp/wireup/wireup.c @@ -1104,15 +1104,17 @@ ucp_wireup_check_config_intersect(ucp_ep_h ep, cm_wireup_ep = ucp_ep_get_cm_wireup_ep(ep); new_key->wireup_msg_lane = new_key->cm_lane; reuse_lane = old_key->wireup_msg_lane; - ucp_wireup_ep_set_aux(cm_wireup_ep, ep->uct_eps[reuse_lane], + ucp_wireup_ep_set_aux(cm_wireup_ep, + ucp_wireup_ep_extract_next_ep(ep->uct_eps[reuse_lane]), old_key->lanes[reuse_lane].rsc_index); ucp_wireup_ep_pending_queue_purge(ep->uct_eps[reuse_lane], ucp_wireup_pending_purge_cb, replay_pending_queue); - /* reset the UCT EP from the previous WIREUP lane to not - * destroy it, since it's not needed anymore in the new - * configuration, but will be used for WIREUP MSG */ + /* reset the UCT EP from the previous WIREUP lane and destroy its WIREUP EP, + * since it's not needed anymore in the new configuration, UCT EP will be + * used for sending WIREUP MSGs in the new configuration */ + uct_ep_destroy(ep->uct_eps[reuse_lane]); ep->uct_eps[reuse_lane] = NULL; } diff --git a/src/ucp/wireup/wireup_ep.c b/src/ucp/wireup/wireup_ep.c index b15f24cc990..26e201fed08 100644 --- a/src/ucp/wireup/wireup_ep.c +++ b/src/ucp/wireup/wireup_ep.c @@ -262,6 +262,7 @@ UCS_CLASS_DEFINE_NAMED_NEW_FUNC(ucp_wireup_ep_create, ucp_wireup_ep_t, uct_ep_t, void ucp_wireup_ep_set_aux(ucp_wireup_ep_t *wireup_ep, uct_ep_h uct_ep, ucp_rsc_index_t rsc_index) { + ucs_assert(!ucp_wireup_ep_test(uct_ep)); wireup_ep->aux_ep = uct_ep; wireup_ep->aux_rsc_index = rsc_index; } @@ -678,6 +679,7 @@ void ucp_wireup_ep_set_next_ep(uct_ep_h uct_ep, uct_ep_h next_ep) ucs_assert(wireup_ep != NULL); ucs_assert(wireup_ep->super.uct_ep == NULL); + ucs_assert(!ucp_wireup_ep_test(next_ep)); wireup_ep->flags |= UCP_WIREUP_EP_FLAG_LOCAL_CONNECTED; ucp_proxy_ep_set_uct_ep(&wireup_ep->super, next_ep, 1); } From d965d79263da1b8661d9d8bde0ef24a26b5ae490 Mon Sep 17 00:00:00 2001 From: dmitrygx Date: Thu, 31 Dec 2020 11:24:21 +0000 Subject: [PATCH 087/107] UCP/CORE: Remove incorrect assertion --- src/ucp/core/ucp_worker.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ucp/core/ucp_worker.c b/src/ucp/core/ucp_worker.c index dcea5adda36..a7ae69972b1 100644 --- a/src/ucp/core/ucp_worker.c +++ b/src/ucp/core/ucp_worker.c @@ -574,7 +574,7 @@ ucp_worker_iface_handle_uct_ep_failure(ucp_ep_h ucp_ep, ucp_lane_index_t lane, * was detected before receiving WIREUP_MSG/ACK from a client or * marking a server's EP as REMOTE_CONNECTED was scheduled on a * progress, but not completed yet (CM_WIREUP_EP/AUX_EP is - * closed when moving am EP to REMOTE_CONNECTED state) */ + * closed when moving an EP to REMOTE_CONNECTED state) */ wireup_ep = ucp_wireup_ep(ucp_ep->uct_eps[lane]); if ((lane == ucp_ep_get_cm_lane(ucp_ep)) && (lane == ucp_ep_get_wireup_msg_lane(ucp_ep)) && @@ -585,7 +585,6 @@ ucp_worker_iface_handle_uct_ep_failure(ucp_ep_h ucp_ep, ucp_lane_index_t lane, /* No need to invoke the error handling flow, just flush and * destroy CM_WIREUP/AUX_EP */ aux_uct_ep = wireup_ep->aux_ep; - ucs_assert(ucp_wireup_ep_test(aux_uct_ep)); ucs_queue_head_init(&tmp_pending_queue); ucp_wireup_ep_disown(ucp_ep->uct_eps[lane], aux_uct_ep); From 0779fde3dbb72f21c8267478063bbe7b8e70bd8e Mon Sep 17 00:00:00 2001 From: Yossi Itigin Date: Tue, 22 Dec 2020 19:24:49 +0200 Subject: [PATCH 088/107] UCS/VFS: Connect to daemon and expose dummy filesystem --- src/ucs/vfs/fuse/vfs_fuse.c | 349 +++++++++++++++++++++++++++++++++++- 1 file changed, 346 insertions(+), 3 deletions(-) diff --git a/src/ucs/vfs/fuse/vfs_fuse.c b/src/ucs/vfs/fuse/vfs_fuse.c index 4552039e238..040948dcfa5 100644 --- a/src/ucs/vfs/fuse/vfs_fuse.c +++ b/src/ucs/vfs/fuse/vfs_fuse.c @@ -5,29 +5,372 @@ */ #ifdef HAVE_CONFIG_H -# include "config.h" +#include "config.h" #endif #include #include #include +#include +#include +#include #include +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_INOTIFY +#include +#endif + + +#define UCS_VFS_DUMMY_FILE_NAME "dummy" +#define UCS_VFS_DUMMY_FILE_DATA "UCX FUSE\n" static struct { - pthread_t thread_id; + pthread_t thread_id; + pthread_mutex_t mutex; + struct fuse *fuse; + int fuse_fd; + int stop; + int inotify_fd; + int watch_desc; } ucs_vfs_fuse_context = { - .thread_id = -1, + .thread_id = -1, + .mutex = PTHREAD_MUTEX_INITIALIZER, + .fuse = NULL, + .fuse_fd = -1, + .stop = 0, + .inotify_fd = -1, + .watch_desc = -1 +}; + +static int ucs_vfs_fuse_getattr(const char *path, struct stat *stbuf, + struct fuse_file_info *fi) +{ + memset(stbuf, 0, sizeof(struct stat)); + stbuf->st_uid = getuid(); + stbuf->st_gid = getgid(); + + if (strcmp(path, "/") == 0) { + stbuf->st_mode = S_IFDIR | S_IRWXU; + stbuf->st_nlink = 2; + return 0; + } + + if (strcmp(path, "/" UCS_VFS_DUMMY_FILE_NAME)) { + return -ENOENT; + } + + stbuf->st_mode = S_IFREG | S_IRUSR; + stbuf->st_size = strlen(UCS_VFS_DUMMY_FILE_DATA); + stbuf->st_nlink = 1; + return 0; +} + +static int +ucs_vfs_fuse_readdir(const char *path, void *buf, fuse_fill_dir_t filler, + off_t offset, struct fuse_file_info *fi, + enum fuse_readdir_flags flags) +{ + filler(buf, ".", NULL, 0, 0); + filler(buf, "..", NULL, 0, 0); + filler(buf, "dummy", NULL, 0, 0); + return 0; +} + +static int ucs_vfs_fuse_open(const char *path, struct fuse_file_info *fi) +{ + if (strcmp(path, "/" UCS_VFS_DUMMY_FILE_NAME)) { + return -ENOENT; + } + + return 0; +} + +static int ucs_vfs_fuse_read(const char *path, char *buf, size_t size, + off_t offset, struct fuse_file_info *fi) +{ + char *data = UCS_VFS_DUMMY_FILE_DATA; + size_t length = strlen(data); + size_t nread; + + if (offset >= length) { + return 0; + } + + if (offset + size <= length) { + nread = size; /* read does not pass end-of-file */ + } else { + nread = length - offset; /* read truncated by end-of-file */ + } + memcpy(buf, data + offset, nread); + + return nread; +} + +struct fuse_operations ucs_vfs_fuse_operations = { + .getattr = ucs_vfs_fuse_getattr, + .open = ucs_vfs_fuse_open, + .read = ucs_vfs_fuse_read, + .readdir = ucs_vfs_fuse_readdir, }; +static void ucs_vfs_fuse_main() +{ + struct fuse_args fargs = FUSE_ARGS_INIT(0, NULL); + char mountpoint_fd[64]; + int ret; + + fuse_opt_add_arg(&fargs, ""); + + pthread_mutex_lock(&ucs_vfs_fuse_context.mutex); + + if (ucs_vfs_fuse_context.stop) { + goto out_unlock; + } + + ucs_vfs_fuse_context.fuse = fuse_new(&fargs, &ucs_vfs_fuse_operations, + sizeof(ucs_vfs_fuse_operations), NULL); + if (ucs_vfs_fuse_context.fuse == NULL) { + ucs_error("fuse_new() failed"); + goto out_unlock; + } + + ucs_snprintf_safe(mountpoint_fd, sizeof(mountpoint_fd), "/dev/fd/%d", + ucs_vfs_fuse_context.fuse_fd); + ret = fuse_mount(ucs_vfs_fuse_context.fuse, mountpoint_fd); + if (ret < 0) { + ucs_error("fuse_mount(%s) failed: %d", mountpoint_fd, ret); + goto out_destroy; + } + + /* Drop the lock and execute main loop */ + pthread_mutex_unlock(&ucs_vfs_fuse_context.mutex); + + fuse_loop(ucs_vfs_fuse_context.fuse); + + pthread_mutex_lock(&ucs_vfs_fuse_context.mutex); +out_destroy: + /* destroy when lock is held */ + fuse_destroy(ucs_vfs_fuse_context.fuse); + ucs_vfs_fuse_context.fuse = NULL; +out_unlock: + pthread_mutex_unlock(&ucs_vfs_fuse_context.mutex); +} + +static ucs_status_t ucs_vfs_fuse_wait_for_path(const char *path) +{ +#ifdef HAVE_INOTIFY + char event_buf[sizeof(struct inotify_event) + NAME_MAX]; + const char *watch_dirname, *watch_filename; + const struct inotify_event *event; + char path_buf[NAME_MAX]; + ucs_status_t status; + ssize_t nread; + size_t offset; + + pthread_mutex_lock(&ucs_vfs_fuse_context.mutex); + + ucs_strncpy_safe(path_buf, path, sizeof(path_buf)); + watch_dirname = dirname(path_buf); + watch_filename = ucs_basename(path); + + /* Create inotify channel */ + ucs_vfs_fuse_context.inotify_fd = inotify_init(); + if (ucs_vfs_fuse_context.inotify_fd < 0) { + ucs_error("inotify_init() failed: %m"); + status = UCS_ERR_IO_ERROR; + goto out; + } + + /* Watch for new files in 'watch_dirname' */ + ucs_vfs_fuse_context.watch_desc = inotify_add_watch( + ucs_vfs_fuse_context.inotify_fd, watch_dirname, IN_CREATE); + if (ucs_vfs_fuse_context.watch_desc < 0) { + ucs_error("inotify_add_watch(%s) failed: %m", watch_dirname); + status = UCS_ERR_IO_ERROR; + goto out_close_inotify_fd; + } + + + /* Read events from inotify channel and exit when either the main thread set + * 'stop' flag, or the file was created + */ + ucs_debug("waiting for creation of '%s' in '%s'", watch_filename, + watch_dirname); + for (;;) { + pthread_mutex_unlock(&ucs_vfs_fuse_context.mutex); + nread = read(ucs_vfs_fuse_context.inotify_fd, event_buf, + sizeof(event_buf)); + pthread_mutex_lock(&ucs_vfs_fuse_context.mutex); + + if (ucs_vfs_fuse_context.stop) { + status = UCS_ERR_CANCELED; + break; + } + + if (nread < 0) { + ucs_error("inotify read() failed: %m"); + status = UCS_ERR_IO_ERROR; + break; + } + + /* Go over new events in the buffer */ + for (offset = 0; offset < nread; + offset += sizeof(*event) + event->len) { + event = UCS_PTR_BYTE_OFFSET(event_buf, offset); + if (!(event->mask & IN_CREATE)) { + ucs_trace("ignoring inotify event with mask 0x%x", event->mask); + continue; + } + + ucs_debug("file '%s' created", event->name); + if (strcmp(event->name, watch_filename)) { + ucs_trace("ignoring inotify create event of '%s'", event->name); + continue; + } + + status = UCS_OK; + goto out_close_watch_id; + } + } + +out_close_watch_id: + inotify_rm_watch(ucs_vfs_fuse_context.inotify_fd, + ucs_vfs_fuse_context.watch_desc); +out_close_inotify_fd: + close(ucs_vfs_fuse_context.inotify_fd); + ucs_vfs_fuse_context.inotify_fd = -1; +out: + pthread_mutex_unlock(&ucs_vfs_fuse_context.mutex); + return status; +#else + return UCS_ERR_UNSUPPORTED; +#endif +} + static void *ucs_vfs_fuse_thread_func(void *arg) { + ucs_vfs_sock_message_t vfs_msg_in, vfs_msg_out; + struct sockaddr_un un_addr; + ucs_status_t status; + int connfd; + int ret; + + connfd = socket(AF_UNIX, SOCK_STREAM, 0); + if (connfd < 0) { + ucs_error("failed to create VFS socket: %m"); + goto out; + } + +again: + ret = ucs_vfs_sock_get_address(&un_addr); + if (ret < 0) { + ucs_warn("failed to get vfs socket path: %s", strerror(-ret)); + goto out_close; + } + + ucs_debug("connecting vfs socket %d to daemon on '%s'", connfd, + un_addr.sun_path); + ret = connect(connfd, (const struct sockaddr*)&un_addr, sizeof(un_addr)); + if (ret < 0) { + /* VFS daemon is not listening. Set up a file watch on the unix socket + * path, to retry when the daemon is started. + */ + if ((errno == ECONNREFUSED) || (errno == ENOENT)) { + ucs_debug("failed to connect to vfs socket '%s': %m", + un_addr.sun_path); + status = ucs_vfs_fuse_wait_for_path(un_addr.sun_path); + if (status == UCS_OK) { + goto again; + } + + ucs_diag("failed to watch on '%s', VFS will be disabled", + un_addr.sun_path); + } else { + ucs_warn("failed to connect to vfs socket '%s': %m", + un_addr.sun_path); + } + goto out_close; + } + + ucs_debug("sending vfs mount request on socket %d", connfd); + vfs_msg_out.action = UCS_VFS_SOCK_ACTION_MOUNT; + ret = ucs_vfs_sock_send(connfd, &vfs_msg_out); + if (ret < 0) { + ucs_warn("failed to send mount action to vfs daemon: %s", + strerror(-ret)); + goto out_close; + } + + ret = ucs_vfs_sock_recv(connfd, &vfs_msg_in); + if (ret < 0) { + ucs_warn("failed to receive mount reply from vfs daemon: %s", + strerror(-ret)); + goto out_close; + } + + ucs_vfs_fuse_context.fuse_fd = vfs_msg_in.fd; + ucs_vfs_fuse_main(); + close(vfs_msg_in.fd); + +out_close: + close(connfd); +out: return NULL; } +static void ucs_fuse_replace_fd_devnull() +{ + int devnull_fd; + + devnull_fd = open("/dev/null", O_RDWR); + if (devnull_fd < 0) { + ucs_warn("failed to open /dev/null: %m"); + return; + } + + /* force exiting from fuse event loop, which reads from fuse_fd */ + ucs_assert(ucs_vfs_fuse_context.fuse_fd != -1); + ucs_debug("dup2(%d, %d)", devnull_fd, ucs_vfs_fuse_context.fuse_fd); + dup2(devnull_fd, ucs_vfs_fuse_context.fuse_fd); + close(devnull_fd); +} + static void ucs_fuse_thread_stop() { + sighandler_t orig_handler; + + orig_handler = signal(SIGUSR1, ucs_empty_function); + + pthread_mutex_lock(&ucs_vfs_fuse_context.mutex); + + ucs_vfs_fuse_context.stop = 1; + + /* If the thread is waiting in inotify loop, wake it */ + if (ucs_vfs_fuse_context.inotify_fd >= 0) { +#ifdef HAVE_INOTIFY + inotify_rm_watch(ucs_vfs_fuse_context.inotify_fd, + ucs_vfs_fuse_context.watch_desc); +#endif + } + + /* If the thread is in fuse loop, terminate it */ + if (ucs_vfs_fuse_context.fuse != NULL) { + fuse_exit(ucs_vfs_fuse_context.fuse); + ucs_fuse_replace_fd_devnull(); + pthread_kill(ucs_vfs_fuse_context.thread_id, SIGUSR1); + } + + pthread_mutex_unlock(&ucs_vfs_fuse_context.mutex); + pthread_join(ucs_vfs_fuse_context.thread_id, NULL); + signal(SIGUSR1, orig_handler); } UCS_STATIC_INIT From d19a639daad3320f1554d760ae59d78c220b0390 Mon Sep 17 00:00:00 2001 From: dmitrygx Date: Wed, 30 Dec 2020 18:59:08 +0000 Subject: [PATCH 089/107] UCT/IB/UD: Introduce EP check (Keep Alive) --- src/uct/ib/ud/accel/ud_mlx5.c | 1 + src/uct/ib/ud/base/ud_ep.c | 21 +++++++++++++++++++++ src/uct/ib/ud/base/ud_ep.h | 2 ++ src/uct/ib/ud/base/ud_iface.c | 1 + src/uct/ib/ud/verbs/ud_verbs.c | 1 + test/gtest/ucp/test_ucp_peer_failure.cc | 1 + 6 files changed, 27 insertions(+) diff --git a/src/uct/ib/ud/accel/ud_mlx5.c b/src/uct/ib/ud/accel/ud_mlx5.c index 0f2d85b2232..31590e0f9f3 100644 --- a/src/uct/ib/ud/accel/ud_mlx5.c +++ b/src/uct/ib/ud/accel/ud_mlx5.c @@ -698,6 +698,7 @@ static uct_ud_iface_ops_t uct_ud_mlx5_iface_ops = { .ep_pending_purge = uct_ud_ep_pending_purge, .ep_flush = uct_ud_ep_flush, .ep_fence = uct_base_ep_fence, + .ep_check = uct_ud_ep_check, .ep_create = uct_ud_mlx5_ep_create, .ep_destroy = uct_ud_ep_disconnect , .ep_get_address = uct_ud_ep_get_address, diff --git a/src/uct/ib/ud/base/ud_ep.c b/src/uct/ib/ud/base/ud_ep.c index a22b960d226..192de1bfb28 100644 --- a/src/uct/ib/ud/base/ud_ep.c +++ b/src/uct/ib/ud/base/ud_ep.c @@ -1063,6 +1063,27 @@ ucs_status_t uct_ud_ep_flush(uct_ep_h ep_h, unsigned flags, return status; } +ucs_status_t uct_ud_ep_check(uct_ep_h tl_ep, unsigned flags, uct_completion_t *comp) +{ + uct_ud_ep_t *ep = ucs_derived_of(tl_ep, uct_ud_ep_t); + uct_ud_iface_t *iface = ucs_derived_of(ep->super.super.iface, uct_ud_iface_t); + char dummy = 0; + + UCT_EP_KEEPALIVE_CHECK_PARAM(flags, comp); + + uct_ud_enter(iface); + if (/* check that no TX resources are available (i.e. there is signaled + * operation which provides actual peer status) */ + !uct_ud_ep_is_connected(ep) || + !uct_ud_ep_is_last_ack_received(ep)) { + uct_ud_leave(iface); + return UCS_OK; + } + uct_ud_leave(iface); + + return uct_ep_put_short(tl_ep, &dummy, 0, 0, 0); +} + static uct_ud_send_skb_t *uct_ud_ep_prepare_crep(uct_ud_ep_t *ep) { uct_ud_send_skb_t *skb; diff --git a/src/uct/ib/ud/base/ud_ep.h b/src/uct/ib/ud/base/ud_ep.h index 1f57fdf3480..7427f8e2951 100644 --- a/src/uct/ib/ud/base/ud_ep.h +++ b/src/uct/ib/ud/base/ud_ep.h @@ -303,6 +303,8 @@ ucs_status_t uct_ud_ep_flush(uct_ep_h ep, unsigned flags, ucs_status_t uct_ud_ep_flush_nolock(uct_ud_iface_t *iface, uct_ud_ep_t *ep, uct_completion_t *comp); +ucs_status_t uct_ud_ep_check(uct_ep_h tl_ep, unsigned flags, uct_completion_t *comp); + ucs_status_t uct_ud_ep_get_address(uct_ep_h tl_ep, uct_ep_addr_t *addr); ucs_status_t uct_ud_ep_create_connected_common(const uct_ep_params_t *params, diff --git a/src/uct/ib/ud/base/ud_iface.c b/src/uct/ib/ud/base/ud_iface.c index 01ec3a52435..912a1d08238 100644 --- a/src/uct/ib/ud/base/ud_iface.c +++ b/src/uct/ib/ud/base/ud_iface.c @@ -669,6 +669,7 @@ ucs_status_t uct_ud_iface_query(uct_ud_iface_t *iface, UCT_IFACE_FLAG_CONNECT_TO_EP | UCT_IFACE_FLAG_CONNECT_TO_IFACE | UCT_IFACE_FLAG_PENDING | + UCT_IFACE_FLAG_EP_CHECK | UCT_IFACE_FLAG_CB_SYNC | UCT_IFACE_FLAG_CB_ASYNC | UCT_IFACE_FLAG_ERRHANDLE_PEER_FAILURE; diff --git a/src/uct/ib/ud/verbs/ud_verbs.c b/src/uct/ib/ud/verbs/ud_verbs.c index aa970039172..5b1891b85ef 100644 --- a/src/uct/ib/ud/verbs/ud_verbs.c +++ b/src/uct/ib/ud/verbs/ud_verbs.c @@ -561,6 +561,7 @@ static uct_ud_iface_ops_t uct_ud_verbs_iface_ops = { .ep_pending_purge = uct_ud_ep_pending_purge, .ep_flush = uct_ud_ep_flush, .ep_fence = uct_base_ep_fence, + .ep_check = uct_ud_ep_check, .ep_create = uct_ud_verbs_ep_create, .ep_destroy = uct_ud_ep_disconnect, .ep_get_address = uct_ud_ep_get_address, diff --git a/test/gtest/ucp/test_ucp_peer_failure.cc b/test/gtest/ucp/test_ucp_peer_failure.cc index e17ddad6dca..6ee8e126d87 100644 --- a/test/gtest/ucp/test_ucp_peer_failure.cc +++ b/test/gtest/ucp/test_ucp_peer_failure.cc @@ -466,6 +466,7 @@ class test_ucp_peer_failure_keepalive : public test_ucp_peer_failure m_rbuf.resize(1 * UCS_MBYTE); m_env.push_back(new ucs::scoped_setenv("UCX_TCP_KEEPIDLE", "inf")); + m_env.push_back(new ucs::scoped_setenv("UCX_UD_TIMEOUT", "3s")); } void init() { From 37d45f9b87525bbfb828b2021d9a3016478f352c Mon Sep 17 00:00:00 2001 From: Peter Rudenko Date: Wed, 30 Dec 2020 16:43:05 +0200 Subject: [PATCH 090/107] UCT: Prevent segfault in uct_rdmacm_cm_ep_str when id is not initialized. --- src/uct/ib/rdmacm/rdmacm_cm_ep.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/uct/ib/rdmacm/rdmacm_cm_ep.c b/src/uct/ib/rdmacm/rdmacm_cm_ep.c index 4aad63ab10b..6f95e9d6ba4 100644 --- a/src/uct/ib/rdmacm/rdmacm_cm_ep.c +++ b/src/uct/ib/rdmacm/rdmacm_cm_ep.c @@ -16,8 +16,8 @@ const char* uct_rdmacm_cm_ep_str(uct_rdmacm_cm_ep_t *cep, char *str, size_t max_len) { - struct sockaddr *local_addr = rdma_get_local_addr(cep->id); - struct sockaddr *remote_addr = rdma_get_peer_addr(cep->id); + struct sockaddr *local_addr = cep->id ? rdma_get_local_addr(cep->id) : NULL; + struct sockaddr *remote_addr = cep->id ? rdma_get_peer_addr(cep->id) : NULL; char flags_buf[UCT_RDMACM_EP_FLAGS_STRING_LEN]; char local_ip_port_str[UCS_SOCKADDR_STRING_LEN]; char remote_ip_port_str[UCS_SOCKADDR_STRING_LEN]; @@ -33,13 +33,13 @@ const char* uct_rdmacm_cm_ep_str(uct_rdmacm_cm_ep_t *cep, char *str, NULL }; - if (ucs_sockaddr_is_known_af(local_addr)) { + if ((local_addr != NULL) && ucs_sockaddr_is_known_af(local_addr)) { ucs_sockaddr_str(local_addr, local_ip_port_str, UCS_SOCKADDR_STRING_LEN); } else { ucs_strncpy_safe(local_ip_port_str, "", UCS_SOCKADDR_STRING_LEN); } - if (ucs_sockaddr_is_known_af(remote_addr)) { + if ((remote_addr != NULL) && ucs_sockaddr_is_known_af(remote_addr)) { ucs_sockaddr_str(remote_addr, remote_ip_port_str, UCS_SOCKADDR_STRING_LEN); } else { ucs_strncpy_safe(remote_ip_port_str, "", UCS_SOCKADDR_STRING_LEN); @@ -314,6 +314,7 @@ static ucs_status_t uct_rdamcm_cm_ep_server_init(uct_rdmacm_cm_ep_t *cep, ucs_status_t status; char ep_str[UCT_RDMACM_EP_STRING_LEN]; + cep->id = event->id; cep->flags |= UCT_RDMACM_CM_EP_ON_SERVER; if (event->listen_id->channel != cm->ev_ch) { @@ -340,7 +341,6 @@ static ucs_status_t uct_rdamcm_cm_ep_server_init(uct_rdmacm_cm_ep_t *cep, goto err; } - cep->id = event->id; cep->id->context = cep; memset(&conn_param, 0, sizeof(conn_param)); @@ -466,6 +466,7 @@ UCS_CLASS_INIT_FUNC(uct_rdmacm_cm_ep_t, const uct_ep_params_t *params) self->qp = NULL; self->flags = 0; self->status = UCS_OK; + self->id = NULL; if (params->field_mask & UCT_EP_PARAM_FIELD_SOCKADDR) { status = uct_rdamcm_cm_ep_client_init(self, params); From 48cd4cd23bc646711b2f22fa96a74887b8f13b5d Mon Sep 17 00:00:00 2001 From: Sergey Oblomov Date: Thu, 24 Dec 2020 11:02:22 +0200 Subject: [PATCH 091/107] UCP/KEEPALIVE: enabled by default - enabled keepalive by default: 60s * 128 EP's - added processing of UCP keepalive in iface_open call --- src/ucp/core/ucp_context.c | 12 +++++++++--- src/ucp/core/ucp_worker.c | 13 +++++++------ src/ucp/core/ucp_worker.inl | 6 ++++++ src/uct/tcp/tcp_iface.c | 13 ++++++++++++- test/gtest/ucp/test_ucp_sockaddr.cc | 3 +++ 5 files changed, 37 insertions(+), 10 deletions(-) diff --git a/src/ucp/core/ucp_context.c b/src/ucp/core/ucp_context.c index 3ac14633a21..6ff29907548 100644 --- a/src/ucp/core/ucp_context.c +++ b/src/ucp/core/ucp_context.c @@ -312,13 +312,13 @@ static ucs_config_field_t ucp_config_table[] = { ucs_offsetof(ucp_config_t, ctx.proto_enable), UCS_CONFIG_TYPE_BOOL}, /* TODO: set for keepalive more reasonable values */ - {"KEEPALIVE_INTERVAL", "0", + {"KEEPALIVE_INTERVAL", "60s", "Time interval between keepalive rounds (0 - disabled).", ucs_offsetof(ucp_config_t, ctx.keepalive_interval), UCS_CONFIG_TYPE_TIME}, - {"KEEPALIVE_NUM_EPS", "0", + {"KEEPALIVE_NUM_EPS", "128", "Maximal number of endpoints to check on every keepalive round\n" - "(0 - disabled, inf - check all endpoints on every round)", + "(inf - check all endpoints on every round, must be greater than 0)", ucs_offsetof(ucp_config_t, ctx.keepalive_num_eps), UCS_CONFIG_TYPE_UINT}, {"PROTO_INDIRECT_ID", "auto", @@ -1543,6 +1543,12 @@ static ucs_status_t ucp_fill_config(ucp_context_h context, } } + if (context->config.ext.keepalive_num_eps == 0) { + ucs_error("UCX_KEEPALIVE_NUM_EPS value must be greater than 0"); + status = UCS_ERR_INVALID_PARAM; + goto err_free_alloc_methods; + } + context->config.keepalive_interval = ucs_time_from_sec(context->config.ext.keepalive_interval); return UCS_OK; diff --git a/src/ucp/core/ucp_worker.c b/src/ucp/core/ucp_worker.c index dcea5adda36..877351b46f7 100644 --- a/src/ucp/core/ucp_worker.c +++ b/src/ucp/core/ucp_worker.c @@ -1245,6 +1245,11 @@ ucs_status_t ucp_worker_iface_open(ucp_worker_h worker, ucp_rsc_index_t tl_id, iface_params->field_mask |= UCT_IFACE_PARAM_FIELD_ASYNC_EVENT_ARG | UCT_IFACE_PARAM_FIELD_ASYNC_EVENT_CB; + if (ucp_worker_keepalive_is_enabled(worker)) { + iface_params->field_mask |= UCT_IFACE_PARAM_FIELD_KEEPALIVE_INTERVAL; + iface_params->keepalive_interval = context->config.keepalive_interval; + } + /* Open UCT interface */ status = uct_iface_open(md, worker->uct, iface_params, iface_config, &wiface->iface); @@ -2658,12 +2663,6 @@ void ucp_worker_print_info(ucp_worker_h worker, FILE *stream) UCP_WORKER_THREAD_CS_EXIT_CONDITIONAL(worker); } -static int ucp_worker_keepalive_is_enabled(ucp_worker_h worker) -{ - return (worker->context->config.keepalive_interval != 0) && - (worker->context->config.ext.keepalive_num_eps != 0); -} - static UCS_F_ALWAYS_INLINE ucp_ep_h ucp_worker_keepalive_current_ep(ucp_worker_h worker) { @@ -2697,6 +2696,8 @@ static unsigned ucp_worker_keepalive_progress(void *arg) ucs_list_link_t *iter_begin; ucp_ep_h ep; + ucs_assert(worker->context->config.ext.keepalive_num_eps != 0); + if (ucs_likely((now - worker->keepalive.last_round) < worker->context->config.keepalive_interval)) { return 0; diff --git a/src/ucp/core/ucp_worker.inl b/src/ucp/core/ucp_worker.inl index 86d83a7d7cb..e804d524b32 100644 --- a/src/ucp/core/ucp_worker.inl +++ b/src/ucp/core/ucp_worker.inl @@ -122,6 +122,12 @@ ucp_worker_extract_request_by_id(ucp_worker_h worker, ucs_ptr_map_key_t id) return request; } +static UCS_F_ALWAYS_INLINE int +ucp_worker_keepalive_is_enabled(ucp_worker_h worker) +{ + return worker->context->config.keepalive_interval != 0; +} + /** * @return worker-iface struct by resource index */ diff --git a/src/uct/tcp/tcp_iface.c b/src/uct/tcp/tcp_iface.c index 2d427d64032..8b5bd95c9fc 100644 --- a/src/uct/tcp/tcp_iface.c +++ b/src/uct/tcp/tcp_iface.c @@ -580,10 +580,21 @@ static UCS_CLASS_INIT_FUNC(uct_tcp_iface_t, uct_md_h md, uct_worker_h worker, self->sockopt.nodelay = config->sockopt_nodelay; self->sockopt.sndbuf = config->sockopt.sndbuf; self->sockopt.rcvbuf = config->sockopt.rcvbuf; - self->config.keepalive.idle = config->keepalive.idle; self->config.keepalive.cnt = config->keepalive.cnt; self->config.keepalive.intvl = config->keepalive.intvl; + if (config->keepalive.idle != UCS_MEMUNITS_AUTO) { + /* TCP iface configuration sets the keepalive interval */ + self->config.keepalive.idle = config->keepalive.idle; + } else if (params->field_mask & UCT_IFACE_PARAM_FIELD_KEEPALIVE_INTERVAL) { + /* User parameters set the keepalive interval */ + self->config.keepalive.idle = params->keepalive_interval; + } else { + /* Use the default keepalive interval */ + self->config.keepalive.idle = + ucs_time_from_sec(UCT_TCP_EP_DEFAULT_KEEPALIVE_IDLE); + } + status = uct_tcp_iface_set_port_range(self, config); if (status != UCS_OK) { goto err; diff --git a/test/gtest/ucp/test_ucp_sockaddr.cc b/test/gtest/ucp/test_ucp_sockaddr.cc index 50b69241c70..4731ebcf1b1 100644 --- a/test/gtest/ucp/test_ucp_sockaddr.cc +++ b/test/gtest/ucp/test_ucp_sockaddr.cc @@ -105,6 +105,7 @@ class test_ucp_sockaddr : public ucp_test { stop_list.push_back("connection request failed on listener"); /* when the "peer failure" error happens, it is followed by: */ stop_list.push_back("received event RDMA_CM_EVENT_UNREACHABLE"); + stop_list.push_back("Connection reset by remote peer"); stop_list.push_back(ucs_status_string(UCS_ERR_UNREACHABLE)); stop_list.push_back(ucs_status_string(UCS_ERR_UNSUPPORTED)); } @@ -481,6 +482,7 @@ class test_ucp_sockaddr : public ucp_test { void one_sided_disconnect(entity &e, enum ucp_ep_close_mode mode) { void *req = e.disconnect_nb(0, 0, mode); ucs_time_t deadline = ucs::get_deadline(); + scoped_log_handler slh(detect_error_logger); while (!is_request_completed(req) && (ucs_get_time() < deadline)) { /* TODO: replace the progress() with e().progress() when async progress is implemented. */ @@ -501,6 +503,7 @@ class test_ucp_sockaddr : public ucp_test { void *receiver_ep_close_req = receiver().disconnect_nb(0, 0, mode); ucs_time_t deadline = ucs::get_deadline(); + scoped_log_handler slh(detect_error_logger); while ((!is_request_completed(sender_ep_close_req) || !is_request_completed(receiver_ep_close_req)) && (ucs_get_time() < deadline)) { From 6779b172de62a8ea51924ef89a11e237e61d1d79 Mon Sep 17 00:00:00 2001 From: Yossi Itigin Date: Sat, 2 Jan 2021 19:31:37 +0200 Subject: [PATCH 092/107] VFS/FUSE: Code style fix --- src/ucs/vfs/fuse/vfs_fuse.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ucs/vfs/fuse/vfs_fuse.c b/src/ucs/vfs/fuse/vfs_fuse.c index 040948dcfa5..281ba638c0a 100644 --- a/src/ucs/vfs/fuse/vfs_fuse.c +++ b/src/ucs/vfs/fuse/vfs_fuse.c @@ -103,7 +103,7 @@ static int ucs_vfs_fuse_read(const char *path, char *buf, size_t size, return 0; } - if (offset + size <= length) { + if ((offset + size) <= length) { nread = size; /* read does not pass end-of-file */ } else { nread = length - offset; /* read truncated by end-of-file */ From 2937bf5e7612e7ed495178b8e7c2336c16c03224 Mon Sep 17 00:00:00 2001 From: Yossi Itigin Date: Tue, 22 Dec 2020 19:55:45 +0200 Subject: [PATCH 093/107] TOOLS/VFS: Implement mount daemon server loop --- src/tools/vfs/Makefile.am | 2 +- src/tools/vfs/vfs_daemon.h | 6 + src/tools/vfs/vfs_main.c | 120 +++++++++++- src/tools/vfs/vfs_server.c | 375 +++++++++++++++++++++++++++++++++++++ 4 files changed, 500 insertions(+), 3 deletions(-) create mode 100644 src/tools/vfs/vfs_server.c diff --git a/src/tools/vfs/Makefile.am b/src/tools/vfs/Makefile.am index a560c0a4de9..62408f8aea7 100644 --- a/src/tools/vfs/Makefile.am +++ b/src/tools/vfs/Makefile.am @@ -9,7 +9,7 @@ if HAVE_FUSE3 bin_PROGRAMS = ucx_vfs ucx_vfs_CPPFLAGS = $(BASE_CPPFLAGS) $(FUSE3_CPPFLAGS) ucx_vfs_CFLAGS = $(BASE_CFLAGS) -ucx_vfs_SOURCES = vfs_main.c +ucx_vfs_SOURCES = vfs_main.c vfs_server.c noinst_HEADERS = vfs_daemon.h ucx_vfs_LDADD = $(FUSE3_LIBS) \ $(top_builddir)/src/ucs/vfs/sock/libucs_vfs_sock.la diff --git a/src/tools/vfs/vfs_daemon.h b/src/tools/vfs/vfs_daemon.h index 523564d9d34..094a1e182c1 100644 --- a/src/tools/vfs/vfs_daemon.h +++ b/src/tools/vfs/vfs_daemon.h @@ -52,4 +52,10 @@ typedef struct { extern vfs_opts_t g_opts; extern const char *vfs_action_names[]; +int vfs_mount(int pid); + +int vfs_unmount(int pid); + +int vfs_server_loop(int listen_fd); + #endif diff --git a/src/tools/vfs/vfs_main.c b/src/tools/vfs/vfs_main.c index 59632087a0b..dc693b7232b 100644 --- a/src/tools/vfs/vfs_main.c +++ b/src/tools/vfs/vfs_main.c @@ -99,6 +99,122 @@ static int vfs_run_fusermount(char **extra_argv) return 0; } +static void vfs_get_mountpoint(pid_t pid, char *mountpoint, size_t max_length) +{ + snprintf(mountpoint, max_length, "%s/%d", g_opts.mountpoint_dir, pid); +} + +static const char *vfs_get_process_name(int pid, char *buf, size_t max_length) +{ + char procfs_comm[NAME_MAX]; + size_t length; + FILE *file; + char *p; + + /* open /proc//comm to read command name */ + snprintf(procfs_comm, sizeof(procfs_comm), "/proc/%d/comm", pid); + file = fopen(procfs_comm, "r"); + if (file == NULL) { + goto err; + } + + /* read command to buffer */ + if (fgets(buf, max_length, file) == NULL) { + goto err_close; + } + + /* remove trailing space/newline */ + length = strlen(buf); + for (p = &buf[length - 1]; (p >= buf) && isspace(*p); --p) { + *p = '\0'; + --length; + } + + /* append process id */ + snprintf(buf + length, max_length - length, "@pid:%d", pid); + goto out; + +err_close: + fclose(file); +err: + snprintf(buf, max_length, "pid:%d", pid); +out: + return buf; +} + +int vfs_mount(int pid) +{ + char mountpoint[PATH_MAX]; + char mountopts[1024]; + char name[NAME_MAX]; + int fuse_fd, ret; + + /* Add common mount options: + * - File system name (source) : process name and pid + * - File system type : ucx_vfs + * - Enable permissions check : yes + * - Direct IO (no caching) : yes + */ + ret = snprintf( + mountopts, sizeof(mountopts), + "fsname=%s,subtype=ucx_vfs,default_permissions,direct_io%s%s", + vfs_get_process_name(pid, name, sizeof(name)), + (strlen(g_opts.mount_opts) > 0) ? "," : "", g_opts.mount_opts); + if (ret >= sizeof(mountopts)) { + return -ENOMEM; + } + + /* Create the mount point directory, and ignore "already exists" error */ + vfs_get_mountpoint(pid, mountpoint, sizeof(mountpoint)); + ret = mkdir(mountpoint, S_IRWXU); + if ((ret < 0) && (errno != EEXIST)) { + ret = -errno; + vfs_error("failed to create directory '%s': %m", mountpoint); + return ret; + } + + /* Mount a new FUSE filesystem in the mount point directory */ + vfs_log("mounting directory '%s' with options '%s'", mountpoint, mountopts); + fuse_fd = fuse_open_channel(mountpoint, mountopts); + if (fuse_fd < 0) { + vfs_error("fuse_open_channel(%s,opts=%s) failed: %m", mountpoint, + mountopts); + return fuse_fd; + } + + vfs_log("mounted directory '%s' with fd %d", mountpoint, fuse_fd); + return fuse_fd; +} + +int vfs_unmount(int pid) +{ + char mountpoint[PATH_MAX]; + char *argv[5]; + int ret; + + /* Unmount FUSE file system */ + vfs_get_mountpoint(pid, mountpoint, sizeof(mountpoint)); + argv[0] = "-u"; + argv[1] = "-z"; + argv[2] = "--"; + argv[3] = mountpoint; + argv[4] = NULL; + ret = vfs_run_fusermount(argv); + if (ret < 0) { + return ret; + } + + /* Remove mount point directory */ + vfs_log("removing directory '%s'", mountpoint); + ret = rmdir(mountpoint); + if (ret < 0) { + vfs_error("failed to remove directory '%s': %m", mountpoint); + return ret; + } + + return 0; +} + static int vfs_unlink_socket(int silent_notexist) { int ret; @@ -155,8 +271,8 @@ static int vfs_listen(int silent_addinuse_err) goto out_unlink; } - vfs_error("closing socket, TODO implement listening for connections"); - close(listen_fd); + vfs_log("listening for connections on '%s'", g_sockaddr.sun_path); + ret = vfs_server_loop(listen_fd); out_unlink: vfs_unlink_socket(0); diff --git a/src/tools/vfs/vfs_server.c b/src/tools/vfs/vfs_server.c new file mode 100644 index 00000000000..ac724b303b2 --- /dev/null +++ b/src/tools/vfs/vfs_server.c @@ -0,0 +1,375 @@ +/** + * Copyright (C) Mellanox Technologies Ltd. 2020. ALL RIGHTS RESERVED. + * + * See file LICENSE for terms. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "vfs_daemon.h" + +#include +#include +#include + + +#define VFS_MAX_FDS 1024 + +typedef enum { + VFS_FD_STATE_LISTENING, + VFS_FD_STATE_ACCEPTED, + VFS_FD_STATE_MOUNTED, + VFS_FD_STATE_FD_SENT, + VFS_FD_STATE_CLOSED +} vfs_socket_state_t; + +typedef struct { + vfs_socket_state_t state; + pid_t pid; + int fuse_fd; +} vfs_serever_fd_state_t; + +typedef struct { + vfs_serever_fd_state_t fd_state[VFS_MAX_FDS]; + struct pollfd poll_fds[VFS_MAX_FDS]; + int nfds; + int stop; +} vfs_server_context_t; + +static vfs_server_context_t vfs_server_context; + +static const char *vfs_server_fd_state_names[] = { + [VFS_FD_STATE_LISTENING] = "LISTENING", + [VFS_FD_STATE_ACCEPTED] = "ACCEPTED", + [VFS_FD_STATE_MOUNTED] = "MOUNTED", + [VFS_FD_STATE_FD_SENT] = "FD_SENT", + [VFS_FD_STATE_CLOSED] = "CLOSED" +}; + +static void vfs_server_log_context(int events) +{ + vfs_serever_fd_state_t *fd_state; + char log_message[1024]; + struct pollfd *pfd; + char *p, *endp; + int idx; + + if (g_opts.verbose < 2) { + return; + } + + p = log_message; + endp = log_message + sizeof(log_message); + + for (idx = 0; idx < vfs_server_context.nfds; ++idx) { + pfd = &vfs_server_context.poll_fds[idx]; + fd_state = &vfs_server_context.fd_state[idx]; + snprintf(p, endp - p, "[%d]{%c %d%s%s %d} ", idx, + vfs_server_fd_state_names[fd_state->state][0], + vfs_server_context.poll_fds[idx].fd, + (events && (pfd->revents & POLLIN)) ? "i" : "", + (events && (pfd->revents & POLLOUT)) ? "o" : "", + fd_state->pid); + p += strlen(p); + } + + *(p - 1) = '\0'; + + vfs_log("%s", log_message); +} + +static int vfs_server_poll_events() +{ + int ret; + + vfs_server_log_context(0); + + ret = poll(vfs_server_context.poll_fds, vfs_server_context.nfds, -1); + if (ret < 0) { + ret = -errno; + if (errno != EINTR) { + vfs_error("poll(nfds=%d) failed: %m", vfs_server_context.nfds) + } + return ret; + } + + vfs_server_log_context(1); + return 0; +} + +static void vfs_server_close_fd(int fd) +{ + int ret = close(fd); + if (ret < 0) { + vfs_error("failed to close fd %d: %m", fd); + } +} + +static void vfs_server_log_fd(int idx, const char *message) +{ + vfs_serever_fd_state_t *fd_state = &vfs_server_context.fd_state[idx]; + struct pollfd *pfd = &vfs_server_context.poll_fds[idx]; + + vfs_log("%s fd[%d]=%d %s, pid: %d fuse_fd: %d", message, idx, pfd->fd, + vfs_server_fd_state_names[fd_state->state], fd_state->fuse_fd, + fd_state->pid); +} + +static int vfs_server_add_fd(int fd, vfs_socket_state_t state) +{ + int idx, ret; + + ret = fcntl(fd, F_GETFL); + if (ret < 0) { + vfs_error("fcntl(%d, F_GETFL) failed: %m", fd); + return -errno; + } + + ret = fcntl(fd, F_SETFL, ret | O_NONBLOCK); + if (ret < 0) { + vfs_error("fcntl(%d, F_SETFL) failed: %m", fd); + return -errno; + } + + idx = vfs_server_context.nfds++; + vfs_server_context.fd_state[idx].state = state; + vfs_server_context.fd_state[idx].pid = -1; + vfs_server_context.fd_state[idx].fuse_fd = -1; + vfs_server_context.poll_fds[idx].events = POLLIN; + vfs_server_context.poll_fds[idx].fd = fd; + vfs_server_context.poll_fds[idx].revents = 0; + + vfs_server_log_fd(idx, "added"); + return 0; +} + +static void vfs_server_remove_fd(int idx) +{ + vfs_server_log_fd(idx, "removing"); + + switch (vfs_server_context.fd_state[idx].state) { + case VFS_FD_STATE_FD_SENT: + case VFS_FD_STATE_MOUNTED: + vfs_server_close_fd(vfs_server_context.fd_state[idx].fuse_fd); + vfs_unmount(vfs_server_context.fd_state[idx].pid); + /* Fall through */ + case VFS_FD_STATE_ACCEPTED: + vfs_server_close_fd(vfs_server_context.poll_fds[idx].fd); + /* Fall through */ + default: + break; + } + + vfs_server_context.fd_state[idx].state = VFS_FD_STATE_CLOSED; + vfs_server_context.fd_state[idx].pid = -1; + vfs_server_context.fd_state[idx].fuse_fd = -1; + vfs_server_context.poll_fds[idx].events = 0; + vfs_server_context.poll_fds[idx].fd = -1; + vfs_server_context.poll_fds[idx].revents = 0; +} + +static void vfs_server_remove_all_fds() +{ + while (vfs_server_context.nfds > 0) { + vfs_server_remove_fd(--vfs_server_context.nfds); + } +} + +static void vfs_server_accept(int listen_fd) +{ + int ret, connfd; + + connfd = accept(listen_fd, NULL, NULL); + if (connfd < 0) { + vfs_error("accept(listen_fd=%d) failed: %m", listen_fd); + return; + } + + ret = ucs_vfs_sock_setopt_passcred(connfd); + if (ret < 0) { + close(connfd); + return; + } + + vfs_server_add_fd(connfd, VFS_FD_STATE_ACCEPTED); +} + +static void vfs_server_mount(int idx, pid_t pid) +{ + int fuse_fd; + + if (pid < 0) { + vfs_error("received invalid pid: %d", pid); + vfs_server_close_fd(idx); + return; + } + + fuse_fd = vfs_mount(pid); + if (fuse_fd < 0) { + vfs_server_close_fd(idx); + return; + } + + vfs_server_context.fd_state[idx].state = VFS_FD_STATE_MOUNTED; + vfs_server_context.fd_state[idx].pid = pid; + vfs_server_context.fd_state[idx].fuse_fd = fuse_fd; + vfs_server_context.poll_fds[idx].events |= POLLOUT; +} + +static void vfs_server_recv(int idx) +{ + ucs_vfs_sock_message_t vfs_msg_in; + char message[64]; + int ret; + + ret = ucs_vfs_sock_recv(vfs_server_context.poll_fds[idx].fd, &vfs_msg_in); + if (ret < 0) { + vfs_error("failed to receive a message: %d (%s)", ret, strerror(-ret)); + vfs_server_remove_fd(idx); + return; + } + + snprintf(message, sizeof(message), "got action '%s' on", + vfs_action_names[vfs_msg_in.action]); + vfs_server_log_fd(idx, message); + + switch (vfs_msg_in.action) { + case UCS_VFS_SOCK_ACTION_STOP: + vfs_server_context.stop = 1; + break; + case UCS_VFS_SOCK_ACTION_MOUNT: + vfs_server_mount(idx, vfs_msg_in.pid); + break; + case UCS_VFS_SOCK_ACTION_NOP: + vfs_server_remove_fd(idx); + break; + default: + vfs_error("ignoring invalid action %d", vfs_msg_in.action); + vfs_server_remove_fd(idx); + break; + } +} + +static void vfs_server_handle_pollin(int idx) +{ + switch (vfs_server_context.fd_state[idx].state) { + case VFS_FD_STATE_LISTENING: + vfs_server_accept(vfs_server_context.poll_fds[idx].fd); + break; + case VFS_FD_STATE_ACCEPTED: + vfs_server_recv(idx); + break; + case VFS_FD_STATE_FD_SENT: + vfs_server_remove_fd(idx); + break; + default: + vfs_server_log_fd(idx, "unexpected POLLIN event on"); + vfs_server_remove_fd(idx); + break; + } +} + +static void vfs_server_handle_pollout(int idx) +{ + ucs_vfs_sock_message_t vfs_msg_out; + int ret; + + if (vfs_server_context.fd_state[idx].state != VFS_FD_STATE_MOUNTED) { + vfs_server_log_fd(idx, "unexpected POLLOUT event on"); + vfs_server_remove_fd(idx); + return; + } + + /* Send reply with file descriptor from fuse mount */ + vfs_msg_out.action = UCS_VFS_SOCK_ACTION_MOUNT_REPLY; + vfs_msg_out.fd = vfs_server_context.fd_state[idx].fuse_fd; + ret = ucs_vfs_sock_send(vfs_server_context.poll_fds[idx].fd, &vfs_msg_out); + if (ret < 0) { + vfs_error("failed to send a message: %d", ret); + vfs_server_remove_fd(idx); + return; + } + + vfs_server_log_fd(idx, "sent fuse_fd on"); + vfs_server_context.fd_state[idx].state = VFS_FD_STATE_FD_SENT; + vfs_server_context.poll_fds[idx].events &= ~POLLOUT; +} + +static void vfs_server_copy_fd_state(int dest_idx, int src_idx) +{ + if (dest_idx != src_idx) { + vfs_server_context.fd_state[dest_idx] = + vfs_server_context.fd_state[src_idx]; + vfs_server_context.poll_fds[dest_idx] = + vfs_server_context.poll_fds[src_idx]; + } +} + +static void vfs_server_sighandler(int signo) +{ + vfs_server_context.stop = 1; +} + +static void vfs_server_set_sighandler() +{ + struct sigaction sigact; + + sigact.sa_handler = vfs_server_sighandler; + sigact.sa_flags = 0; + sigemptyset(&sigact.sa_mask); + + sigaction(SIGINT, &sigact, NULL); + sigaction(SIGHUP, &sigact, NULL); + sigaction(SIGTERM, &sigact, NULL); +} + +int vfs_server_loop(int listen_fd) +{ + int idx, valid_idx; + int ret; + + vfs_server_context.nfds = 0; + vfs_server_context.stop = 0; + + vfs_server_set_sighandler(); + + vfs_server_add_fd(listen_fd, VFS_FD_STATE_LISTENING); + + while (!vfs_server_context.stop) { + ret = vfs_server_poll_events(); + if (ret < 0) { + if (ret == -EINTR) { + continue; + } else { + return ret; + } + } + + valid_idx = 0; + for (idx = 0; idx < vfs_server_context.nfds; ++idx) { + if (vfs_server_context.poll_fds[idx].events == 0) { + vfs_server_copy_fd_state(valid_idx++, idx); + continue; + } + + if (vfs_server_context.poll_fds[idx].revents & POLLIN) { + vfs_server_handle_pollin(idx); + } + if (vfs_server_context.poll_fds[idx].revents & POLLOUT) { + vfs_server_handle_pollout(idx); + } + + if (vfs_server_context.fd_state[idx].state != VFS_FD_STATE_CLOSED) { + vfs_server_copy_fd_state(valid_idx++, idx); + } + } + + vfs_server_context.nfds = valid_idx; + } + + vfs_server_remove_all_fds(); + + return 0; +} From c5ec282c3fd886e11d2310e88e99b3e935a62fd7 Mon Sep 17 00:00:00 2001 From: dmitrygx Date: Sun, 3 Jan 2021 14:59:47 +0000 Subject: [PATCH 094/107] UCP/CORE: Remove unused pending queue --- src/ucp/core/ucp_worker.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/ucp/core/ucp_worker.c b/src/ucp/core/ucp_worker.c index ed6a9669245..e04b8f33830 100644 --- a/src/ucp/core/ucp_worker.c +++ b/src/ucp/core/ucp_worker.c @@ -566,7 +566,6 @@ ucp_worker_iface_handle_uct_ep_failure(ucp_ep_h ucp_ep, ucp_lane_index_t lane, { ucp_worker_h worker = ucp_ep->worker; ucp_wireup_ep_t *wireup_ep; - ucs_queue_head_t tmp_pending_queue; uct_ep_h aux_uct_ep; /* If the failure happened on AUX EP of CM lane on a server EP, @@ -586,7 +585,6 @@ ucp_worker_iface_handle_uct_ep_failure(ucp_ep_h ucp_ep, ucp_lane_index_t lane, * destroy CM_WIREUP/AUX_EP */ aux_uct_ep = wireup_ep->aux_ep; - ucs_queue_head_init(&tmp_pending_queue); ucp_wireup_ep_disown(ucp_ep->uct_eps[lane], aux_uct_ep); ucp_worker_discard_uct_ep(ucp_ep->worker, aux_uct_ep, UCT_FLUSH_FLAG_CANCEL, From b89a4e494fa28c2da4aac9973a6d5245bb170da3 Mon Sep 17 00:00:00 2001 From: Alina Sklarevich Date: Wed, 16 Dec 2020 14:44:45 +0200 Subject: [PATCH 095/107] JENKINS: test CM's fallback in client-server testing --- contrib/test_jenkins.sh | 44 +++++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/contrib/test_jenkins.sh b/contrib/test_jenkins.sh index d8f0dcaee51..c1c18566aba 100755 --- a/contrib/test_jenkins.sh +++ b/contrib/test_jenkins.sh @@ -306,6 +306,26 @@ get_rdma_device_ip_addr() { echo $ipaddr } +get_non_rdma_ip_addr() { + if ! which ibdev2netdev >&/dev/null + then + return + fi + + # get the interface of the ip address that is the default gateway (pure Ethernet IPv4 address). + eth_iface=$(ip route show| sed -n 's/default via \(\S*\) dev \(\S*\).*/\2/p') + + # the pure Ethernet interface should not appear in the ibdev2netdev output. it should not be an IPoIB or + # RoCE interface. + if ibdev2netdev|grep -qw "${eth_iface}" + then + echo "Failed to retrieve an IP of a non IPoIB/RoCE interface" + exit 1 + fi + + get_ifaddr ${eth_iface} +} + # # Prepare build environment # @@ -789,7 +809,7 @@ rename_files() { } run_client_server_app() { - test_name=$1 + test_exe=$1 test_args=$2 server_addr_arg=$3 kill_server=$4 @@ -801,7 +821,7 @@ run_client_server_app() { affinity_server=$(slice_affinity 0) affinity_client=$(slice_affinity 1) - taskset -c $affinity_server ${test_name} ${test_args} ${server_port_arg} & + taskset -c $affinity_server ${test_exe} ${test_args} ${server_port_arg} & server_pid=$! sleep 15 @@ -811,7 +831,7 @@ run_client_server_app() { set +Ee fi - taskset -c $affinity_client ${test_name} ${test_args} ${server_addr_arg} ${server_port_arg} & + taskset -c $affinity_client ${test_exe} ${test_args} ${server_addr_arg} ${server_port_arg} & client_pid=$! wait ${client_pid} @@ -942,7 +962,7 @@ run_client_server() { -Wl,-rpath=${ucx_inst}/lib fi - server_ip=$(get_rdma_device_ip_addr) + server_ip=$1 if [ "$server_ip" == "" ] then return @@ -953,14 +973,17 @@ run_client_server() { run_ucp_client_server() { echo "==== Running UCP client-server ====" - run_client_server + run_client_server $(get_rdma_device_ip_addr) + run_client_server $(get_non_rdma_ip_addr) rm -f ./ucp_client_server } run_io_demo() { - server_ip=$(get_rdma_device_ip_addr) - if [ "$server_ip" == "" ] + server_rdma_addr=$(get_rdma_device_ip_addr) + server_nonrdma_addr=$(get_non_rdma_ip_addr) + + if [ -z "$server_rdma_addr" ] && [ -z "$server_nonrdma_addr" ] then return fi @@ -975,10 +998,11 @@ run_io_demo() { $MAKEP -C test/apps/iodemo ${test_name} fi - export UCX_SOCKADDR_CM_ENABLE=y - run_client_server_app "./test/apps/iodemo/${test_name}" "${test_args}" "${server_ip}" 1 0 + for server_ip in $server_rdma_addr $server_nonrdma_addr + do + run_client_server_app "./test/apps/iodemo/${test_name}" "${test_args}" "${server_ip}" 1 0 + done - unset UCX_SOCKADDR_CM_ENABLE make_clean } From 27444ebcce5284ae697b152e33fba8a7aeb39f56 Mon Sep 17 00:00:00 2001 From: Yossi Itigin Date: Wed, 16 Dec 2020 19:16:05 +0200 Subject: [PATCH 096/107] UCM/BISTRO/CUDA: Add bisro hooks for Cuda In order to catch cuda memory events even when Cuda is statically linked, use bistro. --- config/m4/ucm.m4 | 13 - contrib/test_jenkins.sh | 19 +- src/ucm/api/ucm.h | 2 +- src/ucm/bistro/bistro.c | 47 +++ src/ucm/bistro/bistro.h | 13 + src/ucm/bistro/bistro_aarch64.c | 15 +- src/ucm/bistro/bistro_aarch64.h | 15 +- src/ucm/bistro/bistro_int.h | 13 - src/ucm/bistro/bistro_ppc64.c | 8 +- src/ucm/bistro/bistro_ppc64.h | 29 +- src/ucm/bistro/bistro_x86_64.c | 142 ++++++-- src/ucm/bistro/bistro_x86_64.h | 16 +- src/ucm/cuda/cudamem.c | 559 +++++++++++++------------------- src/ucm/cuda/cudamem.h | 82 +---- src/ucm/mmap/install.c | 11 +- src/ucm/mmap/mmap.h | 11 +- src/ucm/rocm/rocmmem.c | 5 - src/ucm/util/reloc.c | 6 - src/ucm/util/replace.h | 18 + src/ucm/util/sys.c | 2 +- src/ucm/util/sys.h | 24 ++ src/ucs/config/ucm_opts.c | 29 +- test/apps/Makefile.am | 2 +- test/apps/test_cuda_hook.c | 75 ++++- test/gtest/ucm/malloc_hook.cc | 8 +- 25 files changed, 630 insertions(+), 534 deletions(-) diff --git a/config/m4/ucm.m4 b/config/m4/ucm.m4 index 9c7c820d9ff..1e229edc51f 100644 --- a/config/m4/ucm.m4 +++ b/config/m4/ucm.m4 @@ -5,19 +5,6 @@ # -# -# Enable overriding library symbols -# -AC_ARG_ENABLE([symbol-override], - AS_HELP_STRING([--disable-symbol-override], [Disable overriding library symbols, default: NO]), - [], - [enable_symbol_override=yes]) - -AS_IF([test "x$enable_symbol_override" = xyes], - [AC_DEFINE([ENABLE_SYMBOL_OVERRIDE], [1], [Enable symbol override])] - [:] -) - # # Memory allocator selection # diff --git a/contrib/test_jenkins.sh b/contrib/test_jenkins.sh index d8f0dcaee51..93bf29cd55c 100755 --- a/contrib/test_jenkins.sh +++ b/contrib/test_jenkins.sh @@ -1343,21 +1343,28 @@ test_malloc_hook() { then for mode in reloc bistro do - export UCX_MEM_MMAP_HOOK_MODE=${mode} + export UCX_MEM_CUDA_HOOK_MODE=${mode} # Run cuda memory hooks with dynamic link ./test/apps/test_cuda_hook_dynamic # Run cuda memory hooks with static link - if ./test/apps/test_cuda_hook_static + ./test/apps/test_cuda_hook_static && status="pass" || status="fail" + [ ${mode} == "bistro" ] && exp_status="pass" || exp_status="fail" + if [ ${status} == ${exp_status} ] then - echo "Static link with cuda is expected to fail" - exit 1 + echo "Static link with cuda ${status}, as expected" else - echo "Test failed - as expected" + echo "Static link with cuda is expected to ${exp_status}, actual: ${status}" + exit 1 fi - unset UCX_MEM_MMAP_HOOK_MODE + # Test that driver API hooks work in both reloc and bistro modes, + # since we call them directly from the test + ./test/apps/test_cuda_hook_dynamic -d + ./test/apps/test_cuda_hook_static -d + + unset UCX_MEM_CUDA_HOOK_MODE done fi } diff --git a/src/ucm/api/ucm.h b/src/ucm/api/ucm.h index 114f167c86e..65ff4b29a3b 100644 --- a/src/ucm/api/ucm.h +++ b/src/ucm/api/ucm.h @@ -213,7 +213,7 @@ typedef struct ucm_global_config { ucm_mmap_hook_mode_t mmap_hook_mode; /* MMAP hook mode */ int enable_malloc_hooks; /* Enable installing malloc hooks */ int enable_malloc_reloc; /* Enable installing malloc relocations */ - int enable_cuda_reloc; /* Enable installing CUDA relocations */ + ucm_mmap_hook_mode_t cuda_hook_mode; /* Cuda hooks mode */ int enable_dynamic_mmap_thresh; /* Enable adaptive mmap threshold */ size_t alloc_alignment; /* Alignment for memory allocations */ int dlopen_process_rpath; /* Process RPATH section in dlopen hook */ diff --git a/src/ucm/bistro/bistro.c b/src/ucm/bistro/bistro.c index 1eec37ee9ad..c31741aa8d5 100644 --- a/src/ucm/bistro/bistro.c +++ b/src/ucm/bistro/bistro.c @@ -10,9 +10,12 @@ #include #include +#include #include #include +#include + ucs_status_t ucm_bistro_remove_restore_point(ucm_bistro_restore_point_t *rp) { @@ -107,4 +110,48 @@ void *ucm_bistro_restore_addr(ucm_bistro_restore_point_t *rp) return rp->addr; } +void *ucm_bistro_allocate_code(size_t size) +{ + static const size_t mmap_size = 16 * UCS_KBYTE; + static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; + static void *mem_area = MAP_FAILED; + static size_t alloc_offset = 0; + size_t alloc_size; + void *result; + + pthread_mutex_lock(&mutex); + + if (mem_area == MAP_FAILED) { + /* Allocate executable memory block once, and reuse it for + * subsequent allocations. We assume bistro would not really need + * more than 'mmap_size' in total, since it's used for limited number + * of library functions. Also, the memory is never really released, so + * our allocator is very simple. + */ + mem_area = mmap(NULL, ucs_align_up_pow2(mmap_size, ucm_get_page_size()), + PROT_READ | PROT_WRITE | PROT_EXEC, + MAP_PRIVATE | MAP_ANONYMOUS, 0, 0); + if (mem_area == MAP_FAILED) { + ucm_error("failed to allocated executable memory of %zu bytes: %m", + mmap_size); + result = NULL; + goto out; + } + } + + alloc_size = ucs_align_up_pow2(size, UCS_SYS_PARAGRAPH_SIZE); + if ((alloc_size + alloc_offset) > mmap_size) { + result = NULL; + goto out; + } + + /* Allocate next memory block in the mmap-ed area */ + result = UCS_PTR_BYTE_OFFSET(mem_area, alloc_offset); + alloc_offset += alloc_size; + +out: + pthread_mutex_unlock(&mutex); + return result; +} + #endif diff --git a/src/ucm/bistro/bistro.h b/src/ucm/bistro/bistro.h index 16e988700c3..101000455e6 100644 --- a/src/ucm/bistro/bistro.h +++ b/src/ucm/bistro/bistro.h @@ -36,6 +36,7 @@ typedef struct ucm_bistro_restore_point ucm_bistro_restore_point_t; */ ucs_status_t ucm_bistro_restore(ucm_bistro_restore_point_t *rp); + /** * Remove resore point created by @ref ucm_bistro_patch witout * restore original function body @@ -46,6 +47,7 @@ ucs_status_t ucm_bistro_restore(ucm_bistro_restore_point_t *rp); */ ucs_status_t ucm_bistro_remove_restore_point(ucm_bistro_restore_point_t *rp); + /** * Get patch address for restore point * @@ -55,4 +57,15 @@ ucs_status_t ucm_bistro_remove_restore_point(ucm_bistro_restore_point_t *rp); */ void *ucm_bistro_restore_addr(ucm_bistro_restore_point_t *rp); + +/** + * Allocate executable memory which can be used to create trampolines or + * temporary functions. + * + * @param size Memory size to allocated + * + * @return Pointer to allocated memory, or NULL if failed. + */ +void *ucm_bistro_allocate_code(size_t size); + #endif diff --git a/src/ucm/bistro/bistro_aarch64.c b/src/ucm/bistro/bistro_aarch64.c index e161292f9ae..8659c0ee6ab 100644 --- a/src/ucm/bistro/bistro_aarch64.c +++ b/src/ucm/bistro/bistro_aarch64.c @@ -59,12 +59,10 @@ */ #define BR(_reg) ((0xd61f << 16) + ((_reg) << 5)) -ucs_status_t ucm_bistro_patch(const char *symbol, void *hook, +ucs_status_t ucm_bistro_patch(void *func_ptr, void *hook, const char *symbol, + void **orig_func_p, ucm_bistro_restore_point_t **rp) { - void *func; - ucs_status_t status; - ucm_bistro_patch_t patch = { .reg3 = MOVZ(R15, 3, (uintptr_t)hook >> 48), .reg2 = MOVK(R15, 2, (uintptr_t)hook >> 32), @@ -72,15 +70,18 @@ ucs_status_t ucm_bistro_patch(const char *symbol, void *hook, .reg0 = MOVK(R15, 0, (uintptr_t)hook), .br = BR(R15) }; + ucs_status_t status; - UCM_LOOKUP_SYMBOL(func, symbol); + if (orig_func_p != NULL) { + return UCS_ERR_UNSUPPORTED; + } - status = ucm_bistro_create_restore_point(func, sizeof(patch), rp); + status = ucm_bistro_create_restore_point(func_ptr, sizeof(patch), rp); if (UCS_STATUS_IS_ERR(status)) { return status; } - return ucm_bistro_apply_patch(func, &patch, sizeof(patch)); + return ucm_bistro_apply_patch(func_ptr, &patch, sizeof(patch)); } #endif diff --git a/src/ucm/bistro/bistro_aarch64.h b/src/ucm/bistro/bistro_aarch64.h index 487aa923d08..b0770b30fe7 100644 --- a/src/ucm/bistro/bistro_aarch64.h +++ b/src/ucm/bistro/bistro_aarch64.h @@ -28,14 +28,19 @@ typedef struct ucm_bistro_patch { * Set library function call hook using Binary Instrumentation * method (BISTRO): replace function body by user defined call * - * @param symbol function name to replace - * @param hook user-defined function-replacer - * @param rp restore point used to restore original function, - * optional, may be NULL + * @param func_ptr Pointer to function to patch. + * @param hook User-defined function-replacer. + * @param symbol Function name to replace. + * @param orig_func_p Unsupported on this architecture and must be NULL. + * If set to a non-NULL value, this function returns + * @ref UCS_ERR_UNSUPPORTED. + * @param rp Restore point used to restore original function. + * Optional, may be NULL. * * @return Error code as defined by @ref ucs_status_t */ -ucs_status_t ucm_bistro_patch(const char *symbol, void *hook, +ucs_status_t ucm_bistro_patch(void *func_ptr, void *hook, const char *symbol, + void **orig_func_p, ucm_bistro_restore_point_t **rp); #endif diff --git a/src/ucm/bistro/bistro_int.h b/src/ucm/bistro/bistro_int.h index 021dbc10760..80b423b8cd7 100644 --- a/src/ucm/bistro/bistro_int.h +++ b/src/ucm/bistro/bistro_int.h @@ -33,17 +33,4 @@ ucs_status_t ucm_bistro_apply_patch(void *dst, void *patch, size_t len); ucs_status_t ucm_bistro_create_restore_point(void *addr, size_t len, ucm_bistro_restore_point_t **rp); -static inline void *ucm_bistro_lookup(const char *symbol) -{ - void *addr; - - ucm_assert(symbol != NULL); - - addr = dlsym(RTLD_NEXT, symbol); - if (!addr) { - addr = dlsym(RTLD_DEFAULT, symbol); - } - return addr; -} - #endif diff --git a/src/ucm/bistro/bistro_ppc64.c b/src/ucm/bistro/bistro_ppc64.c index 8a08655b365..942346c8c07 100644 --- a/src/ucm/bistro/bistro_ppc64.c +++ b/src/ucm/bistro/bistro_ppc64.c @@ -134,7 +134,7 @@ static void *ucm_bistro_get_text_addr(void *addr) #endif } -ucs_status_t ucm_bistro_patch_toc(const char *symbol, void *hook, +ucs_status_t ucm_bistro_patch_toc(void *func_ptr, void *hook, ucm_bistro_restore_point_t **rp, uint64_t toc) { @@ -143,11 +143,9 @@ ucs_status_t ucm_bistro_patch_toc(const char *symbol, void *hook, ucm_bistro_restore_point_t restore; ucm_bistro_patch_t patch; - UCM_LOOKUP_SYMBOL(func, symbol); + restore.entry = func_ptr; - restore.entry = func; - - func = ucm_bistro_get_text_addr(func); + func = ucm_bistro_get_text_addr(func_ptr); hook = ucm_bistro_get_text_addr(hook); status = ucm_bistro_patch_hook(hook, &restore, toc); diff --git a/src/ucm/bistro/bistro_ppc64.h b/src/ucm/bistro/bistro_ppc64.h index 7b5c3b46e7e..e0d4635ca5a 100644 --- a/src/ucm/bistro/bistro_ppc64.h +++ b/src/ucm/bistro/bistro_ppc64.h @@ -4,13 +4,15 @@ * See file LICENSE for terms. */ - #ifndef UCM_BISTRO_BISTRO_PPC64_H_ #define UCM_BISTRO_BISTRO_PPC64_H_ -#include +#include "bistro.h" #include +#include +#include + /* special processing for ppc64 to save and restore TOC (r2) * Reference: "64-bit PowerPC ELF Application Binary Interface Supplement 1.9" */ @@ -26,26 +28,31 @@ * Set library function call hook using Binary Instrumentation * method (BISTRO): replace function body by user defined call * - * @param symbol function name to replace - * @param hook user-defined function-replacer - * @param rp restore point used to restore original function, - * optional, may be NULL + * @param func_ptr Function to patch. + * @param hook User-defined function-replacer. + * @param rp Restore point used to restore original function. + Optional, may be NULL. * * @return Error code as defined by @ref ucs_status_t */ /* we have to use inline proxy call to save TOC register * value - PPC is very sensible to this register value */ -ucs_status_t ucm_bistro_patch_toc(const char *symbol, void *hook, +ucs_status_t ucm_bistro_patch_toc(void *func_ptr, void *hook, ucm_bistro_restore_point_t **rp, uint64_t toc); -static inline -ucs_status_t ucm_bistro_patch(const char *symbol, void *hook, - ucm_bistro_restore_point_t **rp) +static inline ucs_status_t +ucm_bistro_patch(void *func_ptr, void *hook, const char *symbol, + void **orig_func_p, ucm_bistro_restore_point_t **rp) { uint64_t toc; + + if (orig_func_p != NULL) { + return UCS_ERR_UNSUPPORTED; + } + asm volatile ("std 2, %0" : "=m" (toc)); - return ucm_bistro_patch_toc(symbol, hook, rp, toc); + return ucm_bistro_patch_toc(func_ptr, hook, rp, toc); } #endif diff --git a/src/ucm/bistro/bistro_x86_64.c b/src/ucm/bistro/bistro_x86_64.c index da67dbc517f..c11d21c0493 100644 --- a/src/ucm/bistro/bistro_x86_64.c +++ b/src/ucm/bistro/bistro_x86_64.c @@ -5,7 +5,7 @@ */ #ifdef HAVE_CONFIG_H -# include "config.h" +#include "config.h" #endif /* ******************************************************* @@ -14,7 +14,6 @@ #if defined(__x86_64__) #include -#include #include #include @@ -22,47 +21,152 @@ #include #include #include -#include -#include -ucs_status_t ucm_bistro_patch(const char *symbol, void *hook, +typedef struct { + void *jmp_addr; + char code[]; +} ucm_bistro_orig_func_t; + +typedef struct { + uint8_t opcode; /* 0xff */ + uint8_t modrm; /* 0x25 */ + int32_t displ; +} UCS_S_PACKED ucm_bistro_jmp_indirect_t; + + +/* + * Find the minimal length of initial instructions in the function which can be + * safely executed from any memory location. + * Uses a very simplified disassembler which supports only the typical + * instructions found in function prologue. + */ +static size_t ucm_bistro_detect_pic_prefix(const void *func, size_t min_length) +{ + size_t offset, prev_offset; + uint8_t rex, opcode, modrm; + + offset = 0; + while (offset < min_length) { + prev_offset = offset; + opcode = *(uint8_t*)UCS_PTR_BYTE_OFFSET(func, offset++); + + /* check for REX prefix */ + if ((opcode & 0xF0) == 0x40) { + rex = opcode; + opcode = *(uint8_t*)UCS_PTR_BYTE_OFFSET(func, offset++); + } else { + rex = 0; + } + + /* check the opcode */ + if (((rex == 0) || rex == 0x41) && ((opcode & 0xF0) == 0x50)) { + /* push */ + continue; + } else if ((rex == 0x48) && (opcode == 0x81)) { + /* group 1A operation */ + modrm = *(uint8_t*)UCS_PTR_BYTE_OFFSET(func, offset++); + if (modrm == 0xEC) { + /* sub $imm32, %rsp */ + offset += sizeof(uint32_t); + continue; + } + } else if ((rex == 0x48) && (opcode == 0x89)) { + /* mov %rsp, %rbp */ + modrm = *(uint8_t*)UCS_PTR_BYTE_OFFSET(func, offset++); + if (modrm == 0xE5) { + continue; + } + } + + /* unsupported instruction - bail */ + return prev_offset; + } + + return offset; +} + +static ucs_status_t +ucm_bistro_construct_orig_func(const void *func_ptr, size_t patch_len, + const char *symbol, void **orig_func_p) +{ + ucm_bistro_jmp_indirect_t *jmp_back; + ucm_bistro_orig_func_t *orig_func; + size_t prefix_len, code_size; + + prefix_len = ucm_bistro_detect_pic_prefix(func_ptr, patch_len); + ucm_debug("'%s' at %p prefix length %zu/%zu", symbol, func_ptr, prefix_len, + patch_len); + if (prefix_len < patch_len) { + return UCS_ERR_UNSUPPORTED; + } + + /* Allocate executable page */ + code_size = sizeof(*orig_func) + patch_len + sizeof(*jmp_back); + orig_func = ucm_bistro_allocate_code(code_size); + if (orig_func == NULL) { + return UCS_ERR_NO_MEMORY; + } + + /* Copy code fragment from original function */ + memcpy(orig_func->code, func_ptr, prefix_len); + + /* Indirect jump to *orig_func->jmp_address */ + orig_func->jmp_addr = UCS_PTR_BYTE_OFFSET(func_ptr, prefix_len); + jmp_back = UCS_PTR_BYTE_OFFSET(orig_func->code, prefix_len); + jmp_back->opcode = 0xff; + jmp_back->modrm = 0x25; + jmp_back->displ = UCS_PTR_BYTE_DIFF(jmp_back + 1, &orig_func->jmp_addr); + *orig_func_p = orig_func->code; + + return UCS_OK; +} + +ucs_status_t ucm_bistro_patch(void *func_ptr, void *hook, const char *symbol, + void **orig_func_p, ucm_bistro_restore_point_t **rp) { - ucm_bistro_jmp_r11_patch_t patch_jmp_r11 = { + ucm_bistro_jmp_r11_patch_t jmp_r11 = { .mov_r11 = {0x49, 0xbb}, .jmp_r11 = {0x41, 0xff, 0xe3} }; - ucm_bistro_jmp_near_patch_t patch_jmp_near = { + ucm_bistro_jmp_near_patch_t jmp_near = { .jmp_rel = 0xe9 }; - void *func, *patch, *jmp_base; + void *patch, *jmp_base; ucs_status_t status; ptrdiff_t jmp_disp; size_t patch_len; - UCM_LOOKUP_SYMBOL(func, symbol); - - jmp_base = UCS_PTR_BYTE_OFFSET(func, sizeof(patch_jmp_near)); + jmp_base = UCS_PTR_BYTE_OFFSET(func_ptr, sizeof(jmp_near)); jmp_disp = UCS_PTR_BYTE_DIFF(jmp_base, hook); if (labs(jmp_disp) < INT32_MAX) { /* if 32-bit near jump is possible, use it, since it's a short 5-byte * instruction which reduces the chances of racing with other thread */ - patch_jmp_near.disp = jmp_disp; - patch = &patch_jmp_near; - patch_len = sizeof(patch_jmp_near); + jmp_near.disp = jmp_disp; + patch = &jmp_near; + patch_len = sizeof(jmp_near); } else { - patch_jmp_r11.ptr = hook; - patch = &patch_jmp_r11; - patch_len = sizeof(patch_jmp_r11); + jmp_r11.ptr = hook; + patch = &jmp_r11; + patch_len = sizeof(jmp_r11); } - status = ucm_bistro_create_restore_point(func, patch_len, rp); + if (orig_func_p != NULL) { + status = ucm_bistro_construct_orig_func(func_ptr, patch_len, symbol, + orig_func_p); + if (status != UCS_OK) { + return status; + } + } + + status = ucm_bistro_create_restore_point(func_ptr, patch_len, rp); if (UCS_STATUS_IS_ERR(status)) { return status; } - return ucm_bistro_apply_patch(func, patch, patch_len); + return ucm_bistro_apply_patch(func_ptr, patch, patch_len); } + #endif diff --git a/src/ucm/bistro/bistro_x86_64.h b/src/ucm/bistro/bistro_x86_64.h index 04f09b87415..87ef636cc8a 100644 --- a/src/ucm/bistro/bistro_x86_64.h +++ b/src/ucm/bistro/bistro_x86_64.h @@ -35,14 +35,20 @@ typedef struct ucm_bistro_jmp_near_patch { * Set library function call hook using Binary Instrumentation * method (BISTRO): replace function body by user defined call * - * @param symbol function name to replace - * @param hook user-defined function-replacer - * @param rp restore point used to restore original function, - * optional, may be NULL + * @param func_ptr Pointer to function to patch. + * @param hook User-defined function-replacer. + * @param symbol Function name to replace. + * @param orig_func_p If non-NULL, set to pointer to a trampoline which calls + * the original function (before patching). If it's not + * possible to create such trampoline, the function returns + * @ref UCS_ERR_UNSUPPORTED. + * @param rp Restore point used to restore original function. + * Optional, may be NULL. * * @return Error code as defined by @ref ucs_status_t */ -ucs_status_t ucm_bistro_patch(const char *symbol, void *hook, +ucs_status_t ucm_bistro_patch(void *func_ptr, void *hook, const char *symbol, + void **orig_func_p, ucm_bistro_restore_point_t **rp); #endif diff --git a/src/ucm/cuda/cudamem.c b/src/ucm/cuda/cudamem.c index bef7512399e..25195371fd4 100644 --- a/src/ucm/cuda/cudamem.c +++ b/src/ucm/cuda/cudamem.c @@ -6,392 +6,274 @@ */ #ifdef HAVE_CONFIG_H -# include "config.h" +#include "config.h" #endif -#include +#include "cudamem.h" #include +#include #include #include #include #include -#include #include #include #include -#include #include -#include - - -UCM_DEFINE_REPLACE_DLSYM_FUNC(cuMemFree, CUresult, -1, CUdeviceptr) -UCM_DEFINE_REPLACE_DLSYM_FUNC(cuMemFreeHost, CUresult, -1, void *) -UCM_DEFINE_REPLACE_DLSYM_FUNC(cuMemAlloc, CUresult, -1, CUdeviceptr *, size_t) -UCM_DEFINE_REPLACE_DLSYM_FUNC(cuMemAllocManaged, CUresult, -1, CUdeviceptr *, - size_t, unsigned int) -UCM_DEFINE_REPLACE_DLSYM_FUNC(cuMemAllocPitch, CUresult, -1, CUdeviceptr *, size_t *, - size_t, size_t, unsigned int) -UCM_DEFINE_REPLACE_DLSYM_FUNC(cuMemHostGetDevicePointer, CUresult, -1, CUdeviceptr *, - void *, unsigned int) -UCM_DEFINE_REPLACE_DLSYM_FUNC(cuMemHostUnregister, CUresult, -1, void *) -UCM_DEFINE_REPLACE_DLSYM_FUNC(cudaFree, cudaError_t, -1, void*) -UCM_DEFINE_REPLACE_DLSYM_FUNC(cudaFreeHost, cudaError_t, -1, void*) -UCM_DEFINE_REPLACE_DLSYM_FUNC(cudaMalloc, cudaError_t, -1, void**, size_t) -UCM_DEFINE_REPLACE_DLSYM_FUNC(cudaMallocManaged, cudaError_t, -1, void**, size_t, unsigned int) -UCM_DEFINE_REPLACE_DLSYM_FUNC(cudaMallocPitch, cudaError_t, -1, void**, size_t *, - size_t, size_t) -UCM_DEFINE_REPLACE_DLSYM_FUNC(cudaHostGetDevicePointer, cudaError_t, -1, void**, - void *, unsigned int) -UCM_DEFINE_REPLACE_DLSYM_FUNC(cudaHostUnregister, cudaError_t, -1, void*) - -#if ENABLE_SYMBOL_OVERRIDE -UCM_OVERRIDE_FUNC(cuMemFree, CUresult) -UCM_OVERRIDE_FUNC(cuMemFreeHost, CUresult) -UCM_OVERRIDE_FUNC(cuMemAlloc, CUresult) -UCM_OVERRIDE_FUNC(cuMemAllocManaged, CUresult) -UCM_OVERRIDE_FUNC(cuMemAllocPitch, CUresult) -UCM_OVERRIDE_FUNC(cuMemHostGetDevicePointer, CUresult) -UCM_OVERRIDE_FUNC(cuMemHostUnregister, CUresult) -UCM_OVERRIDE_FUNC(cudaFree, cudaError_t) -UCM_OVERRIDE_FUNC(cudaFreeHost, cudaError_t) -UCM_OVERRIDE_FUNC(cudaMalloc, cudaError_t) -UCM_OVERRIDE_FUNC(cudaMallocManaged, cudaError_t) -UCM_OVERRIDE_FUNC(cudaMallocPitch, cudaError_t) -UCM_OVERRIDE_FUNC(cudaHostGetDevicePointer, cudaError_t) -UCM_OVERRIDE_FUNC(cudaHostUnregister, cudaError_t) -#endif -static void ucm_cuda_set_ptr_attr(CUdeviceptr dptr) -{ - unsigned int value = 1; - CUresult ret; - const char *cu_err_str; +/* Create a body of CUDA memory allocation replacement function */ +#define UCM_CUDA_ALLOC_FUNC(_name, _mem_type, _retval, _success, _size, \ + _ptr_type, _args_fmt, ...) \ + _retval ucm_##_name(_ptr_type *ptr_p, UCM_FUNC_DEFINE_ARGS(__VA_ARGS__)) \ + { \ + _ptr_type ptr; \ + _retval ret; \ + \ + ucm_event_enter(); \ + ret = ucm_orig_##_name(ptr_p, UCM_FUNC_PASS_ARGS(__VA_ARGS__)); \ + if (ret == (_success)) { \ + ptr = *ptr_p; \ + ucm_trace("%s(" _args_fmt ") allocated %p", __FUNCTION__, \ + UCM_FUNC_PASS_ARGS(__VA_ARGS__), (void*)ptr); \ + ucm_cuda_dispatch_mem_alloc((CUdeviceptr)ptr, (_size), \ + (_mem_type)); \ + } \ + ucm_event_leave(); \ + return ret; \ + } - if ((void*)dptr == NULL) { - ucm_trace("skipping cuPointerSetAttribute for null pointer"); - return; +/* Create a body of CUDA memory release replacement function */ +#define UCM_CUDA_FREE_FUNC(_name, _retval, _ptr_type, _mem_type) \ + _retval ucm_##_name(_ptr_type ptr) \ + { \ + _retval ret; \ + \ + ucm_event_enter(); \ + ucm_trace("%s(ptr=%p)", __FUNCTION__, (void*)ptr); \ + ucm_cuda_dispatch_mem_free((CUdeviceptr)ptr, _mem_type, #_name); \ + ret = ucm_orig_##_name(ptr); \ + ucm_event_leave(); \ + return ret; \ } - ret = cuPointerSetAttribute(&value, CU_POINTER_ATTRIBUTE_SYNC_MEMOPS, dptr); - if (ret != CUDA_SUCCESS) { - cuGetErrorString(ret, &cu_err_str); - ucm_warn("cuPointerSetAttribute(%p) failed: %s", (void *) dptr, cu_err_str); +#define UCM_CUDA_FUNC_ENTRY(_func) \ + { \ + {#_func, ucm_override_##_func}, (void**)&ucm_orig_##_func \ } -} -static UCS_F_ALWAYS_INLINE void -ucm_dispatch_mem_type_alloc(void *addr, size_t length, ucs_memory_type_t mem_type) +typedef struct { + ucm_reloc_patch_t patch; + void **orig_func_ptr; +} ucm_cuda_func_t; + + +/* Driver API */ +UCM_DEFINE_REPLACE_DLSYM_PTR_FUNC(cuMemAlloc, CUresult, -1, CUdeviceptr*, + size_t) +UCM_DEFINE_REPLACE_DLSYM_PTR_FUNC(cuMemAlloc_v2, CUresult, -1, CUdeviceptr*, + size_t) +UCM_DEFINE_REPLACE_DLSYM_PTR_FUNC(cuMemAllocManaged, CUresult, -1, CUdeviceptr*, + size_t, unsigned int) +UCM_DEFINE_REPLACE_DLSYM_PTR_FUNC(cuMemAllocPitch, CUresult, -1, CUdeviceptr*, + size_t*, size_t, size_t, unsigned int) +UCM_DEFINE_REPLACE_DLSYM_PTR_FUNC(cuMemAllocPitch_v2, CUresult, -1, + CUdeviceptr*, size_t*, size_t, size_t, + unsigned int) +UCM_DEFINE_REPLACE_DLSYM_PTR_FUNC(cuMemFree, CUresult, -1, CUdeviceptr) +UCM_DEFINE_REPLACE_DLSYM_PTR_FUNC(cuMemFree_v2, CUresult, -1, CUdeviceptr) +UCM_DEFINE_REPLACE_DLSYM_PTR_FUNC(cuMemFreeHost, CUresult, -1, void*) +UCM_DEFINE_REPLACE_DLSYM_PTR_FUNC(cuMemFreeHost_v2, CUresult, -1, void*) + +/* Runtime API */ +UCM_DEFINE_REPLACE_DLSYM_PTR_FUNC(cudaFree, cudaError_t, -1, void*) +UCM_DEFINE_REPLACE_DLSYM_PTR_FUNC(cudaFreeHost, cudaError_t, -1, void*) +UCM_DEFINE_REPLACE_DLSYM_PTR_FUNC(cudaMalloc, cudaError_t, -1, void**, size_t) +UCM_DEFINE_REPLACE_DLSYM_PTR_FUNC(cudaMallocManaged, cudaError_t, -1, void**, + size_t, unsigned int) +UCM_DEFINE_REPLACE_DLSYM_PTR_FUNC(cudaMallocPitch, cudaError_t, -1, void**, + size_t*, size_t, size_t) + +static void ucm_cuda_dispatch_mem_alloc(CUdeviceptr ptr, size_t length, + ucs_memory_type_t mem_type) { + unsigned sync_atr_value = 1; + const char *cu_err_str; ucm_event_t event; + CUresult ret; - event.mem_type.address = addr; + if ((ptr != 0) && (mem_type == UCS_MEMORY_TYPE_CUDA)) { + /* Synchronous operation for GPU direct */ + ret = cuPointerSetAttribute(&sync_atr_value, + CU_POINTER_ATTRIBUTE_SYNC_MEMOPS, ptr); + if (ret != CUDA_SUCCESS) { + cuGetErrorString(ret, &cu_err_str); + ucm_warn("cuPointerSetAttribute(%p) failed: %s", (void*)ptr, + cu_err_str); + } + } + + event.mem_type.address = (void*)ptr; event.mem_type.size = length; event.mem_type.mem_type = mem_type; ucm_event_dispatch(UCM_EVENT_MEM_TYPE_ALLOC, &event); } -static UCS_F_ALWAYS_INLINE void -ucm_dispatch_mem_type_free(void *addr, size_t length, ucs_memory_type_t mem_type) +static void ucm_cuda_dispatch_mem_free(CUdeviceptr ptr, + ucs_memory_type_t mem_type, + const char *func_name) { ucm_event_t event; - - event.mem_type.address = addr; - event.mem_type.size = length; - event.mem_type.mem_type = mem_type; - ucm_event_dispatch(UCM_EVENT_MEM_TYPE_FREE, &event); -} - -static void ucm_cudafree_dispatch_events(CUdeviceptr dptr, const char *func_name) -{ - CUresult ret; CUdeviceptr pbase; - size_t psize; + size_t length; + CUresult ret; - if (dptr == 0) { + if (ptr == 0) { return; } - ret = cuMemGetAddressRange(&pbase, &psize, dptr); + ret = cuMemGetAddressRange(&pbase, &length, ptr); if (ret == CUDA_SUCCESS) { - if (dptr != pbase) { + if (ptr != pbase) { ucm_warn("%s(%p) called with unexpected pointer (expected: %p)", - func_name, (void*)dptr, (void*)pbase); + func_name, (void*)ptr, (void*)pbase); } } else { - ucm_debug("cuMemGetAddressRange(devPtr=%p) failed", (void*)dptr); - psize = 1; /* set minimum length */ + ucm_debug("cuMemGetAddressRange(devPtr=%p) failed", (void*)ptr); + length = 1; /* set minimum length */ } - ucm_dispatch_mem_type_free((void *)dptr, psize, UCS_MEMORY_TYPE_CUDA); -} - -CUresult ucm_cuMemFree(CUdeviceptr dptr) -{ - CUresult ret; - - ucm_event_enter(); - - ucm_trace("ucm_cuMemFree(dptr=%p)",(void*)dptr); - - ucm_cudafree_dispatch_events(dptr, "cuMemFree"); - - ret = ucm_orig_cuMemFree(dptr); - - ucm_event_leave(); - return ret; -} - -CUresult ucm_cuMemFreeHost(void *p) -{ - CUresult ret; - - ucm_event_enter(); - - ucm_trace("ucm_cuMemFreeHost(ptr=%p)", p); - - ucm_dispatch_vm_munmap(p, 0); - - ret = ucm_orig_cuMemFreeHost(p); - - ucm_event_leave(); - return ret; -} - -CUresult ucm_cuMemAlloc(CUdeviceptr *dptr, size_t size) -{ - CUresult ret; - - ucm_event_enter(); - - ret = ucm_orig_cuMemAlloc(dptr, size); - if (ret == CUDA_SUCCESS) { - ucm_trace("ucm_cuMemAlloc(dptr=%p size:%lu)",(void *)*dptr, size); - ucm_dispatch_mem_type_alloc((void *)*dptr, size, UCS_MEMORY_TYPE_CUDA); - ucm_cuda_set_ptr_attr(*dptr); - } - - ucm_event_leave(); - return ret; -} - -CUresult ucm_cuMemAllocManaged(CUdeviceptr *dptr, size_t size, unsigned int flags) -{ - CUresult ret; - - ucm_event_enter(); - - ret = ucm_orig_cuMemAllocManaged(dptr, size, flags); - if (ret == CUDA_SUCCESS) { - ucm_trace("ucm_cuMemAllocManaged(dptr=%p size:%lu, flags:%d)", - (void *)*dptr, size, flags); - ucm_dispatch_mem_type_alloc((void *)*dptr, size, - UCS_MEMORY_TYPE_CUDA_MANAGED); - } - - ucm_event_leave(); - return ret; -} - -CUresult ucm_cuMemAllocPitch(CUdeviceptr *dptr, size_t *pPitch, - size_t WidthInBytes, size_t Height, - unsigned int ElementSizeBytes) -{ - CUresult ret; - - ucm_event_enter(); - - ret = ucm_orig_cuMemAllocPitch(dptr, pPitch, WidthInBytes, Height, ElementSizeBytes); - if (ret == CUDA_SUCCESS) { - ucm_trace("ucm_cuMemAllocPitch(dptr=%p size:%lu)",(void *)*dptr, - (WidthInBytes * Height)); - ucm_dispatch_mem_type_alloc((void *)*dptr, WidthInBytes * Height, - UCS_MEMORY_TYPE_CUDA); - ucm_cuda_set_ptr_attr(*dptr); - } - - ucm_event_leave(); - return ret; -} - -CUresult ucm_cuMemHostGetDevicePointer(CUdeviceptr *pdptr, void *p, unsigned int Flags) -{ - CUresult ret; - - ucm_event_enter(); - - ret = ucm_orig_cuMemHostGetDevicePointer(pdptr, p, Flags); - if (ret == CUDA_SUCCESS) { - ucm_trace("ucm_cuMemHostGetDevicePointer(pdptr=%p p=%p)",(void *)*pdptr, p); - } - - ucm_event_leave(); - return ret; -} - -CUresult ucm_cuMemHostUnregister(void *p) -{ - CUresult ret; - - ucm_event_enter(); - - ucm_trace("ucm_cuMemHostUnregister(ptr=%p)", p); - - ret = ucm_orig_cuMemHostUnregister(p); - - ucm_event_leave(); - return ret; -} - -cudaError_t ucm_cudaFree(void *devPtr) -{ - cudaError_t ret; - - ucm_event_enter(); - - ucm_trace("ucm_cudaFree(devPtr=%p)", devPtr); - - ucm_cudafree_dispatch_events((CUdeviceptr)devPtr, "cudaFree"); - - ret = ucm_orig_cudaFree(devPtr); - - ucm_event_leave(); - - return ret; -} - -cudaError_t ucm_cudaFreeHost(void *ptr) -{ - cudaError_t ret; - - ucm_event_enter(); - - ucm_trace("ucm_cudaFreeHost(ptr=%p)", ptr); - - ucm_dispatch_vm_munmap(ptr, 0); - - ret = ucm_orig_cudaFreeHost(ptr); - - ucm_event_leave(); - return ret; + event.mem_type.address = (void*)ptr; + event.mem_type.size = length; + event.mem_type.mem_type = mem_type; + ucm_event_dispatch(UCM_EVENT_MEM_TYPE_FREE, &event); } -cudaError_t ucm_cudaMalloc(void **devPtr, size_t size) -{ - cudaError_t ret; - - ucm_event_enter(); - - ret = ucm_orig_cudaMalloc(devPtr, size); - if (ret == cudaSuccess) { - ucm_trace("ucm_cudaMalloc(devPtr=%p size:%lu)", *devPtr, size); - ucm_dispatch_mem_type_alloc(*devPtr, size, UCS_MEMORY_TYPE_CUDA); - ucm_cuda_set_ptr_attr((CUdeviceptr) *devPtr); - } - - ucm_event_leave(); +/* Driver API replacements */ +UCM_CUDA_ALLOC_FUNC(cuMemAlloc, UCS_MEMORY_TYPE_CUDA, CUresult, CUDA_SUCCESS, + arg0, CUdeviceptr, "size=%zu", size_t) +UCM_CUDA_ALLOC_FUNC(cuMemAlloc_v2, UCS_MEMORY_TYPE_CUDA, CUresult, CUDA_SUCCESS, + arg0, CUdeviceptr, "size=%zu", size_t) +UCM_CUDA_ALLOC_FUNC(cuMemAllocManaged, UCS_MEMORY_TYPE_CUDA_MANAGED, CUresult, + CUDA_SUCCESS, arg0, CUdeviceptr, "size=%zu flags=0x%x", + size_t, unsigned) +UCM_CUDA_ALLOC_FUNC(cuMemAllocPitch, UCS_MEMORY_TYPE_CUDA, CUresult, + CUDA_SUCCESS, (size_t)arg1 * arg2, CUdeviceptr, + "pitch=%p width=%zu height=%zu elem=%u", size_t*, size_t, + size_t, unsigned) +UCM_CUDA_ALLOC_FUNC(cuMemAllocPitch_v2, UCS_MEMORY_TYPE_CUDA, CUresult, + CUDA_SUCCESS, (size_t)arg1 * arg2, CUdeviceptr, + "pitch=%p width=%zu height=%zu elem=%u", size_t*, size_t, + size_t, unsigned) +UCM_CUDA_FREE_FUNC(cuMemFree, CUresult, CUdeviceptr, UCS_MEMORY_TYPE_CUDA) +UCM_CUDA_FREE_FUNC(cuMemFree_v2, CUresult, CUdeviceptr, UCS_MEMORY_TYPE_CUDA) +UCM_CUDA_FREE_FUNC(cuMemFreeHost, CUresult, void*, UCS_MEMORY_TYPE_HOST) +UCM_CUDA_FREE_FUNC(cuMemFreeHost_v2, CUresult, void*, UCS_MEMORY_TYPE_HOST) + +static ucm_cuda_func_t ucm_cuda_driver_funcs[] = { + UCM_CUDA_FUNC_ENTRY(cuMemAlloc), + UCM_CUDA_FUNC_ENTRY(cuMemAlloc_v2), + UCM_CUDA_FUNC_ENTRY(cuMemAllocManaged), + UCM_CUDA_FUNC_ENTRY(cuMemAllocPitch), + UCM_CUDA_FUNC_ENTRY(cuMemAllocPitch_v2), + UCM_CUDA_FUNC_ENTRY(cuMemFree), + UCM_CUDA_FUNC_ENTRY(cuMemFree_v2), + UCM_CUDA_FUNC_ENTRY(cuMemFreeHost), + UCM_CUDA_FUNC_ENTRY(cuMemFreeHost_v2), + {{NULL}, NULL} +}; - return ret; -} +/* Runtime API replacements */ +UCM_CUDA_ALLOC_FUNC(cudaMalloc, UCS_MEMORY_TYPE_CUDA, cudaError_t, cudaSuccess, + arg0, void*, "size=%zu", size_t) +UCM_CUDA_ALLOC_FUNC(cudaMallocManaged, UCS_MEMORY_TYPE_CUDA_MANAGED, + cudaError_t, cudaSuccess, arg0, void*, + "size=%zu flags=0x%x", size_t, unsigned) +UCM_CUDA_ALLOC_FUNC(cudaMallocPitch, UCS_MEMORY_TYPE_CUDA, cudaError_t, + cudaSuccess, (size_t)arg1 * arg2, void*, + "pitch=%p width=%zu height=%zu", size_t*, size_t, size_t) +UCM_CUDA_FREE_FUNC(cudaFree, cudaError_t, void*, UCS_MEMORY_TYPE_CUDA) +UCM_CUDA_FREE_FUNC(cudaFreeHost, cudaError_t, void*, UCS_MEMORY_TYPE_HOST) + +static ucm_cuda_func_t ucm_cuda_runtime_funcs[] = { + UCM_CUDA_FUNC_ENTRY(cudaFree), + UCM_CUDA_FUNC_ENTRY(cudaFreeHost), + UCM_CUDA_FUNC_ENTRY(cudaMalloc), + UCM_CUDA_FUNC_ENTRY(cudaMallocManaged), + UCM_CUDA_FUNC_ENTRY(cudaMallocPitch), + {{NULL}, NULL} +}; -cudaError_t ucm_cudaMallocManaged(void **devPtr, size_t size, unsigned int flags) +static ucm_mmap_hook_mode_t ucm_cuda_hook_mode() { - cudaError_t ret; - - ucm_event_enter(); - - ret = ucm_orig_cudaMallocManaged(devPtr, size, flags); - if (ret == cudaSuccess) { - ucm_trace("ucm_cudaMallocManaged(devPtr=%p size:%lu flags:%d)", - *devPtr, size, flags); - ucm_dispatch_mem_type_alloc(*devPtr, size, UCS_MEMORY_TYPE_CUDA_MANAGED); - } - - ucm_event_leave(); - - return ret; + return ucm_get_hook_mode(ucm_global_opts.cuda_hook_mode); } -cudaError_t ucm_cudaMallocPitch(void **devPtr, size_t *pitch, - size_t width, size_t height) +static ucs_status_t +ucm_cuda_install_hooks(ucm_cuda_func_t *funcs, int *used_reloc, + const char *name) { - cudaError_t ret; - - ucm_event_enter(); - - ret = ucm_orig_cudaMallocPitch(devPtr, pitch, width, height); - if (ret == cudaSuccess) { - ucm_trace("ucm_cudaMallocPitch(devPtr=%p size:%lu)",*devPtr, (width * height)); - ucm_dispatch_mem_type_alloc(*devPtr, (width * height), UCS_MEMORY_TYPE_CUDA); - ucm_cuda_set_ptr_attr((CUdeviceptr) *devPtr); - } + const char UCS_V_UNUSED *hook_mode; + ucm_cuda_func_t *func; + ucs_status_t status; + void *func_ptr; + + hook_mode = (ucm_cuda_hook_mode() == UCM_MMAP_HOOK_BISTRO) ? + "bistro" : "reloc"; + *used_reloc = 0; + for (func = funcs; func->patch.symbol != NULL; ++func) { + func_ptr = ucm_reloc_get_orig(func->patch.symbol, func->patch.value); + if (func_ptr == NULL) { + continue; + } - ucm_event_leave(); - return ret; -} + status = UCS_ERR_UNSUPPORTED; -cudaError_t ucm_cudaHostGetDevicePointer(void **pDevice, void *pHost, unsigned int flags) -{ - cudaError_t ret; + if (ucm_cuda_hook_mode() == UCM_MMAP_HOOK_BISTRO) { + status = ucm_bistro_patch(func_ptr, func->patch.value, + func->patch.symbol, func->orig_func_ptr, + NULL); + if (status == UCS_OK) { + ucm_trace("installed bistro hook for '%s': %s", + func->patch.symbol, ucs_status_string(status)); + continue; + } + + ucm_debug("failed to install bistro hook for '%s', trying reloc", + func->patch.symbol); + hook_mode = "mixed"; + } - ucm_event_enter(); + status = ucm_reloc_modify(&func->patch); + if (status != UCS_OK) { + ucm_diag("failed to install relocation table entry for '%s'", + func->patch.symbol); + return status; + } - ret = ucm_orig_cudaHostGetDevicePointer(pDevice, pHost, flags); - if (ret == cudaSuccess) { - ucm_trace("ucm_cuMemHostGetDevicePointer(pDevice=%p pHost=%p)", pDevice, pHost); + *used_reloc = 1; + ucm_trace("installed reloc hook on '%s'", func->patch.symbol); } - ucm_event_leave(); - return ret; + ucm_debug("installed cuda memory %s hooks on %s API", hook_mode, name); + return UCS_OK; } -cudaError_t ucm_cudaHostUnregister(void *ptr) -{ - cudaError_t ret; - - ucm_event_enter(); - - ucm_trace("ucm_cudaHostUnregister(ptr=%p)", ptr); - - ret = ucm_orig_cudaHostUnregister(ptr); - - ucm_event_leave(); - return ret; -} - -static ucm_reloc_patch_t patches[] = { - {UCS_PP_MAKE_STRING(cuMemFree), ucm_override_cuMemFree}, - {UCS_PP_MAKE_STRING(cuMemFreeHost), ucm_override_cuMemFreeHost}, - {UCS_PP_MAKE_STRING(cuMemAlloc), ucm_override_cuMemAlloc}, - {UCS_PP_MAKE_STRING(cuMemAllocManaged), ucm_override_cuMemAllocManaged}, - {UCS_PP_MAKE_STRING(cuMemAllocPitch), ucm_override_cuMemAllocPitch}, - {UCS_PP_MAKE_STRING(cuMemHostGetDevicePointer), ucm_override_cuMemHostGetDevicePointer}, - {UCS_PP_MAKE_STRING(cuMemHostUnregister), ucm_override_cuMemHostUnregister}, - {UCS_PP_MAKE_STRING(cudaFree), ucm_override_cudaFree}, - {UCS_PP_MAKE_STRING(cudaFreeHost), ucm_override_cudaFreeHost}, - {UCS_PP_MAKE_STRING(cudaMalloc), ucm_override_cudaMalloc}, - {UCS_PP_MAKE_STRING(cudaMallocManaged), ucm_override_cudaMallocManaged}, - {UCS_PP_MAKE_STRING(cudaMallocPitch), ucm_override_cudaMallocPitch}, - {UCS_PP_MAKE_STRING(cudaHostGetDevicePointer), ucm_override_cudaHostGetDevicePointer}, - {UCS_PP_MAKE_STRING(cudaHostUnregister), ucm_override_cudaHostUnregister}, - {NULL, NULL} -}; - static ucs_status_t ucm_cudamem_install(int events) { - static int ucm_cudamem_installed = 0; + static int ucm_cudamem_installed = 0; static pthread_mutex_t install_mutex = PTHREAD_MUTEX_INITIALIZER; - ucm_reloc_patch_t *patch; - ucs_status_t status = UCS_OK; + ucs_status_t status = UCS_OK; + int used_reloc; if (!(events & (UCM_EVENT_MEM_TYPE_ALLOC | UCM_EVENT_MEM_TYPE_FREE))) { goto out; } - if (!ucm_global_opts.enable_cuda_reloc) { - ucm_debug("installing cudamem relocations is disabled by configuration"); + if (ucm_cuda_hook_mode() == UCM_MMAP_HOOK_NONE) { + ucm_debug("cuda memory hooks are disabled by configuration"); status = UCS_ERR_UNSUPPORTED; goto out; } @@ -402,17 +284,24 @@ static ucs_status_t ucm_cudamem_install(int events) goto out_unlock; } - for (patch = patches; patch->symbol != NULL; ++patch) { - status = ucm_reloc_modify(patch); - if (status != UCS_OK) { - ucm_warn("failed to install relocation table entry for '%s'", patch->symbol); - goto out_unlock; + status = ucm_cuda_install_hooks(ucm_cuda_driver_funcs, &used_reloc, + "driver"); + if (status != UCS_OK) { + ucm_warn("failed to install cuda memory hooks on driver API"); + } else if (!used_reloc) { + ucm_cudamem_installed = 1; + } else if (status == UCS_OK) { + /* Failed to install bistro hooks on all driver APIs, so need to install + hooks on runtime APIs. */ + status = ucm_cuda_install_hooks(ucm_cuda_runtime_funcs, &used_reloc, + "runtime"); + if (status == UCS_OK) { + ucm_cudamem_installed = 1; + } else { + ucm_warn("failed to install cuda memory hooks on runtime API") } } - ucm_debug("cudaFree hooks are ready"); - ucm_cudamem_installed = 1; - out_unlock: pthread_mutex_unlock(&install_mutex); out: @@ -429,13 +318,13 @@ static int ucm_cudamem_scan_regions_cb(void *arg, void *addr, size_t length, /* we are interested in blocks which don't have any access permissions, or * mapped to nvidia device. */ - if ((prot & (PROT_READ|PROT_WRITE|PROT_EXEC)) && + if ((prot & (PROT_READ | PROT_WRITE | PROT_EXEC)) && strncmp(path, cuda_path_pattern, strlen(cuda_path_pattern))) { return 0; } - ucm_debug("dispatching initial memtype allocation for %p..%p %s", - addr, UCS_PTR_BYTE_OFFSET(addr, length), path); + ucm_debug("dispatching initial memtype allocation for %p..%p %s", addr, + UCS_PTR_BYTE_OFFSET(addr, length), path); event.mem_type.address = addr; event.mem_type.size = length; @@ -460,10 +349,12 @@ static ucm_event_installer_t ucm_cuda_initializer = { .get_existing_alloc = ucm_cudamem_get_existing_alloc }; -UCS_STATIC_INIT { +UCS_STATIC_INIT +{ ucs_list_add_tail(&ucm_event_installer_list, &ucm_cuda_initializer.list); } -UCS_STATIC_CLEANUP { +UCS_STATIC_CLEANUP +{ ucs_list_del(&ucm_cuda_initializer.list); } diff --git a/src/ucm/cuda/cudamem.h b/src/ucm/cuda/cudamem.h index 03268231067..cd5a4087e27 100644 --- a/src/ucm/cuda/cudamem.h +++ b/src/ucm/cuda/cudamem.h @@ -7,95 +7,29 @@ #ifndef UCM_CUDAMEM_H_ #define UCM_CUDAMEM_H_ -#include #include #include -/*cuMemFree */ -CUresult ucm_override_cuMemFree(CUdeviceptr dptr); -CUresult ucm_orig_cuMemFree(CUdeviceptr dptr); -CUresult ucm_cuMemFree(CUdeviceptr dptr); - -/*cuMemFreeHost */ -CUresult ucm_override_cuMemFreeHost(void *p); -CUresult ucm_orig_cuMemFreeHost(void *p); -CUresult ucm_cuMemFreeHost(void *p); - -/*cuMemAlloc*/ -CUresult ucm_override_cuMemAlloc(CUdeviceptr *dptr, size_t size); -CUresult ucm_orig_cuMemAlloc(CUdeviceptr *dptr, size_t size); CUresult ucm_cuMemAlloc(CUdeviceptr *dptr, size_t size); - -/*cuMemAllocManaged*/ -CUresult ucm_override_cuMemAllocManaged(CUdeviceptr *dptr, size_t size, - unsigned int flags); -CUresult ucm_orig_cuMemAllocManaged(CUdeviceptr *dptr, size_t size, unsigned int flags); +CUresult ucm_cuMemAlloc_v2(CUdeviceptr *dptr, size_t size); CUresult ucm_cuMemAllocManaged(CUdeviceptr *dptr, size_t size, unsigned int flags); - -/*cuMemAllocPitch*/ -CUresult ucm_override_cuMemAllocPitch(CUdeviceptr *dptr, size_t *pPitch, - size_t WidthInBytes, size_t Height, - unsigned int ElementSizeBytes); -CUresult ucm_orig_cuMemAllocPitch(CUdeviceptr *dptr, size_t *pPitch, - size_t WidthInBytes, size_t Height, - unsigned int ElementSizeBytes); CUresult ucm_cuMemAllocPitch(CUdeviceptr *dptr, size_t *pPitch, size_t WidthInBytes, size_t Height, unsigned int ElementSizeBytes); +CUresult ucm_cuMemAllocPitch_v2(CUdeviceptr *dptr, size_t *pPitch, + size_t WidthInBytes, size_t Height, + unsigned int ElementSizeBytes); +CUresult ucm_cuMemFree(CUdeviceptr dptr); +CUresult ucm_cuMemFree_v2(CUdeviceptr dptr); +CUresult ucm_cuMemFreeHost(void *p); +CUresult ucm_cuMemFreeHost_v2(void *p); -/*cuMemHostGetDevicePointer*/ -CUresult ucm_override_cuMemHostGetDevicePointer(CUdeviceptr *pdptr, void *p, - unsigned int Flags); -CUresult ucm_orig_cuMemHostGetDevicePointer(CUdeviceptr *pdptr, void *p, - unsigned int Flags); -CUresult ucm_cuMemHostGetDevicePointer(CUdeviceptr *pdptr, void *p, unsigned int Flags); - -/*cuMemHostUnregister */ -CUresult ucm_override_cuMemHostUnregister(void *p); -CUresult ucm_orig_cuMemHostUnregister(void *p); -CUresult ucm_cuMemHostUnregister(void *p); - -/*cudaFree*/ -cudaError_t ucm_override_cudaFree(void *devPtr); -cudaError_t ucm_orig_cudaFree(void *devPtr); cudaError_t ucm_cudaFree(void *devPtr); - -/*cudaFreeHost*/ -cudaError_t ucm_override_cudaFreeHost(void *ptr); -cudaError_t ucm_orig_cudaFreeHost(void *ptr); cudaError_t ucm_cudaFreeHost(void *ptr); - -/*cudaMalloc*/ -cudaError_t ucm_override_cudaMalloc(void **devPtr, size_t size); -cudaError_t ucm_orig_cudaMalloc(void **devPtr, size_t size); cudaError_t ucm_cudaMalloc(void **devPtr, size_t size); - -/*cudaMallocManaged*/ -cudaError_t ucm_override_cudaMallocManaged(void **devPtr, size_t size, - unsigned int flags); -cudaError_t ucm_orig_cudaMallocManaged(void **devPtr, size_t size, unsigned int flags); cudaError_t ucm_cudaMallocManaged(void **devPtr, size_t size, unsigned int flags); - -/*cudaMallocPitch*/ -cudaError_t ucm_override_cudaMallocPitch(void **devPtr, size_t *pitch, - size_t width, size_t height); -cudaError_t ucm_orig_cudaMallocPitch(void **devPtr, size_t *pitch, - size_t width, size_t height); cudaError_t ucm_cudaMallocPitch(void **devPtr, size_t *pitch, size_t width, size_t height); -/*cudaHostGetDevicePointer*/ -cudaError_t ucm_override_cudaHostGetDevicePointer(void **pDevice, void *pHost, - unsigned int flags); -cudaError_t ucm_orig_cudaHostGetDevicePointer(void **pDevice, void *pHost, - unsigned int flags); -cudaError_t ucm_cudaHostGetDevicePointer(void **pDevice, void *pHost, unsigned int flags); - - -/*cudaHostUnregister*/ -cudaError_t ucm_override_cudaHostUnregister(void *ptr); -cudaError_t ucm_orig_cudaHostUnregister(void *ptr); -cudaError_t ucm_cudaHostUnregister(void *ptr); - #endif diff --git a/src/ucm/mmap/install.c b/src/ucm/mmap/install.c index c1b284eab62..f2e2bb5ee8d 100644 --- a/src/ucm/mmap/install.c +++ b/src/ucm/mmap/install.c @@ -338,6 +338,7 @@ static ucs_status_t ucs_mmap_install_reloc(int events) static int installed_events = 0; ucm_mmap_func_t *entry; ucs_status_t status; + void *func_ptr; if (ucm_mmap_hook_mode() == UCM_MMAP_HOOK_NONE) { ucm_debug("installing mmap hooks is disabled by configuration"); @@ -363,8 +364,14 @@ static ucs_status_t ucs_mmap_install_reloc(int events) status = ucm_reloc_modify(&entry->patch); } else { ucm_assert(ucm_mmap_hook_mode() == UCM_MMAP_HOOK_BISTRO); - status = ucm_bistro_patch(entry->patch.symbol, entry->patch.value, - NULL); + func_ptr = ucm_reloc_get_orig(entry->patch.symbol, + entry->patch.value); + if (func_ptr == NULL) { + status = UCS_ERR_NO_ELEM; + } else { + status = ucm_bistro_patch(func_ptr, entry->patch.value, + entry->patch.symbol, NULL, NULL); + } } if (status != UCS_OK) { ucm_warn("failed to install %s hook for '%s'", UCM_HOOK_STR, diff --git a/src/ucm/mmap/mmap.h b/src/ucm/mmap/mmap.h index 1d9b04f4cd2..c7b4e4e37b2 100644 --- a/src/ucm/mmap/mmap.h +++ b/src/ucm/mmap/mmap.h @@ -8,6 +8,7 @@ #define UCM_MMAP_H_ #include +#include #include #define UCM_MMAP_HOOK_RELOC_STR "reloc" @@ -40,15 +41,7 @@ void ucm_mmap_init(); static UCS_F_ALWAYS_INLINE ucm_mmap_hook_mode_t ucm_mmap_hook_mode(void) { -#ifdef __SANITIZE_ADDRESS__ - return UCM_MMAP_HOOK_NONE; -#else - if (RUNNING_ON_VALGRIND && (ucm_global_opts.mmap_hook_mode == UCM_MMAP_HOOK_BISTRO)) { - return UCM_MMAP_HOOK_RELOC; - } - - return ucm_global_opts.mmap_hook_mode; -#endif + return ucm_get_hook_mode(ucm_global_opts.mmap_hook_mode); } #endif diff --git a/src/ucm/rocm/rocmmem.c b/src/ucm/rocm/rocmmem.c index bf441d6057d..2a63241e1a3 100644 --- a/src/ucm/rocm/rocmmem.c +++ b/src/ucm/rocm/rocmmem.c @@ -28,11 +28,6 @@ UCM_DEFINE_REPLACE_DLSYM_FUNC(hsa_amd_memory_pool_allocate, hsa_status_t, UCM_DEFINE_REPLACE_DLSYM_FUNC(hsa_amd_memory_pool_free, hsa_status_t, HSA_STATUS_ERROR, void*) -#if ENABLE_SYMBOL_OVERRIDE -UCM_OVERRIDE_FUNC(hsa_amd_memory_pool_allocate, hsa_status_t) -UCM_OVERRIDE_FUNC(hsa_amd_memory_pool_free, hsa_status_t) -#endif - static UCS_F_ALWAYS_INLINE void ucm_dispatch_mem_type_alloc(void *addr, size_t length, ucs_memory_type_t mem_type) { diff --git a/src/ucm/util/reloc.c b/src/ucm/util/reloc.c index 4443a73b5c7..16e9c882a3e 100644 --- a/src/ucm/util/reloc.c +++ b/src/ucm/util/reloc.c @@ -8,12 +8,6 @@ # include "config.h" #endif -#ifndef NVALGRIND -# include -#else -# define RUNNING_ON_VALGRIND 0 -#endif - #include "reloc.h" #include diff --git a/src/ucm/util/replace.h b/src/ucm/util/replace.h index 4b91b037d99..de060d1e048 100644 --- a/src/ucm/util/replace.h +++ b/src/ucm/util/replace.h @@ -79,6 +79,24 @@ extern pthread_t volatile ucm_reloc_get_orig_thread; _UCM_DEFINE_REPLACE_FUNC(ucm_override_##_name, ucm_##_name, \ _rettype, _fail_val, __VA_ARGS__) +/** + * Defines the following: + * - ucm_orig_##_name##_dlsym - calls original function by symbol lookup + * - ucm_orig_##_name - function pointer, initialized by default to + * ucm_orig_##_name##_dlsym + * - ucm_override_##_name - calls ucm_##_name + */ +#define UCM_DEFINE_REPLACE_DLSYM_PTR_FUNC(_name, _rettype, _fail_val, ...) \ + _UCM_DEFINE_DLSYM_FUNC(_name, ucm_orig_##_name##_dlsym, \ + ucm_override_##_name, _rettype, _fail_val, \ + __VA_ARGS__) \ + \ + _rettype (*ucm_orig_##_name)(UCM_FUNC_DEFINE_ARGS(__VA_ARGS__)) = \ + ucm_orig_##_name##_dlsym; \ + \ + _UCM_DEFINE_REPLACE_FUNC(ucm_override_##_name, ucm_##_name, \ + _rettype, _fail_val, __VA_ARGS__) + #define UCM_DEFINE_SYSCALL_FUNC(_name, _rettype, _syscall_id, ...) \ /* Call syscall */ \ _rettype ucm_orig_##_name(UCM_FUNC_DEFINE_ARGS(__VA_ARGS__)) \ diff --git a/src/ucm/util/sys.c b/src/ucm/util/sys.c index 2129ba203fb..9f0bcacdffb 100644 --- a/src/ucm/util/sys.c +++ b/src/ucm/util/sys.c @@ -38,7 +38,7 @@ ucm_global_config_t ucm_global_opts = { .mmap_hook_mode = UCM_DEFAULT_HOOK_MODE, .enable_malloc_hooks = 1, .enable_malloc_reloc = 0, - .enable_cuda_reloc = 1, + .cuda_hook_mode = UCM_DEFAULT_HOOK_MODE, .enable_dynamic_mmap_thresh = 1, .alloc_alignment = 16, .dlopen_process_rpath = 1 diff --git a/src/ucm/util/sys.h b/src/ucm/util/sys.h index 215de109ef7..838fb61a1c5 100644 --- a/src/ucm/util/sys.h +++ b/src/ucm/util/sys.h @@ -8,6 +8,8 @@ #ifndef UCM_UTIL_SYS_H_ #define UCM_UTIL_SYS_H_ +#include +#include #include #include @@ -107,4 +109,26 @@ void *ucm_brk_syscall(void *addr); */ pid_t ucm_get_tid(); + +/** + * Get memory hooks mode to use, based on the configured mode and runtime. + * + * @param config_mode Configured memory hook mode. + * + * @return Memory hook mode to use. + */ +static UCS_F_ALWAYS_INLINE ucm_mmap_hook_mode_t +ucm_get_hook_mode(ucm_mmap_hook_mode_t config_mode) +{ +#ifdef __SANITIZE_ADDRESS__ + return UCM_MMAP_HOOK_NONE; +#else + if (RUNNING_ON_VALGRIND && (config_mode == UCM_MMAP_HOOK_BISTRO)) { + return UCM_MMAP_HOOK_RELOC; + } + + return config_mode; +#endif +} + #endif diff --git a/src/ucs/config/ucm_opts.c b/src/ucs/config/ucm_opts.c index 18abe3228e4..bee035a4a9c 100644 --- a/src/ucs/config/ucm_opts.c +++ b/src/ucs/config/ucm_opts.c @@ -49,12 +49,13 @@ static ucs_config_field_t ucm_global_config_table[] = { {"MMAP_HOOK_MODE", UCM_DEFAULT_HOOK_MODE_STR, "MMAP hook mode\n" - " none - don't set mmap hooks.\n" - " reloc - use ELF relocation table to set hooks.\n" + " none - Don't set mmap hooks.\n" + " reloc - Use ELF relocation table to set hooks.\n" #if UCM_BISTRO_HOOKS - " bistro - use binary instrumentation to set hooks.\n" + " bistro - Use binary instrumentation to set hooks.\n" #endif - ,ucs_offsetof(ucm_global_config_t, mmap_hook_mode), UCS_CONFIG_TYPE_ENUM(ucm_mmap_hook_modes)}, + ,ucs_offsetof(ucm_global_config_t, mmap_hook_mode), + UCS_CONFIG_TYPE_ENUM(ucm_mmap_hook_modes)}, {"MALLOC_HOOKS", "yes", "Enable using glibc malloc hooks", @@ -68,10 +69,24 @@ static ucs_config_field_t ucm_global_config_table[] = { "which would use the original implementation and not ours.", ucs_offsetof(ucm_global_config_t, enable_malloc_reloc), UCS_CONFIG_TYPE_BOOL}, + {"CUDA_HOOK_MODE", UCM_DEFAULT_HOOK_MODE_STR, + "Cuda memory hook mode\n" + " none - Don't set Cuda hooks.\n" + " reloc - Use ELF relocation table to set hooks. In this mode, if any\n" + " part of the application is linked with Cuda runtime statically,\n" + " some memory events may be missed and not reported." +#if UCM_BISTRO_HOOKS + " bistro - Use binary instrumentation to set hooks. In this mode, it's\n" + " possible to intercept calls from the Cuda runtime library to\n" + " Cuda driver APIs, so memory events are reported properly even\n" + " for statically-linked applications." +#endif + ,ucs_offsetof(ucm_global_config_t, cuda_hook_mode), + UCS_CONFIG_TYPE_ENUM(ucm_mmap_hook_modes)}, + {"CUDA_RELOC", "yes", - "Enable installing CUDA symbols in the relocation table", - ucs_offsetof(ucm_global_config_t, enable_cuda_reloc), - UCS_CONFIG_TYPE_BOOL}, + "The configuration parameter replaced by UCX_MEM_CUDA_HOOK_MODE", + UCS_CONFIG_DEPRECATED_FIELD_OFFSET, UCS_CONFIG_TYPE_DEPRECATED}, {"DYNAMIC_MMAP_THRESH", "yes", "Enable dynamic mmap threshold: for every released block, the\n" diff --git a/test/apps/Makefile.am b/test/apps/Makefile.am index 48e2c75d7ab..2ada50f7bef 100644 --- a/test/apps/Makefile.am +++ b/test/apps/Makefile.am @@ -69,7 +69,7 @@ noinst_PROGRAMS += test_cuda_hook_static test_cuda_hook_static_SOURCES = test_cuda_hook.c test_cuda_hook_static_CPPFLAGS = $(BASE_CPPFLAGS) $(CUDA_CPPFLAGS) test_cuda_hook_static_CFLAGS = $(BASE_CFLAGS) -test_cuda_hook_static_LDFLAGS = -lcudart_static -lrt -ldl -lpthread +test_cuda_hook_static_LDFLAGS = -lcudart_static -lcuda -lrt -ldl -lpthread test_cuda_hook_static_LDADD = $(top_builddir)/src/ucp/libucp.la \ $(top_builddir)/src/ucm/libucm.la endif diff --git a/test/apps/test_cuda_hook.c b/test/apps/test_cuda_hook.c index 37391ff06ec..11a1d8423bc 100644 --- a/test/apps/test_cuda_hook.c +++ b/test/apps/test_cuda_hook.c @@ -11,6 +11,8 @@ #include #include #include +#include +#include static void event_cb(ucm_event_type_t event_type, ucm_event_t *event, void *arg) @@ -33,17 +35,74 @@ static void event_cb(ucm_event_type_t event_type, ucm_event_t *event, void *arg) ++(*count_p); } +static void alloc_driver_api() +{ + CUdeviceptr dptr = 0; + CUcontext context; + CUdevice device; + CUresult res; + + res = cuInit(0); + if (res != CUDA_SUCCESS) { + printf("cuInit() failed: %d\n", res); + return; + } + + res = cuDeviceGet(&device, 0); + if (res != CUDA_SUCCESS) { + printf("cuDeviceGet(0) failed: %d\n", res); + return; + } + + res = cuCtxCreate(&context, 0, device); + if (res != CUDA_SUCCESS) { + printf("cuCtxCreate() failed: %d\n", res); + return; + } + + res = cuMemAlloc(&dptr, 4096); + printf("cuMemAlloc() returned 0x%lx result %d\n", (uintptr_t)dptr, res); + cuMemFree(dptr); + + cuCtxDetach(context); +} + +static void alloc_runtime_api() +{ + void *dptr = NULL; + cudaError_t res; + + res = cudaMalloc(&dptr, 4096); + printf("cudaMalloc() returned %p result %d\n", dptr, res); + cudaFree(dptr); +} + int main(int argc, char **argv) { static const ucm_event_type_t memtype_events = UCM_EVENT_MEM_TYPE_ALLOC | UCM_EVENT_MEM_TYPE_FREE; static const int num_expected_events = 2; - ucs_status_t status; ucp_context_h context; + ucs_status_t status; ucp_params_t params; + int use_driver_api; int num_events; - void *dptr; - int res; + int c; + + use_driver_api = 0; + while ((c = getopt(argc, argv, "d")) != -1) { + switch (c) { + case 'd': + use_driver_api = 1; + break; + default: + printf("Usage: test_cuda_hook [options]\n"); + printf("Options are:\n"); + printf(" -d : Use Cuda driver API (Default: use runtime API)\n"); + printf("\n"); + return -1; + } + } params.field_mask = UCP_PARAM_FIELD_FEATURES; params.features = UCP_FEATURE_TAG | UCP_FEATURE_STREAM; @@ -56,11 +115,11 @@ int main(int argc, char **argv) num_events = 0; ucm_set_event_handler(memtype_events, 1000, event_cb, &num_events); - dptr = NULL; - res = cudaMalloc(&dptr, 4096); - printf("cudaMalloc() returned %p result %d\n", dptr, res); - - cudaFree(dptr); + if (use_driver_api) { + alloc_driver_api(); + } else { + alloc_runtime_api(); + } ucm_unset_event_handler(memtype_events, event_cb, &num_events); printf("got %d/%d memory events\n", num_events, num_expected_events); diff --git a/test/gtest/ucm/malloc_hook.cc b/test/gtest/ucm/malloc_hook.cc index 624fda474da..5ee6efd65ac 100644 --- a/test/gtest/ucm/malloc_hook.cc +++ b/test/gtest/ucm/malloc_hook.cc @@ -22,6 +22,7 @@ extern "C" { #include #include #include +#include #include #ifdef HAVE_MALLOC_H #include @@ -207,8 +208,11 @@ class malloc_hook : public ucs::test { bistro_patch(const char* symbol, void *hook) { ucs_status_t status; - - status = ucm_bistro_patch(symbol, hook, &m_rp); + void *func_ptr = ucm_reloc_get_orig(symbol, hook); + if (func_ptr == NULL) { + UCS_TEST_ABORT("could not find " << symbol); + } + status = ucm_bistro_patch(func_ptr, hook, symbol, NULL, &m_rp); ASSERT_UCS_OK(status); EXPECT_NE((intptr_t)m_rp, 0); } From 705eb6259f5f30c6942c2c43b3505434f068b657 Mon Sep 17 00:00:00 2001 From: Yossi Itigin Date: Thu, 31 Dec 2020 17:18:49 +0200 Subject: [PATCH 097/107] JENKINS: Fix distributed task workers count --- contrib/test_jenkins.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/test_jenkins.sh b/contrib/test_jenkins.sh index 93bf29cd55c..6bf9decdba7 100755 --- a/contrib/test_jenkins.sh +++ b/contrib/test_jenkins.sh @@ -1729,13 +1729,13 @@ run_tests() { do_distributed_task 2 4 run_ucx_perftest do_distributed_task 1 4 run_io_demo do_distributed_task 3 4 test_profiling - do_distributed_task 0 3 test_jucx + do_distributed_task 0 4 test_jucx do_distributed_task 1 4 test_ucs_dlopen do_distributed_task 3 4 test_ucs_load do_distributed_task 3 4 test_memtrack do_distributed_task 0 4 test_unused_env_var do_distributed_task 2 4 test_env_var_aliases - do_distributed_task 1 3 test_malloc_hook + do_distributed_task 1 4 test_malloc_hook do_distributed_task 0 4 test_ucp_dlopen do_distributed_task 1 4 test_init_mt From 31c922045fa075339240d205cc22c090165f7e1c Mon Sep 17 00:00:00 2001 From: Artemy Kovalyov Date: Thu, 31 Dec 2020 15:15:55 +0000 Subject: [PATCH 098/107] UCT/IB/MLX5: DV UAR alloc type NC support --- buildlib/azure-pipelines-pr.yml | 5 ++++ src/uct/ib/configure.m4 | 4 ++- src/uct/ib/mlx5/ib_mlx5.c | 49 ++++++++++++++++++++++++++++++--- src/uct/ib/mlx5/ib_mlx5.h | 9 ++++++ 4 files changed, 62 insertions(+), 5 deletions(-) diff --git a/buildlib/azure-pipelines-pr.yml b/buildlib/azure-pipelines-pr.yml index d0abb70c635..17b5cfafd04 100644 --- a/buildlib/azure-pipelines-pr.yml +++ b/buildlib/azure-pipelines-pr.yml @@ -209,6 +209,11 @@ stages: name: new demands: ucx_new -equals yes test_perf: 1 + - template: tests.yml + parameters: + name: hwi + demands: ucx_hwi -equals yes + test_perf: 0 - stage: io_demo dependsOn: [Codestyle] diff --git a/src/uct/ib/configure.m4 b/src/uct/ib/configure.m4 index 21902b5a726..fa431a283e2 100644 --- a/src/uct/ib/configure.m4 +++ b/src/uct/ib/configure.m4 @@ -205,7 +205,9 @@ AS_IF([test "x$with_ib" = "xyes"], mlx5dv_is_supported, mlx5dv_devx_subscribe_devx_event, MLX5DV_CQ_INIT_ATTR_MASK_CQE_SIZE, - MLX5DV_QP_CREATE_ALLOW_SCATTER_TO_CQE], + MLX5DV_QP_CREATE_ALLOW_SCATTER_TO_CQE, + MLX5DV_UAR_ALLOC_TYPE_BF, + MLX5DV_UAR_ALLOC_TYPE_NC], [], [], [[#include ]]) AC_CHECK_MEMBERS([struct mlx5dv_cq.cq_uar], [], [], [[#include ]]) diff --git a/src/uct/ib/mlx5/ib_mlx5.c b/src/uct/ib/mlx5/ib_mlx5.c index bf4a9cce14a..93fdb7d0f2e 100644 --- a/src/uct/ib/mlx5/ib_mlx5.c +++ b/src/uct/ib/mlx5/ib_mlx5.c @@ -400,15 +400,56 @@ int uct_ib_mlx5_devx_uar_cmp(uct_ib_mlx5_devx_uar_t *uar, return uar->ctx == md->super.dev.ibv_context; } +#if HAVE_DEVX +static ucs_status_t +uct_ib_mlx5_devx_alloc_uar(uct_ib_mlx5_md_t *md, unsigned flags, int log_level, + char *title, char *fallback, + struct mlx5dv_devx_uar **uar_p) +{ + struct mlx5dv_devx_uar *uar; + char buf[512]; + + uar = mlx5dv_devx_alloc_uar(md->super.dev.ibv_context, flags); + if (uar == NULL) { + sprintf(buf, "mlx5dv_devx_alloc_uar(device=%s, flags=0x%x(%s)) " + "failed: %m", uct_ib_device_name(&md->super.dev), flags, title); + if (fallback == NULL) { + ucs_log(log_level, "%s", buf); + } else { + ucs_log(log_level, "%s, fallback to %s", buf, fallback); + } + + return UCS_ERR_NO_MEMORY; + } + + *uar_p = uar; + return UCS_OK; +} +#endif + ucs_status_t uct_ib_mlx5_devx_uar_init(uct_ib_mlx5_devx_uar_t *uar, uct_ib_mlx5_md_t *md, uct_ib_mlx5_mmio_mode_t mmio_mode) { #if HAVE_DEVX - uar->uar = mlx5dv_devx_alloc_uar(md->super.dev.ibv_context, 0); - if (uar->uar == NULL) { - ucs_error("mlx5dv_devx_alloc_uar() failed: %m"); - return UCS_ERR_NO_MEMORY; + ucs_status_t status; + +#if HAVE_DECL_MLX5DV_UAR_ALLOC_TYPE_NC + status = uct_ib_mlx5_devx_alloc_uar(md, UCT_IB_MLX5_UAR_ALLOC_TYPE_WC, + UCS_LOG_LEVEL_DEBUG, "WC", "NC", + &uar->uar); + if (status != UCS_OK) { + status = uct_ib_mlx5_devx_alloc_uar(md, UCT_IB_MLX5_UAR_ALLOC_TYPE_NC, + UCS_LOG_LEVEL_ERROR, "NC", NULL, + &uar->uar); + } +#else + status = uct_ib_mlx5_devx_alloc_uar(md, UCT_IB_MLX5_UAR_ALLOC_TYPE_WC, + UCS_LOG_LEVEL_ERROR, "WC", NULL, + &uar->uar); +#endif + if (status != UCS_OK) { + return status; } uar->super.addr.ptr = uar->uar->reg_addr; diff --git a/src/uct/ib/mlx5/ib_mlx5.h b/src/uct/ib/mlx5/ib_mlx5.h index ed6114d2c11..fd2af474b7a 100644 --- a/src/uct/ib/mlx5/ib_mlx5.h +++ b/src/uct/ib/mlx5/ib_mlx5.h @@ -72,6 +72,15 @@ #define UCT_IB_MLX5_MP_RQ_LAST_MSG_FLAG UCS_BIT(30) /* MP last packet indication */ #define UCT_IB_MLX5_MP_RQ_FILLER_FLAG UCS_BIT(31) /* Filler CQE indicator */ +#if HAVE_DECL_MLX5DV_UAR_ALLOC_TYPE_BF +# define UCT_IB_MLX5_UAR_ALLOC_TYPE_WC MLX5DV_UAR_ALLOC_TYPE_BF +#else +# define UCT_IB_MLX5_UAR_ALLOC_TYPE_WC 0 +#endif + +#if HAVE_DECL_MLX5DV_UAR_ALLOC_TYPE_NC +# define UCT_IB_MLX5_UAR_ALLOC_TYPE_NC MLX5DV_UAR_ALLOC_TYPE_NC +#endif #define UCT_IB_MLX5_OPMOD_EXT_ATOMIC(_log_arg_size) \ ((8) | ((_log_arg_size) - 2)) From 515e0167c9f7bca834b978601681bb99639b6a14 Mon Sep 17 00:00:00 2001 From: Yossi Itigin Date: Sat, 2 Jan 2021 19:56:02 +0200 Subject: [PATCH 099/107] UCP/WORKER: Fix mis-spelling --- src/ucp/core/ucp_worker.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ucp/core/ucp_worker.c b/src/ucp/core/ucp_worker.c index e04b8f33830..eaa0b4ae89e 100644 --- a/src/ucp/core/ucp_worker.c +++ b/src/ucp/core/ucp_worker.c @@ -2363,7 +2363,7 @@ ucs_status_t ucp_worker_query(ucp_worker_h worker, if (attr->field_mask & UCP_WORKER_ATTR_FIELD_ADDRESS) { /* If UCP_WORKER_ATTR_FIELD_ADDRESS_FLAGS is not set, - * pack all tl adresses */ + * pack all tl addresses */ tl_bitmap = UINT64_MAX; if (attr->field_mask & UCP_WORKER_ATTR_FIELD_ADDRESS_FLAGS) { From 802409338b6cf634ffcbbf77b1ddba9b6669a856 Mon Sep 17 00:00:00 2001 From: Yossi Itigin Date: Tue, 5 Jan 2021 00:53:46 +0200 Subject: [PATCH 100/107] CUDA/TEST: Check if libcudart_static is present for memory hook test --- config/m4/cuda.m4 | 24 +++++++++++++++++++----- configure.ac | 1 + contrib/test_jenkins.sh | 30 +++++++++++++++++++----------- test/apps/Makefile.am | 8 +++++--- 4 files changed, 44 insertions(+), 19 deletions(-) diff --git a/config/m4/cuda.m4 b/config/m4/cuda.m4 index 1862eb6148c..1e9a8ea31e2 100644 --- a/config/m4/cuda.m4 +++ b/config/m4/cuda.m4 @@ -12,7 +12,10 @@ AS_IF([test "x$cuda_checked" != "xyes"], [], [with_cuda=guess]) AS_IF([test "x$with_cuda" = "xno"], - [cuda_happy=no], + [ + cuda_happy=no + have_cuda_static=no + ], [ save_CPPFLAGS="$CPPFLAGS" save_LDFLAGS="$LDFLAGS" @@ -40,11 +43,21 @@ AS_IF([test "x$cuda_checked" != "xyes"], # Check cuda libraries AS_IF([test "x$cuda_happy" = "xyes"], - [AC_CHECK_LIB([cuda], [cuDeviceGetUuid], - [CUDA_LDFLAGS="$CUDA_LDFLAGS -lcuda"], [cuda_happy="no"])]) + [AC_CHECK_LIB([cuda], [cuDeviceGetUuid], + [CUDA_LDFLAGS="$CUDA_LDFLAGS -lcuda"], [cuda_happy="no"])]) + AS_IF([test "x$cuda_happy" = "xyes"], + [AC_CHECK_LIB([cudart], [cudaGetDeviceCount], + [CUDA_LDFLAGS="$CUDA_LDFLAGS -lcudart"], [cuda_happy="no"])]) + + LDFLAGS="$save_LDFLAGS" + + # Check for cuda static library + have_cuda_static="no" AS_IF([test "x$cuda_happy" = "xyes"], - [AC_CHECK_LIB([cudart], [cudaGetDeviceCount], - [CUDA_LDFLAGS="$CUDA_LDFLAGS -lcudart"], [cuda_happy="no"])]) + [AC_CHECK_LIB([cudart_static], [cudaGetDeviceCount], + [AC_SUBST([CUDA_STATIC_LIB], [-lcudart_static]) + have_cuda_static="yes"], + [], [-ldl -lrt -lpthread])]) CPPFLAGS="$save_CPPFLAGS" LDFLAGS="$save_LDFLAGS" @@ -61,6 +74,7 @@ AS_IF([test "x$cuda_checked" != "xyes"], cuda_checked=yes AM_CONDITIONAL([HAVE_CUDA], [test "x$cuda_happy" != xno]) + AM_CONDITIONAL([HAVE_CUDA_STATIC], [test "X$have_cuda_static" = "Xyes"]) ]) # "x$cuda_checked" != "xyes" diff --git a/configure.ac b/configure.ac index b6f1db0cb4d..426916bad72 100644 --- a/configure.ac +++ b/configure.ac @@ -185,6 +185,7 @@ AS_IF([test "x$with_docs_only" = xyes], AM_CONDITIONAL([HAVE_TL_CM], [false]) AM_CONDITIONAL([HAVE_CRAY_UGNI], [false]) AM_CONDITIONAL([HAVE_CUDA], [false]) + AM_CONDITIONAL([HAVE_CUDA_STATIC], [false]) AM_CONDITIONAL([HAVE_GDR_COPY], [false]) AM_CONDITIONAL([HAVE_ROCM], [false]) AM_CONDITIONAL([HAVE_HIP], [false]) diff --git a/contrib/test_jenkins.sh b/contrib/test_jenkins.sh index 90af015b124..39c1a8d5898 100755 --- a/contrib/test_jenkins.sh +++ b/contrib/test_jenkins.sh @@ -1365,28 +1365,36 @@ test_malloc_hook() { if [ "X$have_cuda" == "Xyes" ] then + cuda_dynamic_exe=./test/apps/test_cuda_hook_dynamic + cuda_static_exe=./test/apps/test_cuda_hook_static + for mode in reloc bistro do export UCX_MEM_CUDA_HOOK_MODE=${mode} # Run cuda memory hooks with dynamic link - ./test/apps/test_cuda_hook_dynamic + ${cuda_dynamic_exe} - # Run cuda memory hooks with static link - ./test/apps/test_cuda_hook_static && status="pass" || status="fail" - [ ${mode} == "bistro" ] && exp_status="pass" || exp_status="fail" - if [ ${status} == ${exp_status} ] + # Run cuda memory hooks with static link, if exists. If the static + # library 'libcudart_static.a' is not present, static test will not + # be built. + if [ -x ${cuda_static_exe} ] then - echo "Static link with cuda ${status}, as expected" - else - echo "Static link with cuda is expected to ${exp_status}, actual: ${status}" - exit 1 + ${cuda_static_exe} && status="pass" || status="fail" + [ ${mode} == "bistro" ] && exp_status="pass" || exp_status="fail" + if [ ${status} == ${exp_status} ] + then + echo "Static link with cuda ${status}, as expected" + else + echo "Static link with cuda is expected to ${exp_status}, actual: ${status}" + exit 1 + fi fi # Test that driver API hooks work in both reloc and bistro modes, # since we call them directly from the test - ./test/apps/test_cuda_hook_dynamic -d - ./test/apps/test_cuda_hook_static -d + ${cuda_dynamic_exe} -d + [ -x ${cuda_static_exe} ] && ${cuda_static_exe} -d unset UCX_MEM_CUDA_HOOK_MODE done diff --git a/test/apps/Makefile.am b/test/apps/Makefile.am index 2ada50f7bef..0a413f7af63 100644 --- a/test/apps/Makefile.am +++ b/test/apps/Makefile.am @@ -55,8 +55,7 @@ test_init_mt_CFLAGS = $(BASE_CFLAGS) $(OPENMP_CFLAGS) test_init_mt_LDADD = $(top_builddir)/src/ucp/libucp.la if HAVE_CUDA -noinst_PROGRAMS += test_cuda_hook_dynamic \ - test_cuda_hook_static +noinst_PROGRAMS += test_cuda_hook_dynamic test_cuda_hook_dynamic_SOURCES = test_cuda_hook.c test_cuda_hook_dynamic_CPPFLAGS = $(BASE_CPPFLAGS) $(CUDA_CPPFLAGS) @@ -65,15 +64,18 @@ test_cuda_hook_dynamic_LDFLAGS = $(CUDA_LDFLAGS) test_cuda_hook_dynamic_LDADD = $(top_builddir)/src/ucp/libucp.la \ $(top_builddir)/src/ucm/libucm.la +if HAVE_CUDA_STATIC noinst_PROGRAMS += test_cuda_hook_static test_cuda_hook_static_SOURCES = test_cuda_hook.c test_cuda_hook_static_CPPFLAGS = $(BASE_CPPFLAGS) $(CUDA_CPPFLAGS) test_cuda_hook_static_CFLAGS = $(BASE_CFLAGS) -test_cuda_hook_static_LDFLAGS = -lcudart_static -lcuda -lrt -ldl -lpthread +test_cuda_hook_static_LDFLAGS = $(CUDA_STATIC_LIB) -lcuda -lrt -ldl -lpthread test_cuda_hook_static_LDADD = $(top_builddir)/src/ucp/libucp.la \ $(top_builddir)/src/ucm/libucm.la endif +endif + if HAVE_TCMALLOC noinst_PROGRAMS += test_tcmalloc test_tcmalloc_SOURCES = test_tcmalloc.c From 219a50007d41f14047fc25fe15128f87dcb062ba Mon Sep 17 00:00:00 2001 From: Peter Rudenko Date: Thu, 31 Dec 2020 13:21:27 +0200 Subject: [PATCH 101/107] UCP/EP: set callback on close request. --- src/ucp/core/ucp_ep.c | 2 +- src/ucp/wireup/wireup_cm.c | 8 +++++--- src/ucp/wireup/wireup_cm.h | 3 ++- test/gtest/ucp/test_ucp_sockaddr.cc | 32 +++++++++++++++++++++++++++++ 4 files changed, 40 insertions(+), 5 deletions(-) diff --git a/src/ucp/core/ucp_ep.c b/src/ucp/core/ucp_ep.c index 53bed9e5a2f..1ae21869291 100644 --- a/src/ucp/core/ucp_ep.c +++ b/src/ucp/core/ucp_ep.c @@ -1063,7 +1063,7 @@ ucs_status_ptr_t ucp_ep_close_nbx(ucp_ep_h ep, const ucp_request_param_t *param) if (ucp_ep_is_cm_local_connected(ep)) { /* lanes already flushed, start disconnect on CM lane */ ucp_ep_cm_disconnect_cm_lane(ep); - close_req = ucp_ep_cm_close_request_get(ep); + close_req = ucp_ep_cm_close_request_get(ep, param); if (close_req != NULL) { request = close_req + 1; ucp_ep_set_close_request(ep, close_req, "close"); diff --git a/src/ucp/wireup/wireup_cm.c b/src/ucp/wireup/wireup_cm.c index 305bb203538..e1e6c327bed 100644 --- a/src/ucp/wireup/wireup_cm.c +++ b/src/ucp/wireup/wireup_cm.c @@ -102,7 +102,7 @@ static int ucp_cm_client_try_fallback_cms(ucp_ep_h ep) static ucp_rsc_index_t ucp_cm_tl_bitmap_get_dev_idx(ucp_context_h context, uint64_t tl_bitmap) -{ +{ ucp_rsc_index_t rsc_index; ucp_rsc_index_t dev_index; @@ -1205,9 +1205,9 @@ void ucp_ep_cm_disconnect_cm_lane(ucp_ep_h ucp_ep) } } -ucp_request_t* ucp_ep_cm_close_request_get(ucp_ep_h ep) +ucp_request_t* ucp_ep_cm_close_request_get(ucp_ep_h ep, const ucp_request_param_t *param) { - ucp_request_t *request = ucp_request_get(ep->worker); + ucp_request_t *request = ucp_request_get_param(ep->worker, param, {return NULL;}); if (request == NULL) { ucs_error("failed to allocate close request for ep %p", ep); @@ -1219,6 +1219,8 @@ ucp_request_t* ucp_ep_cm_close_request_get(ucp_ep_h ep) request->send.ep = ep; request->send.flush.uct_flags = UCT_FLUSH_FLAG_LOCAL; + ucp_request_set_send_callback_param(param, request, send); + return request; } diff --git a/src/ucp/wireup/wireup_cm.h b/src/ucp/wireup/wireup_cm.h index 891b53f70e6..14f5ac0f6f6 100644 --- a/src/ucp/wireup/wireup_cm.h +++ b/src/ucp/wireup/wireup_cm.h @@ -49,7 +49,8 @@ ucp_ep_cm_server_create_connected(ucp_worker_h worker, unsigned ep_init_flags, void ucp_ep_cm_disconnect_cm_lane(ucp_ep_h ucp_ep); -ucp_request_t* ucp_ep_cm_close_request_get(ucp_ep_h ep); +ucp_request_t* ucp_ep_cm_close_request_get(ucp_ep_h ep, + const ucp_request_param_t *param); void ucp_ep_cm_slow_cbq_cleanup(ucp_ep_h ep); diff --git a/test/gtest/ucp/test_ucp_sockaddr.cc b/test/gtest/ucp/test_ucp_sockaddr.cc index 50b69241c70..211a9edecfe 100644 --- a/test/gtest/ucp/test_ucp_sockaddr.cc +++ b/test/gtest/ucp/test_ucp_sockaddr.cc @@ -511,6 +511,11 @@ class test_ucp_sockaddr : public ucp_test { receiver().close_ep_req_free(receiver_ep_close_req); } + static void close_completion(void *request, ucs_status_t status, + void *user_data) { + *reinterpret_cast(user_data) = true; + } + static void err_handler_cb(void *arg, ucp_ep_h ep, ucs_status_t status) { ucp_test::err_handler_cb(arg, ep, status); @@ -616,6 +621,33 @@ UCS_TEST_P(test_ucp_sockaddr, onesided_disconnect_bidi) { one_sided_disconnect(sender(), UCP_EP_CLOSE_MODE_FLUSH); } +UCS_TEST_P(test_ucp_sockaddr, close_callback) { + listen_and_communicate(false, SEND_DIRECTION_BIDI); + + request_wait(receiver().flush_ep_nb()); + request_wait(sender().flush_ep_nb()); + ucp_ep_h ep = receiver().revoke_ep(); + + bool user_data = false; + + ucp_request_param_t param = {0}; + param.op_attr_mask = UCP_OP_ATTR_FIELD_CALLBACK | + UCP_OP_ATTR_FIELD_USER_DATA | + UCP_OP_ATTR_FLAG_NO_IMM_CMPL; + param.cb.send = close_completion; + param.user_data = &user_data; + + ucs_status_ptr_t request = ucp_ep_close_nbx(ep, ¶m); + + bool is_pointer = UCS_PTR_IS_PTR(request); + request_wait(request); + + if (is_pointer) { + ASSERT_TRUE(user_data); + } +} + + UCS_TEST_SKIP_COND_P(test_ucp_sockaddr, onesided_disconnect_bidi_wait_err_cb, no_close_protocol()) { listen_and_communicate(false, SEND_DIRECTION_BIDI); From f2610403da39d57f4edb39b1b5fd09cf0204b912 Mon Sep 17 00:00:00 2001 From: Peter Rudenko Date: Sat, 12 Dec 2020 15:30:11 +0200 Subject: [PATCH 102/107] JUCX: add request status field. --- .../java/org/openucx/jucx/ucp/UcpRequest.java | 10 ++++++++++ .../java/src/main/native/jucx_common_def.cc | 17 +++++++++++++---- bindings/java/src/main/native/jucx_common_def.h | 6 ++++++ .../java/org/openucx/jucx/UcpRequestTest.java | 2 ++ 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/bindings/java/src/main/java/org/openucx/jucx/ucp/UcpRequest.java b/bindings/java/src/main/java/org/openucx/jucx/ucp/UcpRequest.java index 2761f1f4559..63152c68e70 100644 --- a/bindings/java/src/main/java/org/openucx/jucx/ucp/UcpRequest.java +++ b/bindings/java/src/main/java/org/openucx/jucx/ucp/UcpRequest.java @@ -7,6 +7,7 @@ import org.openucx.jucx.UcxCallback; import org.openucx.jucx.UcxNativeStruct; +import org.openucx.jucx.ucs.UcsConstants; import java.io.Closeable; import java.nio.ByteBuffer; @@ -21,6 +22,8 @@ public class UcpRequest extends UcxNativeStruct implements Closeable { private long senderTag; + private int status = UcsConstants.STATUS.UCS_INPROGRESS; + private UcpRequest(long nativeId) { setNativeId(nativeId); } @@ -52,6 +55,13 @@ public boolean isCompleted() { return (getNativeId() == null) || isCompletedNative(getNativeId()); } + /** + * @return status of the current request + */ + public int getStatus() { + return status; + } + /** * This routine releases the non-blocking request back to the library, regardless * of its current state. Communications operations associated with this request diff --git a/bindings/java/src/main/native/jucx_common_def.cc b/bindings/java/src/main/native/jucx_common_def.cc index 913edc66dfd..0ba06503ccb 100644 --- a/bindings/java/src/main/native/jucx_common_def.cc +++ b/bindings/java/src/main/native/jucx_common_def.cc @@ -20,6 +20,7 @@ static jclass jucx_request_cls; static jfieldID native_id_field; static jfieldID recv_size_field; static jfieldID sender_tag_field; +static jfieldID request_status; static jmethodID on_success; static jmethodID jucx_request_constructor; static jclass ucp_rkey_cls; @@ -40,6 +41,7 @@ extern "C" JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *jvm, void* reserved) { jucx_request_cls = (jclass) env->NewGlobalRef(jucx_request_cls_local); jclass jucx_callback_cls = env->FindClass("org/openucx/jucx/UcxCallback"); native_id_field = env->GetFieldID(jucx_request_cls, "nativeId", "Ljava/lang/Long;"); + request_status = env->GetFieldID(jucx_request_cls, "status", "I"); recv_size_field = env->GetFieldID(jucx_request_cls, "recvSize", "J"); sender_tag_field = env->GetFieldID(jucx_request_cls, "senderTag", "J"); on_success = env->GetMethodID(jucx_callback_cls, "onSuccess", @@ -172,10 +174,16 @@ JNIEnv* get_jni_env() return (JNIEnv*)env; } +void jucx_request_update_status(JNIEnv *env, jobject jucx_request, ucs_status_t status) +{ + env->SetIntField(jucx_request, request_status, status); +} + static inline void set_jucx_request_completed(JNIEnv *env, jobject jucx_request, - struct jucx_context *ctx) + struct jucx_context *ctx, ucs_status_t status) { env->SetObjectField(jucx_request, native_id_field, NULL); + jucx_request_update_status(env, jucx_request, status); if (ctx != NULL) { /* sender_tag and length are initialized to 0, * so try to avoid the overhead of setting them again */ @@ -238,7 +246,7 @@ UCS_PROFILE_FUNC_VOID(jucx_request_callback, (request, status), void *request, u } JNIEnv *env = get_jni_env(); - set_jucx_request_completed(env, ctx->jucx_request, ctx); + set_jucx_request_completed(env, ctx->jucx_request, ctx, status); if (ctx->callback != NULL) { jucx_call_callback(ctx->callback, ctx->jucx_request, status); @@ -285,7 +293,7 @@ UCS_PROFILE_FUNC(jobject, process_request, (request, callback), void *request, j } else { // request was completed whether by progress in other thread or inside // ucp_tag_recv_nb function call. - set_jucx_request_completed(env, jucx_request, ctx); + set_jucx_request_completed(env, jucx_request, ctx, ctx->status); if (callback != NULL) { jucx_call_callback(callback, jucx_request, ctx->status); } @@ -296,7 +304,7 @@ UCS_PROFILE_FUNC(jobject, process_request, (request, callback), void *request, j } else { jmethodID empty_constructor = env->GetMethodID(jucx_request_cls, "", "()V"); jucx_request = env->NewObject(jucx_request_cls, empty_constructor); - set_jucx_request_completed(env, jucx_request, NULL); + set_jucx_request_completed(env, jucx_request, NULL, UCS_PTR_RAW_STATUS(request)); if (UCS_PTR_IS_ERR(request)) { JNU_ThrowExceptionByStatus(env, UCS_PTR_STATUS(request)); if (callback != NULL) { @@ -315,6 +323,7 @@ jobject process_completed_stream_recv(size_t length, jobject callback) jobject jucx_request = env->NewObject(jucx_request_cls, jucx_request_constructor, NULL); env->SetObjectField(jucx_request, native_id_field, NULL); env->SetLongField(jucx_request, recv_size_field, length); + jucx_request_update_status(env, jucx_request, UCS_OK); if (callback != NULL) { jucx_call_callback(callback, jucx_request, UCS_OK); } diff --git a/bindings/java/src/main/native/jucx_common_def.h b/bindings/java/src/main/native/jucx_common_def.h index 6e83266d001..392c4bc8497 100644 --- a/bindings/java/src/main/native/jucx_common_def.h +++ b/bindings/java/src/main/native/jucx_common_def.h @@ -91,6 +91,12 @@ void stream_recv_callback(void *request, ucs_status_t status, size_t length); */ jobject process_request(void *request, jobject callback); +/** + * @ingroup JUCX_REQ + * @brief Utility to update status of JUCX request to corresponding ucx request. + */ +void jucx_request_update_status(JNIEnv *env, jobject jucx_request, ucs_status_t status); + /** * @brief Call java callback on completed stream recv operation, that didn't invoke callback. */ diff --git a/bindings/java/src/test/java/org/openucx/jucx/UcpRequestTest.java b/bindings/java/src/test/java/org/openucx/jucx/UcpRequestTest.java index 072852d02f3..bdca94fc67d 100644 --- a/bindings/java/src/test/java/org/openucx/jucx/UcpRequestTest.java +++ b/bindings/java/src/test/java/org/openucx/jucx/UcpRequestTest.java @@ -6,6 +6,7 @@ import org.junit.Test; import org.openucx.jucx.ucp.*; +import org.openucx.jucx.ucs.UcsConstants; import java.nio.ByteBuffer; import static org.junit.Assert.*; @@ -22,6 +23,7 @@ public void testCancelRequest() throws Exception { worker.progress(); } + assertEquals(UcsConstants.STATUS.UCS_ERR_CANCELED, recv.getStatus()); assertTrue(recv.isCompleted()); assertNull(recv.getNativeId()); From c01b78f2f9296d457451bd65c5c2533f9bd067ce Mon Sep 17 00:00:00 2001 From: Yossi Itigin Date: Wed, 6 Jan 2021 02:35:44 +0200 Subject: [PATCH 103/107] BUILD/CUDA: Separate CUDA_LIBS from CUDA_LDFLAGS Allows passing Cuda -L flag to static build example, without passing -lcudart. --- config/m4/cuda.m4 | 12 +++++++++--- examples/Makefile.am | 14 +++++++++----- src/tools/perf/cuda/Makefile.am | 1 + src/ucm/cuda/Makefile.am | 2 +- src/uct/cuda/Makefile.am | 5 +++-- src/uct/cuda/gdr_copy/Makefile.am | 6 ++++-- test/apps/Makefile.am | 8 +++++--- test/gtest/Makefile.am | 6 ++++-- 8 files changed, 36 insertions(+), 18 deletions(-) diff --git a/config/m4/cuda.m4 b/config/m4/cuda.m4 index 1e9a8ea31e2..74bf0e56435 100644 --- a/config/m4/cuda.m4 +++ b/config/m4/cuda.m4 @@ -19,9 +19,12 @@ AS_IF([test "x$cuda_checked" != "xyes"], [ save_CPPFLAGS="$CPPFLAGS" save_LDFLAGS="$LDFLAGS" + save_LIBS="$LIBS" CUDA_CPPFLAGS="" CUDA_LDFLAGS="" + CUDA_LIBS="" + CUDA_STATIC_LIBS="" AS_IF([test ! -z "$with_cuda" -a "x$with_cuda" != "xyes" -a "x$with_cuda" != "xguess"], [ucx_check_cuda_dir="$with_cuda" @@ -44,10 +47,10 @@ AS_IF([test "x$cuda_checked" != "xyes"], # Check cuda libraries AS_IF([test "x$cuda_happy" = "xyes"], [AC_CHECK_LIB([cuda], [cuDeviceGetUuid], - [CUDA_LDFLAGS="$CUDA_LDFLAGS -lcuda"], [cuda_happy="no"])]) + [CUDA_LIBS="$CUDA_LIBS -lcuda"], [cuda_happy="no"])]) AS_IF([test "x$cuda_happy" = "xyes"], [AC_CHECK_LIB([cudart], [cudaGetDeviceCount], - [CUDA_LDFLAGS="$CUDA_LDFLAGS -lcudart"], [cuda_happy="no"])]) + [CUDA_LIBS="$CUDA_LIBS -lcudart"], [cuda_happy="no"])]) LDFLAGS="$save_LDFLAGS" @@ -55,16 +58,19 @@ AS_IF([test "x$cuda_checked" != "xyes"], have_cuda_static="no" AS_IF([test "x$cuda_happy" = "xyes"], [AC_CHECK_LIB([cudart_static], [cudaGetDeviceCount], - [AC_SUBST([CUDA_STATIC_LIB], [-lcudart_static]) + [CUDA_STATIC_LIBS="$CUDA_STATIC_LIBS -lcudart_static" have_cuda_static="yes"], [], [-ldl -lrt -lpthread])]) CPPFLAGS="$save_CPPFLAGS" LDFLAGS="$save_LDFLAGS" + LIBS="$save_LIBS" AS_IF([test "x$cuda_happy" = "xyes"], [AC_SUBST([CUDA_CPPFLAGS], ["$CUDA_CPPFLAGS"]) AC_SUBST([CUDA_LDFLAGS], ["$CUDA_LDFLAGS"]) + AC_SUBST([CUDA_LIBS], ["$CUDA_LIBS"]) + AC_SUBST([CUDA_STATIC_LIBS], ["$CUDA_STATIC_LIBS"]) AC_DEFINE([HAVE_CUDA], 1, [Enable CUDA support])], [AS_IF([test "x$with_cuda" != "xguess"], [AC_MSG_ERROR([CUDA support is requested but cuda packages cannot be found])], diff --git a/examples/Makefile.am b/examples/Makefile.am index 05cde2765dd..318142f331c 100644 --- a/examples/Makefile.am +++ b/examples/Makefile.am @@ -13,18 +13,20 @@ dist_examples_DATA = \ ucp_client_server.c if HAVE_CUDA -EXAMPLE_CUDA_LDFLAGS = $(CUDA_LDFLAGS) +EXAMPLE_CUDA_LD_FLAGS = $(CUDA_LDFLAGS) +EXAMPLE_CUDA_LIBS = $(CUDA_LIBS) # cuda.h couldn't be compiled with -pedantic flag EXAMPLE_CUDA_CFLAGS = EXAMPLE_CUDA_CPPFLAGS = $(CUDA_CPPFLAGS) -DHAVE_CUDA else -EXAMPLE_CUDA_LDFLAGS = +EXAMPLE_CUDA_LD_FLAGS = +EXAMPLE_CUDA_LIBS = EXAMPLE_CUDA_CFLAGS = $(CFLAGS_PEDANTIC) EXAMPLE_CUDA_CPPFLAGS = endif EXAMPLE_CCLD_FLAGS = -lucs -I$(includedir) -L$(libdir) -Wall -Werror -Wl,-rpath,$(libdir) \ - $(EXAMPLE_CUDA_LDFLAGS) $(EXAMPLE_CUDA_CPPFLAGS) + $(EXAMPLE_CUDA_LD_FLAGS) $(EXAMPLE_CUDA_LIBS) $(EXAMPLE_CUDA_CPPFLAGS) installcheck-local: @echo "INSTALLCHECK: Compiling examples with installed library" @@ -43,16 +45,18 @@ bin_PROGRAMS = \ ucp_hello_world_SOURCES = ucp_hello_world.c ucp_hello_world_CFLAGS = $(BASE_CFLAGS) $(EXAMPLE_CUDA_CFLAGS) ucp_hello_world_CPPFLAGS = $(BASE_CPPFLAGS) $(EXAMPLE_CUDA_CPPFLAGS) +ucp_hello_world_LDFLAGS = $(EXAMPLE_CUDA_LD_FLAGS) ucp_hello_world_LDADD = $(top_builddir)/src/ucs/libucs.la \ $(top_builddir)/src/ucp/libucp.la \ - $(EXAMPLE_CUDA_LDFLAGS) + $(EXAMPLE_CUDA_LIBS) uct_hello_world_SOURCES = uct_hello_world.c uct_hello_world_CFLAGS = $(BASE_CFLAGS) $(EXAMPLE_CUDA_CFLAGS) uct_hello_world_CPPFLAGS = $(BASE_CPPFLAGS) $(EXAMPLE_CUDA_CPPFLAGS) +uct_hello_world_LDFLAGS = $(EXAMPLE_CUDA_LD_FLAGS) uct_hello_world_LDADD = $(top_builddir)/src/ucs/libucs.la \ $(top_builddir)/src/uct/libuct.la \ - $(EXAMPLE_CUDA_LDFLAGS) + $(EXAMPLE_CUDA_LIBS) ucp_client_server_SOURCES = ucp_client_server.c ucp_client_server_CFLAGS = $(BASE_CFLAGS) $(CFLAGS_PEDANTIC) diff --git a/src/tools/perf/cuda/Makefile.am b/src/tools/perf/cuda/Makefile.am index aa6cb37065a..ecb7a33c0f7 100644 --- a/src/tools/perf/cuda/Makefile.am +++ b/src/tools/perf/cuda/Makefile.am @@ -10,6 +10,7 @@ module_LTLIBRARIES = libucx_perftest_cuda.la libucx_perftest_cuda_la_CPPFLAGS = $(BASE_CPPFLAGS) $(CUDA_CPPFLAGS) libucx_perftest_cuda_la_CFLAGS = $(BASE_CFLAGS) $(CUDA_CFLAGS) libucx_perftest_cuda_la_LDFLAGS = $(CUDA_LDFLAGS) -version-info $(SOVERSION) +libucx_perftest_cuda_la_LIBADD = $(CUDA_LIBS) libucx_perftest_cuda_la_SOURCES = cuda_alloc.c include $(top_srcdir)/config/module.am diff --git a/src/ucm/cuda/Makefile.am b/src/ucm/cuda/Makefile.am index 438960e2c28..03fd621f97a 100644 --- a/src/ucm/cuda/Makefile.am +++ b/src/ucm/cuda/Makefile.am @@ -9,7 +9,7 @@ if HAVE_CUDA module_LTLIBRARIES = libucm_cuda.la libucm_cuda_la_CPPFLAGS = $(BASE_CPPFLAGS) $(CUDA_CPPFLAGS) libucm_cuda_la_CFLAGS = $(BASE_CFLAGS) $(CUDA_CFLAGS) -libucm_cuda_la_LIBADD = ../libucm.la +libucm_cuda_la_LIBADD = ../libucm.la $(CUDA_LIBS) libucm_cuda_la_LDFLAGS = $(UCM_MODULE_LDFLAGS) \ $(patsubst %, -Xlinker %, $(CUDA_LDFLAGS)) \ -version-info $(SOVERSION) diff --git a/src/uct/cuda/Makefile.am b/src/uct/cuda/Makefile.am index 0992bb4d180..647092eda46 100644 --- a/src/uct/cuda/Makefile.am +++ b/src/uct/cuda/Makefile.am @@ -10,9 +10,10 @@ SUBDIRS = . gdr_copy module_LTLIBRARIES = libuct_cuda.la libuct_cuda_la_CPPFLAGS = $(BASE_CPPFLAGS) $(CUDA_CPPFLAGS) libuct_cuda_la_CFLAGS = $(BASE_CFLAGS) $(CUDA_CFLAGS) -libuct_cuda_la_LIBADD = $(top_builddir)/src/ucs/libucs.la \ - $(top_builddir)/src/uct/libuct.la libuct_cuda_la_LDFLAGS = $(CUDA_LDFLAGS) -version-info $(SOVERSION) +libuct_cuda_la_LIBADD = $(top_builddir)/src/ucs/libucs.la \ + $(top_builddir)/src/uct/libuct.la \ + $(CUDA_LIBS) noinst_HEADERS = \ base/cuda_md.h \ diff --git a/src/uct/cuda/gdr_copy/Makefile.am b/src/uct/cuda/gdr_copy/Makefile.am index e551d5f5862..752fc984523 100644 --- a/src/uct/cuda/gdr_copy/Makefile.am +++ b/src/uct/cuda/gdr_copy/Makefile.am @@ -8,9 +8,11 @@ if HAVE_GDR_COPY module_LTLIBRARIES = libuct_cuda_gdrcopy.la libuct_cuda_gdrcopy_la_CPPFLAGS = $(BASE_CPPFLAGS) $(CUDA_CPPFLAGS) $(GDR_COPY_CPPFLAGS) libuct_cuda_gdrcopy_la_CFLAGS = $(BASE_CFLAGS) +libuct_cuda_gdrcopy_la_LDFLAGS = $(CUDA_LDFLAGS) $(GDR_COPY_LDFLAGS) \ + -version-info $(SOVERSION) libuct_cuda_gdrcopy_la_LIBADD = $(top_builddir)/src/ucs/libucs.la \ - $(top_builddir)/src/uct/cuda/libuct_cuda.la -libuct_cuda_gdrcopy_la_LDFLAGS = $(CUDA_LDFLAGS) $(GDR_COPY_LDFLAGS) -version-info $(SOVERSION) + $(top_builddir)/src/uct/cuda/libuct_cuda.la \ + $(CUDA_LIBS) noinst_HEADERS = \ gdr_copy_md.h \ diff --git a/test/apps/Makefile.am b/test/apps/Makefile.am index 0a413f7af63..2ed94c8a16f 100644 --- a/test/apps/Makefile.am +++ b/test/apps/Makefile.am @@ -62,16 +62,18 @@ test_cuda_hook_dynamic_CPPFLAGS = $(BASE_CPPFLAGS) $(CUDA_CPPFLAGS) test_cuda_hook_dynamic_CFLAGS = $(BASE_CFLAGS) test_cuda_hook_dynamic_LDFLAGS = $(CUDA_LDFLAGS) test_cuda_hook_dynamic_LDADD = $(top_builddir)/src/ucp/libucp.la \ - $(top_builddir)/src/ucm/libucm.la + $(top_builddir)/src/ucm/libucm.la \ + $(CUDA_LIBS) if HAVE_CUDA_STATIC noinst_PROGRAMS += test_cuda_hook_static test_cuda_hook_static_SOURCES = test_cuda_hook.c test_cuda_hook_static_CPPFLAGS = $(BASE_CPPFLAGS) $(CUDA_CPPFLAGS) test_cuda_hook_static_CFLAGS = $(BASE_CFLAGS) -test_cuda_hook_static_LDFLAGS = $(CUDA_STATIC_LIB) -lcuda -lrt -ldl -lpthread +test_cuda_hook_static_LDFLAGS = $(CUDA_LDFLAGS) test_cuda_hook_static_LDADD = $(top_builddir)/src/ucp/libucp.la \ - $(top_builddir)/src/ucm/libucm.la + $(top_builddir)/src/ucm/libucm.la \ + $(CUDA_STATIC_LIBS) -lcuda -lrt -ldl -lpthread endif endif diff --git a/test/gtest/Makefile.am b/test/gtest/Makefile.am index 0f3d24b30c8..202c7103391 100644 --- a/test/gtest/Makefile.am +++ b/test/gtest/Makefile.am @@ -206,7 +206,7 @@ endif if HAVE_TL_RC gtest_SOURCES += \ uct/ib/test_rc.cc -endif +endif if HAVE_TL_DC gtest_SOURCES += \ uct/ib/test_dc.cc @@ -222,8 +222,10 @@ gtest_SOURCES += \ ucm/cuda_hooks.cc gtest_CPPFLAGS += \ $(CUDA_CPPFLAGS) +gtest_LDFLAGS += \ + $(CUDA_LDFLAGS) gtest_LDADD += \ - $(CUDA_LDFLAGS) \ + $(CUDA_LIBS) \ $(top_builddir)/src/uct/cuda/libuct_cuda.la endif From 389768e36f56941bdfb8dad0bb12e31df363f31f Mon Sep 17 00:00:00 2001 From: dmitrygx Date: Sat, 2 Jan 2021 22:52:11 +0000 Subject: [PATCH 104/107] UCP/WIREUP: Allow EP check during WIREUP phase --- src/ucp/core/ucp_ep.c | 16 +++++++- src/ucp/core/ucp_ep.h | 4 +- src/ucp/core/ucp_request.h | 2 - src/ucp/core/ucp_worker.c | 61 +++++++++++++++---------------- src/ucp/core/ucp_worker.h | 2 +- src/ucp/wireup/select.c | 27 +++++++++++++- src/ucp/wireup/wireup.c | 3 +- src/ucp/wireup/wireup_ep.c | 32 +++++++++++++++- test/gtest/ucp/test_ucp_worker.cc | 10 +++-- 9 files changed, 111 insertions(+), 46 deletions(-) diff --git a/src/ucp/core/ucp_ep.c b/src/ucp/core/ucp_ep.c index 4576bd56328..26b338ec8d2 100644 --- a/src/ucp/core/ucp_ep.c +++ b/src/ucp/core/ucp_ep.c @@ -108,6 +108,7 @@ ucs_status_t ucp_ep_create_base(ucp_worker_h worker, const char *peer_name, goto err_free_ep; } + ep->ref_cnt = 1; ep->cfg_index = UCP_WORKER_CFG_INDEX_NULL; ep->worker = worker; ep->am_lane = UCP_NULL_LANE; @@ -157,6 +158,10 @@ ucs_status_t ucp_ep_create_base(ucp_worker_h worker, const char *peer_name, void ucp_ep_destroy_base(ucp_ep_h ep) { + if (--ep->ref_cnt != 0) { + return; + } + UCS_STATS_NODE_FREE(ep->stats); ucs_free(ucp_ep_ext_control(ep)); ucs_strided_alloc_put(&ep->worker->ep_alloc, ep); @@ -204,7 +209,7 @@ ucs_status_t ucp_worker_create_ep(ucp_worker_h worker, unsigned ep_init_flags, return status; } -void ucp_ep_delete(ucp_ep_h ep) +static void ucp_ep_delete(ucp_ep_h ep) { ucs_status_t status; @@ -1011,7 +1016,7 @@ void ucp_ep_discard_lanes(ucp_ep_h ep, ucs_status_t status) ucs_trace("ep %p: discard uct_ep[%d]=%p", ep, lane, ep->uct_eps[lane]); - ucp_worker_discard_uct_ep(ep->worker, ep->uct_eps[lane], + ucp_worker_discard_uct_ep(ep, ep->uct_eps[lane], UCT_FLUSH_FLAG_CANCEL, ucp_ep_err_pending_purge, UCS_STATUS_PTR(status)); @@ -1043,6 +1048,12 @@ ucs_status_ptr_t ucp_ep_close_nbx(ucp_ep_h ep, const ucp_request_param_t *param) UCS_ASYNC_BLOCK(&worker->async); + if (ep->flags & UCP_EP_FLAG_CLOSED) { + ucs_error("ep %p has already been closed", ep); + request = UCS_STATUS_PTR(UCS_ERR_NOT_CONNECTED); + goto out; + } + ep->flags |= UCP_EP_FLAG_CLOSED; if (ucp_request_param_flags(param) & UCP_EP_CLOSE_FLAG_FORCE) { @@ -1076,6 +1087,7 @@ ucs_status_ptr_t ucp_ep_close_nbx(ucp_ep_h ep, const ucp_request_param_t *param) } } +out: UCS_ASYNC_UNBLOCK(&worker->async); return request; } diff --git a/src/ucp/core/ucp_ep.h b/src/ucp/core/ucp_ep.h index c165e8783db..dc96334adbd 100644 --- a/src/ucp/core/ucp_ep.h +++ b/src/ucp/core/ucp_ep.h @@ -351,6 +351,8 @@ struct ucp_ep_config { typedef struct ucp_ep { ucp_worker_h worker; /* Worker this endpoint belongs to */ + uint8_t ref_cnt; /* Reference counter: 0 - it is + allowed to destroy EP */ ucp_worker_cfg_index_t cfg_index; /* Configuration index */ ucp_ep_match_conn_sn_t conn_sn; /* Sequence number for remote connection */ ucp_lane_index_t am_lane; /* Cached value */ @@ -507,8 +509,6 @@ ucs_status_t ucp_worker_create_ep(ucp_worker_h worker, unsigned ep_init_flags, const char *peer_name, const char *message, ucp_ep_h *ep_p); -void ucp_ep_delete(ucp_ep_h ep); - ucs_status_t ucp_ep_init_create_wireup(ucp_ep_h ep, unsigned ep_init_flags, ucp_wireup_ep_t **wireup_ep); diff --git a/src/ucp/core/ucp_request.h b/src/ucp/core/ucp_request.h index a583bbdf284..332023a6cc3 100644 --- a/src/ucp/core/ucp_request.h +++ b/src/ucp/core/ucp_request.h @@ -246,8 +246,6 @@ struct ucp_request { } disconnect; struct { - ucp_worker_h ucp_worker; /* UCP worker where a discard UCT EP - * operation submitted on */ uct_ep_h uct_ep; /* UCT EP that should be flushed and destroyed */ unsigned ep_flush_flags; /* Flags that should be passed into diff --git a/src/ucp/core/ucp_worker.c b/src/ucp/core/ucp_worker.c index eaa0b4ae89e..e0c2e44d85c 100644 --- a/src/ucp/core/ucp_worker.c +++ b/src/ucp/core/ucp_worker.c @@ -586,11 +586,9 @@ ucp_worker_iface_handle_uct_ep_failure(ucp_ep_h ucp_ep, ucp_lane_index_t lane, aux_uct_ep = wireup_ep->aux_ep; ucp_wireup_ep_disown(ucp_ep->uct_eps[lane], aux_uct_ep); - ucp_worker_discard_uct_ep(ucp_ep->worker, aux_uct_ep, - UCT_FLUSH_FLAG_CANCEL, + ucp_worker_discard_uct_ep(ucp_ep, aux_uct_ep, UCT_FLUSH_FLAG_CANCEL, (uct_pending_purge_callback_t) - ucs_empty_function_do_assert, - NULL); + ucs_empty_function_do_assert, NULL); return UCS_OK; } @@ -2161,18 +2159,27 @@ ucs_status_t ucp_worker_create(ucp_context_h context, return status; } +static void ucp_worker_put_flush_req(ucp_request_t *req) +{ + ucp_ep_h ucp_ep = req->send.ep; + + ucp_worker_flush_ops_count_dec(ucp_ep->worker); + ucp_ep_destroy_base(ucp_ep); + ucp_request_put(req); +} + static unsigned ucp_worker_discard_uct_ep_destroy_progress(void *arg) { ucp_request_t *req = (ucp_request_t*)arg; uct_ep_h uct_ep = req->send.discard_uct_ep.uct_ep; - ucp_worker_h worker = req->send.discard_uct_ep.ucp_worker; + ucp_ep_h ucp_ep = req->send.ep; + ucp_worker_h worker = ucp_ep->worker; khiter_t iter; ucp_trace_req(req, "destroy uct_ep=%p", uct_ep); - ucp_request_put(req); UCS_ASYNC_BLOCK(&worker->async); - ucp_worker_flush_ops_count_dec(worker); + ucp_worker_put_flush_req(req); iter = kh_get(ucp_worker_discard_uct_ep_hash, &worker->discard_uct_ep_hash, uct_ep); if (iter == kh_end(&worker->discard_uct_ep_hash)) { @@ -2188,21 +2195,13 @@ static unsigned ucp_worker_discard_uct_ep_destroy_progress(void *arg) return 1; } -static void ucp_worker_put_flush_req(ucp_request_t *req) -{ - ucp_worker_h worker = req->send.discard_uct_ep.ucp_worker; - - ucp_worker_flush_ops_count_dec(worker); - ucp_request_put(req); -} - static void ucp_worker_discard_uct_ep_flush_comp(uct_completion_t *self) { uct_worker_cb_id_t cb_id = UCS_CALLBACKQ_ID_NULL; ucp_request_t *req = ucs_container_of(self, ucp_request_t, send.state.uct_comp); - ucp_worker_h worker = req->send.discard_uct_ep.ucp_worker; + ucp_worker_h worker = req->send.ep->worker; ucp_trace_req(req, "discard_uct_ep flush completion status %s", ucs_status_string(self->status)); @@ -2248,7 +2247,7 @@ static unsigned ucp_worker_discard_uct_ep_progress(void *arg) uct_worker_cb_id_t cb_id = UCS_CALLBACKQ_ID_NULL; ucp_request_t *req = (ucp_request_t*)arg; uct_ep_h uct_ep = req->send.discard_uct_ep.uct_ep; - ucp_worker_h worker = req->send.discard_uct_ep.ucp_worker; + ucp_worker_h worker = req->send.ep->worker; ucs_status_t status; status = ucp_worker_discard_uct_ep_pending_cb(&req->send.uct); @@ -2778,9 +2777,10 @@ void ucp_worker_keepalive_remove_ep(ucp_ep_h ep) } static void -ucp_worker_discard_tl_uct_ep(ucp_worker_h worker, uct_ep_h uct_ep, +ucp_worker_discard_tl_uct_ep(ucp_ep_h ucp_ep, uct_ep_h uct_ep, unsigned ep_flush_flags) { + ucp_worker_h worker = ucp_ep->worker; ucp_request_t *req; int ret; khiter_t iter; @@ -2792,6 +2792,8 @@ ucp_worker_discard_tl_uct_ep(ucp_worker_h worker, uct_ep_h uct_ep, return; } + ucs_assert(ucp_ep->ref_cnt < UINT8_MAX); + ++ucp_ep->ref_cnt; ucp_worker_flush_ops_count_inc(worker); iter = kh_put(ucp_worker_discard_uct_ep_hash, &worker->discard_uct_ep_hash, uct_ep, &ret); @@ -2806,11 +2808,11 @@ ucp_worker_discard_tl_uct_ep(ucp_worker_h worker, uct_ep_h uct_ep, ucs_assert(!ucp_wireup_ep_test(uct_ep)); req->flags = 0; + req->send.ep = ucp_ep; req->send.uct.func = ucp_worker_discard_uct_ep_pending_cb; req->send.state.uct_comp.func = ucp_worker_discard_uct_ep_flush_comp; req->send.state.uct_comp.count = 1; req->send.state.uct_comp.status = UCS_OK; - req->send.discard_uct_ep.ucp_worker = worker; req->send.discard_uct_ep.uct_ep = uct_ep; req->send.discard_uct_ep.ep_flush_flags = ep_flush_flags; @@ -2818,10 +2820,8 @@ ucp_worker_discard_tl_uct_ep(ucp_worker_h worker, uct_ep_h uct_ep, } static void -ucp_worker_discard_wireup_uct_ep(ucp_worker_h worker, - ucp_wireup_ep_t *wireup_ep, - unsigned ep_flush_flags, - uct_ep_h uct_ep) +ucp_worker_discard_wireup_uct_ep(ucp_ep_h ucp_ep, ucp_wireup_ep_t *wireup_ep, + unsigned ep_flush_flags, uct_ep_h uct_ep) { if (uct_ep == NULL) { return; @@ -2829,7 +2829,7 @@ ucp_worker_discard_wireup_uct_ep(ucp_worker_h worker, ucp_wireup_ep_disown(&wireup_ep->super.super, uct_ep); /* discard the WIREUP EP's UCT EP */ - ucp_worker_discard_uct_ep(worker, uct_ep, ep_flush_flags, + ucp_worker_discard_uct_ep(ucp_ep, uct_ep, ep_flush_flags, /* make sure that there are no WIREUP MSGs * anymore that are scheduled on the UCT EP, i.e. * the purge callback hasn't be invoked here */ @@ -2838,8 +2838,7 @@ ucp_worker_discard_wireup_uct_ep(ucp_worker_h worker, } static uct_ep_h -ucp_worker_discard_wireup_ep(ucp_worker_h worker, - ucp_wireup_ep_t *wireup_ep, +ucp_worker_discard_wireup_ep(ucp_ep_h ucp_ep, ucp_wireup_ep_t *wireup_ep, unsigned ep_flush_flags, uct_pending_purge_callback_t purge_cb, void *purge_arg) @@ -2849,9 +2848,9 @@ ucp_worker_discard_wireup_ep(ucp_worker_h worker, ucs_assert(wireup_ep != NULL); - ucp_worker_discard_wireup_uct_ep(worker, wireup_ep, ep_flush_flags, + ucp_worker_discard_wireup_uct_ep(ucp_ep, wireup_ep, ep_flush_flags, wireup_ep->aux_ep); - ucp_worker_discard_wireup_uct_ep(worker, wireup_ep, ep_flush_flags, + ucp_worker_discard_wireup_uct_ep(ucp_ep, wireup_ep, ep_flush_flags, wireup_ep->sockaddr_ep); is_owner = wireup_ep->super.is_owner; @@ -2875,7 +2874,7 @@ int ucp_worker_is_uct_ep_discarding(ucp_worker_h worker, uct_ep_h uct_ep) } /* must be called with async lock held */ -void ucp_worker_discard_uct_ep(ucp_worker_h worker, uct_ep_h uct_ep, +void ucp_worker_discard_uct_ep(ucp_ep_h ucp_ep, uct_ep_h uct_ep, unsigned ep_flush_flags, uct_pending_purge_callback_t purge_cb, void *purge_arg) @@ -2886,7 +2885,7 @@ void ucp_worker_discard_uct_ep(ucp_worker_h worker, uct_ep_h uct_ep, uct_ep_pending_purge(uct_ep, purge_cb, purge_arg); if (ucp_wireup_ep_test(uct_ep)) { - uct_ep = ucp_worker_discard_wireup_ep(worker, ucp_wireup_ep(uct_ep), + uct_ep = ucp_worker_discard_wireup_ep(ucp_ep, ucp_wireup_ep(uct_ep), ep_flush_flags, purge_cb, purge_arg); if (uct_ep == NULL) { @@ -2894,5 +2893,5 @@ void ucp_worker_discard_uct_ep(ucp_worker_h worker, uct_ep_h uct_ep, } } - ucp_worker_discard_tl_uct_ep(worker, uct_ep, ep_flush_flags); + ucp_worker_discard_tl_uct_ep(ucp_ep, uct_ep, ep_flush_flags); } diff --git a/src/ucp/core/ucp_worker.h b/src/ucp/core/ucp_worker.h index 8232f4d3d0d..7eaa261b973 100644 --- a/src/ucp/core/ucp_worker.h +++ b/src/ucp/core/ucp_worker.h @@ -320,7 +320,7 @@ void ucp_worker_keepalive_remove_ep(ucp_ep_h ep); int ucp_worker_is_uct_ep_discarding(ucp_worker_h worker, uct_ep_h uct_ep); /* must be called with async lock held */ -void ucp_worker_discard_uct_ep(ucp_worker_h worker, uct_ep_h uct_ep, +void ucp_worker_discard_uct_ep(ucp_ep_h ucp_ep, uct_ep_h uct_ep, unsigned ep_flush_flags, uct_pending_purge_callback_t purge_cb, void *purge_arg); diff --git a/src/ucp/wireup/select.c b/src/ucp/wireup/select.c index 077dc337a22..89d9268c8a2 100644 --- a/src/ucp/wireup/select.c +++ b/src/ucp/wireup/select.c @@ -225,6 +225,26 @@ static int ucp_wireup_check_amo_flags(const uct_tl_resource_desc_t *resource, return 0; } +static int +ucp_wireup_check_keepalive(const ucp_wireup_select_params_t *select_params, + const uct_tl_resource_desc_t *resource, + uint64_t flags, uint64_t required_flags, + const char *title, const char **flag_descs, + char *reason, size_t max) +{ + ucp_worker_h worker = select_params->ep->worker; + return /* if error handling and keepalive were requested, UCT iface has to + * support peer failure (i.e. UCT_IFACE_FLAG_ERRHANDLE_PEER_FAILURE) + * and either built-in keepalive (i.e. UCT_IFACE_FLAG_EP_KEEPALIVE) + * or EP checking (i.e. UCT_IFACE_FLAG_EP_CHECK) */ + !ucp_worker_keepalive_is_enabled(worker) || + !(select_params->ep_init_flags & UCP_EP_INIT_ERR_MODE_PEER_FAILURE) || + ucp_wireup_check_flags(resource, flags, UCT_IFACE_FLAG_EP_KEEPALIVE, title, + ucp_wireup_iface_flags, reason, max) || + ucp_wireup_check_flags(resource, flags, UCT_IFACE_FLAG_EP_CHECK, title, + ucp_wireup_iface_flags, reason, max); +} + static void ucp_wireup_init_select_info(double score, unsigned addr_index, ucp_rsc_index_t rsc_index, @@ -361,6 +381,10 @@ ucp_wireup_select_transport(const ucp_wireup_select_params_t *select_params, !ucp_wireup_check_flags(resource, iface_attr->cap.flags, criteria->local_iface_flags, criteria->title, ucp_wireup_iface_flags, p, endp - p) || + !ucp_wireup_check_keepalive(select_params, resource, + iface_attr->cap.flags, + criteria->local_iface_flags, criteria->title, + ucp_wireup_iface_flags, p, endp - p) || !ucp_wireup_check_flags(resource, iface_attr->cap.event_flags, criteria->local_event_flags, criteria->title, ucp_wireup_event_flags, p, endp - p) || @@ -722,6 +746,8 @@ static void ucp_wireup_fill_peer_err_criteria(ucp_wireup_criteria_t *criteria, { if (ep_init_flags & UCP_EP_INIT_ERR_MODE_PEER_FAILURE) { criteria->local_iface_flags |= UCT_IFACE_FLAG_ERRHANDLE_PEER_FAILURE; + /* transport selection procedure will check additionally for KA or EP check + * support */ } } @@ -745,7 +771,6 @@ static void ucp_wireup_fill_aux_criteria(ucp_wireup_criteria_t *criteria, criteria->calc_score = ucp_wireup_aux_score_func; criteria->tl_rsc_flags = UCP_TL_RSC_FLAG_AUX; /* Can use aux transports */ - /* TODO: add evaluation for err handling/keepalive mode */ ucp_wireup_fill_peer_err_criteria(criteria, ep_init_flags); } diff --git a/src/ucp/wireup/wireup.c b/src/ucp/wireup/wireup.c index 1d1f2e2c83d..8bfeecf926e 100644 --- a/src/ucp/wireup/wireup.c +++ b/src/ucp/wireup/wireup.c @@ -1061,7 +1061,6 @@ ucp_wireup_check_config_intersect(ucp_ep_h ep, ucp_lane_index_t *connect_lane_bitmap, ucs_queue_head_t *replay_pending_queue) { - ucp_worker_h worker = ep->worker; uct_ep_h new_uct_eps[UCP_MAX_LANES] = { NULL }; ucp_lane_index_t reuse_lane_map[UCP_MAX_LANES] = { UCP_NULL_LANE }; ucp_ep_config_key_t *old_key; @@ -1126,7 +1125,7 @@ ucp_wireup_check_config_intersect(ucp_ep_h ep, if (reuse_lane == UCP_NULL_RESOURCE) { if (ep->uct_eps[lane] != NULL) { ucs_assert(lane != ucp_ep_get_cm_lane(ep)); - ucp_worker_discard_uct_ep(worker, ep->uct_eps[lane], + ucp_worker_discard_uct_ep(ep, ep->uct_eps[lane], UCT_FLUSH_FLAG_LOCAL, ucp_wireup_pending_purge_cb, replay_pending_queue); diff --git a/src/ucp/wireup/wireup_ep.c b/src/ucp/wireup/wireup_ep.c index 26e201fed08..af66930039f 100644 --- a/src/ucp/wireup/wireup_ep.c +++ b/src/ucp/wireup/wireup_ep.c @@ -331,13 +331,43 @@ static ucs_status_t ucp_wireup_ep_flush(uct_ep_h uct_ep, unsigned flags, return UCS_ERR_NO_RESOURCE; } +static ucs_status_t ucp_wireup_ep_check(uct_ep_h uct_ep, unsigned flags, + uct_completion_t *comp) +{ + ucp_wireup_ep_t *wireup_ep = ucp_wireup_ep(uct_ep); + ucp_worker_h worker = wireup_ep->super.ucp_ep->worker; + + if (wireup_ep->flags & UCP_WIREUP_EP_FLAG_LOCAL_CONNECTED) { + return uct_ep_check(wireup_ep->super.uct_ep, flags, comp); + } + + if (wireup_ep->aux_ep != NULL) { + ucs_assert(wireup_ep->aux_rsc_index != UCP_NULL_RESOURCE); + + if (ucp_worker_iface(worker, wireup_ep->aux_rsc_index)->attr.cap.flags & + UCT_IFACE_FLAG_EP_CHECK) { + return uct_ep_check(wireup_ep->aux_ep, flags, comp); + } + + /* if EP_CHECK is not supported by auxiliary transport, it has to support + * a built-in keepalive mechanism to be able to detect peer failure during + * wireup + */ + ucs_assert(ucp_worker_iface(worker, + wireup_ep->aux_rsc_index)->attr.cap.flags & + UCT_IFACE_FLAG_EP_KEEPALIVE); + } + + return UCS_OK; +} + UCS_CLASS_INIT_FUNC(ucp_wireup_ep_t, ucp_ep_h ucp_ep) { static uct_iface_ops_t ops = { .ep_connect_to_ep = ucp_wireup_ep_connect_to_ep, .ep_flush = ucp_wireup_ep_flush, - .ep_check = ucs_empty_function_return_success, + .ep_check = ucp_wireup_ep_check, .ep_destroy = UCS_CLASS_DELETE_FUNC_NAME(ucp_wireup_ep_t), .ep_pending_add = ucp_wireup_ep_pending_add, .ep_pending_purge = ucp_wireup_ep_pending_purge, diff --git a/test/gtest/ucp/test_ucp_worker.cc b/test/gtest/ucp/test_ucp_worker.cc index 16037dd1c20..3fb08a84f67 100644 --- a/test/gtest/ucp/test_ucp_worker.cc +++ b/test/gtest/ucp/test_ucp_worker.cc @@ -75,18 +75,19 @@ class test_ucp_worker_discard : public ucp_test { unsigned wireup_ep_count = 0, unsigned wireup_aux_ep_count = 0) { uct_iface_ops_t ops = {0}; + ucp_ep_t ucp_ep = {0}; unsigned created_wireup_aux_ep_count = 0; unsigned total_ep_count = ep_count + wireup_aux_ep_count; uct_iface_t iface; std::vector eps(total_ep_count); std::vector wireup_eps(wireup_ep_count); - ucp_ep_t ucp_ep; ucs_status_t status; ASSERT_LE(wireup_ep_count, ep_count); ASSERT_LE(wireup_aux_ep_count, wireup_ep_count); - ucp_ep.worker = sender().worker(); + ucp_ep.worker = sender().worker(); + ucp_ep.ref_cnt = 1u; ops.ep_flush = (uct_ep_flush_func_t)ep_flush_func; ops.ep_pending_add = (uct_ep_pending_add_func_t)ep_pending_add_func; @@ -155,8 +156,7 @@ class test_ucp_worker_discard : public ucp_test { uct_ep_h discard_ep = *iter; unsigned purged_reqs_count = 0; - ucp_worker_discard_uct_ep(sender().worker(), discard_ep, - UCT_FLUSH_FLAG_LOCAL, + ucp_worker_discard_uct_ep(&ucp_ep, discard_ep, UCT_FLUSH_FLAG_LOCAL, ep_pending_purge_count_reqs_cb, &purged_reqs_count); @@ -236,6 +236,8 @@ class test_ucp_worker_discard : public ucp_test { for (unsigned i = 0; i < created_wireup_aux_ep_count; i++) { EXPECT_EQ(NULL, eps[i].iface); } + + EXPECT_EQ(1u, ucp_ep.ref_cnt); } static void ep_destroy_func(uct_ep_h ep) { From 677607abc508833d9a506fdd552d76462b0f1273 Mon Sep 17 00:00:00 2001 From: binl Date: Sat, 28 Nov 2020 12:48:15 +0000 Subject: [PATCH 105/107] UCS/UCT/IB: Get device/vendor id of SF from its parent PCI --- src/ucs/sys/string.c | 12 ++++ src/ucs/sys/string.h | 11 ++++ src/uct/ib/base/ib_device.c | 114 +++++++++++++++++++++++++++++------- test/gtest/ucs/test_sys.cc | 10 ++++ 4 files changed, 127 insertions(+), 20 deletions(-) diff --git a/src/ucs/sys/string.c b/src/ucs/sys/string.c index 98a5a705b44..8955d6a75f2 100644 --- a/src/ucs/sys/string.c +++ b/src/ucs/sys/string.c @@ -19,6 +19,7 @@ #include #include #include +#include const char *ucs_memunits_suffixes[] = {"", "K", "M", "G", "T", "P", "E", NULL}; @@ -199,6 +200,17 @@ ucs_status_t ucs_str_to_memunits(const char *buf, void *dest) return UCS_OK; } +char *ucs_dirname(char *path, int num_layers) +{ + while (num_layers-- > 0) { + path = dirname(path); + if (path == NULL) { + return NULL; + } + } + return path; +} + void ucs_snprintf_safe(char *buf, size_t size, const char *fmt, ...) { va_list ap; diff --git a/src/ucs/sys/string.h b/src/ucs/sys/string.h index c398aee7a86..3a17ea6335c 100644 --- a/src/ucs/sys/string.h +++ b/src/ucs/sys/string.h @@ -59,6 +59,17 @@ void ucs_expand_path(const char *path, char *fullpath, size_t max); void ucs_fill_filename_template(const char *tmpl, char *buf, size_t max); +/** + * Strip specified number of last components from file/dir path + * + * @param path The pointer of file path to be stripped + * @param num_layers The number of components to be stripped + * + * @return Pointer of the stripped dir path. + */ +char *ucs_dirname(char *path, int num_layers); + + /** * Format a string to a buffer of given size, and fill the rest of the buffer * with '\0'. Also, guarantee that the last char in the buffer is '\0'. diff --git a/src/uct/ib/base/ib_device.c b/src/uct/ib/base/ib_device.c index 33528e80341..855c892a2e9 100644 --- a/src/uct/ib/base/ib_device.c +++ b/src/uct/ib/base/ib_device.c @@ -482,24 +482,95 @@ void uct_ib_handle_async_event(uct_ib_device_t *dev, uct_ib_async_event_t *event ucs_log(level, "IB Async event on %s: %s", uct_ib_device_name(dev), event_info); } +static ucs_status_t uct_ib_device_get_path_buffer(uct_ib_device_t *dev, + char *path_buffer) +{ + char *resolved_path; + + resolved_path = realpath(dev->ibv_context->device->ibdev_path, path_buffer); + if (resolved_path == NULL) { + return UCS_ERR_IO_ERROR; + } + + /* Make sure there is "/infiniband/" substring in path_buffer */ + if (strstr(path_buffer, "/infiniband/") == NULL) { + return UCS_ERR_IO_ERROR; + } + + return UCS_OK; +} + +static ucs_status_t uct_ib_device_get_ids_from_path(const char *path, + uint16_t *vendor_id, + uint16_t *device_id) +{ + ucs_status_t status; + long value; + + status = ucs_read_file_number(&value, 1, "%s/%s", path, "vendor"); + if (status != UCS_OK) { + return status; + } + *vendor_id = value; + + status = ucs_read_file_number(&value, 1, "%s/%s", path, "device"); + if (status != UCS_OK) { + return status; + } + *device_id = value; + + return UCS_OK; +} + static void uct_ib_device_get_ids(uct_ib_device_t *dev) { - long vendor_id, device_id; - - if ((ucs_read_file_number(&vendor_id, 1, UCT_IB_DEVICE_SYSFS_FMT, - uct_ib_device_name(dev), "vendor") == UCS_OK) && - (ucs_read_file_number(&device_id, 1, UCT_IB_DEVICE_SYSFS_FMT, - uct_ib_device_name(dev), "device") == UCS_OK)) { - dev->pci_id.vendor = vendor_id; - dev->pci_id.device = device_id; - ucs_debug("%s vendor_id: 0x%x device_id: %d", uct_ib_device_name(dev), + char *ids_path; + char path_buffer[PATH_MAX]; + ucs_status_t status; + + /* PF: realpath name is of form /sys/devices/.../0000:03:00.0/infiniband/mlx5_0 */ + /* SF: realpath name is of form /sys/devices/.../0000:03:00.0//infiniband/mlx5_0 */ + + status = uct_ib_device_get_path_buffer(dev, path_buffer); + if (status != UCS_OK) { + goto not_found; + } + + /* PF: strip 2 layers. */ + ids_path = ucs_dirname(path_buffer, 2); + if (ids_path == NULL) { + goto not_found; + } + + status = uct_ib_device_get_ids_from_path(ids_path, + &dev->pci_id.vendor, + &dev->pci_id.device); + if (status == UCS_OK) { + ucs_debug("PF: %s vendor_id: 0x%x device_id: %d", uct_ib_device_name(dev), dev->pci_id.vendor, dev->pci_id.device); - } else { - dev->pci_id.vendor = 0; - dev->pci_id.device = 0; - ucs_warn("%s: could not read device/vendor id from sysfs, " - "performance may be affected", uct_ib_device_name(dev)); + return; } + + /* SF: strip 3 layers (1 more layer than PF). */ + ids_path = ucs_dirname(path_buffer, 1); + if (ids_path == NULL) { + goto not_found; + } + + status = uct_ib_device_get_ids_from_path(ids_path, + &dev->pci_id.vendor, + &dev->pci_id.device); + if (status == UCS_OK) { + ucs_debug("SF: %s vendor_id: 0x%x device_id: %d", uct_ib_device_name(dev), + dev->pci_id.vendor, dev->pci_id.device); + return; + } + +not_found: + dev->pci_id.vendor = 0; + dev->pci_id.device = 0; + ucs_warn("%s: could not read device/vendor id from sysfs, " + "performance may be affected", uct_ib_device_name(dev)); } ucs_status_t uct_ib_device_query(uct_ib_device_t *dev, @@ -1015,7 +1086,7 @@ ucs_status_t uct_ib_modify_qp(struct ibv_qp *qp, enum ibv_qp_state state) static ucs_sys_device_t uct_ib_device_get_sys_dev(uct_ib_device_t *dev) { - char path_buffer[PATH_MAX], *resolved_path; + char path_buffer[PATH_MAX]; ucs_sys_device_t sys_dev; ucs_sys_bus_id_t bus_id; ucs_status_t status; @@ -1025,17 +1096,20 @@ static ucs_sys_device_t uct_ib_device_get_sys_dev(uct_ib_device_t *dev) /* realpath name is of form /sys/devices/.../0000:05:00.0/infiniband/mlx5_0 * and bus_id is constructed from 0000:05:00.0 */ - resolved_path = realpath(dev->ibv_context->device->ibdev_path, path_buffer); - if (resolved_path == NULL) { + status = uct_ib_device_get_path_buffer(dev, path_buffer); + if (status != UCS_OK) { return UCS_SYS_DEVICE_ID_UNKNOWN; } - /* Make sure there is "/infiniband/" substring in path_buffer*/ - if (strstr(path_buffer, "/infiniband/") == NULL) { + pcie_bus = ucs_dirname(path_buffer, 2); + if (pcie_bus == NULL) { + return UCS_SYS_DEVICE_ID_UNKNOWN; + } + pcie_bus = basename(pcie_bus); + if (pcie_bus == NULL) { return UCS_SYS_DEVICE_ID_UNKNOWN; } - pcie_bus = basename(dirname(dirname(path_buffer))); num_fields = sscanf(pcie_bus, "%hx:%hhx:%hhx.%hhx", &bus_id.domain, &bus_id.bus, &bus_id.slot, &bus_id.function); if (num_fields != 4) { diff --git a/test/gtest/ucs/test_sys.cc b/test/gtest/ucs/test_sys.cc index dffcfd59311..03bd31103b2 100644 --- a/test/gtest/ucs/test_sys.cc +++ b/test/gtest/ucs/test_sys.cc @@ -23,6 +23,11 @@ class test_sys : public ucs::test { return ucs_get_mem_prot((uintptr_t)address, (uintptr_t)address + size); } + void test_dirname(char *path, int num_layers, const char *expected) { + path = ucs_dirname(path, num_layers); + EXPECT_EQ(std::string(expected), path); + } + void test_memunits(size_t size, const char *expected) { char buf[256]; @@ -139,6 +144,11 @@ UCS_TEST_F(test_sys, module) { EXPECT_EQ(1, test_module_loaded); } +UCS_TEST_F(test_sys, dirname) { + char path[] = "/sys/devices/pci0000:00/0000:00:00.0"; + test_dirname(path, 3, "/sys"); +} + UCS_TEST_F(test_sys, memunits_to_str) { test_memunits(256, "256"); test_memunits(1256, "1256"); From 8ef1409ace7869c03870a4941f227eca2d81fdf2 Mon Sep 17 00:00:00 2001 From: Leonid Genkin Date: Mon, 7 Dec 2020 08:15:59 +0200 Subject: [PATCH 106/107] UCS/DATASTRUCT: Add allow_list data struct --- src/ucs/config/parser.c | 95 +++++++++++++++++++++++++++++++++-- src/ucs/config/parser.h | 11 ++++ src/ucs/config/types.h | 17 +++++++ test/gtest/ucs/test_config.cc | 63 ++++++++++++++++++----- 4 files changed, 169 insertions(+), 17 deletions(-) diff --git a/src/ucs/config/parser.c b/src/ucs/config/parser.c index 732e73be076..657432c21ef 100644 --- a/src/ucs/config/parser.c +++ b/src/ucs/config/parser.c @@ -25,6 +25,8 @@ /* width of titles in docstring */ #define UCS_CONFIG_PARSER_DOCSTR_WIDTH 10 +/* String literal for allow-list */ +#define UCS_CONFIG_PARSER_ALL "all" /* list of prefixes for a configuration variable, used to dump all possible * aliases. @@ -47,6 +49,7 @@ extern char **environ; UCS_LIST_HEAD(ucs_config_global_list); static khash_t(ucs_config_env_vars) ucs_config_parser_env_vars = {0}; static pthread_mutex_t ucs_config_parser_env_vars_hash_lock = PTHREAD_MUTEX_INITIALIZER; +static char ucs_config_parser_negate = '^'; const char *ucs_async_mode_names[] = { @@ -814,10 +817,14 @@ ucs_status_t ucs_config_clone_array(const void *src, void *dest, const void *arg ucs_status_t status; unsigned i; - dest_array->data = ucs_calloc(src_array->count, array->elem_size, - "config array"); - if (dest_array->data == NULL) { - return UCS_ERR_NO_MEMORY; + if (src_array->count > 0) { + dest_array->data = ucs_calloc(src_array->count, array->elem_size, + "config array"); + if (dest_array->data == NULL) { + return UCS_ERR_NO_MEMORY; + } + } else { + dest_array->data = NULL; } dest_array->count = src_array->count; @@ -855,6 +862,86 @@ void ucs_config_help_array(char *buf, size_t max, const void *arg) array->parser.help(buf + strlen(buf), max - strlen(buf), array->parser.arg); } +int ucs_config_sscanf_allow_list(const char *buf, void *dest, const void *arg) +{ + ucs_config_allow_list_t *field = dest; + unsigned offset = 0; + + if (buf[0] == ucs_config_parser_negate) { + field->mode = UCS_CONFIG_ALLOW_LIST_NEGATE; + offset++; + } else { + field->mode = UCS_CONFIG_ALLOW_LIST_ALLOW; + } + + if (!ucs_config_sscanf_array(&buf[offset], &field->array, arg)) { + return 0; + } + + if ((field->array.count >= 1) && + !strcmp(field->array.names[0], UCS_CONFIG_PARSER_ALL)) { + field->mode = UCS_CONFIG_ALLOW_LIST_ALLOW_ALL; + ucs_config_release_array(&field->array, arg); + if (field->array.count != 1) { + return 0; + } + + field->array.count = 0; + } + + return 1; +} + +int ucs_config_sprintf_allow_list(char *buf, size_t max, const void *src, + const void *arg) +{ + const ucs_config_allow_list_t *allow_list = src; + size_t offset = 0; + + if (allow_list->mode == UCS_CONFIG_ALLOW_LIST_ALLOW_ALL) { + snprintf(buf, max, UCS_CONFIG_PARSER_ALL); + return 1; + } + + if (allow_list->mode == UCS_CONFIG_ALLOW_LIST_NEGATE) { + buf[offset++] = ucs_config_parser_negate; + max--; + } + + return ucs_config_sprintf_array(&buf[offset], max, &allow_list->array, arg); +} + +ucs_status_t ucs_config_clone_allow_list(const void *src, void *dest, const void *arg) +{ + const ucs_config_allow_list_t *src_list = src; + ucs_config_allow_list_t *dest_list = dest; + + dest_list->mode = src_list->mode; + return ucs_config_clone_array(&src_list->array, &dest_list->array, arg); +} + +void ucs_config_release_allow_list(void *ptr, const void *arg) +{ + ucs_config_allow_list_t *allow_list = ptr; + + if (allow_list->mode == UCS_CONFIG_ALLOW_LIST_ALLOW_ALL) { + return; + } + + ucs_config_release_array(&allow_list->array, arg); +} + +void ucs_config_help_allow_list(char *buf, size_t max, const void *arg) +{ + const ucs_config_array_t *array = arg; + + snprintf( + buf, + max, "comma-separated list (use \"all\" for including \ + all items or \'^\' for negation) of: "); + array->parser.help(buf + strlen(buf), max - strlen(buf), array->parser.arg); +} + int ucs_config_sscanf_table(const char *buf, void *dest, const void *arg) { char *tokens; diff --git a/src/ucs/config/parser.h b/src/ucs/config/parser.h index 3c4659bce02..89bf8c4f244 100644 --- a/src/ucs/config/parser.h +++ b/src/ucs/config/parser.h @@ -203,6 +203,13 @@ ucs_status_t ucs_config_clone_array(const void *src, void *dest, const void *arg void ucs_config_release_array(void *ptr, const void *arg); void ucs_config_help_array(char *buf, size_t max, const void *arg); +int ucs_config_sscanf_allow_list(const char *buf, void *dest, const void *arg); +int ucs_config_sprintf_allow_list(char *buf, size_t max, const void *src, + const void *arg); +ucs_status_t ucs_config_clone_allow_list(const void *src, void *dest, const void *arg); +void ucs_config_release_allow_list(void *ptr, const void *arg); +void ucs_config_help_allow_list(char *buf, size_t max, const void *arg); + int ucs_config_sscanf_table(const char *buf, void *dest, const void *arg); ucs_status_t ucs_config_clone_table(const void *src, void *dest, const void *arg); void ucs_config_release_table(void *ptr, const void *arg); @@ -321,6 +328,10 @@ void ucs_config_help_generic(char *buf, size_t max, const void *arg); ucs_config_clone_array, ucs_config_release_array, \ ucs_config_help_array, &ucs_config_array_##a} +#define UCS_CONFIG_TYPE_ALLOW_LIST {ucs_config_sscanf_allow_list, ucs_config_sprintf_allow_list, \ + ucs_config_clone_allow_list, ucs_config_release_allow_list, \ + ucs_config_help_allow_list, &ucs_config_array_string} + #define UCS_CONFIG_TYPE_TABLE(t) {ucs_config_sscanf_table, NULL, \ ucs_config_clone_table, ucs_config_release_table, \ ucs_config_help_table, t} diff --git a/src/ucs/config/types.h b/src/ucs/config/types.h index 30699b21ed5..a23b0f30dd9 100644 --- a/src/ucs/config/types.h +++ b/src/ucs/config/types.h @@ -111,6 +111,23 @@ typedef enum { typedef UCS_CONFIG_STRING_ARRAY_FIELD(names) ucs_config_names_array_t; + +/** + * Enum for representing possible modes of an "allow-list" + */ +typedef enum { + UCS_CONFIG_ALLOW_LIST_ALLOW_ALL, /* Allow all possible options */ + UCS_CONFIG_ALLOW_LIST_ALLOW, /* Allow only the specified options */ + UCS_CONFIG_ALLOW_LIST_NEGATE /* Negate (forbid) the specified options */ +} ucs_config_allow_list_mode_t; + + +typedef struct { + ucs_config_names_array_t array; + ucs_config_allow_list_mode_t mode; +} ucs_config_allow_list_t; + + /** * @ingroup UCS_RESOURCE * BSD socket address specification. diff --git a/test/gtest/ucs/test_config.cc b/test/gtest/ucs/test_config.cc index 3bcf5e9e074..73a92d73621 100644 --- a/test/gtest/ucs/test_config.cc +++ b/test/gtest/ucs/test_config.cc @@ -91,6 +91,7 @@ typedef struct { ucs_time_t time_value; ucs_time_t time_auto; ucs_time_t time_inf; + ucs_config_allow_list_t allow_list; } car_opts_t; @@ -218,6 +219,9 @@ ucs_config_field_t car_opts_table[] = { {"TIME_INF", "inf", "Time value \"inf\"", ucs_offsetof(car_opts_t, time_inf), UCS_CONFIG_TYPE_TIME_UNITS}, + {"ALLOW_LIST", "all", "Allow-list: \"all\" OR \"val1,val2\" OR \"^val1,val2\"", + ucs_offsetof(car_opts_t, allow_list), UCS_CONFIG_TYPE_ALLOW_LIST}, + {NULL} }; @@ -414,6 +418,8 @@ UCS_TEST_F(test_config, parse_default) { EXPECT_EQ(ucs_time_from_sec(1.0), opts->time_value); EXPECT_EQ(UCS_TIME_AUTO, opts->time_auto); EXPECT_EQ(UCS_TIME_INFINITY, opts->time_inf); + EXPECT_EQ(UCS_CONFIG_ALLOW_LIST_ALLOW_ALL, opts->allow_list.mode); + EXPECT_EQ(0, opts->allow_list.array.count); } UCS_TEST_F(test_config, clone) { @@ -436,6 +442,7 @@ UCS_TEST_F(test_config, clone) { } EXPECT_EQ(COLOR_WHITE, (*opts_clone_ptr)->color); + EXPECT_EQ(UCS_CONFIG_ALLOW_LIST_ALLOW_ALL, (*opts_clone_ptr)->allow_list.mode); delete opts_clone_ptr; } @@ -540,27 +547,23 @@ UCS_TEST_F(test_config, unused) { UCS_TEST_F(test_config, dump) { /* aliases must not be counted here */ - test_config_print_opts(UCS_CONFIG_PRINT_CONFIG, 31u); + test_config_print_opts(UCS_CONFIG_PRINT_CONFIG, 32u); } UCS_TEST_F(test_config, dump_hidden) { /* aliases must be counted here */ - test_config_print_opts((UCS_CONFIG_PRINT_CONFIG | - UCS_CONFIG_PRINT_HIDDEN), - 38u); + test_config_print_opts(UCS_CONFIG_PRINT_CONFIG | UCS_CONFIG_PRINT_HIDDEN, 39u); } UCS_TEST_F(test_config, dump_hidden_check_alias_name) { /* aliases must be counted here */ - test_config_print_opts((UCS_CONFIG_PRINT_CONFIG | - UCS_CONFIG_PRINT_HIDDEN | - UCS_CONFIG_PRINT_DOC), - 38u); - - test_config_print_opts((UCS_CONFIG_PRINT_CONFIG | - UCS_CONFIG_PRINT_HIDDEN | - UCS_CONFIG_PRINT_DOC), - 38u, "TEST_"); + test_config_print_opts( + UCS_CONFIG_PRINT_CONFIG | UCS_CONFIG_PRINT_HIDDEN | UCS_CONFIG_PRINT_DOC, + 39u); + + test_config_print_opts( + UCS_CONFIG_PRINT_CONFIG | UCS_CONFIG_PRINT_HIDDEN | UCS_CONFIG_PRINT_DOC, + 39u, "TEST_"); } UCS_TEST_F(test_config, deprecated) { @@ -594,3 +597,37 @@ UCS_TEST_F(test_config, deprecated) { /* reset to not warn about unused env vars */ ucs_global_opts.warn_unused_env_vars = 0; } + +UCS_TEST_F(test_config, test_allow_list) { + const std::string allow_list = "UCX_ALLOW_LIST"; + + { + /* coverity[tainted_string_argument] */ + ucs::scoped_setenv env(allow_list.c_str(), "first,second"); + + car_opts opts(UCS_DEFAULT_ENV_PREFIX, NULL); + EXPECT_EQ(UCS_CONFIG_ALLOW_LIST_ALLOW, opts->allow_list.mode); + EXPECT_EQ(2, opts->allow_list.array.count); + EXPECT_EQ(std::string("first"), opts->allow_list.array.names[0]); + EXPECT_EQ(std::string("second"), opts->allow_list.array.names[1]); + } + + { + /* coverity[tainted_string_argument] */ + ucs::scoped_setenv env(allow_list.c_str(), "^first,second"); + + car_opts opts(UCS_DEFAULT_ENV_PREFIX, NULL); + EXPECT_EQ(UCS_CONFIG_ALLOW_LIST_NEGATE, opts->allow_list.mode); + EXPECT_EQ(2, opts->allow_list.array.count); + EXPECT_EQ(std::string("first"), opts->allow_list.array.names[0]); + EXPECT_EQ(std::string("second"), opts->allow_list.array.names[1]); + } +} + +UCS_TEST_F(test_config, test_allow_list_negative) +{ + ucs_config_allow_list_t field; + + EXPECT_EQ(ucs_config_sscanf_allow_list("all,all", &field, + &ucs_config_array_string), 0); +} From a245659a5ab0cbe0745407416afd53770f46315b Mon Sep 17 00:00:00 2001 From: Alina Sklarevich Date: Tue, 5 Jan 2021 21:51:23 +0200 Subject: [PATCH 107/107] UCP/UCT/TEST/EXAMPLES: Fix several coverity issues. test_sockaddr.cc - NEGATIVE_RETURNS - "index" is passed to a parameter that cannot be negative. [Note: The source code implementation of the function has been overridden by a builtin model.] uct_cm.c - PRINTF_ARGS - Argument "self->iface" to format specifier "%p" was expected to have pointer type but has type "struct uct_base_iface". ucp_hello_world.c - USE_AFTER_FREE - Dereferencing freed pointer "msg" wireup_cm.c - USE_AFTER_FREE - Passing freed pointer "conn_request" as an argument to "ucs_log_dispatch" ("uct_listener_reject" frees "conn_request".). --- examples/ucp_hello_world.c | 1 + src/ucp/wireup/wireup_cm.c | 1 + src/uct/base/uct_cm.c | 2 +- test/gtest/uct/ib/test_sockaddr.cc | 2 +- 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/ucp_hello_world.c b/examples/ucp_hello_world.c index f4a2be5e623..32a0440cb49 100644 --- a/examples/ucp_hello_world.c +++ b/examples/ucp_hello_world.c @@ -402,6 +402,7 @@ static int run_ucx_server(ucp_worker_h ucp_worker) "should detect error by keepalive mechanism\n"); free(msg); raise(SIGKILL); + exit(1); } peer_addr_len = msg->data_len; diff --git a/src/ucp/wireup/wireup_cm.c b/src/ucp/wireup/wireup_cm.c index e1e6c327bed..ff87913d590 100644 --- a/src/ucp/wireup/wireup_cm.c +++ b/src/ucp/wireup/wireup_cm.c @@ -942,6 +942,7 @@ void ucp_cm_server_conn_request_cb(uct_listener_h listener, void *arg, err_reject: status = uct_listener_reject(listener, conn_request); if (status != UCS_OK) { + /* coverity[pass_freed_arg] */ ucs_warn("failed to reject connect request %p on listener %p", conn_request, listener); } diff --git a/src/uct/base/uct_cm.c b/src/uct/base/uct_cm.c index faa9579abd0..d2e0d79f02f 100644 --- a/src/uct/base/uct_cm.c +++ b/src/uct/base/uct_cm.c @@ -284,7 +284,7 @@ UCS_CLASS_INIT_FUNC(uct_cm_t, uct_cm_ops_t* ops, uct_iface_ops_t* iface_ops, return UCS_STATS_NODE_ALLOC(&self->iface.stats, &uct_cm_stats_class, ucs_stats_get_root(), "%s-%p", "iface", - self->iface); + &self->iface); } UCS_CLASS_CLEANUP_FUNC(uct_cm_t) diff --git a/test/gtest/uct/ib/test_sockaddr.cc b/test/gtest/uct/ib/test_sockaddr.cc index 98f1be51b92..0c85432c1a5 100644 --- a/test/gtest/uct/ib/test_sockaddr.cc +++ b/test/gtest/uct/ib/test_sockaddr.cc @@ -1211,7 +1211,7 @@ class test_uct_cm_sockaddr_stress : public test_uct_cm_sockaddr { int index; index = get_ep_index(ep); - ASSERT_GE(index, 0); + ASSERT_TRUE(index >= 0); EXPECT_LT(index, (2 * m_clients_num)); pthread_mutex_lock(&m_lock);