diff --git a/docs/tracing.md b/docs/tracing.md index 92d52fb6a7..92fe080ac9 100644 --- a/docs/tracing.md +++ b/docs/tracing.md @@ -166,6 +166,13 @@ backends/PRs): | `nixl::prepMemView` | `MemoryR` | `mem_type`, `desc_count` | | `nixl::releaseMemView` | `Generic` | - | +Spans cover the synchronous call only; the `nixl::xfer.complete` marker is emitted +when `getXferStatus` first observes success. How a backend renders attributes and +dependencies (`addCtrlDep`/`addDataDep`) is backend-specific and documented with each +backend (e.g. NVTX attaches attributes as typed payloads on the range via +`nvtxRangePopPayload` and ignores dependencies; offline backends such as Chakra +record them). + ### Backend phase spans A backend plugin can record a phase of its own work through @@ -190,12 +197,16 @@ The sink interface is internal (`src/core/tracing/backend_trace.h`, not an installed header), so only in-tree plugins can emit phases. No plugin emits them yet; UCX and libfabric map their internals onto the vocabulary in later PRs. -Spans cover the synchronous call only; the `nixl::xfer.complete` marker is emitted -when `getXferStatus` first observes success. How a backend renders attributes and -dependencies (`addCtrlDep`/`addDataDep`) is backend-specific and documented with each -backend (e.g. NVTX attaches attributes as typed payloads on the range via -`nvtxRangePopPayload` and ignores dependencies; offline backends such as Chakra -record them). +A backend also sees the trace context of the request it is executing, as +`traceContext` in the `nixl_opt_b_args_t` that `prepXfer` and `postXfer` receive. +It is the same context at prep and at every post of that request, reposts +included, and the zeroed, invalid context when tracing is off. It is null in +`prepMemView`, which has no request, and `opt_args` itself may be null when an +engine is called directly. The pointer is valid only for the call: a plugin that +needs the context later, at completion or when building a notification, copies +the value onto its own request handle. A carrier that puts it on the wire calls +`encodeTraceContext` itself. `TraceContext` is internal like the sink, so only +in-tree plugins can read it; none does yet. ## Profiling with NVTX / Nsight Systems @@ -287,7 +298,9 @@ In NIXL tracing, "correlation" can mean two different things: recording the span attributes and dependencies the NVTX backend ignores. - **Cross-rank correlation** — propagate a globally unique request id on the wire so sender and receiver spans can be linked across processes (distributed tracing; a - separate NIXL-architecture effort). + separate NIXL-architecture effort). Backends already receive each request's trace + context (see "Backend phase spans"); the per-plugin carriers that put it on the + wire come next. - **Backend phase emission** — the sink described under "Backend phase spans" exists, but no plugin records a phase yet. diff --git a/src/api/cpp/backend/backend_aux.h b/src/api/cpp/backend/backend_aux.h index 6aadcff5b5..ca2ed53301 100644 --- a/src/api/cpp/backend/backend_aux.h +++ b/src/api/cpp/backend/backend_aux.h @@ -26,6 +26,10 @@ class nixlBackendTraceSink; +namespace nixl::trace { +struct TraceContext; +} + // Might be removed to be decided by backend, or changed to high // level direction or so. typedef std::vector> notif_list_t; @@ -36,6 +40,10 @@ struct nixlBackendOptionalArgs { nixl_blob_t notifMsg; bool hasNotif = false; nixl_blob_t customParam; + // The request's trace context, the same in prepXfer and in every postXfer of + // that request: valid for the call only, so copy it once at prep to keep it. + // Null when no request is involved (prepMemView). + const nixl::trace::TraceContext *traceContext = nullptr; }; using nixl_opt_b_args_t = nixlBackendOptionalArgs; diff --git a/src/core/nixl_agent.cpp b/src/core/nixl_agent.cpp index 50531f0c7e..28aa995c2d 100644 --- a/src/core/nixl_agent.cpp +++ b/src/core/nixl_agent.cpp @@ -910,6 +910,7 @@ nixlAgent::makeXferReq(nixl_xfer_op_t operation, handle->telemetry.totalBytes = total_bytes; handle->telemetry.descCount = handle->initiatorDescs.descCount(); + opt_args.traceContext = &handle->traceContext(); ret = handle->engine->prepXfer(handle->backendOp, handle->initiatorDescs, handle->targetDescs, @@ -1056,6 +1057,7 @@ nixlAgent::createXferReq(const nixl_xfer_op_t &operation, handle->telemetry.totalBytes = total_bytes; handle->telemetry.descCount = handle->initiatorDescs.descCount(); + opt_args.traceContext = &handle->traceContext(); ret1 = handle->engine->prepXfer(handle->backendOp, handle->initiatorDescs, handle->targetDescs, @@ -1201,6 +1203,8 @@ nixlAgent::postXferReq(nixlXferReqH *req_hndl, return NIXL_ERR_BACKEND; } + opt_args.traceContext = &req_hndl->traceContext(); + // If status is not NIXL_IN_PROG we can repost, req_hndl->status = req_hndl->engine->postXfer(req_hndl->backendOp, req_hndl->initiatorDescs, diff --git a/src/core/tracing/trace_context.h b/src/core/tracing/trace_context.h index cd2be9a4f3..02be561efd 100644 --- a/src/core/tracing/trace_context.h +++ b/src/core/tracing/trace_context.h @@ -72,6 +72,9 @@ struct TraceContext { [[nodiscard]] std::uint64_t correlationId64() const noexcept; + + [[nodiscard]] bool + operator==(const TraceContext &) const noexcept = default; }; [[nodiscard]] std::optional diff --git a/src/core/transfer_request.h b/src/core/transfer_request.h index c88850299b..bb5036fb62 100644 --- a/src/core/transfer_request.h +++ b/src/core/transfer_request.h @@ -74,6 +74,11 @@ class nixlXferReqH { return traceContext_.correlationId64(); } + [[nodiscard]] const nixl::trace::TraceContext & + traceContext() const noexcept { + return traceContext_; + } + friend class nixlAgent; private: diff --git a/test/gtest/mocks/gmock_engine.cpp b/test/gtest/mocks/gmock_engine.cpp index 9ed2b1690a..15b0f532e7 100644 --- a/test/gtest/mocks/gmock_engine.cpp +++ b/test/gtest/mocks/gmock_engine.cpp @@ -87,6 +87,34 @@ GMockBackendEngine::SetToParams(nixl_b_params_t ¶ms) const { params[gmock_engine_key] = std::to_string(reinterpret_cast(this)); } +void +GMockBackendEngine::observeTraceSink(const nixlBackendTraceSink *sink) { + const std::lock_guard lock(observedMutex_); + observedTraceSink_ = sink; +} + +void +GMockBackendEngine::observeTraceContext(const nixl_opt_b_args_t *opt_args) { + const std::lock_guard lock(observedMutex_); + if (opt_args != nullptr && opt_args->traceContext != nullptr) { + observedTraceContext_ = *opt_args->traceContext; + } else { + observedTraceContext_.reset(); + } +} + +std::optional +GMockBackendEngine::observedTraceSink() const { + const std::lock_guard lock(observedMutex_); + return observedTraceSink_; +} + +std::optional +GMockBackendEngine::observedTraceContext() const { + const std::lock_guard lock(observedMutex_); + return observedTraceContext_; +} + GMockBackendEngine * GMockBackendEngine::GetFromParams(nixl_b_params_t *params) { try { diff --git a/test/gtest/mocks/gmock_engine.h b/test/gtest/mocks/gmock_engine.h index f4e68be8cd..77b47e73a9 100644 --- a/test/gtest/mocks/gmock_engine.h +++ b/test/gtest/mocks/gmock_engine.h @@ -22,9 +22,11 @@ #include #include +#include #include #include "backend/backend_engine.h" +#include "tracing/trace_context.h" namespace mocks { @@ -67,7 +69,17 @@ class GMockBackendEngine : public nixlBackendEngine { static GMockBackendEngine * GetFromParams(nixl_b_params_t *params); - std::optional observedTraceSink; + void + observeTraceSink(const nixlBackendTraceSink *sink); + + void + observeTraceContext(const nixl_opt_b_args_t *opt_args); + + [[nodiscard]] std::optional + observedTraceSink() const; + + [[nodiscard]] std::optional + observedTraceContext() const; MOCK_METHOD(bool, supportsRemote, (), (const, override)); MOCK_METHOD(bool, supportsLocal, (), (const, override)); @@ -161,6 +173,10 @@ class GMockBackendEngine : public nixlBackendEngine { setDefaults(); void setOptionalDefaults(); + + mutable std::mutex observedMutex_; + std::optional observedTraceSink_; + std::optional observedTraceContext_; }; } // namespace mocks diff --git a/test/gtest/mocks/mock_backend_engine.cpp b/test/gtest/mocks/mock_backend_engine.cpp index 4f1894bc30..1d3a7ca643 100644 --- a/test/gtest/mocks/mock_backend_engine.cpp +++ b/test/gtest/mocks/mock_backend_engine.cpp @@ -23,8 +23,7 @@ MockBackendEngine::MockBackendEngine(const nixlBackendInitParams *init_params) : nixlBackendEngine(init_params), gmock_backend_engine(GMockBackendEngine::GetFromParams(init_params->customParams)), sharedState(1) { - static_cast(gmock_backend_engine)->observedTraceSink = - init_params->traceSink; + gmock_backend_engine->observeTraceSink(init_params->traceSink); } nixl_status_t @@ -67,6 +66,7 @@ MockBackendEngine::prepXfer(const nixl_xfer_op_t &operation, nixlBackendReqH *&handle, const nixl_opt_b_args_t *opt_args) const { assert(sharedState > 0); + gmock_backend_engine->observeTraceContext(opt_args); return gmock_backend_engine->prepXfer(operation, local, remote, remote_agent, handle, opt_args); } @@ -78,6 +78,7 @@ MockBackendEngine::postXfer(const nixl_xfer_op_t &operation, nixlBackendReqH *&handle, const nixl_opt_b_args_t *opt_args) const { assert(sharedState > 0); + gmock_backend_engine->observeTraceContext(opt_args); return gmock_backend_engine->postXfer(operation, local, remote, remote_agent, handle, opt_args); } diff --git a/test/gtest/mocks/mock_backend_engine.h b/test/gtest/mocks/mock_backend_engine.h index f809a31b7f..aef7ff5f97 100644 --- a/test/gtest/mocks/mock_backend_engine.h +++ b/test/gtest/mocks/mock_backend_engine.h @@ -19,6 +19,7 @@ #include "backend/backend_engine.h" #include "backend/backend_plugin.h" +#include "gmock_engine.h" #include #include @@ -26,7 +27,7 @@ namespace mocks { class MockBackendEngine : public nixlBackendEngine { private: - nixlBackendEngine *gmock_backend_engine; + GMockBackendEngine *gmock_backend_engine; public: MockBackendEngine(const nixlBackendInitParams *init_params); diff --git a/test/gtest/unit/agent/agent.cpp b/test/gtest/unit/agent/agent.cpp index a0cd7b98c8..0754ad5157 100644 --- a/test/gtest/unit/agent/agent.cpp +++ b/test/gtest/unit/agent/agent.cpp @@ -26,6 +26,7 @@ #include "nixl.h" #include "plugin_manager.h" #include "transfer_request.h" +#include "tracing/trace_context.h" #include "mocks/gmock_engine.h" namespace gtest { @@ -66,14 +67,23 @@ namespace agent { } }; + struct selfXferSetup { + nixlBackendH *backend = nullptr; + blob localBlob; + nixl_reg_dlist_t regDlist{DRAM_SEG}; + nixl_opt_args_t extraParams; + nixl_xfer_dlist_t xferDlist{DRAM_SEG}; + }; + class agentHelper { protected: ScopedEnv trace_env_; testing::NiceMock gmock_engine_; std::unique_ptr agent_; + const std::string name_; public: - agentHelper(const std::string &name, const std::string &trace_backends = "") { + agentHelper(const std::string &name, const std::string &trace_backends = "") : name_(name) { trace_env_.addVar("NIXL_TRACE_BACKENDS", trace_backends); nixlAgentConfig cfg; cfg.useProgThread = true; @@ -118,6 +128,22 @@ namespace agent { extra_params.backends.push_back(backend); return agent_->registerMem(reg_dlist, &extra_params); } + + void + setupSelfXfer(selfXferSetup &setup) { + nixl_b_params_t params; + EXPECT_EQ(createBackendWithGMock(params, setup.backend), NIXL_SUCCESS); + EXPECT_EQ(initAndRegisterMemory( + setup.localBlob, setup.regDlist, setup.extraParams, setup.backend), + NIXL_SUCCESS); + setup.xferDlist.addDesc(setup.localBlob.getDesc()); + } + + nixl_status_t + createSelfXferReq(selfXferSetup &setup, nixlXferReqH *&req) { + return agent_->createXferReq( + NIXL_WRITE, setup.xferDlist, setup.xferDlist, name_, req, &setup.extraParams); + } }; class tracingEnabledAgentFixture : public testing::Test { @@ -149,11 +175,62 @@ namespace agent { nixl_b_params_t params; nixlBackendH *backend = nullptr; EXPECT_EQ(agent_helper_->createBackendWithGMock(params, backend), NIXL_SUCCESS); - const auto &observed = agent_helper_->getGMockEngine().observedTraceSink; + const auto observed = agent_helper_->getGMockEngine().observedTraceSink(); ASSERT_TRUE(observed.has_value()); EXPECT_NE(*observed, nullptr); } + TEST_F(tracingEnabledAgentFixture, TracingOnHandsPluginEachRequestsContext) { + selfXferSetup setup; + agent_helper_->setupSelfXfer(setup); + nixlAgent *agent = agent_helper_->getAgent(); + const auto &engine = agent_helper_->getGMockEngine(); + + nixlXferReqH *first = nullptr; + ASSERT_EQ(agent_helper_->createSelfXferReq(setup, first), NIXL_SUCCESS); + EXPECT_TRUE(first->traceContext().valid()); + EXPECT_THAT(engine.observedTraceContext(), testing::Optional(first->traceContext())); + + nixlXferReqH *second = nullptr; + ASSERT_EQ(agent_helper_->createSelfXferReq(setup, second), NIXL_SUCCESS); + ASSERT_NE(first->traceContext(), second->traceContext()); + EXPECT_THAT(engine.observedTraceContext(), testing::Optional(second->traceContext())); + + for (nixlXferReqH *req : {first, second, first}) { + EXPECT_EQ(agent->postXferReq(req), NIXL_SUCCESS); + EXPECT_THAT(engine.observedTraceContext(), testing::Optional(req->traceContext())); + } + + EXPECT_EQ(agent->releaseXferReq(first), NIXL_SUCCESS); + EXPECT_EQ(agent->releaseXferReq(second), NIXL_SUCCESS); + } + + TEST_F(tracingEnabledAgentFixture, TracingOnHandsPluginThePreppedRequestsContext) { + selfXferSetup setup; + agent_helper_->setupSelfXfer(setup); + nixlAgent *agent = agent_helper_->getAgent(); + + nixlDlistH *local_side = nullptr; + nixlDlistH *remote_side = nullptr; + ASSERT_EQ(agent->prepXferDlist(setup.xferDlist, local_side), NIXL_SUCCESS); + ASSERT_EQ(agent->prepXferDlist(local_agent_name, setup.xferDlist, remote_side), + NIXL_SUCCESS); + + const std::vector indices{0}; + nixlXferReqH *req = nullptr; + ASSERT_EQ( + agent->makeXferReq( + NIXL_WRITE, *local_side, indices, *remote_side, indices, req, &setup.extraParams), + NIXL_SUCCESS); + EXPECT_TRUE(req->traceContext().valid()); + EXPECT_THAT(agent_helper_->getGMockEngine().observedTraceContext(), + testing::Optional(req->traceContext())); + + EXPECT_EQ(agent->releaseXferReq(req), NIXL_SUCCESS); + EXPECT_EQ(agent->releasedDlistH(local_side), NIXL_SUCCESS); + EXPECT_EQ(agent->releasedDlistH(remote_side), NIXL_SUCCESS); + } + class singleAgentSessionFixture : public testing::Test { protected: std::unique_ptr agent_helper_; @@ -276,11 +353,27 @@ namespace agent { nixl_b_params_t params; nixlBackendH *backend = nullptr; EXPECT_EQ(agent_helper_->createBackendWithGMock(params, backend), NIXL_SUCCESS); - const auto &observed = agent_helper_->getGMockEngine().observedTraceSink; + const auto observed = agent_helper_->getGMockEngine().observedTraceSink(); ASSERT_TRUE(observed.has_value()); EXPECT_EQ(*observed, nullptr); } + TEST_F(singleAgentSessionFixture, TracingOffHandsPluginTheZeroedRequestContext) { + selfXferSetup setup; + agent_helper_->setupSelfXfer(setup); + const auto &engine = agent_helper_->getGMockEngine(); + + nixlXferReqH *req = nullptr; + ASSERT_EQ(agent_helper_->createSelfXferReq(setup, req), NIXL_SUCCESS); + EXPECT_EQ(req->traceContext(), nixl::trace::TraceContext{}); + EXPECT_THAT(engine.observedTraceContext(), testing::Optional(req->traceContext())); + + EXPECT_EQ(agent_->postXferReq(req), NIXL_SUCCESS); + EXPECT_THAT(engine.observedTraceContext(), testing::Optional(req->traceContext())); + + EXPECT_EQ(agent_->releaseXferReq(req), NIXL_SUCCESS); + } + TEST_F(singleAgentSessionFixture, GetNonExistingBackendParamsTest) { nixl_mem_list_t mem; nixl_b_params_t params; diff --git a/test/gtest/unit/tracing/tracing_test.cpp b/test/gtest/unit/tracing/tracing_test.cpp index 64b885f221..c170b042ae 100644 --- a/test/gtest/unit/tracing/tracing_test.cpp +++ b/test/gtest/unit/tracing/tracing_test.cpp @@ -481,11 +481,13 @@ TEST(Tracing, RequestStoresFixedCorrelationContext) { nixl::trace::TraceContext context; context.traceId = {0x4b, 0xf9, 0x2f, 0x35, 0x77, 0xb3, 0x4d, 0xa6}; context.spanId = {0x00, 0xf0, 0x67, 0xaa, 0x0b, 0xa9, 0x02, 0xb7}; + context.flags = 0x03; const nixlXferReqH request( "remote", NIXL_WRITE, DRAM_SEG, DRAM_SEG, 0, nixl_remote_section_weak_t{}, context); EXPECT_EQ(request.traceCorrelationId64(), 0x00f067aa0ba902b7ULL); + EXPECT_EQ(request.traceContext(), context); } TEST(Tracing, RequestContextsAreDistinctAndStable) {