From b0e8d7a57f81acb12c9d6c5bc7463fc4db83e172 Mon Sep 17 00:00:00 2001 From: Ehsan Saei <71217171+esigo@users.noreply.github.com> Date: Wed, 6 Nov 2024 21:20:33 +0100 Subject: [PATCH 1/2] [EXPORTER] bump prometheus to v1.3.0 (#3122) --- CHANGELOG.md | 3 + bazel/repository.bzl | 6 +- docs/maintaining-dependencies.md | 157 +++++++++++++++++++++++++++++++ third_party/prometheus-cpp | 2 +- third_party_release | 2 +- 5 files changed, 165 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 650aebcea5..04b1603c3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,9 @@ Important changes: as the Jaeger propagator can be used without the (now removed) Jaeger exporter. +* Upgrade to prometheus 1.3.0 + [#3122](https://github.com/open-telemetry/opentelemetry-cpp/pull/3122) + ## [1.17 2024-10-07] * [CI] Add a clang-tidy build diff --git a/bazel/repository.bzl b/bazel/repository.bzl index 508b95a39a..4827cffba3 100644 --- a/bazel/repository.bzl +++ b/bazel/repository.bzl @@ -110,10 +110,10 @@ def opentelemetry_cpp_deps(): maybe( http_archive, name = "com_github_jupp0r_prometheus_cpp", - sha256 = "48dbad454d314b836cc667ec4def93ec4a6e4255fc8387c20cacb3b8b6faee30", - strip_prefix = "prometheus-cpp-1.2.4", + sha256 = "ac6e958405a29fbbea9db70b00fa3c420e16ad32e1baf941ab233ba031dd72ee", + strip_prefix = "prometheus-cpp-1.3.0", urls = [ - "https://github.com/jupp0r/prometheus-cpp/archive/refs/tags/v1.2.4.tar.gz", + "https://github.com/jupp0r/prometheus-cpp/archive/refs/tags/v1.3.0.tar.gz", ], ) diff --git a/docs/maintaining-dependencies.md b/docs/maintaining-dependencies.md index 1c79810029..31a50db521 100644 --- a/docs/maintaining-dependencies.md +++ b/docs/maintaining-dependencies.md @@ -442,3 +442,160 @@ Last, in some special case like name collisions for a given symbol, the template itself may need to be adjusted for special logic. See for example how `messaging.client_id` is treated. + +## prometheus-cpp + +### Comments (prometheus-cpp) + +The `prometheus-cpp` library provides a C++ client for Prometheus, +facilitating the creation and registration of metrics that Prometheus scrapes. +`prometheus-cpp` is used as a git submodule under the `third_party` directory +for ease of inclusion in build system. + +### Origin (prometheus-cpp) + +The repository for `prometheus-cpp` can be found here: + +* [repository](https://github.com/jupp0r/prometheus-cpp) + +Check release notes at: + +* [release-notes](https://github.com/jupp0r/prometheus-cpp/releases) + +### Upgrade (prometheus-cpp) + +When upgrading `prometheus-cpp` to a newer release, +you’ll need to update a few key files in the codebase to reflect the new version. + +In this example, we upgrade from `v1.2.3` to `v1.2.4`. + +#### Directory `third_party/prometheus-cpp` + +`prometheus-cpp` is a `git submodule`, +so it needs to be pointed to the new release tag. + +```shell +cd third_party/prometheus-cpp +git log -1 +``` + +The current submodule should show something like: + +```shell +commit abcdef1234567890abcdef1234567890abcdef12 (HEAD, tag: v1.2.3) +Author: John Doe +Date: Fri Apr 25 17:55:35 2024 +0200 + + Minor fixes for performance and compatibility +``` + +Pull new tags: + +```shell +git pull --tag origin +``` + +Upgrade to the new tag: + +```shell +git pull origin v1.2.4 +``` + +Verify the new commit: + +```shell +git log -1 +commit 1234567890abcdef1234567890abcdef12345678 (HEAD, tag: v1.2.4) +Author: Jane Doe +Date: Thu Jun 28 08:19:11 2024 -0500 + + Improved metrics handling for high concurrency +``` + +Return to the root directory: + +```shell +cd ../.. +git status +``` + +The status should display: + +```shell +On branch upgrade_prometheus_1.2.4 +Changes not staged for commit: + (use "git add ..." to update what will be committed) + (use "git restore ..." to discard changes in working directory) + modified: third_party/prometheus-cpp (new commits) +``` + +Add the upgraded submodule: + +```shell +git add third_party/prometheus-cpp +``` + +File third_party_release +Update the line referencing the prometheus-cpp version. + +```shell +prometheus-cpp=v1.2.4 +``` + +Example change: + +```shell +$ git diff third_party_release +diff --git a/third_party_release b/third_party_release +index abc1234..def5678 100644 +--- a/third_party_release ++++ b/third_party_release +@@ -19,7 +19,7 @@ some-dependency=v0.8.3 + another-dependency=1.14.0 + prometheus-cpp=v1.2.3 ++prometheus-cpp=v1.2.4 +``` + +In file bazel/repository.bzl locate the entry for prometheus-cpp: + +```shell +# C++ Prometheus Client library. +maybe( + http_archive, + name = "com_github_jupp0r_prometheus_cpp", + sha256 = "ac6e958405a29fbbea9db70b00fa3c420e16ad32e1baf941ab233ba031dd72ee", + strip_prefix = "prometheus-cpp-1.2.3", + urls = [ + "https://github.com/jupp0r/prometheus-cpp/archive/refs/tags/v1.2.3.tar.gz", + ], +) +``` + +Update the URL to the new tag: + +```shell +urls = [ + "https://github.com/jupp0r/prometheus-cpp/archive/v1.2.4.tar.gz", +], +``` + +Update strip_prefix to match the new version: + +```shell +strip_prefix = "prometheus-cpp-1.2.4", +``` + +Download the new URL: + +```shell +wget https://github.com/jupp0r/prometheus-cpp/archive/v1.2.4.tar.gz +``` + +Calculate the checksum: + +```shell +sha256sum v1.2.4.tar.gz +abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234 v1.2.4.tar.gz +``` + +Update the `sha256`. diff --git a/third_party/prometheus-cpp b/third_party/prometheus-cpp index ad99e21f47..e5fada4313 160000 --- a/third_party/prometheus-cpp +++ b/third_party/prometheus-cpp @@ -1 +1 @@ -Subproject commit ad99e21f4706193670c42b36c9824dc997f4c475 +Subproject commit e5fada43131d251e9c4786b04263ce98b6767ba5 diff --git a/third_party_release b/third_party_release index 7362473f60..2a57541bf0 100644 --- a/third_party_release +++ b/third_party_release @@ -21,5 +21,5 @@ ms-gsl=v3.1.0-67-g6f45293 nlohmann-json=v3.11.3 opentelemetry-proto=v1.3.2 opentracing-cpp=v1.6.0 -prometheus-cpp=v1.2.4 +prometheus-cpp=v1.3.0 vcpkg=2024.02.14 From 84d427070899e4faf75d1a7be175e9c723a01171 Mon Sep 17 00:00:00 2001 From: "Alex E." <36134278+chusitoo@users.noreply.github.com> Date: Wed, 6 Nov 2024 16:50:51 -0500 Subject: [PATCH 2/2] [EXPORTER] Log SSL Connection Information (#3113) --- CMakeLists.txt | 3 + exporters/otlp/src/otlp_http_client.cc | 1 + .../ext/http/client/curl/http_client_curl.h | 3 + .../http/client/curl/http_operation_curl.h | 11 ++- .../ext/http/client/http_client.h | 2 + ext/src/http/client/curl/CMakeLists.txt | 5 + ext/src/http/client/curl/http_client_curl.cc | 8 +- .../http/client/curl/http_operation_curl.cc | 91 ++++++++++++++++++- .../http/client/nosend/http_client_nosend.h | 3 + 9 files changed, 121 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a0710834bf..b2f2e2764d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -215,6 +215,9 @@ option( "Whether to include gzip compression for the OTLP http exporter in the SDK" OFF) +option(WITH_CURL_LOGGING "Whether to enable select CURL verbosity in OTel logs" + OFF) + option(WITH_ZIPKIN "Whether to include the Zipkin exporter in the SDK" OFF) option(WITH_PROMETHEUS "Whether to include the Prometheus Client in the SDK" diff --git a/exporters/otlp/src/otlp_http_client.cc b/exporters/otlp/src/otlp_http_client.cc index 876719fa1e..c330ffae2f 100644 --- a/exporters/otlp/src/otlp_http_client.cc +++ b/exporters/otlp/src/otlp_http_client.cc @@ -989,6 +989,7 @@ OtlpHttpClient::createSession( request->SetBody(body_vec); request->ReplaceHeader("Content-Type", content_type); request->ReplaceHeader("User-Agent", options_.user_agent); + request->EnableLogging(options_.console_debug); if (options_.compression == "gzip") { diff --git a/ext/include/opentelemetry/ext/http/client/curl/http_client_curl.h b/ext/include/opentelemetry/ext/http/client/curl/http_client_curl.h index 902981f392..ef65388fe1 100644 --- a/ext/include/opentelemetry/ext/http/client/curl/http_client_curl.h +++ b/ext/include/opentelemetry/ext/http/client/curl/http_client_curl.h @@ -102,6 +102,8 @@ class Request : public opentelemetry::ext::http::client::Request compression_ = compression; } + void EnableLogging(bool is_log_enabled) noexcept override { is_log_enabled_ = is_log_enabled; } + public: opentelemetry::ext::http::client::Method method_; opentelemetry::ext::http::client::HttpSslOptions ssl_options_; @@ -111,6 +113,7 @@ class Request : public opentelemetry::ext::http::client::Request std::chrono::milliseconds timeout_ms_{5000}; // ms opentelemetry::ext::http::client::Compression compression_{ opentelemetry::ext::http::client::Compression::kNone}; + bool is_log_enabled_{false}; }; class Response : public opentelemetry::ext::http::client::Response diff --git a/ext/include/opentelemetry/ext/http/client/curl/http_operation_curl.h b/ext/include/opentelemetry/ext/http/client/curl/http_operation_curl.h index b6654ce3ad..b94c53b2d0 100644 --- a/ext/include/opentelemetry/ext/http/client/curl/http_operation_curl.h +++ b/ext/include/opentelemetry/ext/http/client/curl/http_operation_curl.h @@ -102,6 +102,12 @@ class HttpOperation static size_t ReadMemoryCallback(char *buffer, size_t size, size_t nitems, void *userp); + static int CurlLoggerCallback(const CURL * /* handle */, + curl_infotype type, + const char *data, + size_t size, + void * /* clientp */) noexcept; + #if LIBCURL_VERSION_NUM >= 0x075000 static int PreRequestCallback(void *clientp, char *conn_primary_ip, @@ -152,7 +158,8 @@ class HttpOperation // Default connectivity and response size options bool is_raw_response = false, std::chrono::milliseconds http_conn_timeout = default_http_conn_timeout, - bool reuse_connection = false); + bool reuse_connection = false, + bool is_log_enabled = false); /** * Destroy CURL instance @@ -300,6 +307,8 @@ class HttpOperation const opentelemetry::ext::http::client::Compression &compression_; + const bool is_log_enabled_; + // Processed response headers and body long response_code_; std::vector response_headers_; diff --git a/ext/include/opentelemetry/ext/http/client/http_client.h b/ext/include/opentelemetry/ext/http/client/http_client.h index d17215da9b..e467f9ef63 100644 --- a/ext/include/opentelemetry/ext/http/client/http_client.h +++ b/ext/include/opentelemetry/ext/http/client/http_client.h @@ -245,6 +245,8 @@ class Request virtual void SetCompression(const Compression &compression) noexcept = 0; + virtual void EnableLogging(bool is_log_enabled) noexcept = 0; + virtual ~Request() = default; }; diff --git a/ext/src/http/client/curl/CMakeLists.txt b/ext/src/http/client/curl/CMakeLists.txt index 6a69c7de51..c812fcefbf 100644 --- a/ext/src/http/client/curl/CMakeLists.txt +++ b/ext/src/http/client/curl/CMakeLists.txt @@ -24,6 +24,11 @@ else() PRIVATE ${CURL_LIBRARIES}) endif() +if(WITH_CURL_LOGGING) + target_compile_definitions(opentelemetry_http_client_curl + PRIVATE ENABLE_CURL_LOGGING) +endif() + if(WITH_OTLP_HTTP_COMPRESSION) if(TARGET ZLIB::ZLIB) target_link_libraries( diff --git a/ext/src/http/client/curl/http_client_curl.cc b/ext/src/http/client/curl/http_client_curl.cc index ae353c5509..6827b9f9c7 100644 --- a/ext/src/http/client/curl/http_client_curl.cc +++ b/ext/src/http/client/curl/http_client_curl.cc @@ -116,10 +116,10 @@ void Session::SendRequest( #endif } - curl_operation_.reset(new HttpOperation(http_request_->method_, url, http_request_->ssl_options_, - callback_ptr, http_request_->headers_, - http_request_->body_, http_request_->compression_, false, - http_request_->timeout_ms_, reuse_connection)); + curl_operation_.reset(new HttpOperation( + http_request_->method_, url, http_request_->ssl_options_, callback_ptr, + http_request_->headers_, http_request_->body_, http_request_->compression_, false, + http_request_->timeout_ms_, reuse_connection, http_request_->is_log_enabled_)); bool success = CURLE_OK == curl_operation_->SendAsync(this, [this, callback](HttpOperation &operation) { if (operation.WasAborted()) diff --git a/ext/src/http/client/curl/http_operation_curl.cc b/ext/src/http/client/curl/http_operation_curl.cc index 4de014fd82..b80624d069 100644 --- a/ext/src/http/client/curl/http_operation_curl.cc +++ b/ext/src/http/client/curl/http_operation_curl.cc @@ -22,6 +22,7 @@ #include "opentelemetry/ext/http/client/curl/http_client_curl.h" #include "opentelemetry/ext/http/client/curl/http_operation_curl.h" #include "opentelemetry/ext/http/client/http_client.h" +#include "opentelemetry/nostd/string_view.h" #include "opentelemetry/sdk/common/global_log_handler.h" #include "opentelemetry/version.h" @@ -261,7 +262,8 @@ HttpOperation::HttpOperation(opentelemetry::ext::http::client::Method method, // Default connectivity and response size options bool is_raw_response, std::chrono::milliseconds http_conn_timeout, - bool reuse_connection) + bool reuse_connection, + bool is_log_enabled) : is_aborted_(false), is_finished_(false), is_cleaned_(false), @@ -281,6 +283,7 @@ HttpOperation::HttpOperation(opentelemetry::ext::http::client::Method method, request_nwrite_(0), session_state_(opentelemetry::ext::http::client::SessionState::Created), compression_(compression), + is_log_enabled_(is_log_enabled), response_code_(0) { /* get a curl handle */ @@ -569,8 +572,77 @@ CURLcode HttpOperation::SetCurlOffOption(CURLoption option, curl_off_t value) return rc; } +int HttpOperation::CurlLoggerCallback(const CURL * /* handle */, + curl_infotype type, + const char *data, + size_t size, + void * /* clientp */) noexcept +{ + nostd::string_view text_to_log{data, size}; + + if (!text_to_log.empty() && text_to_log[size - 1] == '\n') + { + text_to_log = text_to_log.substr(0, size - 1); + } + + if (type == CURLINFO_TEXT) + { + static const auto kTlsInfo = nostd::string_view("SSL connection using"); + static const auto kFailureMsg = nostd::string_view("Recv failure:"); + + if (text_to_log.substr(0, kTlsInfo.size()) == kTlsInfo) + { + OTEL_INTERNAL_LOG_INFO(text_to_log); + } + else if (text_to_log.substr(0, kFailureMsg.size()) == kFailureMsg) + { + OTEL_INTERNAL_LOG_ERROR(text_to_log); + } +// This guard serves as a catch-all block for all other less interesting output that should +// remain available for maintainer internal use and for debugging purposes only. +#ifdef OTEL_CURL_DEBUG + else + { + OTEL_INTERNAL_LOG_DEBUG(text_to_log); + } +#endif // OTEL_CURL_DEBUG + } +// Same as above, this guard is meant only for internal use by maintainers, and should not be used +// in production (information leak). +#ifdef OTEL_CURL_DEBUG + else if (type == CURLINFO_HEADER_OUT) + { + static const auto kHeaderSent = nostd::string_view("Send header => "); + + while (!text_to_log.empty() && !std::iscntrl(text_to_log[0])) + { + const auto pos = text_to_log.find('\n'); + + if (pos != nostd::string_view::npos) + { + OTEL_INTERNAL_LOG_DEBUG(kHeaderSent << text_to_log.substr(0, pos - 1)); + text_to_log = text_to_log.substr(pos + 1); + } + } + } + else if (type == CURLINFO_HEADER_IN) + { + static const auto kHeaderRecv = nostd::string_view("Recv header => "); + OTEL_INTERNAL_LOG_DEBUG(kHeaderRecv << text_to_log); + } +#endif // OTEL_CURL_DEBUG + + return 0; +} + CURLcode HttpOperation::Setup() { +#ifdef ENABLE_CURL_LOGGING + static constexpr auto kEnableCurlLogging = true; +#else + static constexpr auto kEnableCurlLogging = false; +#endif // ENABLE_CURL_LOGGING + if (!curl_resource_.easy_handle) { return CURLE_FAILED_INIT; @@ -581,11 +653,28 @@ CURLcode HttpOperation::Setup() curl_error_message_[0] = '\0'; curl_easy_setopt(curl_resource_.easy_handle, CURLOPT_ERRORBUFFER, curl_error_message_); +// Support for custom debug function callback was added in version 7.9.6 so we guard against +// exposing the default CURL output by keeping verbosity always disabled in lower versions. +#if LIBCURL_VERSION_NUM < CURL_VERSION_BITS(7, 9, 6) rc = SetCurlLongOption(CURLOPT_VERBOSE, 0L); if (rc != CURLE_OK) { return rc; } +#else + rc = SetCurlLongOption(CURLOPT_VERBOSE, (is_log_enabled_ && kEnableCurlLogging) ? 1L : 0L); + if (rc != CURLE_OK) + { + return rc; + } + + rc = SetCurlPtrOption(CURLOPT_DEBUGFUNCTION, + reinterpret_cast(&HttpOperation::CurlLoggerCallback)); + if (rc != CURLE_OK) + { + return rc; + } +#endif // Specify target URL rc = SetCurlStrOption(CURLOPT_URL, url_.c_str()); diff --git a/test_common/include/opentelemetry/test_common/ext/http/client/nosend/http_client_nosend.h b/test_common/include/opentelemetry/test_common/ext/http/client/nosend/http_client_nosend.h index a5f8c2f0e1..211b5b3720 100644 --- a/test_common/include/opentelemetry/test_common/ext/http/client/nosend/http_client_nosend.h +++ b/test_common/include/opentelemetry/test_common/ext/http/client/nosend/http_client_nosend.h @@ -69,6 +69,8 @@ class Request : public opentelemetry::ext::http::client::Request compression_ = compression; } + void EnableLogging(bool is_log_enabled) noexcept override { is_log_enabled_ = is_log_enabled; } + public: opentelemetry::ext::http::client::Method method_; opentelemetry::ext::http::client::HttpSslOptions ssl_options_; @@ -78,6 +80,7 @@ class Request : public opentelemetry::ext::http::client::Request std::chrono::milliseconds timeout_ms_{5000}; // ms opentelemetry::ext::http::client::Compression compression_{ opentelemetry::ext::http::client::Compression::kNone}; + bool is_log_enabled_{false}; }; class Response : public opentelemetry::ext::http::client::Response