opentelemetrytracer: Dynatrace sampler to fetch configuration from an API

api/envoy/extensions/tracers/opentelemetry/samplers/v3/dynatrace_sampler.proto

@@ -33,7 +33,7 @@ message DynatraceSamplerConfig {
   // .. code-block:: yaml
   //
   //    http_uri:
-  //      uri: <tenant>.dev.dynatracelabs.com/api/v2/otlp/v1/traces
+  //      uri: <tenant>.dev.dynatracelabs.com/api/v2/samplingConfiguration
   //      cluster: dynatrace
   //      timeout: 10s
   //

changelogs/current.yaml

@@ -282,6 +282,9 @@ new_features:
 - area: tracing
   change: |
     Added support for variant span attribute type for the OpenTelemetry tracer.
+- area: tracing
+  change: |
+    Dynatrace sampler fetches configuration from Dynatrace API.
 
 deprecated:
 - area: listener

source/extensions/tracers/opentelemetry/samplers/dynatrace/sampler_config.cc

@@ -8,18 +8,19 @@ namespace Extensions {
 namespace Tracers {
 namespace OpenTelemetry {
 
-void SamplerConfig::parse(const std::string& json) {
+bool SamplerConfig::parse(const std::string& json) {
   const auto result = Envoy::Json::Factory::loadFromStringNoThrow(json);
   if (result.ok()) {
     const auto& obj = result.value();
     if (obj->hasObject("rootSpansPerMinute")) {
       const auto value = obj->getInteger("rootSpansPerMinute", default_root_spans_per_minute_);
       root_spans_per_minute_.store(value);
-      return;
+      return true;
     }
   }
   // Didn't get a value, reset to default
   root_spans_per_minute_.store(default_root_spans_per_minute_);
+  return false;
 }
 
 } // namespace OpenTelemetry

source/extensions/tracers/opentelemetry/samplers/dynatrace/sampler_config.h

@@ -22,12 +22,18 @@ class SamplerConfig {
                                            ? default_root_spans_per_minute
                                            : ROOT_SPANS_PER_MINUTE_DEFAULT),
         root_spans_per_minute_(default_root_spans_per_minute_) {}
+
+  SamplerConfig(const SamplerConfig&) = delete;
+  SamplerConfig& operator=(const SamplerConfig&) = delete;
+
   /**
    * @brief Parses a json string containing the expected root spans per minute.
    *
    * @param json A string containing the configuration.
+   *
+   * @return true if parsing was successful, false otherwise
    */
-  void parse(const std::string& json);
+  bool parse(const std::string& json);
 
   /**
    * @brief Returns wanted root spans per minute

source/extensions/tracers/opentelemetry/samplers/dynatrace/sampler_config_provider.cc

@@ -1,5 +1,7 @@
 #include "source/extensions/tracers/opentelemetry/samplers/dynatrace/sampler_config_provider.h"
 
+#include <chrono>
+
 #include "source/common/common/enum_to_int.h"
 #include "source/common/http/utility.h"
 
@@ -8,10 +10,98 @@ namespace Extensions {
 namespace Tracers {
 namespace OpenTelemetry {
 
+static constexpr std::chrono::seconds INITIAL_TIMER_DURATION{10};
+static constexpr std::chrono::minutes TIMER_INTERVAL{5};
+
+namespace {
+
+bool reEnableTimer(Http::Code response_code) {
+  switch (response_code) {
+  case Http::Code::OK:
+  case Http::Code::TooManyRequests:
+  case Http::Code::InternalServerError:
+  case Http::Code::BadGateway:
+  case Http::Code::ServiceUnavailable:
+  case Http::Code::GatewayTimeout:
+    return true;
+  default:
+    return false;
+  }
+}
+
+std::chrono::milliseconds getTimeout(envoy::config::core::v3::HttpUri& http_uri) {
+  auto timeout =
+      std::chrono::milliseconds(DurationUtil::durationToMilliseconds(http_uri.timeout()));
+  // we want to ensure that we don't have more than one pending request
+  if (timeout > (TIMER_INTERVAL / 2)) {
+    timeout = (TIMER_INTERVAL / 2);
+  }
+  return timeout;
+}
+
+} // namespace
+
 SamplerConfigProviderImpl::SamplerConfigProviderImpl(
-    Server::Configuration::TracerFactoryContext& /*context*/,
+    Server::Configuration::TracerFactoryContext& context,
     const envoy::extensions::tracers::opentelemetry::samplers::v3::DynatraceSamplerConfig& config)
-    : sampler_config_(config.root_spans_per_minute()) {}
+    : cluster_manager_(context.serverFactoryContext().clusterManager()),
+      http_uri_(config.http_uri()),
+      authorization_header_value_(absl::StrCat("Api-Token ", config.token())),
+      sampler_config_(config.root_spans_per_minute()), timeout_(getTimeout(http_uri_)) {
+
+  timer_ = context.serverFactoryContext().mainThreadDispatcher().createTimer([this]() -> void {
+    const auto thread_local_cluster = cluster_manager_.getThreadLocalCluster(http_uri_.cluster());
+    if (thread_local_cluster == nullptr) {
+      ENVOY_LOG(error, "SamplerConfigProviderImpl failed: [cluster = {}] is not configured",
+                http_uri_.cluster());
+    } else {
+      Http::RequestMessagePtr message = Http::Utility::prepareHeaders(http_uri_);
+      message->headers().setReferenceMethod(Http::Headers::get().MethodValues.Get);
+      message->headers().setReference(Http::CustomHeaders::get().Authorization,
+                                      authorization_header_value_);
+      active_request_ = thread_local_cluster->httpAsyncClient().send(
+          std::move(message), *this, Http::AsyncClient::RequestOptions().setTimeout(timeout_));
+    }
+  });
+
+  timer_->enableTimer(std::chrono::seconds(INITIAL_TIMER_DURATION));
+}
+
+SamplerConfigProviderImpl::~SamplerConfigProviderImpl() {
+  if (active_request_) {
+    active_request_->cancel();
+  }
+}
+
+void SamplerConfigProviderImpl::onSuccess(const Http::AsyncClient::Request& /*request*/,
+                                          Http::ResponseMessagePtr&& http_response) {
+  active_request_ = nullptr;
+  const auto response_code = Http::Utility::getResponseStatus(http_response->headers());
+  bool json_valid = false;
+  if (response_code == enumToInt(Http::Code::OK)) {
+    ENVOY_LOG(debug, "Received sampling configuration from Dynatrace: {}",
+              http_response->bodyAsString());
+    json_valid = sampler_config_.parse(http_response->bodyAsString());
+    if (!json_valid) {
+      ENVOY_LOG(warn, "Failed to parse sampling configuration received from Dynatrace: {}",
+                http_response->bodyAsString());
+    }
+  } else {
+    ENVOY_LOG(warn, "Failed to get sampling configuration from Dynatrace: {}", response_code);
+  }
+
+  if (json_valid || reEnableTimer(static_cast<Http::Code>(response_code))) {
+    timer_->enableTimer(std::chrono::seconds(TIMER_INTERVAL));
+  }
+}
+
+void SamplerConfigProviderImpl::onFailure(const Http::AsyncClient::Request& /*request*/,
+                                          Http::AsyncClient::FailureReason reason) {
+  active_request_ = nullptr;
+  timer_->enableTimer(std::chrono::seconds(TIMER_INTERVAL));
+  ENVOY_LOG(warn, "Failed to get sampling configuration from Dynatrace. Reason {}",
+            enumToInt(reason));
+}
 
 const SamplerConfig& SamplerConfigProviderImpl::getSamplerConfig() const { return sampler_config_; }
 

source/extensions/tracers/opentelemetry/samplers/dynatrace/sampler_config_provider.h

@@ -40,19 +40,35 @@ class SamplerConfigProvider {
 };
 
 class SamplerConfigProviderImpl : public SamplerConfigProvider,
-                                  public Logger::Loggable<Logger::Id::tracing> {
+                                  public Logger::Loggable<Logger::Id::tracing>,
+                                  public Http::AsyncClient::Callbacks {
 public:
   SamplerConfigProviderImpl(
       Server::Configuration::TracerFactoryContext& context,
       const envoy::extensions::tracers::opentelemetry::samplers::v3::DynatraceSamplerConfig&
           config);
 
+  void onSuccess(const Http::AsyncClient::Request& request,
+                 Http::ResponseMessagePtr&& response) override;
+
+  void onFailure(const Http::AsyncClient::Request& request,
+                 Http::AsyncClient::FailureReason reason) override;
+
+  void onBeforeFinalizeUpstreamSpan(Envoy::Tracing::Span& /*span*/,
+                                    const Http::ResponseHeaderMap* /*response_headers*/) override{};
+
   const SamplerConfig& getSamplerConfig() const override;
 
-  ~SamplerConfigProviderImpl() override = default;
+  ~SamplerConfigProviderImpl() override;
 
 private:
+  Event::TimerPtr timer_;
+  Upstream::ClusterManager& cluster_manager_;
+  envoy::config::core::v3::HttpUri http_uri_;
+  const std::string authorization_header_value_;
+  Http::AsyncClient::Request* active_request_{};
   SamplerConfig sampler_config_;
+  const std::chrono::milliseconds timeout_;
 };
 
 using SamplerConfigProviderPtr = std::unique_ptr<SamplerConfigProvider>;