[EXPORTER] Gzip compression support for OTLP/HTTP and OTLP/gRPC expor…

…ter (#2530)

CHANGELOG.md

@@ -15,6 +15,19 @@ Increment the:
 
 ## [Unreleased]
 
+* [EXPORTER] Gzip compression support for OTLP/HTTP and OTLP/gRPC exporter
+  [#2530](https://github.com/open-telemetry/opentelemetry-cpp/pull/2530)
+
+Important changes:
+
+* [EXPORTER] Gzip compression support for OTLP/HTTP and OTLP/gRPC exporter
+  [#2530](https://github.com/open-telemetry/opentelemetry-cpp/pull/2530)
+  * In the `OtlpHttpExporterOptions` and `OtlpGrpcExporterOptions`, a new
+    field called compression has been introduced. This field can be set
+    to "gzip” to enable gzip compression.
+  * The CMake option `WITH_OTLP_HTTP_COMPRESSION` is introduced to enable
+    gzip compression support for the OTLP HTTP Exporter and includes a
+    dependency on zlib.
 * [SDK] Change OTLP HTTP content_type default to binary
   [#2558](https://github.com/open-telemetry/opentelemetry-cpp/pull/2558)
 

CMakeLists.txt

@@ -204,6 +204,11 @@ option(WITH_OTLP_GRPC "Whether to include the OTLP gRPC exporter in the SDK"
 option(WITH_OTLP_HTTP "Whether to include the OTLP http exporter in the SDK"
        OFF)
 
+option(
+  WITH_OTLP_HTTP_COMPRESSION
+  "Whether to include gzip compression for the OTLP http exporter in the SDK"
+  OFF)
+
 option(WITH_ZIPKIN "Whether to include the Zipkin exporter in the SDK" OFF)
 
 option(WITH_PROMETHEUS "Whether to include the Prometheus Client in the SDK"
@@ -448,6 +453,18 @@ if((NOT WITH_API_ONLY) AND WITH_HTTP_CLIENT_CURL)
   message(STATUS "Found CURL: ${CURL_LIBRARIES}, version ${CURL_VERSION}")
 endif()
 
+#
+# Do we need ZLIB ?
+#
+
+if((NOT WITH_API_ONLY)
+   AND WITH_HTTP_CLIENT_CURL
+   AND WITH_OTLP_HTTP_COMPRESSION)
+  # No specific version required.
+  find_package(ZLIB REQUIRED)
+  message(STATUS "Found ZLIB: ${ZLIB_LIBRARIES}, version ${ZLIB_VERSION}")
+endif()
+
 #
 # Do we need NLOHMANN_JSON ?
 #

api/CMakeLists.txt

@@ -120,6 +120,11 @@ if(WITH_METRICS_EXEMPLAR_PREVIEW)
                              INTERFACE ENABLE_METRICS_EXEMPLAR_PREVIEW)
 endif()
 
+if(WITH_OTLP_HTTP_COMPRESSION)
+  target_compile_definitions(opentelemetry_api
+                             INTERFACE ENABLE_OTLP_COMPRESSION_PREVIEW)
+endif()
+
 include(${PROJECT_SOURCE_DIR}/cmake/pkgconfig.cmake)
 
 if(OPENTELEMETRY_INSTALL)

bazel/repository.bzl

@@ -183,3 +183,16 @@ def opentelemetry_cpp_deps():
             "https://github.com/opentracing/opentracing-cpp/archive/refs/tags/v1.6.0.tar.gz",
         ],
     )
+
+    # Zlib (optional)
+    maybe(
+        http_archive,
+        name = "zlib",
+        build_file = "@io_opentelemetry_cpp//bazel:zlib.BUILD",
+        sha256 = "d14c38e313afc35a9a8760dadf26042f51ea0f5d154b0630a31da0540107fb98",
+        strip_prefix = "zlib-1.2.13",
+        urls = [
+            "https://github.com/madler/zlib/releases/download/v1.2.13/zlib-1.2.13.tar.xz",
+            "https://zlib.net/zlib-1.2.13.tar.xz",
+        ],
+    )

bazel/zlib.BUILD

@@ -0,0 +1,74 @@
+# Copyright The OpenTelemetry Authors
+# SPDX-License-Identifier: Apache-2.0
+
+# Builds ZLIB from a distribution.
+# Copied from https://github.com/protocolbuffers/protobuf/blob/master/third_party/zlib.BUILD
+
+licenses(["notice"])  # BSD/MIT-like license (for zlib)
+
+exports_files(["zlib.BUILD"])
+
+_ZLIB_HEADERS = [
+    "crc32.h",
+    "deflate.h",
+    "gzguts.h",
+    "inffast.h",
+    "inffixed.h",
+    "inflate.h",
+    "inftrees.h",
+    "trees.h",
+    "zconf.h",
+    "zlib.h",
+    "zutil.h",
+]
+
+_ZLIB_PREFIXED_HEADERS = ["zlib/include/" + hdr for hdr in _ZLIB_HEADERS]
+
+# In order to limit the damage from the `includes` propagation
+# via `:zlib`, copy the public headers to a subdirectory and
+# expose those.
+genrule(
+    name = "copy_public_headers",
+    srcs = _ZLIB_HEADERS,
+    outs = _ZLIB_PREFIXED_HEADERS,
+    cmd_bash = "cp $(SRCS) $(@D)/zlib/include/",
+    cmd_bat = " && ".join(
+        ["@copy /Y $(location %s) $(@D)\\zlib\\include\\  >NUL" %
+         s for s in _ZLIB_HEADERS],
+    ),
+)
+
+cc_library(
+    name = "zlib",
+    srcs = [
+        "adler32.c",
+        "compress.c",
+        "crc32.c",
+        "deflate.c",
+        "gzclose.c",
+        "gzlib.c",
+        "gzread.c",
+        "gzwrite.c",
+        "infback.c",
+        "inffast.c",
+        "inflate.c",
+        "inftrees.c",
+        "trees.c",
+        "uncompr.c",
+        "zutil.c",
+        # Include the un-prefixed headers in srcs to work
+        # around the fact that zlib isn't consistent in its
+        # choice of <> or "" delimiter when including itself.
+    ] + _ZLIB_HEADERS,
+    hdrs = _ZLIB_PREFIXED_HEADERS,
+    copts = select({
+        "@platforms//os:windows": [],
+        "//conditions:default": [
+            "-Wno-deprecated-non-prototype",
+            "-Wno-unused-variable",
+            "-Wno-implicit-function-declaration",
+        ],
+    }),
+    includes = ["zlib/include/"],
+    visibility = ["//visibility:public"],
+)

ci/do_ci.sh

@@ -120,6 +120,7 @@ elif [[ "$1" == "cmake.maintainer.sync.test" ]]; then
         -DWITH_ASYNC_EXPORT_PREVIEW=OFF \
         -DOTELCPP_MAINTAINER_MODE=ON \
         -DWITH_NO_DEPRECATED_CODE=ON \
+        -DWITH_OTLP_HTTP_COMPRESSION=ON \
         ${IWYU} \
         "${SRC_DIR}"
   eval "$MAKE_COMMAND"
@@ -140,6 +141,7 @@ elif [[ "$1" == "cmake.maintainer.async.test" ]]; then
         -DWITH_ASYNC_EXPORT_PREVIEW=ON \
         -DOTELCPP_MAINTAINER_MODE=ON \
         -DWITH_NO_DEPRECATED_CODE=ON \
+        -DWITH_OTLP_HTTP_COMPRESSION=ON \
         ${IWYU} \
         "${SRC_DIR}"
   eval "$MAKE_COMMAND"
@@ -161,6 +163,7 @@ elif [[ "$1" == "cmake.maintainer.cpp11.async.test" ]]; then
         -DWITH_ASYNC_EXPORT_PREVIEW=ON \
         -DOTELCPP_MAINTAINER_MODE=ON \
         -DWITH_NO_DEPRECATED_CODE=ON \
+        -DWITH_OTLP_HTTP_COMPRESSION=ON \
         "${SRC_DIR}"
   make -k -j $(nproc)
   make test
@@ -182,6 +185,7 @@ elif [[ "$1" == "cmake.maintainer.abiv2.test" ]]; then
         -DWITH_NO_DEPRECATED_CODE=ON \
         -DWITH_ABI_VERSION_1=OFF \
         -DWITH_ABI_VERSION_2=ON \
+        -DWITH_OTLP_HTTP_COMPRESSION=ON \
         ${IWYU} \
         "${SRC_DIR}"
   eval "$MAKE_COMMAND"

docs/dependencies.md

@@ -63,6 +63,12 @@ Both these dependencies are listed here:
     - protobuf serialized otlp messages are encoded in JSON format using this
       library.
     - License: `MIT License`
+  - [zlib](https://www.zlib.net/): A Massively Spiffy Yet Delicately
+    Unobtrusive Compression Library.
+    - The `http_client` utilizes zlib to compress the message body and send
+      it in gzip format.
+    - License: The library is licensed
+      [here](https://www.zlib.net/zlib_license.html)
 
 - [OTLP/gRPC](/exporters/otlp)
   exporter:

exporters/otlp/include/opentelemetry/exporters/otlp/otlp_environment.h

@@ -152,6 +152,10 @@ inline OtlpHeaders GetOtlpDefaultHeaders()
   return GetOtlpDefaultTracesHeaders();
 }
 
+std::string GetOtlpDefaultTracesCompression();
+std::string GetOtlpDefaultMetricsCompression();
+std::string GetOtlpDefaultLogsCompression();
+
 }  // namespace otlp
 }  // namespace exporter
 OPENTELEMETRY_END_NAMESPACE

exporters/otlp/include/opentelemetry/exporters/otlp/otlp_grpc_client_options.h

@@ -55,6 +55,9 @@ struct OtlpGrpcClientOptions
   /** max number of threads that can be allocated from this */
   std::size_t max_threads;
 
+  /** Compression type. */
+  std::string compression;
+
 #ifdef ENABLE_ASYNC_EXPORT
   // Concurrent requests
   std::size_t max_concurrent_requests;

exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_client.h

@@ -59,6 +59,9 @@ struct OtlpHttpClientOptions
   // This option is ignored if content_type is not kJson
   JsonBytesMappingKind json_bytes_mapping = JsonBytesMappingKind::kHexId;
 
+  // By default, do not compress data
+  std::string compression = "none";
+
   // If using the json name of protobuf field to set the key of json. By default, we will use the
   // field name just like proto files.
   bool use_json_name = false;
@@ -94,6 +97,7 @@ struct OtlpHttpClientOptions
                                nostd::string_view input_ssl_cipher_suite,
                                HttpRequestContentType input_content_type,
                                JsonBytesMappingKind input_json_bytes_mapping,
+                               nostd::string_view input_compression,
                                bool input_use_json_name,
                                bool input_console_debug,
                                std::chrono::system_clock::duration input_timeout,
@@ -116,6 +120,7 @@ struct OtlpHttpClientOptions
                     input_ssl_cipher_suite),
         content_type(input_content_type),
         json_bytes_mapping(input_json_bytes_mapping),
+        compression(input_compression),
         use_json_name(input_use_json_name),
         console_debug(input_console_debug),
         timeout(input_timeout),

exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_exporter_options.h

@@ -100,6 +100,9 @@ struct OPENTELEMETRY_EXPORT OtlpHttpExporterOptions
 
   /** TLS cipher suite. */
   std::string ssl_cipher_suite;
+
+  /** Compression type. */
+  std::string compression;
 };
 
 }  // namespace otlp

exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_log_record_exporter_options.h

@@ -100,6 +100,9 @@ struct OPENTELEMETRY_EXPORT OtlpHttpLogRecordExporterOptions
 
   /** TLS cipher suite. */
   std::string ssl_cipher_suite;
+
+  /** Compression type. */
+  std::string compression;
 };
 
 }  // namespace otlp

exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_metric_exporter_options.h

@@ -103,6 +103,9 @@ struct OPENTELEMETRY_EXPORT OtlpHttpMetricExporterOptions
 
   /** TLS cipher suite. */
   std::string ssl_cipher_suite;
+
+  /** Compression type. */
+  std::string compression;
 };
 
 }  // namespace otlp

exporters/otlp/src/otlp_environment.cc

@@ -1069,6 +1069,57 @@ OtlpHeaders GetOtlpDefaultLogsHeaders()
   return GetHeaders(kSignalEnv, kGenericEnv);
 }
 
+std::string GetOtlpDefaultTracesCompression()
+{
+  constexpr char kSignalEnv[]  = "OTEL_EXPORTER_OTLP_TRACES_COMPRESSION";
+  constexpr char kGenericEnv[] = "OTEL_EXPORTER_OTLP_COMPRESSION";
+
+  std::string value;
+  bool exists;
+
+  exists = GetStringDualEnvVar(kSignalEnv, kGenericEnv, value);
+  if (exists)
+  {
+    return value;
+  }
+
+  return std::string{"none"};
+}
+
+std::string GetOtlpDefaultMetricsCompression()
+{
+  constexpr char kSignalEnv[]  = "OTEL_EXPORTER_OTLP_METRICS_COMPRESSION";
+  constexpr char kGenericEnv[] = "OTEL_EXPORTER_OTLP_COMPRESSION";
+
+  std::string value;
+  bool exists;
+
+  exists = GetStringDualEnvVar(kSignalEnv, kGenericEnv, value);
+  if (exists)
+  {
+    return value;
+  }
+
+  return std::string{"none"};
+}
+
+std::string GetOtlpDefaultLogsCompression()
+{
+  constexpr char kSignalEnv[]  = "OTEL_EXPORTER_OTLP_LOGS_COMPRESSION";
+  constexpr char kGenericEnv[] = "OTEL_EXPORTER_OTLP_COMPRESSION";
+
+  std::string value;
+  bool exists;
+
+  exists = GetStringDualEnvVar(kSignalEnv, kGenericEnv, value);
+  if (exists)
+  {
+    return value;
+  }
+
+  return std::string{"none"};
+}
+
 }  // namespace otlp
 }  // namespace exporter
 OPENTELEMETRY_END_NAMESPACE

exporters/otlp/src/otlp_grpc_client.cc

@@ -297,6 +297,11 @@ std::shared_ptr<grpc::Channel> OtlpGrpcClient::MakeChannel(const OtlpGrpcClientO
     grpc_arguments.SetResourceQuota(quota);
   }
 
+  if (options.compression == "gzip")
+  {
+    grpc_arguments.SetCompressionAlgorithm(GRPC_COMPRESS_GZIP);
+  }
+
   if (options.use_ssl_credentials)
   {
     grpc::SslCredentialsOptions ssl_opts;

exporters/otlp/src/otlp_grpc_exporter_options.cc

@@ -31,6 +31,8 @@ OtlpGrpcExporterOptions::OtlpGrpcExporterOptions()
   user_agent = GetOtlpDefaultUserAgent();
 
   max_threads = 0;
+
+  compression = GetOtlpDefaultTracesCompression();
 #ifdef ENABLE_ASYNC_EXPORT
   max_concurrent_requests = 64;
 #endif

exporters/otlp/src/otlp_grpc_log_record_exporter_options.cc

@@ -29,6 +29,8 @@ OtlpGrpcLogRecordExporterOptions::OtlpGrpcLogRecordExporterOptions()
   user_agent = GetOtlpDefaultUserAgent();
 
   max_threads = 0;
+
+  compression = GetOtlpDefaultLogsCompression();
 #ifdef ENABLE_ASYNC_EXPORT
   max_concurrent_requests = 64;
 #endif

exporters/otlp/src/otlp_grpc_metric_exporter_options.cc

@@ -31,6 +31,8 @@ OtlpGrpcMetricExporterOptions::OtlpGrpcMetricExporterOptions()
   aggregation_temporality = PreferredAggregationTemporality::kCumulative;
 
   max_threads = 0;
+
+  compression = GetOtlpDefaultMetricsCompression();
 #ifdef ENABLE_ASYNC_EXPORT
   max_concurrent_requests = 64;
 #endif

exporters/otlp/src/otlp_http_client.cc

@@ -958,6 +958,11 @@ OtlpHttpClient::createSession(
   request->ReplaceHeader("Content-Type", content_type);
   request->ReplaceHeader("User-Agent", options_.user_agent);
 
+  if (options_.compression == "gzip")
+  {
+    request->SetCompression(opentelemetry::ext::http::client::Compression::kGzip);
+  }
+
   // Returns the created session data
   return HttpSessionData{
       std::move(session),

exporters/otlp/src/otlp_http_exporter.cc

@@ -40,6 +40,7 @@ OtlpHttpExporter::OtlpHttpExporter(const OtlpHttpExporterOptions &options)
                                                             options.ssl_cipher_suite,
                                                             options.content_type,
                                                             options.json_bytes_mapping,
+                                                            options.compression,
                                                             options.use_json_name,
                                                             options.console_debug,
                                                             options.timeout,