diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ff33aa46e3..39207e214c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -284,6 +284,7 @@ jobs: env: CC: /usr/bin/gcc-4.8 CXX: /usr/bin/g++-4.8 + CXX_STANDARD: '11' run: ./ci/do_ci.sh cmake.legacy.test cmake_gcc_48_otlp_exporter_test: @@ -329,8 +330,12 @@ jobs: sudo -E ./ci/setup_ci_environment.sh sudo -E ./ci/setup_cmake.sh - name: run tests + env: + CXX_STANDARD: '20' run: ./ci/do_ci.sh cmake.c++20.test - name: run tests (enable stl) + env: + CXX_STANDARD: '20' run: ./ci/do_ci.sh cmake.c++20.stl.test cmake_test_cxx20_clang: @@ -354,12 +359,14 @@ jobs: CC: /usr/bin/clang CXX: /usr/bin/clang++ CXXFLAGS: "-stdlib=libc++" + CXX_STANDARD: '20' run: ./ci/do_ci.sh cmake.c++20.test - name: run tests (enable stl) env: CC: /usr/bin/clang CXX: /usr/bin/clang++ CXXFLAGS: "-stdlib=libc++" + CXX_STANDARD: '20' run: ./ci/do_ci.sh cmake.c++20.stl.test cmake_otprotocol_test: @@ -378,6 +385,31 @@ jobs: sudo ./ci/setup_grpc.sh ./ci/do_ci.sh cmake.exporter.otprotocol.test + cmake_modern_protobuf_grpc_with_abseil_test: + name: CMake test (with modern protobuf,grpc and abseil) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + - name: setup + env: + PROTOBUF_VERSION: '23.3' + ABSEIL_CPP_VERSION: '20230125.3' + CXX_STANDARD: '14' + run: | + sudo ./ci/setup_cmake.sh + sudo ./ci/setup_ci_environment.sh + sudo -E ./ci/install_abseil.sh + sudo -E ./ci/install_protobuf.sh + - name: run otlp exporter tests + env: + WITH_ABSEIL: 'ON' + CXX_STANDARD: '14' + run: | + sudo -E ./ci/setup_grpc.sh -m -p protobuf -p abseil-cpp + ./ci/do_ci.sh cmake.exporter.otprotocol.test + cmake_do_not_install_test: name: CMake do not install test (with otlp-exporter) runs-on: ubuntu-20.04 diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e95bd0fb0..8b28715280 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ Increment the: ## [Unreleased] +* [EXPORTER] Support protobuf 3.22 or upper + [#2163](https://github.com/open-telemetry/opentelemetry-cpp/pull/2163) * [SDK] Mark logs signal as stable API/SDK [#2229](https://github.com/open-telemetry/opentelemetry-cpp/pull/2229) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3cd71503cf..dcf1034804 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,6 +18,9 @@ project(opentelemetry-cpp) # Mark variables as used so cmake doesn't complain about them mark_as_advanced(CMAKE_TOOLCHAIN_FILE) +# Prefer cmake CONFIG to auto resolve dependencies. +set(CMAKE_FIND_PACKAGE_PREFER_CONFIG TRUE) + # Don't use customized cmake modules if vcpkg is used to resolve dependence. if(NOT DEFINED CMAKE_TOOLCHAIN_FILE) list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/modules/") @@ -125,6 +128,8 @@ endif() option(OPENTELEMETRY_INSTALL "Whether to install opentelemetry targets" ${OPENTELEMETRY_INSTALL_default}) +include("${PROJECT_SOURCE_DIR}/cmake/tools.cmake") + if(NOT DEFINED CMAKE_CXX_STANDARD) if(WITH_STL) # Require at least C++17. C++20 is needed to avoid gsl::span @@ -340,9 +345,21 @@ if(WITH_PROMETHEUS) endif() endif() +if(WITH_ABSEIL) + find_package(absl CONFIG REQUIRED) +endif() + if(WITH_OTLP_GRPC OR WITH_OTLP_HTTP) - set(protobuf_MODULE_COMPATIBLE ON) find_package(Protobuf) + if(Protobuf_VERSION AND Protobuf_VERSION VERSION_GREATER_EQUAL "3.22.0") + if(NOT WITH_ABSEIL) + message( + FATAL_ERROR + "Protobuf 3.22 or upper require abseil-cpp(Recommended version: 20230125 or upper)" + ) + endif() + endif() + if(WITH_OTLP_GRPC) find_package(gRPC) endif() @@ -372,10 +389,14 @@ if(WITH_OTLP_GRPC OR WITH_OTLP_HTTP) ) endif() endif() - # Latest Protobuf uses mixed case instead of uppercase - if(Protobuf_PROTOC_EXECUTABLE) - set(PROTOBUF_PROTOC_EXECUTABLE ${Protobuf_PROTOC_EXECUTABLE}) - endif() + endif() + # Latest Protobuf imported targets and without legacy module support + if(TARGET protobuf::protoc) + project_build_tools_get_imported_location(PROTOBUF_PROTOC_EXECUTABLE + protobuf::protoc) + elseif(Protobuf_PROTOC_EXECUTABLE) + # Some versions of FindProtobuf.cmake uses mixed case instead of uppercase + set(PROTOBUF_PROTOC_EXECUTABLE ${Protobuf_PROTOC_EXECUTABLE}) endif() include(CMakeDependentOption) @@ -524,10 +545,14 @@ if(BUILD_TESTING) set(GTEST_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/googletest/googletest/include ${CMAKE_CURRENT_SOURCE_DIR}/third_party/googletest/googlemock/include) - set(GTEST_BOTH_LIBRARIES - ${CMAKE_BINARY_DIR}/lib/libgtest.a - ${CMAKE_BINARY_DIR}/lib/libgtest_main.a - ${CMAKE_BINARY_DIR}/lib/libgmock.a) + if(TARGET gtest) + set(GTEST_BOTH_LIBRARIES gtest gtest_main gmock) + else() + set(GTEST_BOTH_LIBRARIES + ${CMAKE_BINARY_DIR}/lib/libgtest.a + ${CMAKE_BINARY_DIR}/lib/libgtest_main.a + ${CMAKE_BINARY_DIR}/lib/libgmock.a) + endif() elseif(WIN32) # Make sure we are always bootsrapped with vcpkg on Windows find_package(GTest) @@ -540,7 +565,17 @@ if(BUILD_TESTING) # Prefer GTest installed by OS distro, brew or vcpkg package manager find_package(GTest REQUIRED) endif() - include_directories(SYSTEM ${GTEST_INCLUDE_DIRS}) + if(NOT GTEST_BOTH_LIBRARIES) + # New GTest package names + if(TARGET GTest::gtest) + set(GTEST_BOTH_LIBRARIES GTest::gtest GTest::gtest_main GTest::gmock) + elseif(TARGET GTest::GTest) + set(GTEST_BOTH_LIBRARIES GTest::GTest GTest::Main) + endif() + endif() + if(GTEST_INCLUDE_DIRS) + include_directories(SYSTEM ${GTEST_INCLUDE_DIRS}) + endif() message("GTEST_INCLUDE_DIRS = ${GTEST_INCLUDE_DIRS}") message("GTEST_BOTH_LIBRARIES = ${GTEST_BOTH_LIBRARIES}") enable_testing() diff --git a/api/BUILD b/api/BUILD index 421d26a626..e650d5f7ce 100644 --- a/api/BUILD +++ b/api/BUILD @@ -22,6 +22,7 @@ cc_library( deps = select({ ":with_external_abseil": [ "@com_google_absl//absl/base", + "@com_google_absl//absl/strings", "@com_google_absl//absl/types:variant", ], "//conditions:default": [], diff --git a/api/CMakeLists.txt b/api/CMakeLists.txt index 8a2788ed17..9e32cd24ed 100644 --- a/api/CMakeLists.txt +++ b/api/CMakeLists.txt @@ -42,8 +42,6 @@ endif() if(WITH_ABSEIL) - find_package(absl CONFIG REQUIRED) - target_compile_definitions(opentelemetry_api INTERFACE HAVE_ABSEIL) target_link_libraries( opentelemetry_api INTERFACE absl::bad_variant_access absl::any absl::base diff --git a/ci/do_ci.sh b/ci/do_ci.sh index fce868e37f..c445e0abcd 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -83,13 +83,19 @@ BAZEL_MACOS_TEST_OPTIONS="$BAZEL_MACOS_OPTIONS --test_output=errors" BAZEL_STARTUP_OPTIONS="--output_user_root=$HOME/.cache/bazel" +CMAKE_OPTIONS=(-DCMAKE_BUILD_TYPE=Debug) +if [ ! -z "${CXX_STANDARD}" ]; then + CMAKE_OPTIONS=(${CMAKE_OPTIONS[@]} "-DCMAKE_CXX_STANDARD=${CXX_STANDARD}") +else + CMAKE_OPTIONS=(${CMAKE_OPTIONS[@]} "-DCMAKE_CXX_STANDARD=14") +fi + export CTEST_OUTPUT_ON_FAILURE=1 if [[ "$1" == "cmake.test" ]]; then cd "${BUILD_DIR}" rm -rf * - cmake -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_CXX_STANDARD=14 \ + cmake ${CMAKE_OPTIONS[@]} \ -DWITH_PROMETHEUS=ON \ -DWITH_ZIPKIN=ON \ -DWITH_ELASTICSEARCH=ON \ @@ -102,8 +108,7 @@ if [[ "$1" == "cmake.test" ]]; then elif [[ "$1" == "cmake.maintainer.sync.test" ]]; then cd "${BUILD_DIR}" rm -rf * - cmake -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_CXX_STANDARD=14 \ + cmake ${CMAKE_OPTIONS[@]} \ -DWITH_OTLP_HTTP=ON \ -DWITH_OTLP_HTTP_SSL_PREVIEW=ON \ -DWITH_OTLP_HTTP_SSL_TLS_PREVIEW=ON \ @@ -126,8 +131,7 @@ elif [[ "$1" == "cmake.maintainer.sync.test" ]]; then elif [[ "$1" == "cmake.maintainer.async.test" ]]; then cd "${BUILD_DIR}" rm -rf * - cmake -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_CXX_STANDARD=14 \ + cmake ${CMAKE_OPTIONS[@]} \ -DWITH_OTLP_HTTP=ON \ -DWITH_OTLP_HTTP_SSL_PREVIEW=ON \ -DWITH_OTLP_HTTP_SSL_TLS_PREVIEW=ON \ @@ -150,7 +154,7 @@ elif [[ "$1" == "cmake.maintainer.async.test" ]]; then elif [[ "$1" == "cmake.maintainer.cpp11.async.test" ]]; then cd "${BUILD_DIR}" rm -rf * - cmake -DCMAKE_BUILD_TYPE=Debug \ + cmake ${CMAKE_OPTIONS[@]} \ -DCMAKE_CXX_STANDARD=11 \ -DWITH_OTLP_HTTP=ON \ -DWITH_OTLP_HTTP_SSL_PREVIEW=ON \ @@ -173,8 +177,7 @@ elif [[ "$1" == "cmake.maintainer.cpp11.async.test" ]]; then elif [[ "$1" == "cmake.with_async_export.test" ]]; then cd "${BUILD_DIR}" rm -rf * - cmake -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_CXX_STANDARD=14 \ + cmake ${CMAKE_OPTIONS[@]} \ -DWITH_PROMETHEUS=ON \ -DWITH_ZIPKIN=ON \ -DWITH_ELASTICSEARCH=ON \ @@ -188,8 +191,7 @@ elif [[ "$1" == "cmake.with_async_export.test" ]]; then elif [[ "$1" == "cmake.abseil.test" ]]; then cd "${BUILD_DIR}" rm -rf * - cmake -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_CXX_STANDARD=14 \ + cmake ${CMAKE_OPTIONS[@]} \ -DWITH_METRICS_EXEMPLAR_PREVIEW=ON \ -DCMAKE_CXX_FLAGS="-Werror $CXXFLAGS" \ -DWITH_ASYNC_EXPORT_PREVIEW=ON \ @@ -201,8 +203,7 @@ elif [[ "$1" == "cmake.abseil.test" ]]; then elif [[ "$1" == "cmake.opentracing_shim.test" ]]; then cd "${BUILD_DIR}" rm -rf * - cmake -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_CXX_STANDARD=14 \ + cmake ${CMAKE_OPTIONS[@]} \ -DCMAKE_CXX_FLAGS="-Werror -Wno-error=redundant-move $CXXFLAGS" \ -DWITH_OPENTRACING=ON \ "${SRC_DIR}" @@ -212,8 +213,7 @@ elif [[ "$1" == "cmake.opentracing_shim.test" ]]; then elif [[ "$1" == "cmake.c++20.test" ]]; then cd "${BUILD_DIR}" rm -rf * - cmake -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_CXX_STANDARD=20 \ + cmake ${CMAKE_OPTIONS[@]} \ -DCMAKE_CXX_FLAGS="-Werror $CXXFLAGS" \ -DWITH_ASYNC_EXPORT_PREVIEW=ON \ ${IWYU} \ @@ -224,8 +224,7 @@ elif [[ "$1" == "cmake.c++20.test" ]]; then elif [[ "$1" == "cmake.c++20.stl.test" ]]; then cd "${BUILD_DIR}" rm -rf * - cmake -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_CXX_STANDARD=20 \ + cmake ${CMAKE_OPTIONS[@]} \ -DWITH_METRICS_EXEMPLAR_PREVIEW=ON \ -DCMAKE_CXX_FLAGS="-Werror $CXXFLAGS" \ -DWITH_ASYNC_EXPORT_PREVIEW=ON \ @@ -240,8 +239,7 @@ elif [[ "$1" == "cmake.legacy.test" ]]; then rm -rf * export BUILD_ROOT="${BUILD_DIR}" ${SRC_DIR}/tools/build-benchmark.sh - cmake -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_CXX_STANDARD=11 \ + cmake ${CMAKE_OPTIONS[@]} \ -DCMAKE_CXX_FLAGS="-Werror $CXXFLAGS" \ "${SRC_DIR}" make -j $(nproc) @@ -252,7 +250,7 @@ elif [[ "$1" == "cmake.legacy.exporter.otprotocol.test" ]]; then rm -rf * export BUILD_ROOT="${BUILD_DIR}" ${SRC_DIR}/tools/build-benchmark.sh - cmake -DCMAKE_BUILD_TYPE=Debug \ + cmake ${CMAKE_OPTIONS[@]} \ -DCMAKE_CXX_STANDARD=11 \ -DWITH_OTLP_GRPC=ON \ -DWITH_OTLP_HTTP=ON \ @@ -267,8 +265,10 @@ elif [[ "$1" == "cmake.legacy.exporter.otprotocol.test" ]]; then elif [[ "$1" == "cmake.exporter.otprotocol.test" ]]; then cd "${BUILD_DIR}" rm -rf * - cmake -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_CXX_STANDARD=14 \ + if [[ ! -z "${WITH_ABSEIL}" ]]; then + CMAKE_OPTIONS=(${CMAKE_OPTIONS[@]} "-DWITH_ABSEIL=${WITH_ABSEIL}") + fi + cmake ${CMAKE_OPTIONS[@]} \ -DWITH_OTLP_GRPC=ON \ -DWITH_OTLP_HTTP=ON \ -DWITH_OTLP_GRPC_SSL_MTLS_PREVIEW=ON \ @@ -282,8 +282,7 @@ elif [[ "$1" == "cmake.exporter.otprotocol.test" ]]; then elif [[ "$1" == "cmake.exporter.otprotocol.shared_libs.with_static_grpc.test" ]]; then cd "${BUILD_DIR}" rm -rf * - cmake -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_CXX_STANDARD=14 \ + cmake ${CMAKE_OPTIONS[@]} \ -DWITH_OTLP_GRPC=ON \ -DWITH_OTLP_HTTP=ON \ -DBUILD_SHARED_LIBS=ON \ @@ -297,8 +296,7 @@ elif [[ "$1" == "cmake.exporter.otprotocol.shared_libs.with_static_grpc.test" ]] elif [[ "$1" == "cmake.exporter.otprotocol.with_async_export.test" ]]; then cd "${BUILD_DIR}" rm -rf * - cmake -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_CXX_STANDARD=14 \ + cmake ${CMAKE_OPTIONS[@]} \ -DWITH_OTLP_GRPC=ON \ -DWITH_OTLP_HTTP=ON \ -DWITH_ASYNC_EXPORT_PREVIEW=ON \ @@ -312,8 +310,7 @@ elif [[ "$1" == "cmake.exporter.otprotocol.with_async_export.test" ]]; then elif [[ "$1" == "cmake.do_not_install.test" ]]; then cd "${BUILD_DIR}" rm -rf * - cmake -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_CXX_STANDARD=14 \ + cmake ${CMAKE_OPTIONS[@]} \ -DWITH_OTLP_GRPC=ON \ -DWITH_OTLP_HTTP=ON \ -DWITH_ASYNC_EXPORT_PREVIEW=ON \ @@ -346,7 +343,7 @@ EOF -static-libgcc \ -Wl,--version-script=${PWD}/export.map \ " - cmake -DCMAKE_BUILD_TYPE=Debug \ + cmake ${CMAKE_OPTIONS[@]} \ -DCMAKE_CXX_FLAGS="-Werror $CXXFLAGS" \ -DCMAKE_EXE_LINKER_FLAGS="$LINKER_FLAGS" \ -DCMAKE_SHARED_LINKER_FLAGS="$LINKER_FLAGS" \ @@ -357,7 +354,7 @@ EOF # Verify we can load the plugin cd "${BUILD_DIR}" rm -rf * - cmake -DCMAKE_BUILD_TYPE=Debug \ + cmake ${CMAKE_OPTIONS[@]} \ -DCMAKE_CXX_FLAGS="-Werror $CXXFLAGS" \ "${SRC_DIR}" make load_plugin_example @@ -439,8 +436,7 @@ elif [[ "$1" == "format" ]]; then elif [[ "$1" == "code.coverage" ]]; then cd "${BUILD_DIR}" rm -rf * - cmake -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_CXX_STANDARD=14 \ + cmake ${CMAKE_OPTIONS[@]} \ -DCMAKE_CXX_FLAGS="-Werror --coverage $CXXFLAGS" \ "${SRC_DIR}" make diff --git a/ci/install_abseil.sh b/ci/install_abseil.sh index 7764b4d815..3bc69df89d 100755 --- a/ci/install_abseil.sh +++ b/ci/install_abseil.sh @@ -3,19 +3,27 @@ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 -set -e +set -ex export DEBIAN_FRONTEND=noninteractive +[ -z "${ABSEIL_CPP_VERSION}" ] && export ABSEIL_CPP_VERSION="20220623.1" BUILD_DIR=/tmp/ INSTALL_DIR=/usr/local/ -TAG=20220623.1 pushd $BUILD_DIR -git clone --depth=1 -b ${TAG} https://github.com/abseil/abseil-cpp.git +git clone --depth=1 -b ${ABSEIL_CPP_VERSION} https://github.com/abseil/abseil-cpp.git cd abseil-cpp +ABSEIL_CPP_BUILD_OPTIONS=( + "-DBUILD_TESTING=OFF" + "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" + "-DCMAKE_INSTALL_PREFIX=$INSTALL_DIR" +) + +if [ ! -z "${CXX_STANDARD}" ]; then + ABSEIL_CPP_BUILD_OPTIONS=(${ABSEIL_CPP_BUILD_OPTIONS[@]} "-DCMAKE_CXX_STANDARD=${CXX_STANDARD}") +fi + mkdir build && pushd build -cmake -DBUILD_TESTING=OFF -DCMAKE_CXX_STANDARD=11 \ - -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \ - .. +cmake ${ABSEIL_CPP_BUILD_OPTIONS[@]} .. make -j $(nproc) make install popd diff --git a/ci/install_protobuf.sh b/ci/install_protobuf.sh index dde94111e1..3fb18e7c85 100755 --- a/ci/install_protobuf.sh +++ b/ci/install_protobuf.sh @@ -31,12 +31,41 @@ set -e # when calling this script # -export CPP_PROTOBUF_VERSION="3.${PROTOBUF_VERSION}" - -cd / -wget https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${CPP_PROTOBUF_VERSION}.tar.gz -tar zxf protobuf-cpp-${CPP_PROTOBUF_VERSION}.tar.gz --no-same-owner -cd protobuf-${CPP_PROTOBUF_VERSION} -./configure -make -j $(nproc) && make install +CPP_PROTOBUF_BUILD_OPTIONS=( + "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" + "-Dprotobuf_BUILD_TESTS=OFF" + "-Dprotobuf_BUILD_EXAMPLES=OFF" +) + +if [ ! -z "${CXX_STANDARD}" ]; then + CPP_PROTOBUF_BUILD_OPTIONS=(${CPP_PROTOBUF_BUILD_OPTIONS[@]} "-DCMAKE_CXX_STANDARD=${CXX_STANDARD}") +fi + +# After protobuf 22/4.22, protobuf depends on absl and we can use +# "-Dprotobuf_ABSL_PROVIDER=package" to tell protobuf to find absl from the +# system. Otherwise, it will build absl from source. +# 4.XX.YY and 3.XX.YY are alias of XX.YY, and source pacakges are moved into the +# tag of XX.YY and without -cpp suffix from protobuf v22. +if [[ ${PROTOBUF_VERSION/.*/} -ge 22 ]]; then + export CPP_PROTOBUF_VERSION="${PROTOBUF_VERSION}" + CPP_PROTOBUF_PACKAGE_NAME="protobuf-${CPP_PROTOBUF_VERSION}" + CPP_PROTOBUF_BUILD_OPTIONS=(${CPP_PROTOBUF_BUILD_OPTIONS[@]} "-Dprotobuf_ABSL_PROVIDER=package") +else + export CPP_PROTOBUF_VERSION="3.${PROTOBUF_VERSION}" + CPP_PROTOBUF_PACKAGE_NAME="protobuf-cpp-${CPP_PROTOBUF_VERSION}" +fi + +cd /tmp +wget https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/${CPP_PROTOBUF_PACKAGE_NAME}.tar.gz +tar zxf ${CPP_PROTOBUF_PACKAGE_NAME}.tar.gz --no-same-owner + +mkdir protobuf-${CPP_PROTOBUF_VERSION}/build && pushd protobuf-${CPP_PROTOBUF_VERSION}/build +if [ -e "../CMakeLists.txt" ]; then + cmake .. ${CPP_PROTOBUF_BUILD_OPTIONS[@]} +else + cmake ../cmake ${CPP_PROTOBUF_BUILD_OPTIONS[@]} +fi +cmake --build . -j $(nproc) +cmake --install . +popd ldconfig diff --git a/ci/setup_grpc.sh b/ci/setup_grpc.sh index e1002b6616..9cab7e77fe 100755 --- a/ci/setup_grpc.sh +++ b/ci/setup_grpc.sh @@ -7,24 +7,57 @@ set -e export DEBIAN_FRONTEND=noninteractive old_grpc_version='v1.33.2' new_grpc_version='v1.49.2' +modern_grpc_version='v1.55.0' gcc_version_for_new_grpc='5.1' std_version='14' +if [ ! -z "${CXX_STANDARD}" ]; then + std_version="${CXX_STANDARD}" +fi install_grpc_version=${new_grpc_version} install_dir='/usr/local/' build_shared_libs='' -usage() { echo "Usage: $0 [-v ] [-i "] 1>&2; exit 1;} +build_internal_abseil_cpp=1 +GRPC_BUILD_OPTIONS=() +usage() { + echo "Usage: $0 [options...]" 1>&2; + echo "Available options:" 1>&2; + echo " -v Set GCC version" 1>&2; + echo " -h Show help message and exit" 1>&2; + echo " -i Set installation prefix" 1>&2; + echo " -m Use the modern gRPC version" 1>&2; + echo " -p Let gRPC find protobuf or abseil-cpp by CONFIG package" 1>&2; + echo " -r Specify the version of gRPC" 1>&2; + echo " -s Specify std version(CMAKE_CXX_STANDARD)" 1>&2; + echo " -T Build static libraries" 1>&2; + echo " -H Build shared libraries" 1>&2; +} -while getopts ":v:i:r:s:TH" o; do +while getopts ":v:hi:mp:r:s:TH" o; do case "${o}" in v) gcc_version=${OPTARG} ;; + h) + usage + exit 0; + ;; i) install_dir=${OPTARG} ;; + p) + if [ "${OPTARG}" == "protobuf" ]; then + GRPC_BUILD_OPTIONS=(${GRPC_BUILD_OPTIONS[@]} "-DgRPC_PROTOBUF_PROVIDER=package") + elif [ "${OPTARG}" == "abseil-cpp" ]; then + GRPC_BUILD_OPTIONS=(${GRPC_BUILD_OPTIONS[@]} "-DgRPC_ABSL_PROVIDER=package") + build_internal_abseil_cpp=0 + fi + ;; r) install_grpc_version=${OPTARG} ;; + m) + install_grpc_version=${modern_grpc_version} + ;; s) std_version=${OPTARG} ;; @@ -36,14 +69,16 @@ while getopts ":v:i:r:s:TH" o; do ;; *) usage + exit 1; ;; esac done + if [ -z "${gcc_version}" ]; then gcc_version=`gcc --version | awk '/gcc/ {print $NF}'` fi -if [[ "${gcc_version}" < "${gcc_version_for_new_grpc}" ]]; then - echo "less" +if [[ "${gcc_version}" < "${gcc_version_for_new_grpc}" ]] && [[ "${gcc_version:1:1}" == "." ]]; then + echo "${gcc_version} less than ${gcc_version_for_new_grpc}" std_version='11' install_grpc_version=${old_grpc_version} fi @@ -59,22 +94,26 @@ git clone --depth=1 -b ${install_grpc_version} https://github.com/grpc/grpc pushd grpc git submodule init git submodule update --depth 1 -mkdir -p "third_party/abseil-cpp/build" && pushd "third_party/abseil-cpp/build" -set -x -ABSEIL_CPP_BUILD_OPTIONS=( - -DCMAKE_BUILD_TYPE=Release - -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE - -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR -) -if [ ! -z "$build_shared_libs" ]; then - ABSEIL_CPP_BUILD_OPTIONS=(${ABSEIL_CPP_BUILD_OPTIONS[@]} "-DBUILD_SHARED_LIBS=$build_shared_libs") +if [[ $build_internal_abseil_cpp -ne 0 ]]; then + mkdir -p "third_party/abseil-cpp/build" && pushd "third_party/abseil-cpp/build" + set -x + + ABSEIL_CPP_BUILD_OPTIONS=( + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE + -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR + ) + if [ ! -z "$build_shared_libs" ]; then + ABSEIL_CPP_BUILD_OPTIONS=(${ABSEIL_CPP_BUILD_OPTIONS[@]} "-DBUILD_SHARED_LIBS=$build_shared_libs") + fi + cmake ${ABSEIL_CPP_BUILD_OPTIONS[@]} .. + cmake --build . -j${nproc} --target install && popd fi -cmake ${ABSEIL_CPP_BUILD_OPTIONS[@]} .. -cmake --build . -j${nproc} --target install && popd mkdir -p build && pushd build GRPC_BUILD_OPTIONS=( + ${GRPC_BUILD_OPTIONS[@]} -DgRPC_INSTALL=ON -DCMAKE_CXX_STANDARD=${std_version} -DgRPC_BUILD_TESTS=OFF diff --git a/cmake/opentelemetry-proto.cmake b/cmake/opentelemetry-proto.cmake index 07238ba041..4d64a8479c 100644 --- a/cmake/opentelemetry-proto.cmake +++ b/cmake/opentelemetry-proto.cmake @@ -65,8 +65,6 @@ else() endif() endif() -include(${PROJECT_SOURCE_DIR}/cmake/proto-options-patch.cmake) - set(COMMON_PROTO "${PROTO_PATH}/opentelemetry/proto/common/v1/common.proto") set(RESOURCE_PROTO "${PROTO_PATH}/opentelemetry/proto/resource/v1/resource.proto") @@ -169,64 +167,76 @@ if(WITH_OTLP_GRPC) message(STATUS "gRPC_CPP_PLUGIN_EXECUTABLE=${gRPC_CPP_PLUGIN_EXECUTABLE}") endif() +set(PROTOBUF_COMMON_FLAGS "--proto_path=${PROTO_PATH}" + "--cpp_out=${GENERATED_PROTOBUF_PATH}") +# --experimental_allow_proto3_optional is available from 3.13 and be stable and +# enabled by default from 3.16 +if(Protobuf_VERSION AND Protobuf_VERSION VERSION_LESS "3.16") + list(APPEND PROTOBUF_COMMON_FLAGS "--experimental_allow_proto3_optional") +elseif(PROTOBUF_VERSION AND PROTOBUF_VERSION VERSION_LESS "3.16") + list(APPEND PROTOBUF_COMMON_FLAGS "--experimental_allow_proto3_optional") +endif() + +set(PROTOBUF_GENERATED_FILES + ${COMMON_PB_H_FILE} + ${COMMON_PB_CPP_FILE} + ${RESOURCE_PB_H_FILE} + ${RESOURCE_PB_CPP_FILE} + ${TRACE_PB_H_FILE} + ${TRACE_PB_CPP_FILE} + ${LOGS_PB_H_FILE} + ${LOGS_PB_CPP_FILE} + ${METRICS_PB_H_FILE} + ${METRICS_PB_CPP_FILE} + ${TRACE_SERVICE_PB_H_FILE} + ${TRACE_SERVICE_PB_CPP_FILE} + ${LOGS_SERVICE_PB_H_FILE} + ${LOGS_SERVICE_PB_CPP_FILE} + ${METRICS_SERVICE_PB_H_FILE} + ${METRICS_SERVICE_PB_CPP_FILE}) + if(WITH_OTLP_GRPC) - add_custom_command( - OUTPUT ${COMMON_PB_H_FILE} - ${COMMON_PB_CPP_FILE} - ${RESOURCE_PB_H_FILE} - ${RESOURCE_PB_CPP_FILE} - ${TRACE_PB_H_FILE} - ${TRACE_PB_CPP_FILE} - ${LOGS_PB_H_FILE} - ${LOGS_PB_CPP_FILE} - ${METRICS_PB_H_FILE} - ${METRICS_PB_CPP_FILE} - ${TRACE_SERVICE_PB_H_FILE} - ${TRACE_SERVICE_PB_CPP_FILE} - ${TRACE_SERVICE_GRPC_PB_H_FILE} - ${TRACE_SERVICE_GRPC_PB_CPP_FILE} - ${LOGS_SERVICE_PB_H_FILE} - ${LOGS_SERVICE_PB_CPP_FILE} - ${LOGS_SERVICE_GRPC_PB_H_FILE} - ${LOGS_SERVICE_GRPC_PB_CPP_FILE} - ${METRICS_SERVICE_PB_H_FILE} - ${METRICS_SERVICE_PB_CPP_FILE} - ${METRICS_SERVICE_GRPC_PB_H_FILE} - ${METRICS_SERVICE_GRPC_PB_CPP_FILE} - COMMAND - ${PROTOBUF_PROTOC_EXECUTABLE} ARGS "--experimental_allow_proto3_optional" - "--proto_path=${PROTO_PATH}" ${PROTOBUF_INCLUDE_FLAGS} - "--cpp_out=${GENERATED_PROTOBUF_PATH}" - "--grpc_out=generate_mock_code=true:${GENERATED_PROTOBUF_PATH}" - --plugin=protoc-gen-grpc="${gRPC_CPP_PLUGIN_EXECUTABLE}" ${COMMON_PROTO} - ${RESOURCE_PROTO} ${TRACE_PROTO} ${LOGS_PROTO} ${METRICS_PROTO} - ${TRACE_SERVICE_PROTO} ${LOGS_SERVICE_PROTO} ${METRICS_SERVICE_PROTO}) -else() - add_custom_command( - OUTPUT ${COMMON_PB_H_FILE} - ${COMMON_PB_CPP_FILE} - ${RESOURCE_PB_H_FILE} - ${RESOURCE_PB_CPP_FILE} - ${TRACE_PB_H_FILE} - ${TRACE_PB_CPP_FILE} - ${LOGS_PB_H_FILE} - ${LOGS_PB_CPP_FILE} - ${METRICS_PB_H_FILE} - ${METRICS_PB_CPP_FILE} - ${TRACE_SERVICE_PB_H_FILE} - ${TRACE_SERVICE_PB_CPP_FILE} - ${LOGS_SERVICE_PB_H_FILE} - ${LOGS_SERVICE_PB_CPP_FILE} - ${METRICS_SERVICE_PB_H_FILE} - ${METRICS_SERVICE_PB_CPP_FILE} - COMMAND - ${PROTOBUF_PROTOC_EXECUTABLE} ARGS "--experimental_allow_proto3_optional" - "--proto_path=${PROTO_PATH}" ${PROTOBUF_INCLUDE_FLAGS} - "--cpp_out=${GENERATED_PROTOBUF_PATH}" ${COMMON_PROTO} ${RESOURCE_PROTO} - ${TRACE_PROTO} ${LOGS_PROTO} ${METRICS_PROTO} ${TRACE_SERVICE_PROTO} - ${LOGS_SERVICE_PROTO} ${METRICS_SERVICE_PROTO}) + list(APPEND PROTOBUF_COMMON_FLAGS + "--grpc_out=generate_mock_code=true:${GENERATED_PROTOBUF_PATH}" + --plugin=protoc-gen-grpc="${gRPC_CPP_PLUGIN_EXECUTABLE}") + + list( + APPEND + PROTOBUF_GENERATED_FILES + ${TRACE_SERVICE_GRPC_PB_H_FILE} + ${TRACE_SERVICE_GRPC_PB_CPP_FILE} + ${LOGS_SERVICE_GRPC_PB_H_FILE} + ${LOGS_SERVICE_GRPC_PB_CPP_FILE} + ${METRICS_SERVICE_GRPC_PB_H_FILE} + ${METRICS_SERVICE_GRPC_PB_CPP_FILE}) endif() +set(PROTOBUF_RUN_PROTOC_COMMAND "\"${PROTOBUF_PROTOC_EXECUTABLE}\"") +foreach( + PROTOBUF_RUN_PROTOC_ARG + ${PROTOBUF_COMMON_FLAGS} + ${PROTOBUF_INCLUDE_FLAGS} + ${COMMON_PROTO} + ${RESOURCE_PROTO} + ${TRACE_PROTO} + ${LOGS_PROTO} + ${METRICS_PROTO} + ${TRACE_SERVICE_PROTO} + ${LOGS_SERVICE_PROTO} + ${METRICS_SERVICE_PROTO}) + set(PROTOBUF_RUN_PROTOC_COMMAND + "${PROTOBUF_RUN_PROTOC_COMMAND} \"${PROTOBUF_RUN_PROTOC_ARG}\"") +endforeach() + +add_custom_command( + OUTPUT ${PROTOBUF_GENERATED_FILES} + COMMAND + ${PROTOBUF_PROTOC_EXECUTABLE} ${PROTOBUF_COMMON_FLAGS} + ${PROTOBUF_INCLUDE_FLAGS} ${COMMON_PROTO} ${RESOURCE_PROTO} ${TRACE_PROTO} + ${LOGS_PROTO} ${METRICS_PROTO} ${TRACE_SERVICE_PROTO} ${LOGS_SERVICE_PROTO} + ${METRICS_SERVICE_PROTO} + COMMENT "[Run]: ${PROTOBUF_RUN_PROTOC_COMMAND}") + include_directories("${GENERATED_PROTOBUF_PATH}") unset(OTELCPP_PROTO_TARGET_OPTIONS) @@ -251,6 +261,10 @@ add_library( ${LOGS_SERVICE_PB_CPP_FILE} ${METRICS_SERVICE_PB_CPP_FILE}) +if(WITH_ABSEIL) + target_link_libraries(opentelemetry_proto PUBLIC absl::bad_variant_access) +endif() + if(WITH_OTLP_GRPC) add_library( opentelemetry_proto_grpc diff --git a/cmake/proto-options-patch.cmake b/cmake/tools.cmake similarity index 98% rename from cmake/proto-options-patch.cmake rename to cmake/tools.cmake index 97872050c9..345fc88ff7 100644 --- a/cmake/proto-options-patch.cmake +++ b/cmake/tools.cmake @@ -14,7 +14,7 @@ endmacro() if(NOT PATCH_PROTOBUF_SOURCES_OPTIONS_SET) if(MSVC) unset(PATCH_PROTOBUF_SOURCES_OPTIONS CACHE) - set(PATCH_PROTOBUF_SOURCES_OPTIONS /wd4244 /wd4251 /wd4267 /wd4309) + set(PATCH_PROTOBUF_SOURCES_OPTIONS /wd4244 /wd4251 /wd4267 /wd4309 /wd4668 /wd4946 /wd6001 /wd6244 /wd6246) if(MSVC_VERSION GREATER_EQUAL 1922) # see diff --git a/examples/grpc/CMakeLists.txt b/examples/grpc/CMakeLists.txt index 1ea1c9bee5..938a009e0a 100644 --- a/examples/grpc/CMakeLists.txt +++ b/examples/grpc/CMakeLists.txt @@ -23,7 +23,6 @@ add_custom_command( add_library(example_grpc_proto ${example_grpc_srcs} ${example_grpc_hdrs} ${example_proto_srcs} ${example_proto_hdrs}) -include(${PROJECT_SOURCE_DIR}/cmake/proto-options-patch.cmake) patch_protobuf_targets(example_grpc_proto) include_directories( @@ -33,16 +32,20 @@ include_directories( include_directories(${CMAKE_CURRENT_BINARY_DIR}) if(TARGET protobuf::libprotobuf) - target_link_libraries(example_grpc_proto gRPC::grpc++ protobuf::libprotobuf) + target_link_libraries(example_grpc_proto PUBLIC gRPC::grpc++ + protobuf::libprotobuf) else() - target_include_directories(example_grpc_proto ${Protobuf_INCLUDE_DIRS}) - target_link_libraries(example_grpc_proto ${Protobuf_LIBRARIES}) + target_include_directories(example_grpc_proto PUBLIC ${Protobuf_INCLUDE_DIRS}) + target_link_libraries(example_grpc_proto PUBLIC gRPC::grpc++ + ${Protobuf_LIBRARIES}) +endif() +if(WITH_ABSEIL) + target_link_libraries(example_grpc_proto PUBLIC absl::bad_variant_access) endif() foreach(_target client server) add_executable(${_target} "${_target}.cc") - target_link_libraries( - ${_target} example_grpc_proto protobuf::libprotobuf gRPC::grpc++ - opentelemetry_trace opentelemetry_exporter_ostream_span) + target_link_libraries(${_target} example_grpc_proto opentelemetry_trace + opentelemetry_exporter_ostream_span) patch_protobuf_targets(${_target}) endforeach() diff --git a/exporters/otlp/BUILD b/exporters/otlp/BUILD index 5e52bfd393..5099b43510 100644 --- a/exporters/otlp/BUILD +++ b/exporters/otlp/BUILD @@ -131,7 +131,9 @@ cc_library( "//api", "//ext/src/http/client/curl:http_client_curl", "//sdk:headers", + "//sdk/src/common:base64", "@com_github_opentelemetry_proto//:common_proto_cc", + "@com_google_absl//absl/strings", "@github_nlohmann_json//:json", ], ) diff --git a/exporters/otlp/CMakeLists.txt b/exporters/otlp/CMakeLists.txt index 44715c1753..8a93d34acc 100644 --- a/exporters/otlp/CMakeLists.txt +++ b/exporters/otlp/CMakeLists.txt @@ -114,6 +114,10 @@ if(WITH_OTLP_HTTP) PUBLIC opentelemetry_sdk opentelemetry_ext PRIVATE opentelemetry_proto opentelemetry_http_client_curl nlohmann_json::nlohmann_json) + if(TARGET absl::strings) + target_link_libraries(opentelemetry_exporter_otlp_http_client + PUBLIC absl::strings) + endif() if(nlohmann_json_clone) add_dependencies(opentelemetry_exporter_otlp_http_client nlohmann_json::nlohmann_json) @@ -221,7 +225,9 @@ if(BUILD_TESTING) TEST_PREFIX exporter.otlp. TEST_LIST otlp_metrics_serialization_test) - if(MSVC) + if(NOT GMOCK_LIB AND TARGET GTest::gmock) + set(GMOCK_LIB GTest::gmock) + elseif(MSVC) # Explicitly specify that we consume GTest from shared library. The rest of # code logic below determines whether we link Release or Debug flavor of the # library. These flavors have different prefix on Windows, gmock and gmockd @@ -234,10 +240,12 @@ if(BUILD_TESTING) unset(GMOCK_LIB CACHE) endif() endif() - if(MSVC AND CMAKE_BUILD_TYPE STREQUAL "Debug") - find_library(GMOCK_LIB gmockd PATH_SUFFIXES lib) - else() - find_library(GMOCK_LIB gmock PATH_SUFFIXES lib) + if(NOT GMOCK_LIB) + if(MSVC AND CMAKE_BUILD_TYPE STREQUAL "Debug") + find_library(GMOCK_LIB gmockd PATH_SUFFIXES lib) + else() + find_library(GMOCK_LIB gmock PATH_SUFFIXES lib) + endif() endif() if(WITH_OTLP_GRPC) diff --git a/exporters/otlp/src/otlp_http_client.cc b/exporters/otlp/src/otlp_http_client.cc index a249a842ad..c6bc1e1122 100644 --- a/exporters/otlp/src/otlp_http_client.cc +++ b/exporters/otlp/src/otlp_http_client.cc @@ -17,30 +17,19 @@ #include "google/protobuf/message.h" #include "google/protobuf/reflection.h" #include "google/protobuf/stubs/common.h" -#include "google/protobuf/stubs/stringpiece.h" #include "nlohmann/json.hpp" -#if defined(GOOGLE_PROTOBUF_VERSION) && GOOGLE_PROTOBUF_VERSION >= 3007000 -# include "google/protobuf/stubs/strutil.h" -#else -# include "google/protobuf/stubs/port.h" -namespace google -{ -namespace protobuf -{ -LIBPROTOBUF_EXPORT void Base64Escape(StringPiece src, std::string *dest); -} // namespace protobuf -} // namespace google -#endif - #include "opentelemetry/exporters/otlp/protobuf_include_suffix.h" #include "opentelemetry/common/timestamp.h" +#include "opentelemetry/nostd/string_view.h" +#include "opentelemetry/sdk/common/base64.h" #include "opentelemetry/sdk/common/global_log_handler.h" #include "opentelemetry/sdk_config.h" #include #include +#include #include #include #include @@ -411,16 +400,12 @@ static std::string BytesMapping(const std::string &bytes, } else { - std::string base64_value; - google::protobuf::Base64Escape(bytes, &base64_value); - return base64_value; + return opentelemetry::sdk::common::Base64Escape(bytes); } } case JsonBytesMappingKind::kBase64: { // Base64 is the default bytes mapping of protobuf - std::string base64_value; - google::protobuf::Base64Escape(bytes, &base64_value); - return base64_value; + return opentelemetry::sdk::common::Base64Escape(bytes); } case JsonBytesMappingKind::kHex: return HexEncode(bytes); diff --git a/sdk/include/opentelemetry/sdk/common/base64.h b/sdk/include/opentelemetry/sdk/common/base64.h new file mode 100644 index 0000000000..d88204675c --- /dev/null +++ b/sdk/include/opentelemetry/sdk/common/base64.h @@ -0,0 +1,36 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +#include "opentelemetry/common/macros.h" +#include "opentelemetry/nostd/string_view.h" +#include "opentelemetry/version.h" + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace sdk +{ +namespace common +{ +// Base64Escape() +// +// Encodes a `src` string into a base64-encoded 'dest' string with padding +// characters. This function conforms with RFC 4648 section 4 (base64) and RFC +// 2045. +OPENTELEMETRY_EXPORT void Base64Escape(opentelemetry::nostd::string_view src, std::string *dest); +OPENTELEMETRY_EXPORT std::string Base64Escape(opentelemetry::nostd::string_view src); + +// Base64Unescape() +// +// Converts a `src` string encoded in Base64 (RFC 4648 section 4) to its binary +// equivalent, writing it to a `dest` buffer, returning `true` on success. If +// `src` contains invalid characters, `dest` is cleared and returns `false`. +// If padding is included (note that `Base64Escape()` does produce it), it must +// be correct. In the padding, '=' are treated identically. +OPENTELEMETRY_EXPORT bool Base64Unescape(opentelemetry::nostd::string_view src, std::string *dest); + +} // namespace common +} // namespace sdk +OPENTELEMETRY_END_NAMESPACE diff --git a/sdk/src/common/BUILD b/sdk/src/common/BUILD index cdbd1955f3..fe75f08d7c 100644 --- a/sdk/src/common/BUILD +++ b/sdk/src/common/BUILD @@ -21,6 +21,19 @@ cc_library( ], ) +cc_library( + name = "base64", + srcs = [ + "base64.cc", + ], + include_prefix = "src/common", + deps = [ + "//api", + "//sdk:headers", + "//sdk/src/common/platform:fork", + ], +) + cc_library( name = "env_variables", srcs = [ diff --git a/sdk/src/common/CMakeLists.txt b/sdk/src/common/CMakeLists.txt index 7bb645f5b7..b0f44ac139 100644 --- a/sdk/src/common/CMakeLists.txt +++ b/sdk/src/common/CMakeLists.txt @@ -1,7 +1,8 @@ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 -set(COMMON_SRCS random.cc core.cc global_log_handler.cc env_variables.cc) +set(COMMON_SRCS random.cc core.cc global_log_handler.cc env_variables.cc + base64.cc) if(WIN32) list(APPEND COMMON_SRCS platform/fork_windows.cc) else() @@ -17,6 +18,10 @@ target_link_libraries( opentelemetry_common PUBLIC opentelemetry_api opentelemetry_sdk Threads::Threads) +if(WITH_ABSEIL) + target_link_libraries(opentelemetry_common PUBLIC absl::strings) +endif() + if(OPENTELEMETRY_INSTALL) install( TARGETS opentelemetry_common diff --git a/sdk/src/common/base64.cc b/sdk/src/common/base64.cc new file mode 100644 index 0000000000..04a865ffe8 --- /dev/null +++ b/sdk/src/common/base64.cc @@ -0,0 +1,351 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#include "opentelemetry/sdk/common/base64.h" + +#if defined(HAVE_GSL) +# include +#else +# include +#endif +#include +#include +#include + +#if defined(HAVE_ABSEIL) +# include "absl/strings/escaping.h" +#endif + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace sdk +{ +namespace common +{ +#if !defined(HAVE_ABSEIL) +namespace +{ +using Base64EscapeChars = const unsigned char[64]; +using Base64UnescapeChars = const unsigned char[128]; + +static constexpr Base64EscapeChars kBase64EscapeCharsBasic = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', + 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', + 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'}; + +static constexpr Base64UnescapeChars kBase64UnescapeCharsBasic = { + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 62, 127, 127, 127, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, + 61, 127, 127, 127, 127, 127, 127, 127, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 127, 127, 127, 127, + 127, 127, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, + 43, 44, 45, 46, 47, 48, 49, 50, 51, 127, 127, 127, 127, 127}; + +static int Base64EscapeInternal(unsigned char *dest, + std::size_t dlen, + std::size_t *olen, + const unsigned char *src, + std::size_t slen, + Base64EscapeChars &base64_enc_map, + unsigned char padding_char) +{ + std::size_t i, n, nopadding; + int C1, C2, C3; + unsigned char *p; + + if (slen == 0) + { + *olen = 0; + return 0; + } + + n = (slen + 2) / 3; + + if (n > (std::numeric_limits::max() - 1) / 4) + { + *olen = std::numeric_limits::max(); + return -1; + } + + n *= 4; + + // no padding + if (0 == padding_char) + { + nopadding = slen % 3; + if (0 != nopadding) + { + n -= 3 - nopadding; + } + } + + if ((dlen < n + 1) || (nullptr == dest)) + { + *olen = n + 1; + return -1; + } + + n = (slen / 3) * 3; + + for (i = 0, p = dest; i < n; i += 3) + { + C1 = *src++; + C2 = *src++; + C3 = *src++; + + *p++ = base64_enc_map[(C1 >> 2) & 0x3F]; + *p++ = base64_enc_map[(((C1 & 3) << 4) + (C2 >> 4)) & 0x3F]; + *p++ = base64_enc_map[(((C2 & 15) << 2) + (C3 >> 6)) & 0x3F]; + *p++ = base64_enc_map[C3 & 0x3F]; + } + + if (i < slen) + { + C1 = *src++; + C2 = ((i + 1) < slen) ? *src++ : 0; + + *p++ = base64_enc_map[(C1 >> 2) & 0x3F]; + *p++ = base64_enc_map[(((C1 & 3) << 4) + (C2 >> 4)) & 0x3F]; + + if ((i + 1) < slen) + { + *p++ = base64_enc_map[((C2 & 15) << 2) & 0x3F]; + } + else if (padding_char) + { + *p++ = padding_char; + } + + if (padding_char) + { + *p++ = padding_char; + } + } + + *olen = static_cast(p - dest); + *p = 0; + + return 0; +} + +static inline int Base64EscapeInternal(std::string &dest, + const unsigned char *src, + std::size_t slen, + Base64EscapeChars &base64_enc_map, + unsigned char padding_char) +{ + std::size_t olen = 0; + Base64EscapeInternal(nullptr, 0, &olen, src, slen, base64_enc_map, padding_char); + dest.resize(olen); + + if (nullptr == src || 0 == slen) + { + return 0; + } + + int ret = Base64EscapeInternal(reinterpret_cast(&dest[0]), dest.size(), &olen, + src, slen, base64_enc_map, padding_char); +# if defined(HAVE_GSL) + Expects(0 != ret || dest.size() == olen + 1); +# else + assert(0 != ret || dest.size() == olen + 1); +# endif + // pop back last zero + if (!dest.empty() && *dest.rbegin() == 0) + { + dest.resize(dest.size() - 1); + } + return ret; +} + +static int Base64UnescapeInternal(unsigned char *dst, + std::size_t dlen, + std::size_t *olen, + const unsigned char *src, + std::size_t slen, + Base64UnescapeChars &base64_dec_map, + unsigned char padding_char) +{ + std::size_t i, n; + std::size_t j, x; + std::size_t valid_slen, line_len; + unsigned char *p; + + /* First pass: check for validity and get output length */ + for (i = n = j = valid_slen = line_len = 0; i < slen; i++) + { + /* Skip spaces before checking for EOL */ + x = 0; + while (i < slen && (src[i] == ' ' || src[i] == '\t')) + { + ++i; + ++x; + } + + /* Spaces at end of buffer are OK */ + if (i == slen) + break; + + if (src[i] == '\r' || src[i] == '\n') + { + line_len = 0; + continue; + } + + /* Space inside a line is an error */ + if (x != 0 && line_len != 0) + return -2; + + ++valid_slen; + ++line_len; + if (src[i] == padding_char) + { + if (++j > 2) + { + return -2; + } + else if ((valid_slen & 3) == 1 || (valid_slen & 3) == 2) + { + // First and second char of every group can not be padding char + return -2; + } + } + else + { + if (src[i] > 127 || base64_dec_map[src[i]] == 127) + return -2; + } + + if (base64_dec_map[src[i]] < 64 && j != 0) + return -2; + + n++; + } + + if (n == 0) + { + *olen = 0; + return 0; + } + + // no padding, add j to padding length + if (valid_slen & 3) + { + j += 4 - (valid_slen & 3); + n += 4 - (valid_slen & 3); + } + + /* The following expression is to calculate the following formula without + * risk of integer overflow in n: + * n = ( ( n * 6 ) + 7 ) >> 3; + */ + n = (6 * (n >> 3)) + ((6 * (n & 0x7) + 7) >> 3); + n -= j; + + if (dst == nullptr || dlen < n) + { + *olen = n; + return -1; + } + + for (n = x = 0, p = dst; i > 0; i--, src++) + { + if (*src == '\r' || *src == '\n' || *src == ' ' || *src == '\t') + continue; + if (*src == padding_char) + continue; + + x = (x << 6) | (base64_dec_map[*src] & 0x3F); + + if (++n == 4) + { + n = 0; + *p++ = (unsigned char)(x >> 16); + *p++ = (unsigned char)(x >> 8); + *p++ = (unsigned char)(x); + } + } + + // no padding, the tail code + if (n == 2) + { + *p++ = (unsigned char)(x >> 4); + } + else if (n == 3) + { + *p++ = (unsigned char)(x >> 10); + *p++ = (unsigned char)(x >> 2); + } + + *olen = static_cast(p - dst); + + return 0; +} + +} // namespace +#endif + +// Base64Escape() +// +// Encodes a `src` string into a base64-encoded 'dest' string with padding +// characters. This function conforms with RFC 4648 section 4 (base64) and RFC +// 2045. +OPENTELEMETRY_EXPORT void Base64Escape(opentelemetry::nostd::string_view src, std::string *dest) +{ + if (nullptr == dest || src.empty()) + { + return; + } + +#if defined(HAVE_ABSEIL) + absl::Base64Escape(absl::string_view{src.data(), src.size()}, dest); +#else + Base64EscapeInternal(*dest, reinterpret_cast(src.data()), src.size(), + kBase64EscapeCharsBasic, '='); +#endif +} + +OPENTELEMETRY_EXPORT std::string Base64Escape(opentelemetry::nostd::string_view src) +{ + std::string result; + + Base64Escape(src, &result); + return result; +} + +OPENTELEMETRY_EXPORT bool Base64Unescape(opentelemetry::nostd::string_view src, std::string *dest) +{ + if (nullptr == dest) + { + return false; + } + +#if defined(HAVE_ABSEIL) + return absl::Base64Unescape(absl::string_view{src.data(), src.size()}, dest); +#else + if (src.empty()) + { + return true; + } + + std::size_t olen = 0; + + if (-2 == Base64UnescapeInternal(nullptr, 0, &olen, + reinterpret_cast(src.data()), src.size(), + kBase64UnescapeCharsBasic, '=')) + { + return false; + } + + dest->resize(olen); + Base64UnescapeInternal(reinterpret_cast(&(*dest)[0]), dest->size(), &olen, + reinterpret_cast(src.data()), src.size(), + kBase64UnescapeCharsBasic, '='); + return true; +#endif +} + +} // namespace common +} // namespace sdk +OPENTELEMETRY_END_NAMESPACE diff --git a/sdk/test/common/BUILD b/sdk/test/common/BUILD index 0f886ec569..b08bcc0976 100644 --- a/sdk/test/common/BUILD +++ b/sdk/test/common/BUILD @@ -15,6 +15,18 @@ cc_test( ], ) +cc_test( + name = "base64_test", + srcs = [ + "base64_test.cc", + ], + tags = ["test"], + deps = [ + "//sdk/src/common:base64", + "@com_google_googletest//:gtest_main", + ], +) + cc_test( name = "fast_random_number_generator_test", srcs = [ @@ -27,6 +39,16 @@ cc_test( ], ) +otel_cc_benchmark( + name = "base64_benchmark", + srcs = ["base64_benchmark.cc"], + tags = [ + "benchmark", + "test", + ], + deps = ["//sdk/src/common:base64"], +) + otel_cc_benchmark( name = "random_benchmark", srcs = ["random_benchmark.cc"], diff --git a/sdk/test/common/CMakeLists.txt b/sdk/test/common/CMakeLists.txt index 0205061911..00ad67a5bf 100644 --- a/sdk/test/common/CMakeLists.txt +++ b/sdk/test/common/CMakeLists.txt @@ -24,11 +24,22 @@ foreach( TEST_LIST ${testname}) endforeach() +add_executable(base64_test base64_test.cc) +target_link_libraries(base64_test ${GTEST_BOTH_LIBRARIES} opentelemetry_common) +gtest_add_tests( + TARGET base64_test + TEST_PREFIX common. + TEST_LIST base64_test) + add_executable(random_fork_test random_fork_test.cc) target_link_libraries(random_fork_test opentelemetry_common) add_test(random_fork_test random_fork_test) if(WITH_BENCHMARK) + add_executable(base64_benchmark base64_benchmark.cc) + target_link_libraries(base64_benchmark benchmark::benchmark + ${CMAKE_THREAD_LIBS_INIT} opentelemetry_common) + add_executable(random_benchmark random_benchmark.cc) target_link_libraries(random_benchmark benchmark::benchmark ${CMAKE_THREAD_LIBS_INIT} opentelemetry_common) diff --git a/sdk/test/common/base64_benchmark.cc b/sdk/test/common/base64_benchmark.cc new file mode 100644 index 0000000000..acb8b01ff4 --- /dev/null +++ b/sdk/test/common/base64_benchmark.cc @@ -0,0 +1,49 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#include + +#include "opentelemetry/nostd/string_view.h" +#include "opentelemetry/sdk/common/base64.h" + +#include "benchmark/benchmark.h" + +namespace +{ + +static const unsigned char base64_test_dec[64] = { + 0x24, 0x48, 0x6E, 0x56, 0x87, 0x62, 0x5A, 0xBD, 0xBF, 0x17, 0xD9, 0xA2, 0xC4, 0x17, 0x1A, 0x01, + 0x94, 0xED, 0x8F, 0x1E, 0x11, 0xB3, 0xD7, 0x09, 0x0C, 0xB6, 0xE9, 0x10, 0x6F, 0x22, 0xEE, 0x13, + 0xCA, 0xB3, 0x07, 0x05, 0x76, 0xC9, 0xFA, 0x31, 0x6C, 0x08, 0x34, 0xFF, 0x8D, 0xC2, 0x6C, 0x38, + 0x00, 0x43, 0xE9, 0x54, 0x97, 0xAF, 0x50, 0x4B, 0xD1, 0x41, 0xBA, 0x95, 0x31, 0x5A, 0x0B, 0x97}; + +static const unsigned char base64_test_enc_rfc_2045[] = + "JEhuVodiWr2/F9mixBcaAZTtjx4Rs9cJDLbpEG8i7hPK" + "swcFdsn6MWwINP+Nwmw4AEPpVJevUEvRQbqVMVoLlw=="; + +void BM_Base64Escape(benchmark::State &state) +{ + while (state.KeepRunning()) + { + benchmark::DoNotOptimize( + opentelemetry::sdk::common::Base64Escape(opentelemetry::nostd::string_view{ + reinterpret_cast(base64_test_dec), sizeof(base64_test_dec)})); + } +} +BENCHMARK(BM_Base64Escape); + +void BM_Base64Unescape(benchmark::State &state) +{ + std::string dest; + while (state.KeepRunning()) + { + benchmark::DoNotOptimize(opentelemetry::sdk::common::Base64Unescape( + opentelemetry::nostd::string_view{reinterpret_cast(base64_test_enc_rfc_2045), + 88}, + &dest)); + } +} +BENCHMARK(BM_Base64Unescape); + +} // namespace +BENCHMARK_MAIN(); diff --git a/sdk/test/common/base64_test.cc b/sdk/test/common/base64_test.cc new file mode 100644 index 0000000000..74ebe5cf27 --- /dev/null +++ b/sdk/test/common/base64_test.cc @@ -0,0 +1,92 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#include "opentelemetry/sdk/common/base64.h" + +#include + +#include + +static const unsigned char base64_test_dec[64] = { + 0x24, 0x48, 0x6E, 0x56, 0x87, 0x62, 0x5A, 0xBD, 0xBF, 0x17, 0xD9, 0xA2, 0xC4, 0x17, 0x1A, 0x01, + 0x94, 0xED, 0x8F, 0x1E, 0x11, 0xB3, 0xD7, 0x09, 0x0C, 0xB6, 0xE9, 0x10, 0x6F, 0x22, 0xEE, 0x13, + 0xCA, 0xB3, 0x07, 0x05, 0x76, 0xC9, 0xFA, 0x31, 0x6C, 0x08, 0x34, 0xFF, 0x8D, 0xC2, 0x6C, 0x38, + 0x00, 0x43, 0xE9, 0x54, 0x97, 0xAF, 0x50, 0x4B, 0xD1, 0x41, 0xBA, 0x95, 0x31, 0x5A, 0x0B, 0x97}; + +static const unsigned char base64_test_enc_rfc_2045[] = + "JEhuVodiWr2/F9mixBcaAZTtjx4Rs9cJDLbpEG8i7hPK" + "swcFdsn6MWwINP+Nwmw4AEPpVJevUEvRQbqVMVoLlw=="; + +TEST(Base64Test, EscapeRfc2045) +{ + std::string encoded = opentelemetry::sdk::common::Base64Escape(opentelemetry::nostd::string_view{ + reinterpret_cast(base64_test_dec), sizeof(base64_test_dec)}); + opentelemetry::nostd::string_view expected{ + reinterpret_cast(base64_test_enc_rfc_2045), 88}; + + EXPECT_EQ(encoded, expected); +} + +TEST(Base64Test, UnescapeRfc2045) +{ + std::string decoded; + EXPECT_TRUE(opentelemetry::sdk::common::Base64Unescape( + opentelemetry::nostd::string_view{reinterpret_cast(base64_test_enc_rfc_2045), + 88}, + &decoded)); + opentelemetry::nostd::string_view expected{reinterpret_cast(base64_test_dec), + sizeof(base64_test_dec)}; + + EXPECT_EQ(decoded, expected); +} + +TEST(Base64Test, UnescapeRfc2045InvalidInput) +{ + std::string std_str_out; + std::string std_str_in = "="; + + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + + std_str_in = "J="; + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + std_str_in = "JEhu="; + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + std_str_in = "JEhuV="; + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + std_str_in = "JEhuVodi="; + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + std_str_in = "JEhuVodiW="; + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + std_str_in = " J="; + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + std_str_in = " JEhu="; + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + std_str_in = " JEhuV="; + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + std_str_in = " JEhuVodi="; + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + std_str_in = " JEhuVodiW="; + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + + std_str_in = "JE=="; + EXPECT_TRUE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + std_str_in = "JE==huVo"; + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + std_str_in = "JE=huVo="; + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + + std_str_in = " JE=="; + EXPECT_TRUE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + std_str_in = " JE==huVo"; + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + std_str_in = " JE=huVo="; + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + + std_str_in = "JE==\r\n"; + EXPECT_TRUE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + + std_str_in = "J E="; + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); +}