diff --git a/WORKSPACE b/WORKSPACE index 5545e2ac4fed0..3ae0dba60b645 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -4,6 +4,9 @@ load("//bazel:repositories.bzl", "envoy_dependencies") load("//bazel:cc_configure.bzl", "cc_configure") envoy_dependencies() +load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependencies") +rules_foreign_cc_dependencies() + cc_configure() load("@envoy_api//bazel:repositories.bzl", "api_dependencies") diff --git a/bazel/EXTERNAL_DEPS.md b/bazel/EXTERNAL_DEPS.md index 4c125f523ea90..c09c7f7fe93b3 100644 --- a/bazel/EXTERNAL_DEPS.md +++ b/bazel/EXTERNAL_DEPS.md @@ -9,6 +9,18 @@ build process. `external_deps` attribute. 3. `bazel test //test/...` +# Adding external dependencies to Envoy (extern cmake) + +This is the preferred style of adding dependencies that use cmake for their build system. + +1. Define a the source Bazel repository in [`bazel/repositories.bzl`](repositories.bzl), in the + `envoy_dependencies()` function. +2. Add a `cmake_external` rule to [`bazel/foreign_cc/BUILD`](bazel/foreign_cc/BUILD). This will + reference the source repository in step 1. +3. Reference your new external dependency in some `envoy_cc_library` via Y in the + `external_deps` attribute. +4. `bazel test //test/...` + # Adding external dependencies to Envoy (genrule repository) This is the newer style of adding dependencies with no upstream Bazel configs. diff --git a/bazel/foreign_cc/BUILD b/bazel/foreign_cc/BUILD new file mode 100644 index 0000000000000..1b98f6f52ed7f --- /dev/null +++ b/bazel/foreign_cc/BUILD @@ -0,0 +1,39 @@ +licenses(["notice"]) # Apache 2 + +load("//bazel:envoy_build_system.bzl", "envoy_package") +load("@rules_foreign_cc//tools/build_defs:cmake.bzl", "cmake_external") +load("@foreign_cc_platform_utils//:tools.bzl", "NINJA_COMMAND", "NINJA_DEP") + +envoy_package() + +cmake_external( + name = "ares", + cache_entries = { + "CARES_SHARED": "no", + "CARES_STATIC": "on", + }, + cmake_options = ["-GNinja"], + lib_source = "@com_github_c_ares_c_ares//:all", + make_commands = [ + NINJA_COMMAND, + NINJA_COMMAND + " install", + ], + static_libraries = ["libcares.a"], + tools_deps = NINJA_DEP, +) + +cmake_external( + name = "nghttp2", + cache_entries = { + "ENABLE_STATIC_LIB": "on", + "ENABLE_LIB_ONLY": "on", + }, + cmake_options = ["-GNinja"], + lib_source = "@com_github_nghttp2_nghttp2//:all", + make_commands = [ + NINJA_COMMAND, + NINJA_COMMAND + " install", + ], + static_libraries = ["libnghttp2.a"], + tools_deps = NINJA_DEP, +) diff --git a/bazel/foreign_cc/nghttp2.patch b/bazel/foreign_cc/nghttp2.patch new file mode 100644 index 0000000000000..4cc4ea7b32ae0 --- /dev/null +++ b/bazel/foreign_cc/nghttp2.patch @@ -0,0 +1,12 @@ +diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt +index 17e422b2..e58070f5 100644 +--- a/lib/CMakeLists.txt ++++ b/lib/CMakeLists.txt +@@ -52,6 +52,7 @@ + COMPILE_FLAGS "${WARNCFLAGS}" + VERSION ${LT_VERSION} SOVERSION ${LT_SOVERSION} + ARCHIVE_OUTPUT_NAME nghttp2 ++ ARCHIVE_OUTPUT_DIRECTORY static + ) + target_compile_definitions(nghttp2_static PUBLIC "-DNGHTTP2_STATICLIB") + if(ENABLE_STATIC_LIB) diff --git a/bazel/patched_http_archive.bzl b/bazel/patched_http_archive.bzl deleted file mode 100644 index 8a6d54881cdfe..0000000000000 --- a/bazel/patched_http_archive.bzl +++ /dev/null @@ -1,30 +0,0 @@ -def _patched_http_archive(ctx): - ctx.download_and_extract( - ctx.attr.urls, - "", # output - ctx.attr.sha256, - "", # type - ctx.attr.strip_prefix, - ) - for ii, patch in enumerate(ctx.attr.patches): - patch_input = "patch-input-%d.patch" % (ii,) - ctx.symlink(patch, patch_input) - patch_result = ctx.execute(["patch", "-p0", "--input", patch_input]) - if patch_result.return_code != 0: - fail("Failed to apply patch %r: %s" % (patch, patch_result.stderr)) - -patched_http_archive = repository_rule( - attrs = { - "urls": attr.string_list( - mandatory = True, - allow_empty = False, - ), - "sha256": attr.string(), - "strip_prefix": attr.string(), - "patches": attr.label_list( - allow_files = [".patch"], - allow_empty = True, - ), - }, - implementation = _patched_http_archive, -) diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 5f0d3e3689d57..941ac333541ce 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -3,8 +3,8 @@ load( "git_repository", "new_git_repository", ) +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load(":genrule_repository.bzl", "genrule_repository") -load(":patched_http_archive.bzl", "patched_http_archive") load(":repository_locations.bzl", "REPOSITORY_LOCATIONS") load(":target_recipes.bzl", "TARGET_RECIPES") load( @@ -17,6 +17,10 @@ load("@bazel_tools//tools/cpp:lib_cc_configure.bzl", "get_env_var") # dict of {build recipe name: longform extension name,} PPC_SKIP_TARGETS = {"luajit": "envoy.filters.http.lua"} +# Make all contents of an external repository accessible under a filegroup. Used for external HTTP +# archives, e.g. cares. +BUILD_ALL_CONTENT = """filegroup(name = "all", srcs = glob(["**"]), visibility = ["//visibility:public"])""" + def _repository_impl(name, **kwargs): # `existing_rule_keys` contains the names of repositories that have already # been defined in the Bazel workspace. By skipping repos with existing keys, @@ -278,6 +282,9 @@ def envoy_dependencies(path = "@envoy_deps//", skip_targets = []): if "envoy_build_config" not in native.existing_rules().keys(): _default_envoy_build_config(name = "envoy_build_config") + # Setup rules_foreign_cc + _foreign_cc_dependencies() + # The long repo names (`com_github_fmtlib_fmt` instead of `fmtlib`) are # semi-standard in the Bazel community, intended to avoid both duplicate # dependencies and name conflicts. @@ -285,6 +292,7 @@ def envoy_dependencies(path = "@envoy_deps//", skip_targets = []): _com_google_absl() _com_github_bombela_backward() _com_github_circonus_labs_libcircllhist() + _com_github_c_ares_c_ares() _com_github_cyan4973_xxhash() _com_github_eile_tclap() _com_github_fmtlib_fmt() @@ -296,6 +304,7 @@ def envoy_dependencies(path = "@envoy_deps//", skip_targets = []): _com_github_grpc_grpc() _com_github_google_jwt_verify() _com_github_nanopb_nanopb() + _com_github_nghttp2_nghttp2() _com_github_nodejs_http_parser() _com_github_tencent_rapidjson() _com_google_googletest() @@ -336,6 +345,18 @@ def _com_github_circonus_labs_libcircllhist(): actual = "@com_github_circonus_labs_libcircllhist//:libcircllhist", ) +def _com_github_c_ares_c_ares(): + location = REPOSITORY_LOCATIONS["com_github_c_ares_c_ares"] + http_archive( + name = "com_github_c_ares_c_ares", + build_file_content = BUILD_ALL_CONTENT, + **location + ) + native.bind( + name = "ares", + actual = "//bazel/foreign_cc:ares", + ) + def _com_github_cyan4973_xxhash(): _repository_impl( name = "com_github_cyan4973_xxhash", @@ -396,6 +417,21 @@ def _com_github_google_libprotobuf_mutator(): actual = "@com_github_google_libprotobuf_mutator//:libprotobuf_mutator", ) +def _com_github_nghttp2_nghttp2(): + location = REPOSITORY_LOCATIONS["com_github_nghttp2_nghttp2"] + http_archive( + name = "com_github_nghttp2_nghttp2", + build_file_content = BUILD_ALL_CONTENT, + patch_args = ["-p1"], + patch_cmds = ["find . -name '*.sh' -exec sed -i.orig '1s|#!/usr/bin/env sh\$|/bin/sh\$|' {} +"], + patches = ["//bazel/foreign_cc:nghttp2.patch"], + **location + ) + native.bind( + name = "nghttp2", + actual = "//bazel/foreign_cc:nghttp2", + ) + def _io_opentracing_cpp(): _repository_impl("io_opentracing_cpp") native.bind( @@ -542,6 +578,14 @@ def _com_github_google_jwt_verify(): actual = "@com_github_google_jwt_verify//:jwt_verify_lib", ) +def _foreign_cc_dependencies(): + http_archive( + name = "rules_foreign_cc", + strip_prefix = "rules_foreign_cc-master", + # TODO(htuch): Pin to SHA or release. + url = "https://github.com/bazelbuild/rules_foreign_cc/archive/master.zip", + ) + def _apply_dep_blacklist(ctxt, recipes): newlist = [] skip_list = dict() diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index e4078aad79536..bab8ea7aebf82 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -18,6 +18,11 @@ REPOSITORY_LOCATIONS = dict( strip_prefix = "backward-cpp-1.4", urls = ["https://github.com/bombela/backward-cpp/archive/v1.4.tar.gz"], ), + com_github_c_ares_c_ares = dict( + sha256 = "62dd12f0557918f89ad6f5b759f0bf4727174ae9979499f5452c02be38d9d3e8", + strip_prefix = "c-ares-cares-1_14_0", + urls = ["https://github.com/c-ares/c-ares/archive/cares-1_14_0.tar.gz"], + ), com_github_circonus_labs_libcircllhist = dict( commit = "050da53a44dede7bda136b93a9aeef47bd91fa12", # 2018-07-02 remote = "https://github.com/circonus-labs/libcircllhist", @@ -62,6 +67,11 @@ REPOSITORY_LOCATIONS = dict( strip_prefix = "nanopb-f8ac463766281625ad710900479130c7fcb4d63b", urls = ["https://github.com/nanopb/nanopb/archive/f8ac463766281625ad710900479130c7fcb4d63b.tar.gz"], ), + com_github_nghttp2_nghttp2 = dict( + sha256 = "42fff7f290100c02234ac3b0095852e4392e6bfd95ebed900ca8bd630850d88c", + strip_prefix = "nghttp2-1.33.0", + urls = ["https://github.com/nghttp2/nghttp2/releases/download/v1.33.0/nghttp2-1.33.0.tar.gz"], + ), io_opentracing_cpp = dict( sha256 = "4455ca507936bc4b658ded10a90d8ebbbd61c58f06207be565a4ffdc885687b5", strip_prefix = "opentracing-cpp-1.5.0", diff --git a/bazel/target_recipes.bzl b/bazel/target_recipes.bzl index 6260336887927..1c010014d346c 100644 --- a/bazel/target_recipes.bzl +++ b/bazel/target_recipes.bzl @@ -2,12 +2,10 @@ # target in //ci/prebuilt/BUILD to the underlying build recipe in # ci/build_container/build_recipes. TARGET_RECIPES = { - "ares": "cares", "benchmark": "benchmark", "event": "libevent", "tcmalloc_and_profiler": "gperftools", "luajit": "luajit", - "nghttp2": "nghttp2", "yaml_cpp": "yaml-cpp", "zlib": "zlib", } diff --git a/ci/WORKSPACE b/ci/WORKSPACE index 823c4ea3a1db9..221fe13702765 100644 --- a/ci/WORKSPACE +++ b/ci/WORKSPACE @@ -12,6 +12,9 @@ local_repository( envoy_dependencies( path = "@envoy//ci/prebuilt", ) +# TODO(htuch): Roll this into envoy_dependencies() +load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependencies") +rules_foreign_cc_dependencies() cc_configure() diff --git a/ci/WORKSPACE.filter.example b/ci/WORKSPACE.filter.example index 539b8eac217fc..c81dcc9b0ee1f 100644 --- a/ci/WORKSPACE.filter.example +++ b/ci/WORKSPACE.filter.example @@ -11,6 +11,9 @@ load("//bazel:cc_configure.bzl", "cc_configure") envoy_dependencies( path = "@envoy//ci/prebuilt", ) +# TODO(htuch): Roll this into envoy_dependencies() +load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependencies") +rules_foreign_cc_dependencies() cc_configure() diff --git a/ci/build_container/build_recipes/cares.sh b/ci/build_container/build_recipes/cares.sh deleted file mode 100755 index 05bb6ea5ab051..0000000000000 --- a/ci/build_container/build_recipes/cares.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -set -e - -VERSION=cares-1_14_0 -SHA256=62dd12f0557918f89ad6f5b759f0bf4727174ae9979499f5452c02be38d9d3e8 - -# cares is fussy over whether -D appears inside CFLAGS vs. CPPFLAGS, oss-fuzz -# sets CFLAGS with -D, so we need to impedance match here. In turn, OS X automake -# is fussy about newlines in CFLAGS/CPPFLAGS, so translate them into spaces. -CPPFLAGS="$(for f in $CXXFLAGS; do if [[ $f =~ -D.* ]]; then echo $f; fi; done | tr '\n' ' ')" -CFLAGS="$(for f in $CXXFLAGS; do if [[ ! $f =~ -D.* ]]; then echo $f; fi; done | tr '\n' ' ')" - -curl https://github.com/c-ares/c-ares/archive/"$VERSION".tar.gz -sLo c-ares-"$VERSION".tar.gz \ - && echo "$SHA256" c-ares-"$VERSION".tar.gz | sha256sum --check -tar xf c-ares-"$VERSION".tar.gz -cd c-ares-"$VERSION" - -mkdir build -cd build - -build_type=RelWithDebInfo -if [[ "${OS}" == "Windows_NT" ]]; then - # On Windows, every object file in the final executable needs to be compiled to use the - # same version of the C Runtime Library. If Envoy is built with '-c dbg', then it will - # use the Debug C Runtime Library. Setting CMAKE_BUILD_TYPE to Debug will cause c-ares - # to use the debug version as well - # TODO: when '-c fastbuild' and '-c opt' work for Windows builds, set this appropriately - build_type=Debug -fi - -cmake -G "Ninja" -DCMAKE_INSTALL_PREFIX="$THIRDPARTY_BUILD" \ - -DCARES_SHARED=no \ - -DCARES_STATIC=on \ - -DCMAKE_BUILD_TYPE="$build_type" \ - .. -ninja -ninja install - -if [[ "${OS}" == "Windows_NT" ]]; then - cp "CMakeFiles/c-ares.dir/c-ares.pdb" "$THIRDPARTY_BUILD/lib/c-ares.pdb" -fi diff --git a/ci/build_container/build_recipes/nghttp2.sh b/ci/build_container/build_recipes/nghttp2.sh deleted file mode 100755 index ce999fff00e3f..0000000000000 --- a/ci/build_container/build_recipes/nghttp2.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -set -e - -VERSION=1.33.0 -SHA256=42fff7f290100c02234ac3b0095852e4392e6bfd95ebed900ca8bd630850d88c - -curl https://github.com/nghttp2/nghttp2/releases/download/v"$VERSION"/nghttp2-"$VERSION".tar.gz -sLo nghttp2-"$VERSION".tar.gz \ - && echo "$SHA256" nghttp2-"$VERSION".tar.gz | sha256sum --check -tar xf nghttp2-"$VERSION".tar.gz -cd nghttp2-"$VERSION" - -# Allow nghttp2 to build as static lib on Windows -# TODO: remove once https://github.com/nghttp2/nghttp2/pull/1198 is merged -cat > nghttp2_cmakelists.diff << 'EOF' -diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt -index 17e422b2..e58070f5 100644 ---- a/lib/CMakeLists.txt -+++ b/lib/CMakeLists.txt -@@ -56,6 +56,7 @@ if(HAVE_CUNIT OR ENABLE_STATIC_LIB) - COMPILE_FLAGS "${WARNCFLAGS}" - VERSION ${LT_VERSION} SOVERSION ${LT_SOVERSION} - ARCHIVE_OUTPUT_NAME nghttp2 -+ ARCHIVE_OUTPUT_DIRECTORY static - ) - target_compile_definitions(nghttp2_static PUBLIC "-DNGHTTP2_STATICLIB") - if(ENABLE_STATIC_LIB) -EOF - -if [[ "${OS}" == "Windows_NT" ]]; then - git apply nghttp2_cmakelists.diff -fi - -mkdir build -cd build - -cmake -G "Ninja" -DCMAKE_INSTALL_PREFIX="$THIRDPARTY_BUILD" \ - -DCMAKE_INSTALL_LIBDIR="$THIRDPARTY_BUILD/lib" \ - -DENABLE_STATIC_LIB=on \ - -DENABLE_LIB_ONLY=on \ - .. -ninja -ninja install - -if [[ "${OS}" == "Windows_NT" ]]; then - cp "lib/CMakeFiles/nghttp2_static.dir/nghttp2_static.pdb" "$THIRDPARTY_BUILD/lib/nghttp2_static.pdb" -fi diff --git a/ci/prebuilt/BUILD b/ci/prebuilt/BUILD index 8997736ea30ae..dbcd36a79e6fc 100644 --- a/ci/prebuilt/BUILD +++ b/ci/prebuilt/BUILD @@ -7,16 +7,6 @@ config_setting( values = {"cpu": "x64_windows"}, ) -cc_library( - name = "ares", - srcs = select({ - ":windows_x86_64": ["thirdparty_build/lib/cares.lib"], - "//conditions:default": ["thirdparty_build/lib/libcares.a"], - }), - hdrs = glob(["thirdparty_build/include/ares*.h"]), - includes = ["thirdparty_build/include"], -) - cc_library( name = "benchmark", srcs = select({ @@ -49,16 +39,6 @@ cc_library( # the headers get included using -I vs. -isystem which then causes old-style-cast warnings. ) -cc_library( - name = "nghttp2", - srcs = select({ - ":windows_x86_64": ["thirdparty_build/lib/nghttp2.lib"], - "//conditions:default": ["thirdparty_build/lib/libnghttp2.a"], - }), - hdrs = glob(["thirdparty_build/include/nghttp2/**/*.h"]), - includes = ["thirdparty_build/include"], -) - cc_library( name = "tcmalloc_and_profiler", srcs = ["thirdparty_build/lib/libtcmalloc_and_profiler.a"], diff --git a/test/run_envoy_bazel_coverage.sh b/test/run_envoy_bazel_coverage.sh index a336d6949e44b..9e77b81627461 100755 --- a/test/run_envoy_bazel_coverage.sh +++ b/test/run_envoy_bazel_coverage.sh @@ -35,6 +35,7 @@ BAZEL_TEST_OPTIONS="${BAZEL_TEST_OPTIONS} -c dbg --copt=-DNDEBUG" # a single coverage test binary and do not require the "bazel coverage" support # for collecting multiple traces and glueing them together. "${BAZEL_COVERAGE}" --batch test "${COVERAGE_TARGET}" ${BAZEL_TEST_OPTIONS} \ + --experimental_cc_skylark_api_enabled_packages=@rules_foreign_cc//tools/build_defs,tools/build_defs \ --cache_test_results=no --cxxopt="--coverage" --cxxopt="-DENVOY_CONFIG_COVERAGE=1" \ --linkopt="--coverage" --define ENVOY_CONFIG_COVERAGE=1 --test_output=streamed \ --strategy=Genrule=standalone --spawn_strategy=standalone --test_timeout=1000 \ diff --git a/tools/bazel.rc b/tools/bazel.rc index 3891aeab3e673..d1e041ee91f01 100644 --- a/tools/bazel.rc +++ b/tools/bazel.rc @@ -1,6 +1,7 @@ # Envoy specific Bazel build/test options. build --workspace_status_command=bazel/get_workspace_status +build --experimental_cc_skylark_api_enabled_packages=@rules_foreign_cc//tools/build_defs,tools/build_defs # Basic ASAN/UBSAN that works for gcc build:asan --define ENVOY_CONFIG_ASAN=1