build: external_cmake for cmake deps.

.circleci/config.yml

@@ -4,7 +4,7 @@ executors:
   ubuntu-build:
     description: "A regular build executor based on ubuntu image"
     docker:
-      - image: envoyproxy/envoy-build:d1646d643b492ea070c96d0b67caf5bb2c8ca854
+      - image: envoyproxy/envoy-build:111781fa2823535762e9c514db0c5c41a119b4b1
     resource_class: xlarge
     working_directory: /source
 

WORKSPACE

@@ -5,6 +5,10 @@ load("//bazel:cc_configure.bzl", "cc_configure")
 
 envoy_dependencies()
 
+load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependencies")
+
+rules_foreign_cc_dependencies()
+
 cc_configure()
 
 load("@envoy_api//bazel:repositories.bzl", "api_dependencies")

bazel/EXTERNAL_DEPS.md

@@ -17,6 +17,18 @@ build process.
    `external_deps` attribute.
 3. `bazel test //test/...`
 
+# Adding external dependencies to Envoy (external CMake)
+
+This is the preferred style of adding dependencies that use CMake for their build system.
+
+1. Define a the source Bazel repository in [`bazel/repositories.bzl`](repositories.bzl), in the
+   `envoy_dependencies()` function.
+2. Add a `cmake_external` rule to [`bazel/foreign_cc/BUILD`](bazel/foreign_cc/BUILD). This will
+   reference the source repository in step 1.
+3. Reference your new external dependency in some `envoy_cc_library` via the name bound in step 1
+   `external_deps` attribute.
+4. `bazel test //test/...`
+
 # Adding external dependencies to Envoy (genrule repository)
 
 This is the newer style of adding dependencies with no upstream Bazel configs.
@@ -44,8 +56,9 @@ to binaries, libraries, headers, etc.
 
 # Adding external dependencies to Envoy (build recipe)
 
-This is the older style of adding dependencies. It uses shell scripts to build
-and install dependencies into a shared directory prefix.
+This is the older style of adding dependencies. It uses shell scripts to build and install
+dependencies into a shared directory prefix. This should no longer be used unless there are
+extenuating circumstances.
 
 1. Add a build recipe X in [`ci/build_container/build_recipes`](../ci/build_container/build_recipes)
    for developer-local and CI external dependency build flows.

bazel/foreign_cc/BUILD

@@ -0,0 +1,83 @@
+licenses(["notice"])  # Apache 2
+
+load("//bazel:envoy_build_system.bzl", "envoy_package")
+load("@rules_foreign_cc//tools/build_defs:cmake.bzl", "cmake_external")
+
+envoy_package()
+
+cmake_external(
+    name = "ares",
+    cache_entries = {
+        "CARES_SHARED": "no",
+        "CARES_STATIC": "on",
+        "CMAKE_BUILD_TYPE": "RelWithDebInfo",
+    },
+    cmake_options = ["-GNinja"],
+    lib_source = "@com_github_c_ares_c_ares//:all",
+    make_commands = [
+        "ninja",
+        "ninja install",
+    ],
+    static_libraries = ["libcares.a"],
+)
+
+cmake_external(
+    name = "event",
+    cache_entries = {
+        "EVENT__DISABLE_OPENSSL": "on",
+        "EVENT__DISABLE_REGRESS": "on",
+        "CMAKE_BUILD_TYPE": "Release",
+    },
+    cmake_options = ["-GNinja"],
+    lib_source = "@com_github_libevent_libevent//:all",
+    make_commands = [
+        "ninja",
+        "ninja install",
+    ],
+    static_libraries = ["libevent.a"],
+)
+
+cmake_external(
+    name = "nghttp2",
+    cache_entries = {
+        "ENABLE_STATIC_LIB": "on",
+        "ENABLE_LIB_ONLY": "on",
+        "CMAKE_BUILD_TYPE": "RelWithDebInfo",
+    },
+    cmake_options = ["-GNinja"],
+    lib_source = "@com_github_nghttp2_nghttp2//:all",
+    make_commands = [
+        "ninja",
+        "ninja install",
+    ],
+    static_libraries = ["libnghttp2.a"],
+)
+
+cmake_external(
+    name = "yaml",
+    cache_entries = {
+        "YAML_CPP_BUILD_TESTS": "off",
+        "CMAKE_BUILD_TYPE": "RelWithDebInfo",
+    },
+    cmake_options = ["-GNinja"],
+    lib_source = "@com_github_jbeder_yaml_cpp//:all",
+    make_commands = [
+        "ninja",
+        "ninja install",
+    ],
+    static_libraries = ["libyaml-cpp.a"],
+)
+
+cmake_external(
+    name = "zlib",
+    cache_entries = {
+        "CMAKE_BUILD_TYPE": "RelWithDebInfo",
+    },
+    cmake_options = ["-GNinja"],
+    lib_source = "@com_github_madler_zlib//:all",
+    make_commands = [
+        "ninja",
+        "ninja install",
+    ],
+    static_libraries = ["libz.a"],
+)

bazel/foreign_cc/nghttp2.patch

@@ -0,0 +1,12 @@
+diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt
+index 17e422b2..e58070f5 100644
+--- a/lib/CMakeLists.txt
++++ b/lib/CMakeLists.txt
+@@ -52,6 +52,7 @@
+     COMPILE_FLAGS "${WARNCFLAGS}"
+     VERSION ${LT_VERSION} SOVERSION ${LT_SOVERSION}
+     ARCHIVE_OUTPUT_NAME nghttp2
++    ARCHIVE_OUTPUT_DIRECTORY static
+   )
+   target_compile_definitions(nghttp2_static PUBLIC "-DNGHTTP2_STATICLIB")
+   if(ENABLE_STATIC_LIB)

bazel/repositories.bzl

@@ -15,6 +15,10 @@ PPC_SKIP_TARGETS = {"luajit": "envoy.filters.http.lua"}
 # go version for rules_go
 GO_VERSION = "1.10.4"
 
+# Make all contents of an external repository accessible under a filegroup.  Used for external HTTP
+# archives, e.g. cares.
+BUILD_ALL_CONTENT = """filegroup(name = "all", srcs = glob(["**"]), visibility = ["//visibility:public"])"""
+
 def _repository_impl(name, **kwargs):
     # `existing_rule_keys` contains the names of repositories that have already
     # been defined in the Bazel workspace. By skipping repos with existing keys,
@@ -265,6 +269,9 @@ def envoy_dependencies(path = "@envoy_deps//", skip_targets = []):
     if "envoy_build_config" not in native.existing_rules().keys():
         _default_envoy_build_config(name = "envoy_build_config")
 
+    # Setup rules_foreign_cc
+    _foreign_cc_dependencies()
+
     # Binding to an alias pointing to the selected version of BoringSSL:
     # - BoringSSL FIPS from @boringssl_fips//:ssl,
     # - non-FIPS BoringSSL from @boringssl//:ssl.
@@ -281,6 +288,7 @@ def envoy_dependencies(path = "@envoy_deps//", skip_targets = []):
     _com_google_absl()
     _com_github_bombela_backward()
     _com_github_circonus_labs_libcircllhist()
+    _com_github_c_ares_c_ares()
     _com_github_cyan4973_xxhash()
     _com_github_eile_tclap()
     _com_github_fmtlib_fmt()
@@ -291,8 +299,13 @@ def envoy_dependencies(path = "@envoy_deps//", skip_targets = []):
     _com_lightstep_tracer_cpp()
     _com_github_datadog_dd_opentracing_cpp()
     _com_github_grpc_grpc()
+    _com_github_google_benchmark()
     _com_github_google_jwt_verify()
+    _com_github_jbeder_yaml_cpp()
+    _com_github_libevent_libevent()
+    _com_github_madler_zlib()
     _com_github_nanopb_nanopb()
+    _com_github_nghttp2_nghttp2()
     _com_github_nodejs_http_parser()
     _com_github_tencent_rapidjson()
     _com_google_googletest()
@@ -341,6 +354,18 @@ def _com_github_circonus_labs_libcircllhist():
         actual = "@com_github_circonus_labs_libcircllhist//:libcircllhist",
     )
 
+def _com_github_c_ares_c_ares():
+    location = REPOSITORY_LOCATIONS["com_github_c_ares_c_ares"]
+    http_archive(
+        name = "com_github_c_ares_c_ares",
+        build_file_content = BUILD_ALL_CONTENT,
+        **location
+    )
+    native.bind(
+        name = "ares",
+        actual = "//bazel/foreign_cc:ares",
+    )
+
 def _com_github_cyan4973_xxhash():
     _repository_impl(
         name = "com_github_cyan4973_xxhash",
@@ -401,6 +426,17 @@ def _com_github_gcovr_gcovr():
         actual = "@com_github_gcovr_gcovr//:gcovr",
     )
 
+def _com_github_google_benchmark():
+    location = REPOSITORY_LOCATIONS["com_github_google_benchmark"]
+    http_archive(
+        name = "com_github_google_benchmark",
+        **location
+    )
+    native.bind(
+        name = "benchmark",
+        actual = "@com_github_google_benchmark//:benchmark",
+    )
+
 def _com_github_google_libprotobuf_mutator():
     _repository_impl(
         name = "com_github_google_libprotobuf_mutator",
@@ -411,6 +447,57 @@ def _com_github_google_libprotobuf_mutator():
         actual = "@com_github_google_libprotobuf_mutator//:libprotobuf_mutator",
     )
 
+def _com_github_jbeder_yaml_cpp():
+    location = REPOSITORY_LOCATIONS["com_github_jbeder_yaml_cpp"]
+    http_archive(
+        name = "com_github_jbeder_yaml_cpp",
+        build_file_content = BUILD_ALL_CONTENT,
+        **location
+    )
+    native.bind(
+        name = "yaml_cpp",
+        actual = "//bazel/foreign_cc:yaml",
+    )
+
+def _com_github_libevent_libevent():
+    location = REPOSITORY_LOCATIONS["com_github_libevent_libevent"]
+    http_archive(
+        name = "com_github_libevent_libevent",
+        build_file_content = BUILD_ALL_CONTENT,
+        **location
+    )
+    native.bind(
+        name = "event",
+        actual = "//bazel/foreign_cc:event",
+    )
+
+def _com_github_madler_zlib():
+    location = REPOSITORY_LOCATIONS["com_github_madler_zlib"]
+    http_archive(
+        name = "com_github_madler_zlib",
+        build_file_content = BUILD_ALL_CONTENT,
+        **location
+    )
+    native.bind(
+        name = "zlib",
+        actual = "//bazel/foreign_cc:zlib",
+    )
+
+def _com_github_nghttp2_nghttp2():
+    location = REPOSITORY_LOCATIONS["com_github_nghttp2_nghttp2"]
+    http_archive(
+        name = "com_github_nghttp2_nghttp2",
+        build_file_content = BUILD_ALL_CONTENT,
+        patch_args = ["-p1"],
+        patch_cmds = ["find . -name '*.sh' -exec sed -i.orig '1s|#!/usr/bin/env sh\$|/bin/sh\$|' {} +"],
+        patches = ["//bazel/foreign_cc:nghttp2.patch"],
+        **location
+    )
+    native.bind(
+        name = "nghttp2",
+        actual = "//bazel/foreign_cc:nghttp2",
+    )
+
 def _io_opentracing_cpp():
     _repository_impl("io_opentracing_cpp")
     native.bind(
@@ -641,6 +728,9 @@ def _com_github_google_jwt_verify():
         actual = "@com_github_google_jwt_verify//:jwt_verify_lib",
     )
 
+def _foreign_cc_dependencies():
+    _repository_impl("rules_foreign_cc")
+
 def _apply_dep_blacklist(ctxt, recipes):
     newlist = []
     skip_list = []

bazel/repository_locations.bzl

@@ -31,6 +31,11 @@ REPOSITORY_LOCATIONS = dict(
         strip_prefix = "backward-cpp-1.4",
         urls = ["https://github.com/bombela/backward-cpp/archive/v1.4.tar.gz"],
     ),
+    com_github_c_ares_c_ares = dict(
+        sha256 = "7deb7872cbd876c29036d5f37e30c4cbc3cc068d59d8b749ef85bb0736649f04",
+        strip_prefix = "c-ares-cares-1_15_0",
+        urls = ["https://github.com/c-ares/c-ares/archive/cares-1_15_0.tar.gz"],
+    ),
     com_github_circonus_labs_libcircllhist = dict(
         sha256 = "9949e2864b8ad00ee5c3e9c1c3c01e51b6b68bb442a919652fc66b9776477987",
         strip_prefix = "libcircllhist-fd8a14463739d247b414825cc56ca3946792a3b9",
@@ -83,6 +88,11 @@ REPOSITORY_LOCATIONS = dict(
         strip_prefix = "nanopb-0.3.9.2",
         urls = ["https://github.com/nanopb/nanopb/archive/0.3.9.2.tar.gz"],
     ),
+    com_github_nghttp2_nghttp2 = dict(
+        sha256 = "cb70261634c33dc5adbe780afcfc5dab17838ee303631a02b983c6a217bc16ba",
+        strip_prefix = "nghttp2-1.35.1",
+        urls = ["https://github.com/nghttp2/nghttp2/releases/download/v1.35.1/nghttp2-1.35.1.tar.gz"],
+    ),
     io_opentracing_cpp = dict(
         sha256 = "015c4187f7a6426a2b5196f0ccd982aa87f010cf61f507ae3ce5c90523f92301",
         strip_prefix = "opentracing-cpp-1.5.1",
@@ -104,6 +114,27 @@ REPOSITORY_LOCATIONS = dict(
         strip_prefix = "dd-opentracing-cpp-0.4.1",
         urls = ["https://github.com/DataDog/dd-opentracing-cpp/archive/v0.4.1.tar.gz"],
     ),
+    com_github_google_benchmark = dict(
+        # TODO (moderation) change back to tarball method on next benchmark release
+        sha256 = "0de43b6eaddd356f1d6cd164f73f37faf2f6c96fd684e1f7ea543ce49c1d144e",
+        strip_prefix = "benchmark-505be96ab23056580a3a2315abba048f4428b04e",
+        urls = ["https://github.com/google/benchmark/archive/505be96ab23056580a3a2315abba048f4428b04e.tar.gz"],
+    ),
+    com_github_libevent_libevent = dict(
+        sha256 = "316ddb401745ac5d222d7c529ef1eada12f58f6376a66c1118eee803cb70f83d",
+        strip_prefix = "libevent-release-2.1.8-stable",
+        urls = ["https://github.com/libevent/libevent/archive/release-2.1.8-stable.tar.gz"],
+    ),
+    com_github_madler_zlib = dict(
+        sha256 = "629380c90a77b964d896ed37163f5c3a34f6e6d897311f1df2a7016355c45eff",
+        strip_prefix = "zlib-1.2.11",
+        urls = ["https://github.com/madler/zlib/archive/v1.2.11.tar.gz"],
+    ),
+    com_github_jbeder_yaml_cpp = dict(
+        sha256 = "53dcffd55f3433b379fcc694f45c54898711c0e29159a7bd02e82a3e0253bac3",
+        strip_prefix = "yaml-cpp-0f9a586ca1dc29c2ecb8dd715a315b93e3f40f79",
+        urls = ["https://github.com/jbeder/yaml-cpp/archive/0f9a586ca1dc29c2ecb8dd715a315b93e3f40f79.tar.gz"],
+    ),
     com_github_msgpack_msgpack_c = dict(
         sha256 = "bda49f996a73d2c6080ff0523e7b535917cd28c8a79c3a5da54fc29332d61d1e",
         strip_prefix = "msgpack-c-cpp-3.1.1",
@@ -187,6 +218,11 @@ REPOSITORY_LOCATIONS = dict(
         sha256 = "7be7dc01f1e0afdba6c8eb2b43d2fa01c743be1b9273ab1eaf6c233df078d705",
         urls = ["https://github.com/bazelbuild/rules_go/releases/download/0.16.5/rules_go-0.16.5.tar.gz"],
     ),
+    rules_foreign_cc = dict(
+        sha256 = "78cbd1a8134b2f0ead8e637228d8ac1ac7c0ab3f0fbcd149a85e55330697d9a3",
+        strip_prefix = "rules_foreign_cc-216ded8acb95d81e312b228dce3c39872c7a7c34",
+        urls = ["https://github.com/bazelbuild/rules_foreign_cc/archive/216ded8acb95d81e312b228dce3c39872c7a7c34.tar.gz"],
+    ),
     six_archive = dict(
         sha256 = "105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a",
         urls = ["https://pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz#md5=34eed507548117b2ab523ab14b2f8b55"],

bazel/target_recipes.bzl

@@ -2,13 +2,7 @@
 # target in //ci/prebuilt/BUILD to the underlying build recipe in
 # ci/build_container/build_recipes.
 TARGET_RECIPES = {
-    "ares": "cares",
-    "benchmark": "benchmark",
-    "event": "libevent",
     "tcmalloc_and_profiler": "gperftools",
     "tcmalloc_debug": "gperftools",
     "luajit": "luajit",
-    "nghttp2": "nghttp2",
-    "yaml_cpp": "yaml-cpp",
-    "zlib": "zlib",
 }

ci/WORKSPACE

@@ -13,6 +13,11 @@ envoy_dependencies(
     path = "@envoy//ci/prebuilt",
 )
 
+# TODO(htuch): Roll this into envoy_dependencies()
+load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependencies")
+
+rules_foreign_cc_dependencies()
+
 cc_configure()
 
 load("@envoy_api//bazel:repositories.bzl", "api_dependencies")

ci/WORKSPACE.filter.example

@@ -11,6 +11,9 @@ load("//bazel:cc_configure.bzl", "cc_configure")
 envoy_dependencies(
     path = "@envoy//ci/prebuilt",
 )
+# TODO(htuch): Roll this into envoy_dependencies()
+load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependencies")
+rules_foreign_cc_dependencies()
 
 cc_configure()