From bdad76509d3d061d5f8b4c174cb8b5794ee0bbcf Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Tue, 1 Nov 2022 01:13:13 +0000 Subject: [PATCH] bazel: remove genrule_repository This rule seems to have been used a lot more in the past, likely pre rules_foreign_cc. At this point it was only used for boringssl_fips, and we can easily replace that use with a normal genrule that just calls the same script. The script had to be updated slightly to remove the few dependencies on bazel specifics. Signed-off-by: Keith Smiley --- bazel/EXTERNAL_DEPS.md | 26 ---- bazel/external/BUILD | 2 + bazel/external/boringssl_fips.BUILD | 5 +- bazel/external/boringssl_fips.genrule_cmd | 53 +++++---- bazel/genrule_repository.bzl | 138 ---------------------- bazel/repositories.bzl | 12 +- 6 files changed, 32 insertions(+), 204 deletions(-) mode change 100644 => 100755 bazel/external/boringssl_fips.genrule_cmd delete mode 100644 bazel/genrule_repository.bzl diff --git a/bazel/EXTERNAL_DEPS.md b/bazel/EXTERNAL_DEPS.md index f5ea35a7f1b73..b7919b5b06c80 100644 --- a/bazel/EXTERNAL_DEPS.md +++ b/bazel/EXTERNAL_DEPS.md @@ -31,32 +31,6 @@ This is the preferred style of adding dependencies that use CMake for their buil `external_deps` attribute. 4. `bazel test //test/...` - -## genrule repository - -This is the newer style of adding dependencies with no upstream Bazel configs. -It wraps the dependency's native build tooling in a Bazel-aware shell script, -installing to a Bazel-managed prefix. - -The shell script is executed by Bash, with a few Bazel-specific extensions. -See the [Bazel docs for "genrule"](https://docs.bazel.build/versions/master/be/general.html#genrule) -for details on Bazel's shell extensions. - -1. Add a BUILD file in [`bazel/external/`](external/), using a `genrule` target - to build the dependency. Please do not add BUILD logic that replaces the - dependency's upstream build tooling. -2. Define a new Bazel repository in [`bazel/repositories.bzl`](repositories.bzl), - in the `envoy_dependencies()` function. The repository may use `genrule_repository` - from [`bazel/genrule_repository.bzl`](genrule_repository.bzl) to place large - genrule shell commands into a separate file. -3. Reference your new external dependency in some `envoy_cc_library` via Y in the - `external_deps` attribute. -4. `bazel test //test/...` - -Dependencies between external libraries can use the standard Bazel dependency -resolution logic, using the `$(location)` shell extension to resolve paths -to binaries, libraries, headers, etc. - # Adding external dependencies to Envoy (Python) Python dependencies should be added via `pip` and `rules_python`. The process diff --git a/bazel/external/BUILD b/bazel/external/BUILD index add40009318dc..62a6ca994d26a 100644 --- a/bazel/external/BUILD +++ b/bazel/external/BUILD @@ -1,5 +1,7 @@ licenses(["notice"]) # Apache 2 +exports_files(["boringssl_fips.genrule_cmd"]) + # Use a wrapper cc_library with an empty source source file to force # compilation of other cc_library targets that only list *.a sources. cc_library( diff --git a/bazel/external/boringssl_fips.BUILD b/bazel/external/boringssl_fips.BUILD index 94fca2ac4c89a..1af9f34b1f020 100644 --- a/bazel/external/boringssl_fips.BUILD +++ b/bazel/external/boringssl_fips.BUILD @@ -1,5 +1,3 @@ -load(":genrule_cmd.bzl", "genrule_cmd") - licenses(["notice"]) # Apache 2 cc_library( @@ -31,5 +29,6 @@ genrule( "crypto/libcrypto.a", "ssl/libssl.a", ], - cmd = genrule_cmd("@envoy//bazel/external:boringssl_fips.genrule_cmd"), + cmd = "$(location {}) $(location crypto/libcrypto.a) $(location ssl/libssl.a)".format("@envoy//bazel/external:boringssl_fips.genrule_cmd"), + exec_tools = ["@envoy//bazel/external:boringssl_fips.genrule_cmd"], ) diff --git a/bazel/external/boringssl_fips.genrule_cmd b/bazel/external/boringssl_fips.genrule_cmd old mode 100644 new mode 100755 index 25455c91e564e..b4036e9bb9f0d --- a/bazel/external/boringssl_fips.genrule_cmd +++ b/bazel/external/boringssl_fips.genrule_cmd @@ -12,31 +12,32 @@ if [[ `uname` != "Linux" || `uname -m` != "x86_64" ]]; then fi # Bazel magic. -ROOT=$$(dirname $(rootpath boringssl/BUILDING.md))/.. -pushd $$ROOT +# ROOT=$(dirname $(rootpath boringssl/BUILDING.md))/.. +ROOT=./external/boringssl_fips +pushd "$ROOT" # Build tools requirements: # - Clang compiler version 7.0.1 (https://releases.llvm.org/download.html) # - Go programming language version 1.12.7 (https://golang.org/dl/) # - Ninja build system version 1.9.0 (https://github.com/ninja-build/ninja/releases) -# Override $$PATH for build tools, to avoid picking up anything else. -export PATH="$$(dirname `which cmake`):/usr/bin:/bin" +# Override $PATH for build tools, to avoid picking up anything else. +export PATH="$(dirname `which cmake`):/usr/bin:/bin" # Clang 7.0.1 VERSION=7.0.1 SHA256=02ad925add5b2b934d64c3dd5cbd1b2002258059f7d962993ba7f16524c3089c PLATFORM="x86_64-linux-gnu-ubuntu-16.04" -curl -sLO https://releases.llvm.org/"$$VERSION"/clang+llvm-"$$VERSION"-"$$PLATFORM".tar.xz \ - && echo "$$SHA256" clang+llvm-"$$VERSION"-"$$PLATFORM".tar.xz | sha256sum --check -tar xf clang+llvm-"$$VERSION"-"$$PLATFORM".tar.xz +curl -sLO https://releases.llvm.org/"$VERSION"/clang+llvm-"$VERSION"-"$PLATFORM".tar.xz \ + && echo "$SHA256" clang+llvm-"$VERSION"-"$PLATFORM".tar.xz | sha256sum --check +tar xf clang+llvm-"$VERSION"-"$PLATFORM".tar.xz -export HOME="$$PWD" -printf "set(CMAKE_C_COMPILER \"clang\")\nset(CMAKE_CXX_COMPILER \"clang++\")\n" > $${HOME}/toolchain -export PATH="$$PWD/clang+llvm-$$VERSION-$$PLATFORM/bin:$$PATH" +export HOME="$PWD" +printf "set(CMAKE_C_COMPILER \"clang\")\nset(CMAKE_CXX_COMPILER \"clang++\")\n" > ${HOME}/toolchain +export PATH="$PWD/clang+llvm-$VERSION-$PLATFORM/bin:$PATH" -if [[ `clang --version | head -1 | awk '{print $$3}'` != "$$VERSION" ]]; then +if [[ `clang --version | head -1 | awk '{print $3}'` != "$VERSION" ]]; then echo "ERROR: Clang version doesn't match." exit 1 fi @@ -46,15 +47,15 @@ VERSION=1.12.7 SHA256=66d83bfb5a9ede000e33c6579a91a29e6b101829ad41fffb5c5bb6c900e109d9 PLATFORM="linux-amd64" -curl -sLO https://dl.google.com/go/go"$$VERSION"."$$PLATFORM".tar.gz \ - && echo "$$SHA256" go"$$VERSION"."$$PLATFORM".tar.gz | sha256sum --check -tar xf go"$$VERSION"."$$PLATFORM".tar.gz +curl -sLO https://dl.google.com/go/go"$VERSION"."$PLATFORM".tar.gz \ + && echo "$SHA256" go"$VERSION"."$PLATFORM".tar.gz | sha256sum --check +tar xf go"$VERSION"."$PLATFORM".tar.gz -export GOPATH="$$PWD/gopath" -export GOROOT="$$PWD/go" -export PATH="$$GOPATH/bin:$$GOROOT/bin:$$PATH" +export GOPATH="$PWD/gopath" +export GOROOT="$PWD/go" +export PATH="$GOPATH/bin:$GOROOT/bin:$PATH" -if [[ `go version | awk '{print $$3}'` != "go$$VERSION" ]]; then +if [[ `go version | awk '{print $3}'` != "go$VERSION" ]]; then echo "ERROR: Go version doesn't match." exit 1 fi @@ -64,13 +65,13 @@ VERSION=1.9.0 SHA256=1b1235f2b0b4df55ac6d80bbe681ea3639c9d2c505c7ff2159a3daf63d196305 PLATFORM="linux" -curl -sLO https://github.com/ninja-build/ninja/releases/download/v"$$VERSION"/ninja-"$$PLATFORM".zip \ - && echo "$$SHA256" ninja-"$$PLATFORM".zip | sha256sum --check -unzip -o ninja-"$$PLATFORM".zip +curl -sLO https://github.com/ninja-build/ninja/releases/download/v"$VERSION"/ninja-"$PLATFORM".zip \ + && echo "$SHA256" ninja-"$PLATFORM".zip | sha256sum --check +unzip -o ninja-"$PLATFORM".zip -export PATH="$$PWD:$$PATH" +export PATH="$PWD:$PATH" -if [[ `ninja --version` != "$$VERSION" ]]; then +if [[ `ninja --version` != "$VERSION" ]]; then echo "ERROR: Ninja version doesn't match." exit 1 fi @@ -80,7 +81,7 @@ rm -rf boringssl/build # Build BoringSSL. cd boringssl -mkdir build && cd build && cmake -GNinja -DCMAKE_TOOLCHAIN_FILE=$${HOME}/toolchain -DFIPS=1 -DCMAKE_BUILD_TYPE=Release .. +mkdir build && cd build && cmake -GNinja -DCMAKE_TOOLCHAIN_FILE=${HOME}/toolchain -DFIPS=1 -DCMAKE_BUILD_TYPE=Release .. ninja ninja run_tests @@ -92,5 +93,5 @@ fi # Move compiled libraries to the expected destinations. popd -mv $$ROOT/boringssl/build/crypto/libcrypto.a $(execpath crypto/libcrypto.a) -mv $$ROOT/boringssl/build/ssl/libssl.a $(execpath ssl/libssl.a) +mv $ROOT/boringssl/build/crypto/libcrypto.a $1 +mv $ROOT/boringssl/build/ssl/libssl.a $2 diff --git a/bazel/genrule_repository.bzl b/bazel/genrule_repository.bzl deleted file mode 100644 index e263c43d4689f..0000000000000 --- a/bazel/genrule_repository.bzl +++ /dev/null @@ -1,138 +0,0 @@ -def _genrule_repository(ctx): - ctx.download_and_extract( - ctx.attr.urls, - "", # output - ctx.attr.sha256, - "", # type - ctx.attr.strip_prefix, - ) - for ii, patch in enumerate(ctx.attr.patches): - patch_input = "patch-input-%d.patch" % (ii,) - ctx.symlink(patch, patch_input) - patch_result = ctx.execute(["patch", "-p0", "--input", patch_input]) - if patch_result.return_code != 0: - fail("Failed to apply patch %r: %s, %s" % (patch, patch_result.stderr, patch_result.stdout)) - - genrule_cmd = ctx.read(ctx.attr.genrule_cmd_file) - ctx.file("WORKSPACE", "workspace(name=%r)" % (ctx.name,)) - ctx.delete("BUILD.bazel") - ctx.symlink(ctx.attr.build_file, "BUILD.bazel") - - # Inject the genrule_cmd content into a .bzl file that can be loaded - # from the repository BUILD file. We force the user to look up the - # command content "by label" so the inclusion source is obvious. - ctx.file("genrule_cmd.bzl", """ -_GENRULE_CMD = {%r: %r} -def genrule_cmd(label): - return _GENRULE_CMD[Label(label)] -""" % (ctx.attr.genrule_cmd_file, genrule_cmd)) - -genrule_repository = repository_rule( - attrs = { - "urls": attr.string_list( - mandatory = True, - allow_empty = False, - ), - "sha256": attr.string(), - "strip_prefix": attr.string(), - "patches": attr.label_list( - allow_files = [".patch"], - allow_empty = True, - ), - "genrule_cmd_file": attr.label( - mandatory = True, - allow_single_file = [".genrule_cmd"], - ), - "build_file": attr.label( - mandatory = True, - allow_single_file = [".BUILD"], - ), - }, - implementation = _genrule_repository, -) - -def _genrule_cc_deps(ctx): - outs = depset() - for dep in ctx.attr.deps: - outs = dep.cc.transitive_headers + dep.cc.libs + outs - return DefaultInfo(files = outs) - -genrule_cc_deps = rule( - attrs = { - "deps": attr.label_list( - providers = [], # CcStarlarkApiProvider - mandatory = True, - allow_empty = False, - ), - }, - implementation = _genrule_cc_deps, -) - -def _absolute_bin(path): - # If the binary path looks like it's relative to the current directory, - # transform it to be absolute by appending "${PWD}". - if "/" in path and not path.startswith("/"): - return '"${PWD}"/%r' % (path,) - return "%r" % (path,) - -def _genrule_environment(ctx): - lines = [] - - # Bazel uses the same command for C and C++ compilation. - c_compiler = ctx.var["CC"] - - # Bare minimum cflags to get included test binaries to link. - # - # See .bazelrc for the full set. - asan_flags = ["-fsanitize=address,undefined"] - tsan_flags = ["-fsanitize=thread"] - - # Older versions of GCC in Ubuntu, including GCC 5 used in CI images, - # incorrectly invoke the older `/usr/bin/ld` with gold-specific options when - # building with sanitizers enabled. Work around this by forcing use of gold - # in sanitize mode. - # - # This is not a great solution because it doesn't detect GCC when Bazel has - # wrapped it in an intermediate script, but it works well enough to keep CI - # running. - # - # https://stackoverflow.com/questions/37603238/fsanitize-not-using-gold-linker-in-gcc-6-1 - force_ld = [] - if "clang" in c_compiler: - force_ld = ["-fuse-ld=lld"] - elif "gcc" in c_compiler or "g++" in c_compiler: - force_ld = ["-fuse-ld=gold"] - - cc_flags = [] - ld_flags = [] - ld_libs = [] - if ctx.var.get("ENVOY_CONFIG_COVERAGE"): - ld_libs.append("-lgcov") - if ctx.var.get("ENVOY_CONFIG_ASAN"): - cc_flags += asan_flags - ld_flags += asan_flags - ld_flags += force_ld - if ctx.var.get("ENVOY_CONFIG_TSAN"): - cc_flags += tsan_flags - ld_flags += tsan_flags - ld_flags += force_ld - - lines.append("export CFLAGS=%r" % (" ".join(cc_flags),)) - lines.append("export LDFLAGS=%r" % (" ".join(ld_flags),)) - lines.append("export LIBS=%r" % (" ".join(ld_libs),)) - lines.append("export CC=%s" % (_absolute_bin(c_compiler),)) - lines.append("export CXX=%s" % (_absolute_bin(c_compiler),)) - - # Some Autoconf helper binaries leak, which makes ./configure think the - # system is unable to do anything. Turn off leak checking during part of - # the build. - lines.append("export ASAN_OPTIONS=detect_leaks=0") - - lines.append("") - out = ctx.actions.declare_file(ctx.attr.name + ".sh") - ctx.actions.write(out, "\n".join(lines)) - return DefaultInfo(files = depset([out])) - -genrule_environment = rule( - implementation = _genrule_environment, -) diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 2bef75a728746..dcb8d10ecba3d 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -1,5 +1,4 @@ load(":dev_binding.bzl", "envoy_dev_binding") -load(":genrule_repository.bzl", "genrule_repository") load("@envoy_api//bazel:envoy_http_archive.bzl", "envoy_http_archive") load("@envoy_api//bazel:external_deps.bzl", "load_repository_locations") load(":repository_locations.bzl", "PROTOC_VERSIONS", "REPOSITORY_LOCATIONS_SPEC") @@ -33,14 +32,6 @@ def external_http_archive(name, **kwargs): **kwargs ) -# Use this macro to reference any genrule_repository sourced from bazel/repository_locations.bzl. -def external_genrule_repository(name, **kwargs): - location = REPOSITORY_LOCATIONS[name] - genrule_repository( - name = name, - **dict(location, **kwargs) - ) - def _default_envoy_build_config_impl(ctx): ctx.file("WORKSPACE", "") ctx.file("BUILD.bazel", "") @@ -271,9 +262,8 @@ def _boringssl(): ) def _boringssl_fips(): - external_genrule_repository( + external_http_archive( name = "boringssl_fips", - genrule_cmd_file = "@envoy//bazel/external:boringssl_fips.genrule_cmd", build_file = "@envoy//bazel/external:boringssl_fips.BUILD", patches = ["@envoy//bazel/external:boringssl_fips.patch"], )