ci: compile_time_options in Azure RBE

.azure-pipelines/linux.yml

@@ -6,6 +6,10 @@ jobs:
         CI_TARGET: 'bazel.release'
       tsan:
         CI_TARGET: 'bazel.tsan'
+      gcc:
+        CI_TARGET: 'bazel.gcc'
+      compile_time_options:
+        CI_TARGET: 'bazel.compile_time_options'
   dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel.
   timeoutInMinutes: 360
   pool:
@@ -29,7 +33,8 @@ jobs:
     workingDirectory: $(Build.SourcesDirectory)
     env:
       ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
-      BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-clang --config=remote-ci --jobs=100 --curses=no"
+      ENVOY_RBE: "true"
+      BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=remote --jobs=100 --curses=no"
       BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
       BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
       GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)

.bazelrc

@@ -92,10 +92,20 @@ build --test_env=HEAPCHECK=normal --test_env=PPROF_PATH
 build:rbe-toolchain --host_platform=@envoy//bazel/toolchains:rbe_ubuntu_clang_platform
 build:rbe-toolchain --platforms=@envoy//bazel/toolchains:rbe_ubuntu_clang_platform
 build:rbe-toolchain --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
-build:rbe-toolchain --crosstool_top=@rbe_ubuntu_clang//cc:toolchain
-build:rbe-toolchain --extra_toolchains=@rbe_ubuntu_clang//config:cc-toolchain
-build:rbe-toolchain --linkopt=-fuse-ld=lld
-build:rbe-toolchain --action_env=CC=clang --action_env=CXX=clang++ --action_env=PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin
+
+build:rbe-toolchain-clang --config=rbe-toolchain
+build:rbe-toolchain-clang --crosstool_top=@rbe_ubuntu_clang//cc:toolchain
+build:rbe-toolchain-clang --extra_toolchains=@rbe_ubuntu_clang//config:cc-toolchain
+build:rbe-toolchain-clang --action_env=CC=clang --action_env=CXX=clang++ --action_env=PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin
+
+build:rbe-toolchain-clang-libc++ --config=rbe-toolchain
+build:rbe-toolchain-clang-libc++ --crosstool_top=@rbe_ubuntu_clang_libcxx//cc:toolchain
+build:rbe-toolchain-clang-libc++ --extra_toolchains=@rbe_ubuntu_clang_libcxx//config:cc-toolchain
+build:rbe-toolchain-clang-libc++ --action_env=CC=clang --action_env=CXX=clang++ --action_env=PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin
+
+build:rbe-toolchain-gcc --config=rbe-toolchain
+build:rbe-toolchain-gcc --crosstool_top=@rbe_ubuntu_gcc//cc:toolchain
+build:rbe-toolchain-gcc --extra_toolchains=@rbe_ubuntu_gcc//config:cc-toolchain
 
 build:remote --spawn_strategy=remote,sandboxed,local
 build:remote --strategy=Javac=remote,sandboxed,local
@@ -109,7 +119,7 @@ build:remote --experimental_remote_download_outputs=toplevel
 test:remote --experimental_remote_download_outputs=minimal
 
 build:remote-clang --config=remote
-build:remote-clang --config=rbe-toolchain
+build:remote-clang --config=rbe-toolchain-clang
 
 # Docker sandbox
 build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build:8246167b9d238797cbc6c03dccc9e3921c37617d
@@ -122,7 +132,7 @@ build:docker-sandbox --experimental_docker_verbose
 build:docker-sandbox --experimental_enable_docker_sandbox
 
 build:docker-clang --config=docker-sandbox
-build:docker-clang --config=rbe-toolchain
+build:docker-clang --config=rbe-toolchain-clang
 
 # CI configurations
 build:remote-ci --remote_cache=grpcs://remotebuildexecution.googleapis.com

.circleci/config.yml

@@ -54,15 +54,6 @@ jobs:
            path: /build/envoy/generated
            destination: /
 
-   compile_time_options:
-     executor: ubuntu-build
-     steps:
-       - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken
-       - checkout
-       - run: ci/do_circle_ci.sh bazel.compile_time_options
-       - store_artifacts:
-           path: /build/envoy/generated
-           destination: /
    api:
      executor: ubuntu-build
      steps:
@@ -156,7 +147,6 @@ workflows:
             tags:
               only: /^v.*/
       - asan
-      - compile_time_options
       - api
       - filter_example_mirror
       - coverage

bazel/toolchains/configs/gcc/bazel_0.28.1/cc/BUILD

@@ -116,8 +116,7 @@ cc_toolchain_config(
     "-B/usr/bin",
     "-pass-exit-codes",
     "-lm",
-    "-static-libgcc",
-    "-fuse-ld=lld"],
+    "-static-libgcc"],
     link_libs = ["-l:libstdc++.a"],
     opt_link_flags = ["-Wl,--gc-sections"],
     unfiltered_compile_flags = ["-fno-canonical-system-headers",

bazel/toolchains/configs/versions.bzl

@@ -3,7 +3,7 @@
 """Definitions to be used in rbe_repo attr of an rbe_autoconf rule  """
 toolchain_config_spec0 = struct(config_repos = [], create_cc_configs = True, create_java_configs = False, env = {"BAZEL_COMPILER": "clang", "BAZEL_LINKLIBS": "-l%:libstdc++.a", "BAZEL_LINKOPTS": "-lm:-static-libgcc:-fuse-ld=lld", "BAZEL_USE_LLVM_NATIVE_COVERAGE": "1", "GCOV": "llvm-profdata", "CC": "clang", "CXX": "clang++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin"}, java_home = None, name = "clang")
 toolchain_config_spec1 = struct(config_repos = [], create_cc_configs = True, create_java_configs = False, env = {"BAZEL_COMPILER": "clang", "BAZEL_LINKLIBS": "-l%:libc++.a:-l%:libc++abi.a", "BAZEL_LINKOPTS": "-lm:-static-libgcc:-pthread:-fuse-ld=lld", "BAZEL_USE_LLVM_NATIVE_COVERAGE": "1", "GCOV": "llvm-profdata", "CC": "clang", "CXX": "clang++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin", "BAZEL_CXXOPTS": "-stdlib=libc++", "CXXFLAGS": "-stdlib=libc++"}, java_home = None, name = "clang_libcxx")
-toolchain_config_spec2 = struct(config_repos = [], create_cc_configs = True, create_java_configs = False, env = {"BAZEL_COMPILER": "gcc", "BAZEL_LINKLIBS": "-l%:libstdc++.a", "BAZEL_LINKOPTS": "-lm:-static-libgcc:-fuse-ld=lld", "CC": "gcc", "CXX": "g++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin"}, java_home = None, name = "gcc")
+toolchain_config_spec2 = struct(config_repos = [], create_cc_configs = True, create_java_configs = False, env = {"BAZEL_COMPILER": "gcc", "BAZEL_LINKLIBS": "-l%:libstdc++.a", "BAZEL_LINKOPTS": "-lm:-static-libgcc", "CC": "gcc", "CXX": "g++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin"}, java_home = None, name = "gcc")
 _TOOLCHAIN_CONFIG_SPECS = [toolchain_config_spec0,toolchain_config_spec1,toolchain_config_spec2]
 _BAZEL_TO_CONFIG_SPEC_NAMES = {"0.28.1": ["clang", "clang_libcxx", "gcc"]}
 LATEST = "sha256:d1f6087fdeb6a6e5d4fd52a5dc06b15f43f49e2c20fc813bcaaa12333485a70b"

bazel/toolchains/rbe_toolchains_config.bzl

@@ -28,7 +28,7 @@ _CLANG_LIBCXX_ENV = dicts.add(_CLANG_ENV, {
 _GCC_ENV = {
     "BAZEL_COMPILER": "gcc",
     "BAZEL_LINKLIBS": "-l%:libstdc++.a",
-    "BAZEL_LINKOPTS": "-lm:-static-libgcc:-fuse-ld=lld",
+    "BAZEL_LINKOPTS": "-lm:-static-libgcc",
     "CC": "gcc",
     "CXX": "g++",
     "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin",

ci/build_setup.sh

@@ -11,19 +11,41 @@ export PPROF_PATH=/thirdparty_build/bin/pprof
 echo "ENVOY_SRCDIR=${ENVOY_SRCDIR}"
 
 function setup_gcc_toolchain() {
-  export CC=gcc
-  export CXX=g++
-  export BAZEL_COMPILER=gcc
-  echo "$CC/$CXX toolchain configured"
+  if [[ -z "${ENVOY_RBE}" ]]; then
+    export CC=gcc
+    export CXX=g++
+    export BAZEL_COMPILER=gcc
+    echo "$CC/$CXX toolchain configured"
+  else
+    export BAZEL_BUILD_OPTIONS="--config=rbe-toolchain-gcc ${BAZEL_BUILD_OPTIONS}"
+  fi
 }
 
 function setup_clang_toolchain() {
-  export PATH=/usr/lib/llvm-8/bin:$PATH
-  export CC=clang
-  export CXX=clang++
-  export BAZEL_COMPILER=clang
-  export ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-8/bin/llvm-symbolizer
-  echo "$CC/$CXX toolchain configured"
+  if [[ -z "${ENVOY_RBE}" ]]; then
+    export PATH=/usr/lib/llvm-8/bin:$PATH
+    export CC=clang
+    export CXX=clang++
+    export BAZEL_COMPILER=clang
+    export ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-8/bin/llvm-symbolizer
+    echo "$CC/$CXX toolchain configured"
+  else
+    export BAZEL_BUILD_OPTIONS="--config=rbe-toolchain-clang ${BAZEL_BUILD_OPTIONS}"
+  fi
+}
+
+function setup_clang_libcxx_toolchain() {
+  if [[ -z "${ENVOY_RBE}" ]]; then
+    export PATH=/usr/lib/llvm-8/bin:$PATH
+    export CC=clang
+    export CXX=clang++
+    export BAZEL_COMPILER=clang
+    export ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-8/bin/llvm-symbolizer
+    export BAZEL_BUILD_OPTIONS="--config=libc++ ${BAZEL_BUILD_OPTIONS}"
+    echo "$CC/$CXX toolchain with libc++ configured"
+  else
+    export BAZEL_BUILD_OPTIONS="--config=rbe-toolchain-clang-libc++ ${BAZEL_BUILD_OPTIONS}"
+  fi
 }
 
 # Create a fake home. Python site libs tries to do getpwuid(3) if we don't and the CI

ci/do_ci.sh

@@ -123,6 +123,11 @@ elif [[ "$CI_TARGET" == "bazel.sizeopt" ]]; then
   echo "Testing ${TEST_TARGETS}"
   bazel test ${BAZEL_BUILD_OPTIONS} --config=sizeopt ${TEST_TARGETS}
   exit 0
+elif [[ "$CI_TARGET" == "bazel.gcc" ]]; then
+  setup_gcc_toolchain
+  echo "bazel fastbuild build..."
+  bazel_binary_build fastbuild
+  exit 0
 elif [[ "$CI_TARGET" == "bazel.debug" ]]; then
   setup_clang_toolchain
   echo "bazel debug build with tests..."
@@ -185,7 +190,6 @@ elif [[ "$CI_TARGET" == "bazel.compile_time_options" ]]; then
   # changes, this build type may need to be broken up.
   # TODO(mpwarres): remove quiche=enabled once QUICHE is built by default.
   COMPILE_TIME_OPTIONS="\
-    --config libc++ \
     --define signal_trace=disabled \
     --define hot_restart=disabled \
     --define google_grpc=disabled \
@@ -194,7 +198,7 @@ elif [[ "$CI_TARGET" == "bazel.compile_time_options" ]]; then
     --define quiche=enabled \
     --define path_normalization_by_default=true \
   "
-  setup_clang_toolchain
+  setup_clang_libcxx_toolchain
   # This doesn't go into CI but is available for developer convenience.
   echo "bazel with different compiletime options build with tests..."
   # Building all the dependencies from scratch to link them against libc++.

ci/run_envoy_docker.sh

@@ -25,7 +25,7 @@ mkdir -p "${ENVOY_DOCKER_BUILD_DIR}"
 docker run --rm ${DOCKER_TTY_OPTION} -e HTTP_PROXY=${http_proxy} -e HTTPS_PROXY=${https_proxy} \
   -u "${USER}":"${USER_GROUP}" -v "${ENVOY_DOCKER_BUILD_DIR}":/build ${GIT_VOLUME_OPTION} \
   -e BAZEL_BUILD_EXTRA_OPTIONS -e BAZEL_EXTRA_TEST_OPTIONS -e BAZEL_REMOTE_CACHE \
-  -e BAZEL_REMOTE_INSTANCE -e GCP_SERVICE_ACCOUNT_KEY -e NUM_CPUS \
+  -e BAZEL_REMOTE_INSTANCE -e GCP_SERVICE_ACCOUNT_KEY -e NUM_CPUS -e ENVOY_RBE \
   -v "$PWD":/source --cap-add SYS_PTRACE --cap-add NET_RAW --cap-add NET_ADMIN "${IMAGE_NAME}":"${IMAGE_ID}" \
   /bin/bash -lc "groupadd --gid $(id -g) -f envoygroup && useradd -o --uid $(id -u) --gid $(id -g) --no-create-home \
   --home-dir /source envoybuild && usermod -a -G pcap envoybuild && su envoybuild -c \"cd source && $*\""