Merge attempt to upstream/36cf26b9d6ac32f5671c24b92f9ea22ab70bc6c1

.azure-pipelines/linux.yml

@@ -1,14 +1,62 @@
-resources:
-  containers:
-  - container: envoy-build
-    image: envoyproxy/envoy-build:ec38ecb88fd1abe55ab1daa2f6bd239ffccc9d98
-
 jobs:
-- job: BuildEnvoy
+- job: bazel
+  strategy:
+    matrix:
+      release:
+        CI_TARGET: 'bazel.release'
+      tsan:
+        CI_TARGET: 'bazel.tsan'
+      gcc:
+        CI_TARGET: 'bazel.gcc'
+      compile_time_options:
+        CI_TARGET: 'bazel.compile_time_options'
+      fuzz:
+        CI_TARGET: 'bazel.fuzz'
+  dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel.
   timeoutInMinutes: 360
   pool:
     vmImage: 'Ubuntu 16.04'
-  container: envoy-build
   steps:
-  - script: bazel build //source/exe:envoy-static
+  - bash: |
+      echo "disk space at beginning of build:"
+      df -h
+    displayName: "Check disk space at beginning"
+
+  - bash: |
+      sudo mkdir -p /etc/docker
+      echo '{
+        "ipv6": true,
+        "fixed-cidr-v6": "2001:db8:1::/64"
+      }' | sudo tee /etc/docker/daemon.json
+      sudo service docker restart
+    displayName: "Enable IPv6"
+
+  - script: ci/run_envoy_docker.sh 'ci/do_ci.sh $(CI_TARGET)'
+    workingDirectory: $(Build.SourcesDirectory)
+    env:
+      ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
+      ENVOY_RBE: "true"
+      BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=remote --jobs=100 --curses=no"
+      BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
+      BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
+      GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
+    displayName: "Run CI script"
+
+  - bash: |
+      echo "disk space at end of build:"
+      df -h
+    displayName: "Check disk space at end"
+    condition: always()
+
+  - task: PublishTestResults@2
+    inputs:
+      testResultsFiles: '**/bazel-out/**/testlogs/**/test.xml'
+      testRunTitle: '$(CI_TARGET)'
+      searchFolder: $(Build.StagingDirectory)/tmp
+    condition: always()
 
+  - task: PublishBuildArtifacts@1
+    inputs:
+      pathtoPublish: "$(Build.StagingDirectory)/envoy"
+      artifactName: $(CI_TARGET)
+    condition: always()

.azure-pipelines/macos.yml

@@ -15,7 +15,7 @@ jobs:
   - script: ./ci/mac_ci_steps.sh
     displayName: 'Run Mac CI'
     env:
-      BAZEL_REMOTE_CACHE: remotebuildexecution.googleapis.com
+      BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
       BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
       GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
 

.bazelci/presubmit.yml

@@ -1,8 +1,11 @@
 ---
 tasks:
-  ubuntu_1804:
+  release:
     platform: ubuntu1804
     build_targets:
-      - "//source/..."
+      - "//source/exe:envoy-static"
     test_targets:
       - "//test/..."
+    test_flags:
+    # Workaround for https://github.com/envoyproxy/envoy/issues/7647
+      - "--deleted_packages=//test/extensions/tracers/dynamic_ot"

.bazelrc

@@ -1,25 +1,48 @@
 # Envoy specific Bazel build/test options.
 
-# Bazel doesn't need more than 200MB of memory based on memory profiling:
+# Bazel doesn't need more than 200MB of memory for local build based on memory profiling:
 # https://docs.bazel.build/versions/master/skylark/performance.html#memory-profiling
+# The default JVM max heapsize is 1/4 of physical memory up to 32GB which could be large
+# enough to consume all memory constrained by cgroup in large host, which is the case in CircleCI.
 # Limiting JVM heapsize here to let it do GC more when approaching the limit to
 # leave room for compiler/linker.
-startup --host_jvm_args=-Xmx512m
+# The number 2G is choosed heuristically to both support in CircleCI and large enough for RBE.
+# Startup options cannot be selected via config.
+startup --host_jvm_args=-Xmx2g
+
 build --workspace_status_command=bazel/get_workspace_status
 build --experimental_remap_main_repo
+build --experimental_local_memory_estimate
+build --experimental_strict_action_env=true
+build --host_force_python=PY2
+build --action_env=BAZEL_LINKLIBS=-l%:libstdc++.a
+build --action_env=BAZEL_LINKOPTS=-lm
+build --host_javabase=@bazel_tools//tools/jdk:remote_jdk11
+build --javabase=@bazel_tools//tools/jdk:remote_jdk11
+
+# We already have absl in the build, define absl=1 to tell googletest to use absl for backtrace.
+build --define absl=1
+
+# Pass PATH, CC and CXX variables from the environment.
+build --action_env=CC
+build --action_env=CXX
+build --action_env=PATH
+
+# Common flags for sanitizers
+build:sanitizer --define tcmalloc=disabled
+build:sanitizer --linkopt -ldl
+build:sanitizer --build_tag_filters=-no_san
+build:sanitizer --test_tag_filters=-no_san
 
 # Basic ASAN/UBSAN that works for gcc
+build:asan --config=sanitizer
+# ASAN install its signal handler, disable ours so the stacktrace will be printed by ASAN
+build:asan --define signal_trace=disabled
 build:asan --define ENVOY_CONFIG_ASAN=1
 build:asan --copt -fsanitize=address,undefined
 build:asan --linkopt -fsanitize=address,undefined
 build:asan --copt -fno-sanitize=vptr
 build:asan --linkopt -fno-sanitize=vptr
-build:asan --linkopt -fuse-ld=lld
-build:asan --linkopt -ldl
-build:asan --define tcmalloc=disabled
-build:asan --build_tag_filters=-no_asan
-build:asan --test_tag_filters=-no_asan
-build:asan --define signal_trace=disabled
 build:asan --copt -DADDRESS_SANITIZER=1
 build:asan --copt -D__SANITIZE_ADDRESS__
 build:asan --test_env=ASAN_OPTIONS=handle_abort=1:allow_addr2line=true:check_initialization_order=true:strict_init_order=true:detect_odr_violation=1
@@ -28,41 +51,104 @@ build:asan --test_env=ASAN_SYMBOLIZER_PATH
 
 # Clang ASAN/UBSAN
 build:clang-asan --config=asan
+build:clang-asan --linkopt -fuse-ld=lld
 
 # macOS ASAN/UBSAN
 build:macos-asan --config=asan
 # Workaround, see https://github.com/bazelbuild/bazel/issues/6932
 build:macos-asan --copt -Wno-macro-redefined
 build:macos-asan --copt -D_FORTIFY_SOURCE=0
+# Workaround, see https://github.com/bazelbuild/bazel/issues/4341
+build:macos-asan --copt -DGRPC_BAZEL_BUILD
+# Dynamic link cause issues like: `dyld: malformed mach-o: load commands size (59272) > 32768`
+build:macos-asan --dynamic_mode=off
 
 # Clang TSAN
+build:clang-tsan --config=sanitizer
 build:clang-tsan --define ENVOY_CONFIG_TSAN=1
 build:clang-tsan --copt -fsanitize=thread
 build:clang-tsan --linkopt -fsanitize=thread
 build:clang-tsan --linkopt -fuse-ld=lld
-build:clang-tsan --define tcmalloc=disabled
 # Needed due to https://github.com/libevent/libevent/issues/777
 build:clang-tsan --copt -DEVENT__DISABLE_DEBUG_MODE
 
 # Clang MSAN - broken today since we need to rebuild lib[std]c++ and external deps with MSAN
 # support (see https://github.com/envoyproxy/envoy/issues/443).
+build:clang-msan --config=sanitizer
 build:clang-msan --define ENVOY_CONFIG_MSAN=1
 build:clang-msan --copt -fsanitize=memory
 build:clang-msan --linkopt -fsanitize=memory
-build:clang-msan --define tcmalloc=disabled
 build:clang-msan --copt -fsanitize-memory-track-origins=2
 
 # Clang with libc++
 # TODO(cmluciano) fix and re-enable _LIBCPP_VERSION testing for TCMALLOC in Envoy::Stats::TestUtil::hasDeterministicMallocStats
 # and update stats_integration_test with appropriate m_per_cluster value
-build:libc++ --action_env=CC
-build:libc++ --action_env=CXX
 build:libc++ --action_env=CXXFLAGS=-stdlib=libc++
-build:libc++ --action_env=PATH
+build:libc++ --action_env=BAZEL_CXXOPTS=-stdlib=libc++
+build:libc++ --action_env=BAZEL_LINKLIBS=-l%:libc++.a:-l%:libc++abi.a:-lm
+build:libc++ --host_linkopt=-fuse-ld=lld
 build:libc++ --define force_libcpp=enabled
 
 # Optimize build for binary size reduction.
 build:sizeopt -c opt --copt -Os
 
 # Test options
-test --test_env=HEAPCHECK=normal --test_env=PPROF_PATH
+build --test_env=HEAPCHECK=normal --test_env=PPROF_PATH
+
+# Remote execution: https://docs.bazel.build/versions/master/remote-execution.html
+build:rbe-toolchain --host_platform=@envoy//bazel/toolchains:rbe_ubuntu_clang_platform
+build:rbe-toolchain --platforms=@envoy//bazel/toolchains:rbe_ubuntu_clang_platform
+build:rbe-toolchain --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
+
+build:rbe-toolchain-clang --config=rbe-toolchain
+build:rbe-toolchain-clang --crosstool_top=@rbe_ubuntu_clang//cc:toolchain
+build:rbe-toolchain-clang --extra_toolchains=@rbe_ubuntu_clang//config:cc-toolchain
+build:rbe-toolchain-clang --action_env=CC=clang --action_env=CXX=clang++ --action_env=PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin
+
+build:rbe-toolchain-clang-libc++ --config=rbe-toolchain
+build:rbe-toolchain-clang-libc++ --crosstool_top=@rbe_ubuntu_clang_libcxx//cc:toolchain
+build:rbe-toolchain-clang-libc++ --extra_toolchains=@rbe_ubuntu_clang_libcxx//config:cc-toolchain
+build:rbe-toolchain-clang-libc++ --action_env=CC=clang --action_env=CXX=clang++ --action_env=PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin
+build:rbe-toolchain-clang-libc++ --define force_libcpp=enabled
+
+build:rbe-toolchain-gcc --config=rbe-toolchain
+build:rbe-toolchain-gcc --crosstool_top=@rbe_ubuntu_gcc//cc:toolchain
+build:rbe-toolchain-gcc --extra_toolchains=@rbe_ubuntu_gcc//config:cc-toolchain
+
+build:remote --spawn_strategy=remote,sandboxed,local
+build:remote --strategy=Javac=remote,sandboxed,local
+build:remote --strategy=Closure=remote,sandboxed,local
+build:remote --strategy=Genrule=remote,sandboxed,local
+build:remote --remote_timeout=3600
+build:remote --auth_enabled=true
+build:remote --experimental_inmemory_jdeps_files
+build:remote --experimental_inmemory_dotd_files
+build:remote --experimental_remote_download_outputs=toplevel
+
+build:remote-clang --config=remote
+build:remote-clang --config=rbe-toolchain-clang
+
+# Docker sandbox
+build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build@sha256:9236915d10004a35f2439ce4a1c33c1dbb06f95f84c4a4497d4e4f95cdc9e07f
+build:docker-sandbox --spawn_strategy=docker
+build:docker-sandbox --strategy=Javac=docker
+build:docker-sandbox --strategy=Closure=docker
+build:docker-sandbox --strategy=Genrule=docker
+build:docker-sandbox --define=EXECUTOR=remote
+build:docker-sandbox --experimental_docker_verbose
+build:docker-sandbox --experimental_enable_docker_sandbox
+
+build:docker-clang --config=docker-sandbox
+build:docker-clang --config=rbe-toolchain-clang
+
+# CI configurations
+build:remote-ci --remote_cache=grpcs://remotebuildexecution.googleapis.com
+build:remote-ci --remote_executor=grpcs://remotebuildexecution.googleapis.com
+
+# Fuzz builds
+build:asan-fuzzer --config=asan
+build:asan-fuzzer --define=FUZZING_ENGINE=libfuzzer
+build:asan-fuzzer --copt=-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+build:asan-fuzzer --copt=-fsanitize-coverage=trace-pc-guard
+# Remove UBSAN halt_on_error to avoid crashing on protobuf errors.
+build:asan-fuzzer --test_env=UBSAN_OPTIONS=print_stacktrace=1

.bazelversion

@@ -0,0 +1 @@
+0.29.1

.circleci/config.yml

@@ -4,7 +4,8 @@ executors:
   ubuntu-build:
     description: "A regular build executor based on ubuntu image"
     docker:
-      - image: envoyproxy/envoy-build:ec38ecb88fd1abe55ab1daa2f6bd239ffccc9d98
+      # NOTE: Update bazel/toolchains/rbe_toolchains_config.bzl with sha256 digest to match the image here.
+      - image: envoyproxy/envoy-build:cb15cc3d2010aff77d6e022ddf6d723fa8becbc0
     resource_class: xlarge
     working_directory: /source
 
@@ -52,25 +53,7 @@ jobs:
        - store_artifacts:
            path: /build/envoy/generated
            destination: /
-   tsan:
-     executor: ubuntu-build
-     steps:
-       - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken
-       - checkout
-       - run: ci/do_circle_ci.sh bazel.tsan
-       - store_artifacts:
-           path: /build/envoy/generated
-           destination: /
 
-   compile_time_options:
-     executor: ubuntu-build
-     steps:
-       - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken
-       - checkout
-       - run: ci/do_circle_ci.sh bazel.compile_time_options
-       - store_artifacts:
-           path: /build/envoy/generated
-           destination: /
    api:
      executor: ubuntu-build
      steps:
@@ -93,28 +76,6 @@ jobs:
            fingerprints:
              - "f6:f9:df:90:9c:4b:5f:9c:f4:69:fd:42:94:ff:88:24"
        - run: ci/filter_example_mirror.sh
-   ipv6_tests:
-     machine: true
-     steps:
-     - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken
-     - checkout
-     - run:
-         name: enable ipv6
-         command: |
-           echo '{
-             "ipv6": true,
-             "fixed-cidr-v6": "2001:db8:1::/64"
-           }' | sudo tee /etc/docker/daemon.json
-
-           sudo service docker restart
-     - run: dig go.googlesource.com A go.googlesource.com AAAA # Debug IPv6 network issues
-     - run: ifconfig
-     - run: route -A inet -A inet6
-     - run: curl -v https://go.googlesource.com
-     - run: curl -6 -v https://go.googlesource.com || true
-     - run: ./ci/do_circle_ci_ipv6_tests.sh
-     - store_artifacts:
-         path: /tmp/envoy-docker/envoy/generated/failed-testlogs
 
    coverage:
      executor: ubuntu-build
@@ -125,11 +86,24 @@ jobs:
            command:
              ci/do_circle_ci.sh bazel.coverage
            no_output_timeout: 60m
-       - run: ci/coverage_publish.sh
+       - persist_to_workspace:
+           root: /build/envoy/generated
+           paths:
+             - coverage
        - store_artifacts:
            path: /build/envoy/generated
            destination: /
 
+   coverage_publish:
+     docker:
+       - image: google/cloud-sdk
+     steps:
+       - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken
+       - checkout
+       - attach_workspace:
+           at: /build/envoy/generated
+       - run: ci/coverage_publish.sh
+
    clang_tidy:
      executor: ubuntu-build
      steps:
@@ -153,11 +127,7 @@ jobs:
        - run: ci/do_circle_ci.sh check_spelling_pedantic
    build_image:
      docker:
-       - image: circleci/python:3.7
-         environment:
-           # See comment in do_circle_ci.sh for why we do this.
-           NUM_CPUS: 8
-     resource_class: xlarge
+       - image: google/cloud-sdk
      steps:
        - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken
        - checkout
@@ -190,12 +160,11 @@ workflows:
             tags:
               only: /^v.*/
       - asan
-      - tsan
-      - compile_time_options
       - api
       - filter_example_mirror
-      - ipv6_tests
       - coverage
+      - coverage_publish:
+          requires: [coverage]
       - format
       - clang_tidy
       - build_image