Update Envoy to 31128e7dc22355876020188bc8feb99304663041

.bazelrc

@@ -13,7 +13,7 @@
 # Startup options cannot be selected via config.
 startup --host_jvm_args=-Xmx2g
 
-build --workspace_status_command=bazel/get_workspace_status
+build --workspace_status_command="bash bazel/get_workspace_status"
 build --experimental_local_memory_estimate
 build --experimental_strict_action_env=true
 build --host_force_python=PY3
@@ -113,6 +113,28 @@ build:sizeopt -c opt --copt -Os
 # Test options
 build --test_env=HEAPCHECK=normal --test_env=PPROF_PATH
 
+# Coverage options
+coverage --config=coverage
+build:coverage --action_env=BAZEL_USE_LLVM_NATIVE_COVERAGE=1
+build:coverage --action_env=GCOV=llvm-profdata
+build:coverage --copt=-DNDEBUG
+build:coverage --test_timeout=900
+build:coverage --define=ENVOY_CONFIG_COVERAGE=1
+build:coverage --cxxopt="-DENVOY_CONFIG_COVERAGE=1"
+build:coverage --coverage_support=@envoy//bazel/coverage:coverage_support
+build:coverage --test_env=CC_CODE_COVERAGE_SCRIPT=external/envoy/bazel/coverage/collect_cc_coverage.sh
+build:coverage --test_env=HEAPCHECK=
+build:coverage --combined_report=lcov
+build:coverage --strategy=TestRunner=sandboxed,local
+build:coverage --strategy=CoverageReport=sandboxed,local
+build:coverage --experimental_use_llvm_covmap
+build:coverage --collect_code_coverage
+build:coverage --instrumentation_filter="//source(?!/common/chromium_url|/extensions/quic_listeners/quiche/platform)[/:],//include[/:]"
+coverage:test-coverage --test_arg="--log-path /dev/null"
+coverage:test-coverage --test_arg="-l trace"
+coverage:fuzz-coverage --config=asan-fuzzer
+coverage:fuzz-coverage --run_under=@envoy//bazel/coverage:fuzz_coverage_wrapper.sh
+
 # Remote execution: https://docs.bazel.build/versions/master/remote-execution.html
 build:rbe-toolchain --host_platform=@envoy_build_tools//toolchains:rbe_ubuntu_clang_platform
 build:rbe-toolchain --platforms=@envoy_build_tools//toolchains:rbe_ubuntu_clang_platform
@@ -162,7 +184,7 @@ build:remote-msan --config=rbe-toolchain-msan
 
 # Docker sandbox
 # NOTE: Update this from https://github.com/envoyproxy/envoy-build-tools/blob/master/toolchains/rbe_toolchains_config.bzl#L8
-build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu:09a5a914c904faa39dbc641181cb43b68cabf626
+build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu:04f06115b6ee7cfea74930353fb47a41149cbec3
 build:docker-sandbox --spawn_strategy=docker
 build:docker-sandbox --strategy=Javac=docker
 build:docker-sandbox --strategy=Closure=docker
@@ -193,6 +215,7 @@ build:asan-fuzzer --config=clang-asan
 build:asan-fuzzer --define=FUZZING_ENGINE=libfuzzer
 build:asan-fuzzer --copt=-fsanitize=fuzzer-no-link
 build:asan-fuzzer --copt=-fno-omit-frame-pointer
+build:asan-fuzzer --copt=-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
 # Remove UBSAN halt_on_error to avoid crashing on protobuf errors.
 build:asan-fuzzer --test_env=UBSAN_OPTIONS=print_stacktrace=1
 
@@ -208,5 +231,46 @@ build:compdb --strip=always
 build:compdb --build_tag_filters=-nocompdb
 build:compdb --define=ENVOY_CONFIG_COMPILATION_DATABASE=1
 
+# Windows build quirks
+build:windows --action_env=TMPDIR
+build:windows --define signal_trace=disabled
+build:windows --define hot_restart=disabled
+build:windows --define tcmalloc=disabled
+build:windows --define manual_stamp=manual_stamp
+
+# Should not be required after upstream fix to bazel,
+# and already a no-op to linux/macos builds
+# see issue https://github.com/bazelbuild/rules_foreign_cc/issues/301
+build:windows --copt="-DCARES_STATICLIB"
+build:windows --copt="-DNGHTTP2_STATICLIB"
+build:windows --copt="-DCURL_STATICLIB"
+
+# Required to work around build defects on Windows MSVC cl
+# Unguarded gcc pragmas in quiche are not recognized by MSVC
+build:msvc-cl --copt="/wd4068"
+# Allows 'nodiscard' function return values to be discarded
+build:msvc-cl --copt="/wd4834"
+# Allows inline functions to be undefined
+build:msvc-cl --copt="/wd4506"
+build:msvc-cl --copt="-D_SILENCE_EXPERIMENTAL_FILESYSTEM_DEPRECATION_WARNING"
+
+# Required to work around Windows clang-cl build defects
+# Ignore conflicting definitions of _WIN32_WINNT
+# Overriding __TIME__ etc is problematic (and is actually an invalid no-op)
+build:clang-cl --copt="-Wno-macro-redefined"
+build:clang-cl --copt="-Wno-builtin-macro-redefined"
+build:clang-cl --action_env=USE_CLANG_CL=1
+
+# Defaults to 'auto' - Off for windows, so override to linux behavior
+build:windows --enable_runfiles=yes
+
+# This should become adopted by bazel as the default
+build:windows --features=compiler_param_file
+
+# These options attempt to force a monolithic binary including the CRT
+build:windows --features=fully_static_link
+build:windows --features=static_link_msvcrt
+build:windows --dynamic_mode=off
+
 try-import %workspace%/clang.bazelrc
 try-import %workspace%/user.bazelrc

.bazelversion

@@ -1 +1 @@
-2.2.0
+3.1.0

.circleci/config.yml

@@ -1,7 +1,7 @@
 references:
   envoy-build-image: &envoy-build-image
     # Jan 9th, 2020
-    envoyproxy/envoy-build-ubuntu@sha256:3788a87461f2b3dc8048ad0ce5df40438a56e0a8f1a4ab0f61b4ef0d8c11ff1f
+    envoyproxy/envoy-build-ubuntu:04f06115b6ee7cfea74930353fb47a41149cbec3
 version: 2
 jobs:
   build:

WORKSPACE

@@ -16,6 +16,10 @@ load("@envoy//bazel:repositories.bzl", "envoy_dependencies")
 
 envoy_dependencies()
 
+load("@envoy//bazel:repositories_extra.bzl", "envoy_dependencies_extra")
+
+envoy_dependencies_extra()
+
 load("@envoy//bazel:dependency_imports.bzl", "envoy_dependency_imports")
 
 envoy_dependency_imports()

bazel/repositories.bzl

@@ -1,10 +1,7 @@
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
-ENVOY_COMMIT = "888e0e28900a470df448c65d7b99d8065fd60251"  # May 9th, 2020
-ENVOY_SHA = "9a4e2342d1ddaf2c9ff6f819f2010d35871f8c19a93b58ee63f2e0fc57fc9fe6"
-
-RULES_PYTHON_COMMIT = "dd7f9c5f01bafbfea08c44092b6b0c8fc8fcb77f"  # Feb 22nd, 2020
-RULES_PYTHON_SHA = "0aa9ec790a58053e3ab5af397879b267a625955f8297c239b2d8559c6773397b"
+ENVOY_COMMIT = "31128e7dc22355876020188bc8feb99304663041"  # May 17th, 2020
+ENVOY_SHA = "30e150207239a0b08beb6944120c1c29b2fa9e9b47c900dfa020d00a36e1eedb"
 
 HDR_HISTOGRAM_C_VERSION = "0.9.13"  # Feb 22nd, 2020
 HDR_HISTOGRAM_C_SHA = "2bd4a4631b64f2f8cf968ef49dd03ff3c51b487c3c98a01217ae4cf4a35b8310"
@@ -55,10 +52,3 @@ cc_library(
         strip_prefix = "HdrHistogram_c-%s" % HDR_HISTOGRAM_C_VERSION,
         url = "https://github.com/HdrHistogram/HdrHistogram_c/archive/%s.tar.gz" % HDR_HISTOGRAM_C_VERSION,
     )
-
-    http_archive(
-        name = "io_bazel_rules_python",
-        sha256 = RULES_PYTHON_SHA,
-        strip_prefix = "rules_python-%s" % RULES_PYTHON_COMMIT,
-        url = "https://github.com/bazelbuild/rules_python/archive/%s.tar.gz" % RULES_PYTHON_COMMIT,
-    )

ci/do_ci.sh

@@ -122,9 +122,12 @@ if [ -n "$CIRCLECI" ]; then
     fi
     # We constrain parallelism in CI to avoid running out of memory.	
     NUM_CPUS=8
-    if [ "$1" == "asan" ]; then
+    if [[ "$1" == "asan" ]]; then
         NUM_CPUS=5
     fi
+    if [[ "$1" == "coverage" ]]; then
+        NUM_CPUS=3
+    fi
 fi
 
 if grep 'docker\|lxc' /proc/1/cgroup; then

source/client/process_impl.cc

@@ -107,7 +107,7 @@ ProcessImpl::ProcessImpl(const Options& options, Envoy::Event::TimeSystem& time_
       singleton_manager_(std::make_unique<Envoy::Singleton::ManagerImpl>(api_->threadFactory())),
       access_log_manager_(std::chrono::milliseconds(1000), *api_, *dispatcher_, access_log_lock_,
                           store_root_),
-      init_watcher_("Nighthawk", []() {}), validation_context_(false, false) {
+      init_watcher_("Nighthawk", []() {}), validation_context_(false, false, false) {
   // Any dispatchers created after the following call will use hr timers.
   setupForHRTimers();
   std::string lower = absl::AsciiStrToLower(

source/server/README.md

@@ -107,6 +107,7 @@ bazel-bin/nighthawk_test_server  [--disable-extensions <string>]
 <string>] [--local-address-ip-version
 <string>] [--admin-address-path
 <string>]
+[--ignore-unknown-dynamic-fields]
 [--reject-unknown-dynamic-fields]
 [--allow-unknown-static-fields]
 [--allow-unknown-fields]
@@ -199,6 +200,9 @@ The local IP address version (v4 or v6).
 --admin-address-path <string>
 Admin address path
 
+--ignore-unknown-dynamic-fields
+ignore unknown fields in dynamic configuration
+
 --reject-unknown-dynamic-fields
 reject unknown fields in dynamic configuration
 

test/run_nighthawk_bazel_coverage.sh

@@ -24,39 +24,23 @@ else
   COVERAGE_TARGETS=//test/...
 fi
 
-# Make sure //test/coverage:coverage_tests is up-to-date.
-SCRIPT_DIR="$(realpath "$(dirname "$0")")"
-"${SCRIPT_DIR}"/coverage/gen_build.sh ${COVERAGE_TARGETS}
-
-BAZEL_USE_LLVM_NATIVE_COVERAGE=1 GCOV=llvm-profdata bazel coverage ${BAZEL_BUILD_OPTIONS} \
-    -c fastbuild --copt=-DNDEBUG --instrumentation_filter=//source/...,//include/... \
-    --test_timeout=2000 --cxxopt="-DENVOY_CONFIG_COVERAGE=1" --test_output=errors \
-    --test_arg="--log-path /dev/null" --test_arg="-l trace" --test_env=HEAPCHECK= \
-    --test_env=ENVOY_IP_TEST_VERSIONS=v4only //test/coverage:coverage_tests
+BAZEL_BUILD_OPTIONS+=" --config=test-coverage --test_tag_filters=-nocoverage --test_env=ENVOY_IP_TEST_VERSIONS=v4only"
+bazel coverage ${BAZEL_BUILD_OPTIONS} --test_output=all ${COVERAGE_TARGETS}
 
 COVERAGE_DIR="${SRCDIR}"/generated/coverage
 mkdir -p "${COVERAGE_DIR}"
 
-COVERAGE_IGNORE_REGEX="(/external/|pb\.(validate\.)?(h|cc)|/chromium_url/|/test/|/tmp)"
-COVERAGE_BINARY="bazel-bin/test/coverage/coverage_tests"
 COVERAGE_DATA="${COVERAGE_DIR}/coverage.dat"
 
-echo "Merging coverage data..."
-llvm-profdata merge -sparse -o ${COVERAGE_DATA} $(find -L bazel-out/k8-fastbuild/testlogs/test/coverage/coverage_tests/ -name coverage.dat)
+cp bazel-out/_coverage/_coverage_report.dat "${COVERAGE_DATA}"
 
-echo "Generating report..."
-llvm-cov show "${COVERAGE_BINARY}" -instr-profile="${COVERAGE_DATA}" -Xdemangler=c++filt \
-  -ignore-filename-regex="${COVERAGE_IGNORE_REGEX}" -output-dir=${COVERAGE_DIR} -format=html
-sed -i -e 's|>proc/self/cwd/|>|g' "${COVERAGE_DIR}/index.html"
-sed -i -e 's|>bazel-out/[^/]*/bin/\([^/]*\)/[^<]*/_virtual_includes/[^/]*|>\1|g' "${COVERAGE_DIR}/index.html"
+COVERAGE_VALUE=$(genhtml --prefix ${PWD} --output "${COVERAGE_DIR}" "${COVERAGE_DATA}" | tee /dev/stderr | grep lines... | cut -d ' ' -f 4)
+COVERAGE_VALUE=${COVERAGE_VALUE%?}
 
 [[ -z "${ENVOY_COVERAGE_DIR}" ]] || rsync -av "${COVERAGE_DIR}"/ "${ENVOY_COVERAGE_DIR}"
 
 if [ "$VALIDATE_COVERAGE" == "true" ]
 then
-  COVERAGE_VALUE=$(llvm-cov export "${COVERAGE_BINARY}" -instr-profile="${COVERAGE_DATA}" \
-    -ignore-filename-regex="${COVERAGE_IGNORE_REGEX}" -summary-only | \
-    python3 -c "import sys, json; print(json.load(sys.stdin)['data'][0]['totals']['lines']['percent'])")
   COVERAGE_THRESHOLD=98.2
   COVERAGE_FAILED=$(echo "${COVERAGE_VALUE}<${COVERAGE_THRESHOLD}" | bc)
   if test ${COVERAGE_FAILED} -eq 1; then

test/test_data/output_formatter.dotted.gold

@@ -10,7 +10,7 @@ worker_0..min: 0
 worker_0..max: 0
 worker_0.foo_size.samples: 4
 worker_0.foo_size.mean: 15.5
-worker_0.foo_size.pstdev: 1.11803
+worker_0.foo_size.pstdev: 1.118033988749895
 worker_0.foo_size.min: 14
 worker_0.foo_size.max: 17
 worker_0.foo_size.permilles-0.count: 1
@@ -48,7 +48,7 @@ worker_1..min: 0
 worker_1..max: 0
 worker_1.foo_size.samples: 4
 worker_1.foo_size.mean: 15.5
-worker_1.foo_size.pstdev: 1.11803
+worker_1.foo_size.pstdev: 1.118033988749895
 worker_1.foo_size.min: 14
 worker_1.foo_size.max: 17
 worker_1.foo_size.permilles-0.count: 1
@@ -86,7 +86,7 @@ global..min: 0
 global..max: 0
 global.foo_size.samples: 4
 global.foo_size.mean: 15.5
-global.foo_size.pstdev: 1.11803
+global.foo_size.pstdev: 1.118033988749895
 global.foo_size.min: 14
 global.foo_size.max: 17
 global.foo_size.permilles-0.count: 1

test/test_data/output_formatter.txt.gold

@@ -4,7 +4,7 @@ stat_id (3 samples)
   min: 0s 001ms 000us | mean: 0s 002ms 000us | max: 0s 003ms 000us | pstdev: 0s 000ms 816us
 
 foo_size (4 samples)
-  min: 14 | mean: 15.5 | max: 17 | pstdev: 1.11803
+  min: 14 | mean: 15.5 | max: 17 | pstdev: 1.118033988749895
 
   Percentile  Count       Value          
   0.5         2           15             

tools/shell_utils.sh

@@ -1,11 +1,25 @@
 source_venv() {
   VENV_DIR=$1
-  if [[ "$VIRTUAL_ENV" == "" ]]; then
+  if [[ "${VIRTUAL_ENV}" == "" ]]; then
     if [[ ! -d "${VENV_DIR}"/venv ]]; then
-      virtualenv "${VENV_DIR}"/venv --no-site-packages --python=python2.7
+      virtualenv "${VENV_DIR}"/venv --python=python3
     fi
     source "${VENV_DIR}"/venv/bin/activate
   else
     echo "Found existing virtualenv"
   fi
 }
+
+python_venv() {
+  SCRIPT_DIR=$(realpath "$(dirname "$0")")
+
+  BUILD_DIR=build_tools
+  PY_NAME="$1"
+  VENV_DIR="${BUILD_DIR}/${PY_NAME}"
+
+  source_venv "${VENV_DIR}"
+  pip install -r "${SCRIPT_DIR}"/requirements.txt
+
+  shift
+  python3 "${SCRIPT_DIR}/${PY_NAME}.py" $*
+}