Proxy-Wasm mega backport

.bazelrc

@@ -192,8 +192,6 @@ build:remote --spawn_strategy=remote,sandboxed,local
 build:remote --strategy=Javac=remote,sandboxed,local
 build:remote --strategy=Closure=remote,sandboxed,local
 build:remote --strategy=Genrule=remote,sandboxed,local
-# rules_rust is not remote runnable (yet)
-build:remote --strategy=Rustc=sandboxed,local
 build:remote --remote_timeout=7200
 build:remote --auth_enabled=true
 build:remote --remote_download_toplevel

bazel/external/wee8.BUILD

@@ -9,12 +9,18 @@ cc_library(
     srcs = [
         "libwee8.a",
     ],
-    hdrs = [
-        "wee8/include/v8-version.h",
-        "wee8/third_party/wasm-api/wasm.hh",
+    hdrs =
+        glob([
+            "wee8/include/**/*.h",
+            "wee8/src/**/*.h",
+            "wee8/third_party/wasm-api/wasm.hh",
+        ]),
+    copts = [
+        "-Wno-range-loop-analysis",
     ],
     defines = ["ENVOY_WASM_V8"],
     includes = [
+        "wee8",
         "wee8/include",
         "wee8/third_party",
     ],
@@ -23,7 +29,10 @@ cc_library(
 
 genrule(
     name = "build",
-    srcs = glob(["wee8/**"]),
+    srcs = glob(
+        ["wee8/**"],
+        exclude = ["wee8/out/**"],
+    ),
     outs = [
         "libwee8.a",
     ],

bazel/external/wee8.genrule_cmd

@@ -2,12 +2,12 @@
 
 set -e
 
-# This works only on Linux-{x86_64,s390x,aarch64} and macOS-x86_64.
+# This works only on Linux-{x86_64,s390x,aarch64,ppc64le} and macOS-x86_64.
 case "$$(uname -s)-$$(uname -m)" in
-Linux-x86_64|Linux-s390x|Linux-aarch64|Darwin-x86_64)
+Linux-x86_64|Linux-s390x|Linux-aarch64|Linux-ppc64le|Darwin-x86_64)
   ;;
 *)
-  echo "ERROR: wee8 is currently supported only on Linux-{x86_64,s390x,aarch64} and macOS-x86_64." >&2
+  echo "ERROR: wee8 is currently supported only on Linux-{x86_64,s390x,aarch64,ppc64le} and macOS-x86_64." >&2
   exit 1
 esac
 
@@ -19,7 +19,7 @@ pushd $$ROOT/wee8
 rm -rf out/wee8
 
 # Export compiler configuration.
-export CXXFLAGS="$${CXXFLAGS-} -Wno-sign-compare -Wno-deprecated-copy -Wno-unknown-warning-option"
+export CXXFLAGS="$${CXXFLAGS-} -Wno-sign-compare -Wno-deprecated-copy -Wno-unknown-warning-option -Wno-range-loop-analysis -Wno-shorten-64-to-32"
 if [[ ( `uname` == "Darwin" && $${CXX-} == "" ) || $${CXX-} == *"clang"* ]]; then
   export IS_CLANG=true
   export CC=$${CC:-clang}
@@ -88,6 +88,11 @@ WEE8_BUILD_ARGS+=" v8_enable_shared_ro_heap=false"
 if [[ `uname -m` == "aarch64" ]]; then
   WEE8_BUILD_ARGS+=" target_cpu=\"arm64\""
 fi
+# Support ppc64
+# Only tests with gn 5da62d5
+if [[ `uname -m` == "ppc64le" ]]; then
+  WEE8_BUILD_ARGS+=" target_cpu=\"ppc64\""
+fi
 
 # Build wee8.
 if [[ -f /etc/centos-release ]] && [[ $$(cat /etc/centos-release) =~ "CentOS Linux release 7" ]] && [[ -x "$$(command -v gn)" ]]; then

bazel/external/wee8.patch

@@ -1,9 +1,10 @@
 # 1. Fix linking with unbundled toolchain on macOS.
 # 2. Increase VSZ limit to 4TiB (allows us to start up to 409 VMs).
 # 3. Fix MSAN linking.
+# 4. Fix Wasm module deserialization (http://crbug.com/v8/11024).
 --- wee8/build/toolchain/gcc_toolchain.gni
 +++ wee8/build/toolchain/gcc_toolchain.gni
-@@ -329,6 +329,8 @@ template("gcc_toolchain") {
+@@ -348,6 +348,8 @@ template("gcc_toolchain") {
          # AIX does not support either -D (deterministic output) or response
          # files.
          command = "$ar -X64 {{arflags}} -r -c -s {{output}} {{inputs}}"
@@ -12,7 +13,7 @@
        } else {
          rspfile = "{{output}}.rsp"
          rspfile_content = "{{inputs}}"
-@@ -507,7 +509,7 @@ template("gcc_toolchain") {
+@@ -543,7 +545,7 @@ template("gcc_toolchain") {
 
        start_group_flag = ""
        end_group_flag = ""
@@ -51,5 +52,17 @@
 -    is_msan && (msan_track_origins == 0 || msan_track_origins == 2)
 +prebuilt_instrumented_libraries_available = false
 
- if (use_libfuzzer && is_linux) {
+ if (use_libfuzzer && (is_linux || is_chromeos)) {
    if (is_asan) {
+--- wee8/src/wasm/module-compiler.cc
++++ wee8/src/wasm/module-compiler.cc
+@@ -2901,6 +2901,9 @@ void CompilationStateImpl::InitializeCompilationProgressAfterDeserialization() {
+       RequiredBaselineTierField::encode(ExecutionTier::kTurbofan) |
+       RequiredTopTierField::encode(ExecutionTier::kTurbofan) |
+       ReachedTierField::encode(ExecutionTier::kTurbofan);
++  finished_events_.Add(CompilationEvent::kFinishedExportWrappers);
++  finished_events_.Add(CompilationEvent::kFinishedBaselineCompilation);
++  finished_events_.Add(CompilationEvent::kFinishedTopTierCompilation);
+   compilation_progress_.assign(module->num_declared_functions,
+                                kProgressAfterDeserialization);
+ }

bazel/foreign_cc/BUILD

@@ -351,10 +351,10 @@ envoy_cmake_external(
 envoy_cmake_external(
     name = "zlib",
     cache_entries = {
-        "BUILD_SHARED_LIBS": "off",
         "CMAKE_CXX_COMPILER_FORCED": "on",
         "CMAKE_C_COMPILER_FORCED": "on",
         "SKIP_BUILD_EXAMPLES": "on",
+        "BUILD_SHARED_LIBS": "off",
 
         # The following entries are for zlib-ng. Since zlib and zlib-ng are compatible source
         # codes and CMake ignores unknown cache entries, it is fine to combine it into one
@@ -382,7 +382,7 @@ envoy_cmake_external(
         "//conditions:default": "@net_zlib//:all",
     }),
     static_libraries = select({
-        "//bazel:windows_x86_64": ["zlibstatic.lib"],
+        "//bazel:windows_x86_64": ["zlib.lib"],
         "//conditions:default": ["libz.a"],
     }),
 )

bazel/foreign_cc/zlib.patch

@@ -1,8 +1,59 @@
 diff --git a/CMakeLists.txt b/CMakeLists.txt
-index 0fe939d..2f0475a 100644
+index e108c16..2cd82ef 100644
 --- a/CMakeLists.txt
 +++ b/CMakeLists.txt
-@@ -229,21 +229,22 @@ endif()
+@@ -183,10 +183,18 @@ if(MINGW)
+     set(ZLIB_DLL_SRCS ${CMAKE_CURRENT_BINARY_DIR}/zlib1rc.obj)
+ endif(MINGW)
+
+-add_library(zlib SHARED ${ZLIB_SRCS} ${ZLIB_ASMS} ${ZLIB_DLL_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS})
+-add_library(zlibstatic STATIC ${ZLIB_SRCS} ${ZLIB_ASMS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS})
+-set_target_properties(zlib PROPERTIES DEFINE_SYMBOL ZLIB_DLL)
+-set_target_properties(zlib PROPERTIES SOVERSION 1)
++if(NOT DEFINED BUILD_SHARED_LIBS)
++    add_library(zlib SHARED ${ZLIB_SRCS} ${ZLIB_ASMS} ${ZLIB_DLL_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS})
++    add_library(zlibstatic STATIC ${ZLIB_SRCS} ${ZLIB_ASMS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS})
++    set_target_properties(zlib PROPERTIES DEFINE_SYMBOL ZLIB_DLL)
++    set_target_properties(zlib PROPERTIES SOVERSION 1)
++
++    set(ZLIB_INSTALL_LIBRARIES zlib zlibstatic)
++else()
++    add_library(zlib ${ZLIB_SRCS} ${ZLIB_ASMS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS})
++
++    set(ZLIB_INSTALL_LIBRARIES zlib)
++endif()
+
+ if(NOT CYGWIN)
+     # This property causes shared libraries on Linux to have the full version
+@@ -196,22 +204,22 @@ if(NOT CYGWIN)
+     #
+     # This has no effect with MSVC, on that platform the version info for
+     # the DLL comes from the resource file win32/zlib1.rc
+-    set_target_properties(zlib PROPERTIES VERSION ${ZLIB_FULL_VERSION})
++    set_target_properties(${ZLIB_INSTALL_LIBRARIES} PROPERTIES VERSION ${ZLIB_FULL_VERSION})
+ endif()
+
+ if(UNIX)
+     # On unix-like platforms the library is almost always called libz
+-   set_target_properties(zlib zlibstatic PROPERTIES OUTPUT_NAME z)
++   set_target_properties(${ZLIB_INSTALL_LIBRARIES} PROPERTIES OUTPUT_NAME z)
+    if(NOT APPLE)
+-     set_target_properties(zlib PROPERTIES LINK_FLAGS "-Wl,--version-script,\"${CMAKE_CURRENT_SOURCE_DIR}/zlib.map\"")
++     set_target_properties(${ZLIB_INSTALL_LIBRARIES} PROPERTIES LINK_FLAGS "-Wl,--version-script,\"${CMAKE_CURRENT_SOURCE_DIR}/zlib.map\"")
+    endif()
+ elseif(BUILD_SHARED_LIBS AND WIN32)
+     # Creates zlib1.dll when building shared library version
+-    set_target_properties(zlib PROPERTIES SUFFIX "1.dll")
++    set_target_properties(${ZLIB_INSTALL_LIBRARIES} PROPERTIES SUFFIX "1.dll")
+ endif()
+
+ if(NOT SKIP_INSTALL_LIBRARIES AND NOT SKIP_INSTALL_ALL )
+-    install(TARGETS zlib zlibstatic
++    install(TARGETS ${ZLIB_INSTALL_LIBRARIES}
+         RUNTIME DESTINATION "${INSTALL_BIN_DIR}"
+         ARCHIVE DESTINATION "${INSTALL_LIB_DIR}"
+         LIBRARY DESTINATION "${INSTALL_LIB_DIR}" )
+@@ -229,21 +237,22 @@ endif()
  #============================================================================
  # Example binaries
  #============================================================================

bazel/repository_locations.bzl

@@ -689,10 +689,10 @@ DEPENDENCY_REPOSITORIES_SPEC = dict(
         project_name = "V8",
         project_desc = "Google’s open source high-performance JavaScript and WebAssembly engine, written in C++",
         project_url = "https://v8.dev",
-        version = "8.5.210.20",
+        version = "8.7.220.10",
         # This archive was created using https://storage.googleapis.com/envoyproxy-wee8/wee8-archive.sh
         # and contains complete checkout of V8 with all dependencies necessary to build wee8.
-        sha256 = "ef404643d7da6854b76b9fb9950a79a1acbd037b7a26f02c585ac379b0f7dee1",
+        sha256 = "f22734640e0515bc34d1ca3772513aef24374fafa44d0489d3a9a57cadec69fb",
         urls = ["https://storage.googleapis.com/envoyproxy-wee8/wee8-{version}.tar.gz"],
         use_category = ["dataplane_ext"],
         extensions = [
@@ -702,7 +702,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict(
             "envoy.filters.network.wasm",
             "envoy.stat_sinks.wasm",
         ],
-        last_updated = "2020-08-31",
+        last_updated = "2020-10-27",
         cpe = "cpe:2.3:a:google:v8:*",
     ),
     com_googlesource_quiche = dict(
@@ -869,9 +869,8 @@ DEPENDENCY_REPOSITORIES_SPEC = dict(
         project_name = "WebAssembly for Proxies (C++ host implementation)",
         project_desc = "WebAssembly for Proxies (C++ host implementation)",
         project_url = "https://github.com/proxy-wasm/proxy-wasm-cpp-host",
-        # 2020-09-10
-        version = "49ed20e895b728aae6b811950a2939ecbaf76f7c",
-        sha256 = "fa03293d01450b9164f8f56ef9227301f7d1af4f373f996400f75c93f6ebc822",
+        version = "a044a3a5bec75ce57c12d9e2b0e95e2a14f9f944",
+        sha256 = "619e61997682931e07e92f5b64a4268715598d3aa22a41cadeeca816103d731f",
         strip_prefix = "proxy-wasm-cpp-host-{version}",
         urls = ["https://github.com/proxy-wasm/proxy-wasm-cpp-host/archive/{version}.tar.gz"],
         use_category = ["dataplane_ext"],
@@ -882,7 +881,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict(
             "envoy.filters.network.wasm",
             "envoy.stat_sinks.wasm",
         ],
-        last_updated = "2020-07-29",
+        last_updated = "2020-11-12",
         cpe = "N/A",
     ),
     # TODO: upgrade to the latest version (1.41 currently fails tests)
@@ -904,14 +903,14 @@ DEPENDENCY_REPOSITORIES_SPEC = dict(
         project_name = "Bazel rust rules",
         project_desc = "Bazel rust rules (used by Wasm)",
         project_url = "https://github.com/bazelbuild/rules_rust",
-        version = "fda9a1ce6482973adfda022cadbfa6b300e269c3",
-        sha256 = "484a2b2b67cd2d1fa1054876de7f8d291c4b203fd256bc8cbea14d749bb864ce",
+        version = "fb90a7484800157fbb8a5904fbeb608dc1effc0c",
+        sha256 = "cbb253b8c5ab1a3c1787790f900e7d6774e95ba038714fc0f710935e62f30f5f",
         # Last commit where "out_binary = True" works.
         # See: https://github.com/bazelbuild/rules_rust/issues/386
         strip_prefix = "rules_rust-{version}",
         urls = ["https://github.com/bazelbuild/rules_rust/archive/{version}.tar.gz"],
-        use_category = ["build"],
-        last_updated = "2020-07-29",
+        use_category = ["test_only"],
+        last_updated = "2020-10-15",
     ),
     rules_antlr = dict(
         project_name = "ANTLR Rules for Bazel",

bazel/wasm/wasm.bzl

@@ -88,9 +88,9 @@ def wasm_cc_binary(name, tags = [], repository = "", **kwargs):
     kwargs.setdefault("additional_linker_inputs", ["@proxy_wasm_cpp_sdk//:jslib", "@envoy//source/extensions/common/wasm/ext:jslib"])
 
     if repository == "@envoy":
-        envoy_js = "--js-library source/extensions/common/wasm/ext/envoy_wasm_intrinsics.js"
-    else:
         envoy_js = "--js-library external/envoy/source/extensions/common/wasm/ext/envoy_wasm_intrinsics.js"
+    else:
+        envoy_js = "--js-library source/extensions/common/wasm/ext/envoy_wasm_intrinsics.js"
     kwargs.setdefault("linkopts", [
         envoy_js,
         "--js-library external/proxy_wasm_cpp_sdk/proxy_wasm_intrinsics.js",
@@ -112,10 +112,10 @@ def wasm_cc_binary(name, tags = [], repository = "", **kwargs):
     )
 
 def envoy_wasm_cc_binary(name, tags = [], **kwargs):
-    wasm_cc_binary(name, tags, repository = "@envoy", **kwargs)
+    wasm_cc_binary(name, tags, repository = "", **kwargs)
 
 def wasm_rust_binary(name, tags = [], **kwargs):
-    wasm_name = "_wasm_" + (name if not ".wasm" in name else name.strip(".wasm"))
+    wasm_name = "_wasm_" + name.replace(".", "_")
     kwargs.setdefault("visibility", ["//visibility:public"])
 
     rust_binary(

docs/root/version_history/current.rst

@@ -78,6 +78,7 @@ Bug Fixes
 * rocketmq_proxy: fixed an issue involving incorrect header lengths. In debug mode it causes crash and in release mode it causes underflow.
 * thrift_proxy: fixed crashing bug on request overflow.
 * udp_proxy: fixed a crash due to UDP packets being processed after listener removal.
+* tls: fix detection of the upstream connection close event.
 
 Removed Config or Runtime
 -------------------------

source/extensions/common/wasm/context.cc

@@ -853,11 +853,7 @@ BufferInterface* Context::getBuffer(WasmBufferType type) {
 void Context::onDownstreamConnectionClose(CloseType close_type) {
   ContextBase::onDownstreamConnectionClose(close_type);
   downstream_closed_ = true;
-  // Call close on TCP connection, if upstream connection closed or there was a failure seen in
-  // this connection.
-  if (upstream_closed_ || getRequestStreamInfo()->hasAnyResponseFlag()) {
-    onCloseTCP();
-  }
+  onCloseTCP();
 }
 
 void Context::onUpstreamConnectionClose(CloseType close_type) {
@@ -1488,12 +1484,14 @@ WasmResult Context::continueStream(WasmStreamType stream_type) {
   switch (stream_type) {
   case WasmStreamType::Request:
     if (decoder_callbacks_) {
-      decoder_callbacks_->continueDecoding();
+      // We are in a reentrant call, so defer.
+      wasm()->addAfterVmCallAction([this] { decoder_callbacks_->continueDecoding(); });
     }
     break;
   case WasmStreamType::Response:
     if (encoder_callbacks_) {
-      encoder_callbacks_->continueEncoding();
+      // We are in a reentrant call, so defer.
+      wasm()->addAfterVmCallAction([this] { encoder_callbacks_->continueEncoding(); });
     }
     break;
   default:

source/extensions/transport_sockets/tls/ssl_handshaker.h

@@ -67,7 +67,7 @@ class SslHandshakerImpl : public Ssl::ConnectionInfo, public Ssl::Handshaker {
   // Ssl::Handshaker
   Network::PostIoAction doHandshake() override;
 
-  Ssl::SocketState state() { return state_; }
+  Ssl::SocketState state() const { return state_; }
   void setState(Ssl::SocketState state) { state_ = state; }
   SSL* ssl() const { return ssl_.get(); }
   Ssl::HandshakeCallbacks* handshakeCallbacks() { return handshake_callbacks_; }

source/extensions/transport_sockets/tls/ssl_socket.cc

@@ -140,10 +140,18 @@ Network::IoResult SslSocket::doRead(Buffer::Instance& read_buffer) {
         case SSL_ERROR_WANT_READ:
           break;
         case SSL_ERROR_ZERO_RETURN:
+          // Graceful shutdown using close_notify TLS alert.
           end_stream = true;
           break;
+        case SSL_ERROR_SYSCALL:
+          if (result.error_.value() == 0) {
+            // Non-graceful shutdown by closing the underlying socket.
+            end_stream = true;
+            break;
+          }
+          FALLTHRU;
         case SSL_ERROR_WANT_WRITE:
-        // Renegotiation has started. We don't handle renegotiation so just fall through.
+          // Renegotiation has started. We don't handle renegotiation so just fall through.
         default:
           drainErrorQueue();
           action = PostIoAction::Close;

test/extensions/bootstrap/wasm/wasm_test.cc

@@ -222,21 +222,6 @@ TEST_P(WasmTest, DivByZero) {
   wasm_->isFailed();
 }
 
-TEST_P(WasmTest, EmscriptenVersion) {
-  createWasm();
-  const auto code = TestEnvironment::readFileToStringForTest(TestEnvironment::substitute(
-      "{{ test_rundir }}/test/extensions/bootstrap/wasm/test_data/segv_cpp.wasm"));
-  EXPECT_FALSE(code.empty());
-  EXPECT_TRUE(wasm_->initialize(code, false));
-  uint32_t major = 9, minor = 9, abi_major = 9, abi_minor = 9;
-  EXPECT_TRUE(wasm_->getEmscriptenVersion(&major, &minor, &abi_major, &abi_minor));
-  EXPECT_EQ(major, 0);
-  EXPECT_LE(minor, 3);
-  // Up to (at least) emsdk 1.39.6.
-  EXPECT_EQ(abi_major, 0);
-  EXPECT_LE(abi_minor, 20);
-}
-
 TEST_P(WasmTest, IntrinsicGlobals) {
   createWasm();
   const auto code = TestEnvironment::readFileToStringForTest(TestEnvironment::substitute(