From dfe4170df99ce662a88b86499c7f555a589bdde9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=BE=84=E6=BD=AD?= Date: Tue, 24 May 2022 00:52:16 +0800 Subject: [PATCH 1/6] wasm: fix potential crash of extensions for failed remote code fetch (#20843) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: johnlanni ztywto@qq.com Commit Message: Handle null plugins handle when skipping config canarying for duplicate filters and also when the remote code fetch is in progress or fails. Risk Level: low Testing: ok Signed-off-by: 澄潭 Signed-off-by: Ryan Northey --- .../wasm/wasm_access_log_impl.h | 6 + .../filters/http/wasm/wasm_filter.h | 6 + .../filters/network/wasm/wasm_filter.h | 6 + .../access_loggers/wasm/config_test.cc | 219 ++++++++++++++++-- .../access_loggers/wasm/test_data/test_cpp.cc | 9 + .../filters/http/wasm/config_test.cc | 55 +++++ .../filters/http/wasm/test_data/test_cpp.cc | 6 +- .../filters/network/wasm/config_test.cc | 149 ++++++++++++ .../network/wasm/test_data/test_cpp.cc | 20 +- 9 files changed, 455 insertions(+), 21 deletions(-) diff --git a/source/extensions/access_loggers/wasm/wasm_access_log_impl.h b/source/extensions/access_loggers/wasm/wasm_access_log_impl.h index 64962aaff52ab..6faf2c2fc6211 100644 --- a/source/extensions/access_loggers/wasm/wasm_access_log_impl.h +++ b/source/extensions/access_loggers/wasm/wasm_access_log_impl.h @@ -31,7 +31,13 @@ class WasmAccessLog : public AccessLog::Instance { } } + if (!tls_slot_) { + return; + } auto handle = tls_slot_->get()->handle(); + if (!handle) { + return; + } if (handle->wasmHandle()) { handle->wasmHandle()->wasm()->log(plugin_, request_headers, response_headers, response_trailers, stream_info); diff --git a/source/extensions/filters/http/wasm/wasm_filter.h b/source/extensions/filters/http/wasm/wasm_filter.h index 8032153dd1dba..a01b86a9db1d7 100644 --- a/source/extensions/filters/http/wasm/wasm_filter.h +++ b/source/extensions/filters/http/wasm/wasm_filter.h @@ -28,7 +28,13 @@ class FilterConfig : Logger::Loggable { std::shared_ptr createFilter() { Wasm* wasm = nullptr; + if (!tls_slot_->currentThreadRegistered()) { + return nullptr; + } PluginHandleSharedPtr handle = tls_slot_->get()->handle(); + if (!handle) { + return nullptr; + } if (handle->wasmHandle()) { wasm = handle->wasmHandle()->wasm().get(); } diff --git a/source/extensions/filters/network/wasm/wasm_filter.h b/source/extensions/filters/network/wasm/wasm_filter.h index 7fd906a808073..e1fa293150888 100644 --- a/source/extensions/filters/network/wasm/wasm_filter.h +++ b/source/extensions/filters/network/wasm/wasm_filter.h @@ -28,7 +28,13 @@ class FilterConfig : Logger::Loggable { std::shared_ptr createFilter() { Wasm* wasm = nullptr; + if (!tls_slot_->currentThreadRegistered()) { + return nullptr; + } PluginHandleSharedPtr handle = tls_slot_->get()->handle(); + if (!handle) { + return nullptr; + } if (handle->wasmHandle()) { wasm = handle->wasmHandle()->wasm().get(); } diff --git a/test/extensions/access_loggers/wasm/config_test.cc b/test/extensions/access_loggers/wasm/config_test.cc index ea4425810622a..1c354178b5728 100644 --- a/test/extensions/access_loggers/wasm/config_test.cc +++ b/test/extensions/access_loggers/wasm/config_test.cc @@ -2,6 +2,8 @@ #include "envoy/registry/registry.h" #include "source/common/access_log/access_log_impl.h" +#include "source/common/crypto/utility.h" +#include "source/common/http/message_impl.h" #include "source/common/protobuf/protobuf.h" #include "source/extensions/access_loggers/wasm/config.h" #include "source/extensions/access_loggers/wasm/wasm_access_log_impl.h" @@ -16,24 +18,48 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" +using testing::_; +using testing::ReturnRef; + namespace Envoy { namespace Extensions { namespace AccessLoggers { namespace Wasm { -class TestFactoryContext : public NiceMock { -public: - TestFactoryContext(Api::Api& api, Stats::Scope& scope) : api_(api), scope_(scope) {} - Api::Api& api() override { return api_; } - Stats::Scope& scope() override { return scope_; } +class WasmAccessLogConfigTest + : public testing::TestWithParam> { +protected: + WasmAccessLogConfigTest() : api_(Api::createApiForTest(stats_store_)) { + ON_CALL(context_, api()).WillByDefault(ReturnRef(*api_)); + ON_CALL(context_, scope()).WillByDefault(ReturnRef(stats_store_)); + ON_CALL(context_, listenerMetadata()).WillByDefault(ReturnRef(listener_metadata_)); + ON_CALL(context_, initManager()).WillByDefault(ReturnRef(init_manager_)); + ON_CALL(context_, clusterManager()).WillByDefault(ReturnRef(cluster_manager_)); + ON_CALL(context_, mainThreadDispatcher()).WillByDefault(ReturnRef(dispatcher_)); + } + + void SetUp() override { Envoy::Extensions::Common::Wasm::clearCodeCacheForTesting(); } -private: - Api::Api& api_; - Stats::Scope& scope_; -}; + void initializeForRemote() { + retry_timer_ = new Event::MockTimer(); -class WasmAccessLogConfigTest - : public testing::TestWithParam> {}; + EXPECT_CALL(dispatcher_, createTimer_(_)).WillOnce(Invoke([this](Event::TimerCb timer_cb) { + retry_timer_cb_ = timer_cb; + return retry_timer_; + })); + } + + NiceMock context_; + Stats::IsolatedStoreImpl stats_store_; + Api::ApiPtr api_; + envoy::config::core::v3::Metadata listener_metadata_; + Init::ManagerImpl init_manager_{"init_manager"}; + NiceMock cluster_manager_; + Init::ExpectableWatcherImpl init_watcher_; + NiceMock dispatcher_; + Event::MockTimer* retry_timer_; + Event::TimerCb retry_timer_cb_; +}; INSTANTIATE_TEST_SUITE_P(Runtimes, WasmAccessLogConfigTest, Envoy::Extensions::Common::Wasm::runtime_and_cpp_values, @@ -49,11 +75,10 @@ TEST_P(WasmAccessLogConfigTest, CreateWasmFromEmpty) { ASSERT_NE(nullptr, message); AccessLog::FilterPtr filter; - NiceMock context; AccessLog::InstanceSharedPtr instance; EXPECT_THROW_WITH_MESSAGE( - instance = factory->createAccessLogInstance(*message, std::move(filter), context), + instance = factory->createAccessLogInstance(*message, std::move(filter), context_), Common::Wasm::WasmException, "Unable to create Wasm access log "); } @@ -80,16 +105,13 @@ TEST_P(WasmAccessLogConfigTest, CreateWasmFromWASM) { config.mutable_config()->mutable_vm_config()->mutable_configuration()->PackFrom(some_proto); AccessLog::FilterPtr filter; - Stats::IsolatedStoreImpl stats_store; - Api::ApiPtr api = Api::createApiForTest(stats_store); - TestFactoryContext context(*api, stats_store); AccessLog::InstanceSharedPtr instance = - factory->createAccessLogInstance(config, std::move(filter), context); + factory->createAccessLogInstance(config, std::move(filter), context_); EXPECT_NE(nullptr, instance); EXPECT_NE(nullptr, dynamic_cast(instance.get())); // Check if the custom stat namespace is registered during the initialization. - EXPECT_TRUE(api->customStatNamespaces().registered("wasmcustom")); + EXPECT_TRUE(api_->customStatNamespaces().registered("wasmcustom")); Http::TestRequestHeaderMapImpl request_header; Http::TestResponseHeaderMapImpl response_header; @@ -99,7 +121,166 @@ TEST_P(WasmAccessLogConfigTest, CreateWasmFromWASM) { filter = std::make_unique>(); AccessLog::InstanceSharedPtr filter_instance = - factory->createAccessLogInstance(config, std::move(filter), context); + factory->createAccessLogInstance(config, std::move(filter), context_); + filter_instance->log(&request_header, &response_header, &response_trailer, log_stream_info); +} + +TEST_P(WasmAccessLogConfigTest, YamlLoadFromFileWasmInvalidConfig) { + if (std::get<0>(GetParam()) == "null") { + return; + } + auto factory = + Registry::FactoryRegistry::getFactory( + "envoy.access_loggers.wasm"); + ASSERT_NE(factory, nullptr); + + const std::string invalid_yaml = + TestEnvironment::substitute(absl::StrCat(R"EOF( + config: + vm_config: + runtime: "envoy.wasm.runtime.)EOF", + std::get<0>(GetParam()), R"EOF(" + configuration: + "@type": "type.googleapis.com/google.protobuf.StringValue" + value: "some configuration" + code: + local: + filename: "{{ test_rundir }}/test/extensions/access_loggers/wasm/test_data/test_cpp.wasm" + configuration: + "@type": "type.googleapis.com/google.protobuf.StringValue" + value: "invalid" + )EOF")); + + envoy::extensions::access_loggers::wasm::v3::WasmAccessLog proto_config; + TestUtility::loadFromYaml(invalid_yaml, proto_config); + EXPECT_THROW_WITH_MESSAGE(factory->createAccessLogInstance(proto_config, nullptr, context_), + Envoy::Extensions::Common::Wasm::WasmException, + "Unable to create Wasm access log "); + const std::string valid_yaml = + TestEnvironment::substitute(absl::StrCat(R"EOF( + config: + vm_config: + runtime: "envoy.wasm.runtime.)EOF", + std::get<0>(GetParam()), R"EOF(" + configuration: + "@type": "type.googleapis.com/google.protobuf.StringValue" + value: "some configuration" + code: + local: + filename: "{{ test_rundir }}/test/extensions/access_loggers/wasm/test_data/test_cpp.wasm" + configuration: + "@type": "type.googleapis.com/google.protobuf.StringValue" + value: "valid" + )EOF")); + TestUtility::loadFromYaml(valid_yaml, proto_config); + AccessLog::InstanceSharedPtr filter_instance = + factory->createAccessLogInstance(proto_config, nullptr, context_); + StreamInfo::MockStreamInfo log_stream_info; + filter_instance = factory->createAccessLogInstance(proto_config, nullptr, context_); + filter_instance->log(nullptr, nullptr, nullptr, log_stream_info); + + TestUtility::loadFromYaml(invalid_yaml, proto_config); + filter_instance = factory->createAccessLogInstance(proto_config, nullptr, context_); + filter_instance->log(nullptr, nullptr, nullptr, log_stream_info); +} + +TEST_P(WasmAccessLogConfigTest, YamlLoadFromRemoteWasmCreateFilter) { + if (std::get<0>(GetParam()) == "null") { + return; + } + const std::string code = TestEnvironment::readFileToStringForTest(TestEnvironment::substitute( + "{{ test_rundir }}/test/extensions/access_loggers/wasm/test_data/test_cpp.wasm")); + const std::string sha256 = Hex::encode( + Envoy::Common::Crypto::UtilitySingleton::get().getSha256Digest(Buffer::OwnedImpl(code))); + const std::string yaml = TestEnvironment::substitute(absl::StrCat(R"EOF( + config: + vm_config: + runtime: "envoy.wasm.runtime.)EOF", + std::get<0>(GetParam()), R"EOF(" + code: + remote: + http_uri: + uri: https://example.com/data + cluster: cluster_1 + timeout: 5s + sha256: )EOF", + sha256)); + WasmAccessLogFactory factory; + envoy::extensions::access_loggers::wasm::v3::WasmAccessLog proto_config; + TestUtility::loadFromYaml(yaml, proto_config); + NiceMock client; + NiceMock request(&client); + + cluster_manager_.initializeThreadLocalClusters({"cluster_1"}); + EXPECT_CALL(cluster_manager_.thread_local_cluster_, httpAsyncClient()) + .WillOnce(ReturnRef(cluster_manager_.thread_local_cluster_.async_client_)); + Http::AsyncClient::Callbacks* async_callbacks = nullptr; + EXPECT_CALL(cluster_manager_.thread_local_cluster_.async_client_, send_(_, _, _)) + .WillOnce( + Invoke([&](Http::RequestMessagePtr&, Http::AsyncClient::Callbacks& callbacks, + const Http::AsyncClient::RequestOptions&) -> Http::AsyncClient::Request* { + if (!async_callbacks) { + async_callbacks = &callbacks; + } + return &request; + })); + StreamInfo::MockStreamInfo log_stream_info; + AccessLog::InstanceSharedPtr filter_instance = + factory.createAccessLogInstance(proto_config, nullptr, context_); + filter_instance->log(nullptr, nullptr, nullptr, log_stream_info); + EXPECT_CALL(init_watcher_, ready()); + context_.initManager().initialize(init_watcher_); + auto response = Http::ResponseMessagePtr{new Http::ResponseMessageImpl( + Http::ResponseHeaderMapPtr{new Http::TestResponseHeaderMapImpl{{":status", "200"}}})}; + response->body().add(code); + async_callbacks->onSuccess(request, std::move(response)); + EXPECT_EQ(context_.initManager().state(), Init::Manager::State::Initialized); + filter_instance->log(nullptr, nullptr, nullptr, log_stream_info); +} + +TEST_P(WasmAccessLogConfigTest, FailedToGetThreadLocalPlugin) { + if (std::get<0>(GetParam()) == "null") { + return; + } + auto factory = + Registry::FactoryRegistry::getFactory( + "envoy.access_loggers.wasm"); + ASSERT_NE(factory, nullptr); + + NiceMock threadlocal; + const std::string yaml = TestEnvironment::substitute(absl::StrCat(R"EOF( + config: + vm_config: + runtime: "envoy.wasm.runtime.)EOF", + std::get<0>(GetParam()), R"EOF(" + configuration: + "@type": "type.googleapis.com/google.protobuf.StringValue" + value: "some configuration" + code: + local: + filename: "{{ test_rundir }}/test/extensions/access_loggers/wasm/test_data/test_cpp.wasm" + configuration: + "@type": "type.googleapis.com/google.protobuf.StringValue" + value: "valid" + )EOF")); + + envoy::extensions::access_loggers::wasm::v3::WasmAccessLog proto_config; + TestUtility::loadFromYaml(yaml, proto_config); + EXPECT_CALL(context_, threadLocal()).WillOnce(ReturnRef(threadlocal)); + threadlocal.registered_ = true; + AccessLog::InstanceSharedPtr filter_instance = + factory->createAccessLogInstance(proto_config, nullptr, context_); + ASSERT_EQ(threadlocal.current_slot_, 1); + + Http::TestRequestHeaderMapImpl request_header; + Http::TestResponseHeaderMapImpl response_header; + Http::TestResponseTrailerMapImpl response_trailer; + StreamInfo::MockStreamInfo log_stream_info; + + filter_instance->log(&request_header, &response_header, &response_trailer, log_stream_info); + // Even if the thread local plugin handle returns nullptr, `log` should not raise error or + // exception. + threadlocal.data_[0] = std::make_shared(nullptr); filter_instance->log(&request_header, &response_header, &response_trailer, log_stream_info); } diff --git a/test/extensions/access_loggers/wasm/test_data/test_cpp.cc b/test/extensions/access_loggers/wasm/test_data/test_cpp.cc index 18e59d57ddfcd..9e9136ecda5ac 100644 --- a/test/extensions/access_loggers/wasm/test_data/test_cpp.cc +++ b/test/extensions/access_loggers/wasm/test_data/test_cpp.cc @@ -13,11 +13,20 @@ START_WASM_PLUGIN(AccessLoggerTestCpp) class TestRootContext : public RootContext { public: using RootContext::RootContext; + bool onConfigure(size_t) override; void onLog() override; }; static RegisterContextFactory register_ExampleContext(ROOT_FACTORY(TestRootContext)); +bool TestRootContext::onConfigure(size_t size) { + if (size > 0 && + getBufferBytes(WasmBufferType::PluginConfiguration, 0, size)->toString() == "invalid") { + return false; + } + return true; +} + void TestRootContext::onLog() { auto path = getRequestHeader(":path"); logWarn("onLog " + std::to_string(id()) + " " + std::string(path->view())); diff --git a/test/extensions/filters/http/wasm/config_test.cc b/test/extensions/filters/http/wasm/config_test.cc index 7a56307760222..29108ee426315 100644 --- a/test/extensions/filters/http/wasm/config_test.cc +++ b/test/extensions/filters/http/wasm/config_test.cc @@ -9,6 +9,7 @@ #include "source/common/stats/isolated_store_impl.h" #include "source/extensions/common/wasm/wasm.h" #include "source/extensions/filters/http/wasm/config.h" +#include "source/extensions/filters/http/wasm/wasm_filter.h" #include "test/extensions/common/wasm/wasm_runtime.h" #include "test/mocks/http/mocks.h" @@ -173,6 +174,60 @@ TEST_P(WasmFilterConfigTest, YamlLoadFromFileWasmFailOpenOk) { cb(filter_callback); } +TEST_P(WasmFilterConfigTest, YamlLoadFromFileWasmInvalidConfig) { + const std::string invalid_yaml = + TestEnvironment::substitute(absl::StrCat(R"EOF( + config: + vm_config: + runtime: "envoy.wasm.runtime.)EOF", + std::get<0>(GetParam()), R"EOF(" + configuration: + "@type": "type.googleapis.com/google.protobuf.StringValue" + value: "some configuration" + code: + local: + filename: "{{ test_rundir }}/test/extensions/filters/http/wasm/test_data/test_cpp.wasm" + configuration: + "@type": "type.googleapis.com/google.protobuf.StringValue" + value: "invalid" + )EOF")); + + envoy::extensions::filters::http::wasm::v3::Wasm proto_config; + TestUtility::loadFromYaml(invalid_yaml, proto_config); + WasmFilterConfig factory; + EXPECT_THROW_WITH_MESSAGE(factory.createFilterFactoryFromProto(proto_config, "stats", context_), + WasmException, "Unable to create Wasm HTTP filter "); + const std::string valid_yaml = + TestEnvironment::substitute(absl::StrCat(R"EOF( + config: + vm_config: + runtime: "envoy.wasm.runtime.)EOF", + std::get<0>(GetParam()), R"EOF(" + configuration: + "@type": "type.googleapis.com/google.protobuf.StringValue" + value: "some configuration" + code: + local: + filename: "{{ test_rundir }}/test/extensions/filters/http/wasm/test_data/test_cpp.wasm" + configuration: + "@type": "type.googleapis.com/google.protobuf.StringValue" + value: "valid" + )EOF")); + TestUtility::loadFromYaml(valid_yaml, proto_config); + Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context_); + EXPECT_CALL(init_watcher_, ready()); + context_.initManager().initialize(init_watcher_); + EXPECT_EQ(context_.initManager().state(), Init::Manager::State::Initialized); + Http::MockFilterChainFactoryCallbacks filter_callback; + EXPECT_CALL(filter_callback, addStreamFilter(_)); + EXPECT_CALL(filter_callback, addAccessLogHandler(_)); + cb(filter_callback); + + TestUtility::loadFromYaml(invalid_yaml, proto_config); + auto filter_config = std::make_unique(proto_config, context_); + EXPECT_EQ(filter_config->createFilter(), nullptr); +} + TEST_P(WasmFilterConfigTest, YamlLoadInlineWasm) { const std::string code = TestEnvironment::readFileToStringForTest(TestEnvironment::substitute( "{{ test_rundir }}/test/extensions/filters/http/wasm/test_data/test_cpp.wasm")); diff --git a/test/extensions/filters/http/wasm/test_data/test_cpp.cc b/test/extensions/filters/http/wasm/test_data/test_cpp.cc index 039713e0e76a3..a78db4148322f 100644 --- a/test/extensions/filters/http/wasm/test_data/test_cpp.cc +++ b/test/extensions/filters/http/wasm/test_data/test_cpp.cc @@ -54,7 +54,11 @@ bool TestRootContext::onStart(size_t configuration_size) { return true; } -bool TestRootContext::onConfigure(size_t) { +bool TestRootContext::onConfigure(size_t size) { + if (size > 0 && + getBufferBytes(WasmBufferType::PluginConfiguration, 0, size)->toString() == "invalid") { + return false; + } if (test_ == "property") { { // Many properties are not available in the root context. diff --git a/test/extensions/filters/network/wasm/config_test.cc b/test/extensions/filters/network/wasm/config_test.cc index c686af0f7f506..d6a357311a6f2 100644 --- a/test/extensions/filters/network/wasm/config_test.cc +++ b/test/extensions/filters/network/wasm/config_test.cc @@ -3,6 +3,7 @@ #include "source/common/common/base64.h" #include "source/common/common/hex.h" #include "source/common/crypto/utility.h" +#include "source/common/http/message_impl.h" #include "source/extensions/common/wasm/wasm.h" #include "source/extensions/filters/network/wasm/config.h" #include "source/extensions/filters/network/wasm/wasm_filter.h" @@ -301,6 +302,154 @@ TEST_P(WasmNetworkFilterConfigTest, FilterConfigAllowOnVmStart) { cb(connection); } +TEST_P(WasmNetworkFilterConfigTest, YamlLoadFromFileWasmInvalidConfig) { + if (std::get<0>(GetParam()) == "null") { + return; + } + const std::string invalid_yaml = + TestEnvironment::substitute(absl::StrCat(R"EOF( + config: + vm_config: + runtime: "envoy.wasm.runtime.)EOF", + std::get<0>(GetParam()), R"EOF(" + configuration: + "@type": "type.googleapis.com/google.protobuf.StringValue" + value: "some configuration" + code: + local: + filename: "{{ test_rundir }}/test/extensions/filters/network/wasm/test_data/test_cpp.wasm" + configuration: + "@type": "type.googleapis.com/google.protobuf.StringValue" + value: "invalid" + )EOF")); + + envoy::extensions::filters::network::wasm::v3::Wasm proto_config; + TestUtility::loadFromYaml(invalid_yaml, proto_config); + WasmFilterConfig factory; + EXPECT_THROW_WITH_MESSAGE(factory.createFilterFactoryFromProto(proto_config, context_), + Envoy::Extensions::Common::Wasm::WasmException, + "Unable to create Wasm network filter "); + const std::string valid_yaml = + TestEnvironment::substitute(absl::StrCat(R"EOF( + config: + vm_config: + runtime: "envoy.wasm.runtime.)EOF", + std::get<0>(GetParam()), R"EOF(" + configuration: + "@type": "type.googleapis.com/google.protobuf.StringValue" + value: "some configuration" + code: + local: + filename: "{{ test_rundir }}/test/extensions/filters/network/wasm/test_data/test_cpp.wasm" + configuration: + "@type": "type.googleapis.com/google.protobuf.StringValue" + value: "valid" + )EOF")); + TestUtility::loadFromYaml(valid_yaml, proto_config); + Network::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, context_); + EXPECT_CALL(init_watcher_, ready()); + context_.initManager().initialize(init_watcher_); + EXPECT_EQ(context_.initManager().state(), Init::Manager::State::Initialized); + Network::MockConnection connection; + EXPECT_CALL(connection, addFilter(_)); + cb(connection); + + TestUtility::loadFromYaml(invalid_yaml, proto_config); + auto filter_config = std::make_unique(proto_config, context_); + EXPECT_EQ(filter_config->createFilter(), nullptr); +} + +TEST_P(WasmNetworkFilterConfigTest, YamlLoadFromRemoteWasmCreateFilter) { + if (std::get<0>(GetParam()) == "null") { + return; + } + const std::string code = TestEnvironment::readFileToStringForTest(TestEnvironment::substitute( + "{{ test_rundir }}/test/extensions/filters/network/wasm/test_data/test_cpp.wasm")); + const std::string sha256 = Hex::encode( + Envoy::Common::Crypto::UtilitySingleton::get().getSha256Digest(Buffer::OwnedImpl(code))); + const std::string yaml = TestEnvironment::substitute(absl::StrCat(R"EOF( + config: + vm_config: + runtime: "envoy.wasm.runtime.)EOF", + std::get<0>(GetParam()), R"EOF(" + code: + remote: + http_uri: + uri: https://example.com/data + cluster: cluster_1 + timeout: 5s + sha256: )EOF", + sha256)); + envoy::extensions::filters::network::wasm::v3::Wasm proto_config; + TestUtility::loadFromYaml(yaml, proto_config); + WasmFilterConfig factory; + NiceMock client; + NiceMock request(&client); + + cluster_manager_.initializeThreadLocalClusters({"cluster_1"}); + EXPECT_CALL(cluster_manager_.thread_local_cluster_, httpAsyncClient()) + .WillOnce(ReturnRef(cluster_manager_.thread_local_cluster_.async_client_)); + Http::AsyncClient::Callbacks* async_callbacks = nullptr; + EXPECT_CALL(cluster_manager_.thread_local_cluster_.async_client_, send_(_, _, _)) + .WillOnce( + Invoke([&](Http::RequestMessagePtr&, Http::AsyncClient::Callbacks& callbacks, + const Http::AsyncClient::RequestOptions&) -> Http::AsyncClient::Request* { + if (!async_callbacks) { + async_callbacks = &callbacks; + } + return &request; + })); + NiceMock threadlocal; + EXPECT_CALL(context_, threadLocal()).WillRepeatedly(ReturnRef(threadlocal)); + threadlocal.registered_ = false; + auto filter_config = std::make_unique(proto_config, context_); + EXPECT_EQ(filter_config->createFilter(), nullptr); + EXPECT_CALL(init_watcher_, ready()); + context_.initManager().initialize(init_watcher_); + auto response = Http::ResponseMessagePtr{new Http::ResponseMessageImpl( + Http::ResponseHeaderMapPtr{new Http::TestResponseHeaderMapImpl{{":status", "200"}}})}; + response->body().add(code); + async_callbacks->onSuccess(request, std::move(response)); + EXPECT_EQ(context_.initManager().state(), Init::Manager::State::Initialized); + threadlocal.registered_ = true; + EXPECT_NE(filter_config->createFilter(), nullptr); +} + +TEST_P(WasmNetworkFilterConfigTest, FailedToGetThreadLocalPlugin) { + if (std::get<0>(GetParam()) == "null") { + return; + } + + NiceMock threadlocal; + const std::string yaml = TestEnvironment::substitute(absl::StrCat(R"EOF( + config: + vm_config: + runtime: "envoy.wasm.runtime.)EOF", + std::get<0>(GetParam()), R"EOF(" + configuration: + "@type": "type.googleapis.com/google.protobuf.StringValue" + value: "some configuration" + code: + local: + filename: "{{ test_rundir }}/test/extensions/filters/network/wasm/test_data/test_cpp.wasm" + configuration: + "@type": "type.googleapis.com/google.protobuf.StringValue" + value: "valid" + )EOF")); + + envoy::extensions::filters::network::wasm::v3::Wasm proto_config; + TestUtility::loadFromYaml(yaml, proto_config); + EXPECT_CALL(context_, threadLocal()).WillOnce(ReturnRef(threadlocal)); + threadlocal.registered_ = true; + auto filter_config = std::make_unique(proto_config, context_); + ASSERT_EQ(threadlocal.current_slot_, 1); + ASSERT_NE(filter_config->createFilter(), nullptr); + + // If the thread local plugin handle returns nullptr, `createFilter` should return nullptr + threadlocal.data_[0] = std::make_shared(nullptr); + EXPECT_EQ(filter_config->createFilter(), nullptr); +} + } // namespace Wasm } // namespace NetworkFilters } // namespace Extensions diff --git a/test/extensions/filters/network/wasm/test_data/test_cpp.cc b/test/extensions/filters/network/wasm/test_data/test_cpp.cc index 644b52eb61742..599bfc944713a 100644 --- a/test/extensions/filters/network/wasm/test_data/test_cpp.cc +++ b/test/extensions/filters/network/wasm/test_data/test_cpp.cc @@ -11,6 +11,15 @@ START_WASM_PLUGIN(NetworkTestCpp) static int* badptr = nullptr; +class TestRootContext : public RootContext { +public: + explicit TestRootContext(uint32_t id, std::string_view root_id) : RootContext(id, root_id) {} + bool onConfigure(size_t) override; + + std::string test_; + uint32_t stream_context_id_; +}; + class ExampleContext : public Context { public: explicit ExampleContext(uint32_t id, RootContext* root) : Context(id, root) {} @@ -22,7 +31,16 @@ class ExampleContext : public Context { void onDownstreamConnectionClose(CloseType close_type) override; void onUpstreamConnectionClose(CloseType close_type) override; }; -static RegisterContextFactory register_ExampleContext(CONTEXT_FACTORY(ExampleContext)); +static RegisterContextFactory register_ExampleContext(CONTEXT_FACTORY(ExampleContext), + ROOT_FACTORY(TestRootContext)); + +bool TestRootContext::onConfigure(size_t size) { + if (size > 0 && + getBufferBytes(WasmBufferType::PluginConfiguration, 0, size)->toString() == "invalid") { + return false; + } + return true; +} FilterStatus ExampleContext::onNewConnection() { logTrace("onNewConnection " + std::to_string(id())); From 00302c4a30033759f10b6ba4869a36f74063387b Mon Sep 17 00:00:00 2001 From: Ingwon Song <102102227+ingwonsong@users.noreply.github.com> Date: Tue, 2 Aug 2022 08:15:09 -0700 Subject: [PATCH 2/6] wasm: update proxy-wasm to apply canary always (#22469) Signed-off-by: Ingwon Song Signed-off-by: Ingwon Song <102102227+ingwonsong@users.noreply.github.com> Signed-off-by: Ryan Northey --- bazel/repository_locations.bzl | 6 +- .../filters/http/wasm/wasm_filter.h | 7 +- .../filters/network/wasm/wasm_filter.h | 7 +- .../access_loggers/wasm/config_test.cc | 4 - .../filters/http/wasm/config_test.cc | 86 ++++++++++++++++++- .../filters/network/wasm/config_test.cc | 4 - 6 files changed, 91 insertions(+), 23 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 05d1ae99dd281..070ce494e4b4a 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1090,8 +1090,8 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "WebAssembly for Proxies (C++ host implementation)", project_desc = "WebAssembly for Proxies (C++ host implementation)", project_url = "https://github.com/proxy-wasm/proxy-wasm-cpp-host", - version = "694a0b073912ff3bd00b6ca70d16ca43b2aebbf2", - sha256 = "d8b8ed52c48a4847a76c2192095362972e4ce07b95311f36da844128d5b35ab5", + version = "4ddbed3c8c8c1aa76db8157e22ae3be676e76ac1", + sha256 = "e9c05ed5e827e256f4725735c2ed3d5420fc95f5bce052b42f93aa37248a1f42", strip_prefix = "proxy-wasm-cpp-host-{version}", urls = ["https://github.com/proxy-wasm/proxy-wasm-cpp-host/archive/{version}.tar.gz"], use_category = ["dataplane_ext"], @@ -1107,7 +1107,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( "envoy.wasm.runtime.wavm", "envoy.wasm.runtime.wasmtime", ], - release_date = "2022-04-07", + release_date = "2022-07-30", cpe = "N/A", ), proxy_wasm_rust_sdk = dict( diff --git a/source/extensions/filters/http/wasm/wasm_filter.h b/source/extensions/filters/http/wasm/wasm_filter.h index a01b86a9db1d7..6dd63140e9e83 100644 --- a/source/extensions/filters/http/wasm/wasm_filter.h +++ b/source/extensions/filters/http/wasm/wasm_filter.h @@ -40,11 +40,10 @@ class FilterConfig : Logger::Loggable { } if (!wasm || wasm->isFailed()) { if (handle->plugin()->fail_open_) { - // Fail open skips adding this filter to callbacks. - return nullptr; + return nullptr; // Fail open skips adding this filter to callbacks. } else { - // Fail closed is handled by an empty Context. - return std::make_shared(nullptr, 0, handle); + return std::make_shared(nullptr, 0, + handle); // Fail closed is handled by an empty Context. } } return std::make_shared(wasm, handle->rootContextId(), handle); diff --git a/source/extensions/filters/network/wasm/wasm_filter.h b/source/extensions/filters/network/wasm/wasm_filter.h index e1fa293150888..dffd08b0c6209 100644 --- a/source/extensions/filters/network/wasm/wasm_filter.h +++ b/source/extensions/filters/network/wasm/wasm_filter.h @@ -40,11 +40,10 @@ class FilterConfig : Logger::Loggable { } if (!wasm || wasm->isFailed()) { if (handle->plugin()->fail_open_) { - // Fail open skips adding this filter to callbacks. - return nullptr; + return nullptr; // Fail open skips adding this filter to callbacks. } else { - // Fail closed is handled by an empty Context. - return std::make_shared(nullptr, 0, handle); + return std::make_shared(nullptr, 0, + handle); // Fail closed is handled by an empty Context. } } return std::make_shared(wasm, handle->rootContextId(), handle); diff --git a/test/extensions/access_loggers/wasm/config_test.cc b/test/extensions/access_loggers/wasm/config_test.cc index 1c354178b5728..de30cbeafb1d4 100644 --- a/test/extensions/access_loggers/wasm/config_test.cc +++ b/test/extensions/access_loggers/wasm/config_test.cc @@ -178,10 +178,6 @@ TEST_P(WasmAccessLogConfigTest, YamlLoadFromFileWasmInvalidConfig) { StreamInfo::MockStreamInfo log_stream_info; filter_instance = factory->createAccessLogInstance(proto_config, nullptr, context_); filter_instance->log(nullptr, nullptr, nullptr, log_stream_info); - - TestUtility::loadFromYaml(invalid_yaml, proto_config); - filter_instance = factory->createAccessLogInstance(proto_config, nullptr, context_); - filter_instance->log(nullptr, nullptr, nullptr, log_stream_info); } TEST_P(WasmAccessLogConfigTest, YamlLoadFromRemoteWasmCreateFilter) { diff --git a/test/extensions/filters/http/wasm/config_test.cc b/test/extensions/filters/http/wasm/config_test.cc index 29108ee426315..21fe97cd08ccb 100644 --- a/test/extensions/filters/http/wasm/config_test.cc +++ b/test/extensions/filters/http/wasm/config_test.cc @@ -222,10 +222,6 @@ TEST_P(WasmFilterConfigTest, YamlLoadFromFileWasmInvalidConfig) { EXPECT_CALL(filter_callback, addStreamFilter(_)); EXPECT_CALL(filter_callback, addAccessLogHandler(_)); cb(filter_callback); - - TestUtility::loadFromYaml(invalid_yaml, proto_config); - auto filter_config = std::make_unique(proto_config, context_); - EXPECT_EQ(filter_config->createFilter(), nullptr); } TEST_P(WasmFilterConfigTest, YamlLoadInlineWasm) { @@ -901,6 +897,88 @@ TEST_P(WasmFilterConfigTest, YamlLoadFromRemoteSuccessBadcodeFailOpen) { cb(filter_callback); } +TEST_P(WasmFilterConfigTest, YamlLoadFromRemoteWasmCreateFilter) { + const std::string code = TestEnvironment::readFileToStringForTest(TestEnvironment::substitute( + "{{ test_rundir }}/test/extensions/filters/http/wasm/test_data/test_cpp.wasm")); + const std::string sha256 = Hex::encode( + Envoy::Common::Crypto::UtilitySingleton::get().getSha256Digest(Buffer::OwnedImpl(code))); + const std::string yaml = TestEnvironment::substitute(absl::StrCat(R"EOF( + config: + vm_config: + runtime: "envoy.wasm.runtime.)EOF", + std::get<0>(GetParam()), R"EOF(" + code: + remote: + http_uri: + uri: https://example.com/data + cluster: cluster_1 + timeout: 5s + sha256: )EOF", + sha256)); + envoy::extensions::filters::http::wasm::v3::Wasm proto_config; + TestUtility::loadFromYaml(yaml, proto_config); + WasmFilterConfig factory; + NiceMock client; + NiceMock request(&client); + + cluster_manager_.initializeThreadLocalClusters({"cluster_1"}); + EXPECT_CALL(cluster_manager_.thread_local_cluster_, httpAsyncClient()) + .WillOnce(ReturnRef(cluster_manager_.thread_local_cluster_.async_client_)); + Http::AsyncClient::Callbacks* async_callbacks = nullptr; + EXPECT_CALL(cluster_manager_.thread_local_cluster_.async_client_, send_(_, _, _)) + .WillOnce( + Invoke([&](Http::RequestMessagePtr&, Http::AsyncClient::Callbacks& callbacks, + const Http::AsyncClient::RequestOptions&) -> Http::AsyncClient::Request* { + if (!async_callbacks) { + async_callbacks = &callbacks; + } + return &request; + })); + NiceMock threadlocal; + EXPECT_CALL(context_, threadLocal()).WillRepeatedly(ReturnRef(threadlocal)); + threadlocal.registered_ = false; + auto filter_config = std::make_unique(proto_config, context_); + EXPECT_EQ(filter_config->createFilter(), nullptr); + EXPECT_CALL(init_watcher_, ready()); + context_.initManager().initialize(init_watcher_); + auto response = Http::ResponseMessagePtr{new Http::ResponseMessageImpl( + Http::ResponseHeaderMapPtr{new Http::TestResponseHeaderMapImpl{{":status", "200"}}})}; + response->body().add(code); + async_callbacks->onSuccess(request, std::move(response)); + EXPECT_EQ(context_.initManager().state(), Init::Manager::State::Initialized); + threadlocal.registered_ = true; + EXPECT_NE(filter_config->createFilter(), nullptr); +} + +TEST_P(WasmFilterConfigTest, FailedToGetThreadLocalPlugin) { + NiceMock threadlocal; + const std::string yaml = TestEnvironment::substitute(absl::StrCat(R"EOF( + config: + fail_open: true + vm_config: + runtime: "envoy.wasm.runtime.)EOF", + std::get<0>(GetParam()), R"EOF(" + configuration: + "@type": "type.googleapis.com/google.protobuf.StringValue" + value: "some configuration" + code: + local: + filename: "{{ test_rundir }}/test/extensions/filters/http/wasm/test_data/test_cpp.wasm" + )EOF")); + + envoy::extensions::filters::http::wasm::v3::Wasm proto_config; + TestUtility::loadFromYaml(yaml, proto_config); + EXPECT_CALL(context_, threadLocal()).WillOnce(ReturnRef(threadlocal)); + threadlocal.registered_ = true; + auto filter_config = std::make_unique(proto_config, context_); + ASSERT_EQ(threadlocal.current_slot_, 1); + ASSERT_NE(filter_config->createFilter(), nullptr); + + // If the thread local plugin handle returns nullptr, `createFilter` should return nullptr + threadlocal.data_[0] = std::make_shared(nullptr); + EXPECT_EQ(filter_config->createFilter(), nullptr); +} + } // namespace Wasm } // namespace HttpFilters } // namespace Extensions diff --git a/test/extensions/filters/network/wasm/config_test.cc b/test/extensions/filters/network/wasm/config_test.cc index d6a357311a6f2..1557621ada608 100644 --- a/test/extensions/filters/network/wasm/config_test.cc +++ b/test/extensions/filters/network/wasm/config_test.cc @@ -353,10 +353,6 @@ TEST_P(WasmNetworkFilterConfigTest, YamlLoadFromFileWasmInvalidConfig) { Network::MockConnection connection; EXPECT_CALL(connection, addFilter(_)); cb(connection); - - TestUtility::loadFromYaml(invalid_yaml, proto_config); - auto filter_config = std::make_unique(proto_config, context_); - EXPECT_EQ(filter_config->createFilter(), nullptr); } TEST_P(WasmNetworkFilterConfigTest, YamlLoadFromRemoteWasmCreateFilter) { From b18b9f6588fb8a0aafd828046a1cabeecbe3cd45 Mon Sep 17 00:00:00 2001 From: Piotr Sikora Date: Wed, 3 Aug 2022 12:09:59 -0500 Subject: [PATCH 3/6] wasm: update Wasmtime to v0.39.1. (#22490) Signed-off-by: Piotr Sikora Signed-off-by: Ryan Northey --- bazel/repository_locations.bzl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 070ce494e4b4a..9b5c15eb67491 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -801,11 +801,11 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "wasmtime", project_desc = "A standalone runtime for WebAssembly", project_url = "https://github.com/bytecodealliance/wasmtime", - version = "0.35.2", - sha256 = "9dcb51313f9d6a67169f70759411cddf511000b0372e57532e638441100aac9c", + version = "0.39.1", + sha256 = "6ef70886da14245f575c6ff8c7c999ae22579257eba5ebf382e066598c1e381c", strip_prefix = "wasmtime-{version}", urls = ["https://github.com/bytecodealliance/wasmtime/archive/v{version}.tar.gz"], - release_date = "2022-03-31", + release_date = "2022-07-21", use_category = ["dataplane_ext"], extensions = ["envoy.wasm.runtime.wasmtime"], cpe = "cpe:2.3:a:bytecodealliance:wasmtime:*", @@ -1090,8 +1090,8 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "WebAssembly for Proxies (C++ host implementation)", project_desc = "WebAssembly for Proxies (C++ host implementation)", project_url = "https://github.com/proxy-wasm/proxy-wasm-cpp-host", - version = "4ddbed3c8c8c1aa76db8157e22ae3be676e76ac1", - sha256 = "e9c05ed5e827e256f4725735c2ed3d5420fc95f5bce052b42f93aa37248a1f42", + version = "9387a5b77418e41bbf2b0909c68c14db42a1e743", + sha256 = "cc67ec35fe58e214ff19fb07435e6cfbf62493dbb543ca65017a8162c2baa137", strip_prefix = "proxy-wasm-cpp-host-{version}", urls = ["https://github.com/proxy-wasm/proxy-wasm-cpp-host/archive/{version}.tar.gz"], use_category = ["dataplane_ext"], @@ -1107,7 +1107,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( "envoy.wasm.runtime.wavm", "envoy.wasm.runtime.wasmtime", ], - release_date = "2022-07-30", + release_date = "2022-08-01", cpe = "N/A", ), proxy_wasm_rust_sdk = dict( From b0d4ff9b99f7223067a7bdec3f3a59e49c061661 Mon Sep 17 00:00:00 2001 From: Piotr Sikora Date: Thu, 14 Jul 2022 15:32:57 -0500 Subject: [PATCH 4/6] wasm: update V8 to v10.4.132.18. (#22154) Signed-off-by: Piotr Sikora Signed-off-by: Ryan Northey --- bazel/repository_locations.bzl | 12 +- bazel/v8.patch | 307 +++++++++++++++++++++++- test/tools/wee8_compile/BUILD | 1 + test/tools/wee8_compile/wee8_compile.cc | 7 + 4 files changed, 309 insertions(+), 18 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 9b5c15eb67491..0df27f5e468b4 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -862,13 +862,13 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_desc = "Google’s open source high-performance JavaScript and WebAssembly engine, written in C++", project_url = "https://v8.dev", # NOTE: Update together with com_googlesource_chromium_base_trace_event_common and com_googlesource_chromium_zlib. - version = "10.0.139.6", + version = "10.4.132.18", # Static snapshot created using https://storage.googleapis.com/envoyproxy-wee8/wee8-fetch-deps.sh. - sha256 = "3585ebc8db95e61b65260b3d141ed5392b843bd066744fa62ce92dab3708e2f8", + sha256 = "154223731091f531a3cb1a95ab9c14225983891240220a9e232c729358badd2c", urls = ["https://storage.googleapis.com/envoyproxy-wee8/v8-{version}.tar.gz"], use_category = ["dataplane_ext"], extensions = ["envoy.wasm.runtime.v8"], - release_date = "2022-03-07", + release_date = "2022-07-11", cpe = "cpe:2.3:a:google:v8:*", ), com_googlesource_chromium_base_trace_event_common = dict( @@ -892,13 +892,13 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_url = "https://chromium.googlesource.com/chromium/src/third_party/zlib/", # NOTE: Update together with v8 and com_googlesource_chromium_base_trace_event_common. # Use version and sha256 from https://storage.googleapis.com/envoyproxy-wee8/v8--deps.sha256. - version = "9538f4194f6e5eff1bd59f2396ed9d05b1a8d801", + version = "64bbf988543996eb8df9a86877b32917187eba8f", # Static snapshot created using https://storage.googleapis.com/envoyproxy-wee8/wee8-fetch-deps.sh. - sha256 = "7045c6808267f3e803e3196fa06346647c2a060cfd1a2f96ef6325fba6eaa956", + sha256 = "6e70bbb25ad1c567e3b44f4ad76f50fcc8fb7d34c6e2e8858e033ffd579ac1af", urls = ["https://storage.googleapis.com/envoyproxy-wee8/chromium-zlib-{version}.tar.gz"], use_category = ["dataplane_ext"], extensions = ["envoy.wasm.runtime.v8"], - release_date = "2022-02-15", + release_date = "2022-05-31", cpe = "N/A", ), com_github_google_quiche = dict( diff --git a/bazel/v8.patch b/bazel/v8.patch index a55ec77a3c933..6d64274a66f8c 100644 --- a/bazel/v8.patch +++ b/bazel/v8.patch @@ -1,8 +1,11 @@ # 1. Use already imported python dependencies. # 2. Fix the include path for //external:zlib. # 3. Add support for --define=no_debug_info=1. -# 4. Disable pointer compression (https://crbug.com/v8/12592). -# 5. Add M1 CPU support https://chromium-review.googlesource.com/c/v8/v8/+/3502848 +# 4. Disable pointer compression (limits the maximum number of WasmVMs). +# 5. Don't expose Wasm C API (only Wasm C++ API). +# 6. Fix cross-compilation (https://crrev.com/c/3735165). +# 7. Fix build errors in SIMD IndexOf/includes (https://crrev.com/c/3749192). +# 8. Fix build on arm64. diff --git a/BUILD.bazel b/BUILD.bazel index 13f2a5bebf..2197568c48 100644 @@ -27,10 +30,18 @@ index 13f2a5bebf..2197568c48 100644 # Default setting for v8_enable_pointer_compression. diff --git a/bazel/defs.bzl b/bazel/defs.bzl -index dee5e69cc4..d0b5a3c49a 100644 +index e957c0fad3..eee285ab60 100644 --- a/bazel/defs.bzl +++ b/bazel/defs.bzl -@@ -151,6 +151,11 @@ def _default_args(): +@@ -116,6 +116,7 @@ def _default_args(): + }) + select({ + "@v8//bazel/config:is_clang": [ + "-Wno-invalid-offsetof", ++ "-Wno-unneeded-internal-declaration", + "-std=c++17", + ], + "@v8//bazel/config:is_gcc": [ +@@ -151,6 +152,11 @@ def _default_args(): "-fno-integrated-as", ], "//conditions:default": [], @@ -42,14 +53,6 @@ index dee5e69cc4..d0b5a3c49a 100644 }), includes = ["include"], linkopts = select({ -@@ -383,6 +388,7 @@ def _v8_target_cpu_transition_impl(settings, attr): - "k8": "x64", - "x86_64": "x64", - "darwin": "x64", -+ "darwin_arm64": "arm64", - "darwin_x86_64": "x64", - "x64_windows": "x64", - "x86": "ia32", diff --git a/src/snapshot/snapshot-utils.cc b/src/snapshot/snapshot-utils.cc index 6db6698d7e..b56d31085f 100644 --- a/src/snapshot/snapshot-utils.cc @@ -63,3 +66,283 @@ index 6db6698d7e..b56d31085f 100644 namespace v8 { namespace internal { +diff --git a/src/wasm/c-api.cc b/src/wasm/c-api.cc +index ce3f569fd5..dc8a4c4f6a 100644 +--- a/src/wasm/c-api.cc ++++ b/src/wasm/c-api.cc +@@ -2238,6 +2238,8 @@ auto Instance::exports() const -> ownvec { + + } // namespace wasm + ++#if 0 ++ + // BEGIN FILE wasm-c.cc + + extern "C" { +@@ -3257,3 +3259,5 @@ wasm_instance_t* wasm_frame_instance(const wasm_frame_t* frame) { + #undef WASM_DEFINE_SHARABLE_REF + + } // extern "C" ++ ++#endif +diff --git a/src/execution/clobber-registers.cc b/src/execution/clobber-registers.cc +index 8f7fba765f..a7f5bf80cf 100644 +--- a/src/execution/clobber-registers.cc ++++ b/src/execution/clobber-registers.cc +@@ -5,19 +5,22 @@ + + #include "src/base/build_config.h" + +-#if V8_HOST_ARCH_ARM ++// Check both {HOST_ARCH} and {TARGET_ARCH} to disable the functionality of this ++// file for cross-compilation. The reason is that the inline assembly code below ++// does not work for cross-compilation. ++#if V8_HOST_ARCH_ARM && V8_TARGET_ARCH_ARM + #include "src/codegen/arm/register-arm.h" +-#elif V8_HOST_ARCH_ARM64 ++#elif V8_HOST_ARCH_ARM64 && V8_TARGET_ARCH_ARM64 + #include "src/codegen/arm64/register-arm64.h" +-#elif V8_HOST_ARCH_IA32 ++#elif V8_HOST_ARCH_IA32 && V8_TARGET_ARCH_IA32 + #include "src/codegen/ia32/register-ia32.h" +-#elif V8_HOST_ARCH_X64 ++#elif V8_HOST_ARCH_X64 && V8_TARGET_ARCH_X64 + #include "src/codegen/x64/register-x64.h" +-#elif V8_HOST_ARCH_LOONG64 ++#elif V8_HOST_ARCH_LOONG64 && V8_TARGET_ARCH_LOONG64 + #include "src/codegen/loong64/register-loong64.h" +-#elif V8_HOST_ARCH_MIPS ++#elif V8_HOST_ARCH_MIPS && V8_TARGET_ARCH_MIPS + #include "src/codegen/mips/register-mips.h" +-#elif V8_HOST_ARCH_MIPS64 ++#elif V8_HOST_ARCH_MIPS64 && V8_TARGET_ARCH_MIPS64 + #include "src/codegen/mips64/register-mips64.h" + #endif + +@@ -26,14 +29,15 @@ namespace internal { + + #if V8_CC_MSVC + // msvc only support inline assembly on x86 +-#if V8_HOST_ARCH_IA32 ++#if V8_HOST_ARCH_IA32 && V8_TARGET_ARCH_IA32 + #define CLOBBER_REGISTER(R) __asm xorps R, R + + #endif + + #else // !V8_CC_MSVC + +-#if V8_HOST_ARCH_X64 || V8_HOST_ARCH_IA32 ++#if (V8_HOST_ARCH_X64 && V8_TARGET_ARCH_X64) || \ ++ (V8_HOST_ARCH_IA32 && V8_TARGET_ARCH_IA32) + #define CLOBBER_REGISTER(R) \ + __asm__ volatile( \ + "xorps " \ +@@ -42,20 +46,19 @@ namespace internal { + "%%" #R :: \ + :); + +-#elif V8_HOST_ARCH_ARM64 ++#elif V8_HOST_ARCH_ARM64 && V8_TARGET_ARCH_ARM64 + #define CLOBBER_REGISTER(R) __asm__ volatile("fmov " #R ",xzr" :::); + +-#elif V8_HOST_ARCH_LOONG64 ++#elif V8_HOST_ARCH_LOONG64 && V8_TARGET_ARCH_LOONG64 + #define CLOBBER_REGISTER(R) __asm__ volatile("movgr2fr.d $" #R ",$zero" :::); + +-#elif V8_HOST_ARCH_MIPS ++#elif V8_HOST_ARCH_MIPS && V8_TARGET_ARCH_MIPS + #define CLOBBER_USE_REGISTER(R) __asm__ volatile("mtc1 $zero,$" #R :::); + +-#elif V8_HOST_ARCH_MIPS64 ++#elif V8_HOST_ARCH_MIPS64 && V8_TARGET_ARCH_MIPS64 + #define CLOBBER_USE_REGISTER(R) __asm__ volatile("dmtc1 $zero,$" #R :::); + +-#endif // V8_HOST_ARCH_X64 || V8_HOST_ARCH_IA32 || V8_HOST_ARCH_ARM64 || +- // V8_HOST_ARCH_LOONG64 || V8_HOST_ARCH_MIPS || V8_HOST_ARCH_MIPS64 ++#endif // V8_HOST_ARCH_XXX && V8_TARGET_ARCH_XXX + + #endif // V8_CC_MSVC + +diff --git a/src/objects/simd.cc b/src/objects/simd.cc +index 0a73b9c686..be6b72d157 100644 +--- a/src/objects/simd.cc ++++ b/src/objects/simd.cc +@@ -354,8 +354,13 @@ Address ArrayIndexOfIncludes(Address array_start, uintptr_t array_len, + if (reinterpret_cast(array) % sizeof(double) != 0) { + // Slow scalar search for unaligned double array. + for (; from_index < array_len; from_index++) { +- if (fixed_array.get_representation(static_cast(from_index)) == +- *reinterpret_cast(&search_num)) { ++ if (fixed_array.is_the_hole(static_cast(from_index))) { ++ // |search_num| cannot be NaN, so there is no need to check against ++ // holes. ++ continue; ++ } ++ if (fixed_array.get_scalar(static_cast(from_index)) == ++ search_num) { + return from_index; + } + } +diff --git a/src/objects/simd.cc b/src/objects/simd.cc +index d3cedfe330..0a73b9c686 100644 +--- a/src/objects/simd.cc ++++ b/src/objects/simd.cc +@@ -95,24 +95,21 @@ inline int extract_first_nonzero_index(T v) { + } + + template <> +-inline int extract_first_nonzero_index(int32x4_t v) { +- int32x4_t mask = {4, 3, 2, 1}; ++inline int extract_first_nonzero_index(uint32x4_t v) { ++ uint32x4_t mask = {4, 3, 2, 1}; + mask = vandq_u32(mask, v); + return 4 - vmaxvq_u32(mask); + } + + template <> +-inline int extract_first_nonzero_index(int64x2_t v) { +- int32x4_t mask = {2, 0, 1, 0}; // Could also be {2,2,1,1} or {0,2,0,1} +- mask = vandq_u32(mask, vreinterpretq_s32_s64(v)); ++inline int extract_first_nonzero_index(uint64x2_t v) { ++ uint32x4_t mask = {2, 0, 1, 0}; // Could also be {2,2,1,1} or {0,2,0,1} ++ mask = vandq_u32(mask, vreinterpretq_u32_u64(v)); + return 2 - vmaxvq_u32(mask); + } + +-template <> +-inline int extract_first_nonzero_index(float64x2_t v) { +- int32x4_t mask = {2, 0, 1, 0}; // Could also be {2,2,1,1} or {0,2,0,1} +- mask = vandq_u32(mask, vreinterpretq_s32_f64(v)); +- return 2 - vmaxvq_u32(mask); ++inline int32_t reinterpret_vmaxvq_u64(uint64x2_t v) { ++ return vmaxvq_u32(vreinterpretq_u32_u64(v)); + } + #endif + +@@ -204,14 +201,14 @@ inline uintptr_t fast_search_noavx(T* array, uintptr_t array_len, + } + #elif defined(NEON64) + if constexpr (std::is_same::value) { +- VECTORIZED_LOOP_Neon(int32x4_t, int32x4_t, vdupq_n_u32, vceqq_u32, ++ VECTORIZED_LOOP_Neon(uint32x4_t, uint32x4_t, vdupq_n_u32, vceqq_u32, + vmaxvq_u32) + } else if constexpr (std::is_same::value) { +- VECTORIZED_LOOP_Neon(int64x2_t, int64x2_t, vdupq_n_u64, vceqq_u64, +- vmaxvq_u32) ++ VECTORIZED_LOOP_Neon(uint64x2_t, uint64x2_t, vdupq_n_u64, vceqq_u64, ++ reinterpret_vmaxvq_u64) + } else if constexpr (std::is_same::value) { +- VECTORIZED_LOOP_Neon(float64x2_t, float64x2_t, vdupq_n_f64, vceqq_f64, +- vmaxvq_f64) ++ VECTORIZED_LOOP_Neon(float64x2_t, uint64x2_t, vdupq_n_f64, vceqq_f64, ++ reinterpret_vmaxvq_u64) + } + #else + UNREACHABLE(); +diff --git a/src/objects/simd.cc b/src/objects/simd.cc +index be6b72d157..a71968fd10 100644 +--- a/src/objects/simd.cc ++++ b/src/objects/simd.cc +@@ -148,9 +148,14 @@ inline int32_t reinterpret_vmaxvq_u64(uint64x2_t v) { + template + inline uintptr_t fast_search_noavx(T* array, uintptr_t array_len, + uintptr_t index, T search_element) { +- static_assert(std::is_same::value || +- std::is_same::value || +- std::is_same::value); ++ static constexpr bool is_uint32 = ++ sizeof(T) == sizeof(uint32_t) && std::is_integral::value; ++ static constexpr bool is_uint64 = ++ sizeof(T) == sizeof(uint64_t) && std::is_integral::value; ++ static constexpr bool is_double = ++ sizeof(T) == sizeof(double) && std::is_floating_point::value; ++ ++ static_assert(is_uint32 || is_uint64 || is_double); + + #if !(defined(__SSE3__) || defined(NEON64)) + // No SIMD available. +@@ -178,14 +183,14 @@ inline uintptr_t fast_search_noavx(T* array, uintptr_t array_len, + + // Inserting one of the vectorized loop + #ifdef __SSE3__ +- if constexpr (std::is_same::value) { ++ if constexpr (is_uint32) { + #define MOVEMASK(x) _mm_movemask_ps(_mm_castsi128_ps(x)) + #define EXTRACT(x) base::bits::CountTrailingZeros32(x) + VECTORIZED_LOOP_x86(__m128i, __m128i, _mm_set1_epi32, _mm_cmpeq_epi32, + MOVEMASK, EXTRACT) + #undef MOVEMASK + #undef EXTRACT +- } else if constexpr (std::is_same::value) { ++ } else if constexpr (is_uint64) { + #define SET1(x) _mm_castsi128_ps(_mm_set1_epi64x(x)) + #define CMP(a, b) _mm_cmpeq_pd(_mm_castps_pd(a), _mm_castps_pd(b)) + #define EXTRACT(x) base::bits::CountTrailingZeros32(x) +@@ -193,20 +198,20 @@ inline uintptr_t fast_search_noavx(T* array, uintptr_t array_len, + #undef SET1 + #undef CMP + #undef EXTRACT +- } else if constexpr (std::is_same::value) { ++ } else if constexpr (is_double) { + #define EXTRACT(x) base::bits::CountTrailingZeros32(x) + VECTORIZED_LOOP_x86(__m128d, __m128d, _mm_set1_pd, _mm_cmpeq_pd, + _mm_movemask_pd, EXTRACT) + #undef EXTRACT + } + #elif defined(NEON64) +- if constexpr (std::is_same::value) { ++ if constexpr (is_uint32) { + VECTORIZED_LOOP_Neon(uint32x4_t, uint32x4_t, vdupq_n_u32, vceqq_u32, + vmaxvq_u32) +- } else if constexpr (std::is_same::value) { ++ } else if constexpr (is_uint64) { + VECTORIZED_LOOP_Neon(uint64x2_t, uint64x2_t, vdupq_n_u64, vceqq_u64, + reinterpret_vmaxvq_u64) +- } else if constexpr (std::is_same::value) { ++ } else if constexpr (is_double) { + VECTORIZED_LOOP_Neon(float64x2_t, uint64x2_t, vdupq_n_f64, vceqq_f64, + reinterpret_vmaxvq_u64) + } +@@ -240,9 +245,14 @@ template + TARGET_AVX2 inline uintptr_t fast_search_avx(T* array, uintptr_t array_len, + uintptr_t index, + T search_element) { +- static_assert(std::is_same::value || +- std::is_same::value || +- std::is_same::value); ++ static constexpr bool is_uint32 = ++ sizeof(T) == sizeof(uint32_t) && std::is_integral::value; ++ static constexpr bool is_uint64 = ++ sizeof(T) == sizeof(uint64_t) && std::is_integral::value; ++ static constexpr bool is_double = ++ sizeof(T) == sizeof(double) && std::is_floating_point::value; ++ ++ static_assert(is_uint32 || is_uint64 || is_double); + + const int target_align = 32; + // Scalar loop to reach desired alignment +@@ -256,21 +266,21 @@ TARGET_AVX2 inline uintptr_t fast_search_avx(T* array, uintptr_t array_len, + } + + // Generating vectorized loop +- if constexpr (std::is_same::value) { ++ if constexpr (is_uint32) { + #define MOVEMASK(x) _mm256_movemask_ps(_mm256_castsi256_ps(x)) + #define EXTRACT(x) base::bits::CountTrailingZeros32(x) + VECTORIZED_LOOP_x86(__m256i, __m256i, _mm256_set1_epi32, _mm256_cmpeq_epi32, + MOVEMASK, EXTRACT) + #undef MOVEMASK + #undef EXTRACT +- } else if constexpr (std::is_same::value) { ++ } else if constexpr (is_uint64) { + #define MOVEMASK(x) _mm256_movemask_pd(_mm256_castsi256_pd(x)) + #define EXTRACT(x) base::bits::CountTrailingZeros32(x) + VECTORIZED_LOOP_x86(__m256i, __m256i, _mm256_set1_epi64x, + _mm256_cmpeq_epi64, MOVEMASK, EXTRACT) + #undef MOVEMASK + #undef EXTRACT +- } else if constexpr (std::is_same::value) { ++ } else if constexpr (is_double) { + #define CMP(a, b) _mm256_cmp_pd(a, b, _CMP_EQ_OQ) + #define EXTRACT(x) base::bits::CountTrailingZeros32(x) + VECTORIZED_LOOP_x86(__m256d, __m256d, _mm256_set1_pd, CMP, diff --git a/test/tools/wee8_compile/BUILD b/test/tools/wee8_compile/BUILD index d35416c42783d..f52efac7f61bb 100644 --- a/test/tools/wee8_compile/BUILD +++ b/test/tools/wee8_compile/BUILD @@ -18,6 +18,7 @@ envoy_cc_library( name = "wee8_compile_lib", srcs = ["wee8_compile.cc"], copts = [ + "-Wno-comments", "-Wno-non-virtual-dtor", "-Wno-unused-parameter", ], diff --git a/test/tools/wee8_compile/wee8_compile.cc b/test/tools/wee8_compile/wee8_compile.cc index 50e2aeb602f33..fee38179b2026 100644 --- a/test/tools/wee8_compile/wee8_compile.cc +++ b/test/tools/wee8_compile/wee8_compile.cc @@ -11,6 +11,11 @@ #include "v8-version.h" #include "wasm-api/wasm.hh" +namespace v8::internal { +extern bool FLAG_liftoff; +extern unsigned int FLAG_wasm_max_mem_pages; +} // namespace v8::internal + uint32_t parseVarint(const byte_t*& pos, const byte_t* end) { uint32_t n = 0; uint32_t shift = 0; @@ -149,6 +154,8 @@ wasm::vec stripWasmModule(const wasm::vec& module) { } wasm::vec serializeWasmModule(const char* path, const wasm::vec& content) { + ::v8::internal::FLAG_liftoff = false; + ::v8::internal::FLAG_wasm_max_mem_pages = 16384; /* 16,384 * 64 KiB pages == 1 GiB limit */ const auto engine = wasm::Engine::make(); if (engine == nullptr) { std::cerr << "ERROR: Failed to start V8." << std::endl; From f96cccddd3ef9e914227821aa064d2f7330eb95a Mon Sep 17 00:00:00 2001 From: Piotr Sikora Date: Wed, 13 Jul 2022 17:09:37 -0500 Subject: [PATCH 5/6] wasm: set upper limit for maximum Wasm memory size when using V8. (#22117) Signed-off-by: Piotr Sikora Signed-off-by: Ryan Northey --- tools/spelling/spelling_dictionary.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/spelling/spelling_dictionary.txt b/tools/spelling/spelling_dictionary.txt index b8b012c6b4f99..d86155c32566a 100644 --- a/tools/spelling/spelling_dictionary.txt +++ b/tools/spelling/spelling_dictionary.txt @@ -24,6 +24,9 @@ BACKTRACE BBR BSON BPF +Bdecoded +Bencoded +GiB Repick TRA btree @@ -1349,4 +1352,3 @@ ElastiCache pinterest NSS SSLKEYLOGFILE - From ab91bc3f21e6dcf2590a711606951c6ed57befcf Mon Sep 17 00:00:00 2001 From: Piotr Sikora Date: Fri, 15 Jul 2022 01:41:43 -0500 Subject: [PATCH 6/6] wasm: update WAMR to WAMR-05-18-2022. (#22157) Signed-off-by: Piotr Sikora --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 0df27f5e468b4..ea52f931e9b2b 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -775,11 +775,11 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Webassembly Micro Runtime", project_desc = "A standalone runtime with a small footprint for WebAssembly", project_url = "https://github.com/bytecodealliance/wasm-micro-runtime", - version = "WAMR-12-30-2021", - sha256 = "ab6e8643ec553347b0e129a355dc723969f49355d1d5bfa74d724d984c238037", + version = "WAMR-05-18-2022", + sha256 = "c71f571f99c7028fd7595126ec25d6df4bc9ba62702bb88cbd5ea8a29e477f12", strip_prefix = "wasm-micro-runtime-{version}", urls = ["https://github.com/bytecodealliance/wasm-micro-runtime/archive/{version}.tar.gz"], - release_date = "2021-12-30", + release_date = "2022-05-18", use_category = ["dataplane_ext"], extensions = ["envoy.wasm.runtime.wamr"], cpe = "N/A",