http: configurable ignore of HTTP/1.1 upgrades (#37642)

api/envoy/config/core/v3/protocol.proto

@@ -3,6 +3,7 @@ syntax = "proto3";
 package envoy.config.core.v3;
 
 import "envoy/config/core/v3/extension.proto";
+import "envoy/type/matcher/v3/string.proto";
 import "envoy/type/v3/percent.proto";
 
 import "google/protobuf/duration.proto";
@@ -302,7 +303,7 @@ message HttpProtocolOptions {
   google.protobuf.UInt32Value max_requests_per_connection = 6;
 }
 
-// [#next-free-field: 11]
+// [#next-free-field: 12]
 message Http1ProtocolOptions {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.api.v2.core.Http1ProtocolOptions";
@@ -414,6 +415,14 @@ message Http1ProtocolOptions {
   // <envoy_v3_api_field_extensions.http.header_validators.envoy_default.v3.HeaderValidatorConfig.restrict_http_methods>`
   // to reject custom methods.
   bool allow_custom_methods = 10 [(xds.annotations.v3.field_status).work_in_progress = true];
+
+  // Ignore HTTP/1.1 upgrade values matching any of the supplied matchers.
+  //
+  // .. note::
+  //
+  //   ``h2c`` upgrades are always removed for backwards compatibility, regardless of the
+  //   value in this setting.
+  repeated type.matcher.v3.StringMatcher ignore_http_11_upgrade = 11;
 }
 
 message KeepaliveSettings {

changelogs/current.yaml

@@ -13,5 +13,10 @@ removed_config_or_runtime:
 # *Normally occurs at the end of the* :ref:`deprecation period <deprecated>`
 
 new_features:
+- area: http
+  change: |
+    Added :ref:`ignore_http_11_upgrade
+    <envoy_v3_api_field_config.core.v3.Http1ProtocolOptions.ignore_http_11_upgrade>`
+    to ignore HTTP/1.1 Upgrade values matching any of the supplied matchers.
 
 deprecated:

envoy/http/BUILD

@@ -50,6 +50,7 @@ envoy_cc_library(
         ":stream_reset_handler_interface",
         "//envoy/access_log:access_log_interface",
         "//envoy/buffer:buffer_interface",
+        "//envoy/common:matchers_interface",
         "//envoy/grpc:status",
         "//envoy/network:address_interface",
         "//envoy/stream_info:stream_info_interface",

envoy/http/codec.h

@@ -6,6 +6,7 @@
 
 #include "envoy/access_log/access_log.h"
 #include "envoy/buffer/buffer.h"
+#include "envoy/common/matchers.h"
 #include "envoy/common/pure.h"
 #include "envoy/grpc/status.h"
 #include "envoy/http/header_formatter.h"
@@ -497,6 +498,9 @@ struct Http1Settings {
   // headers set. By default such messages are rejected, but if option is enabled - Envoy will
   // remove Content-Length header and process message.
   bool allow_chunked_length_{false};
+  // Remove HTTP/1.1 Upgrade header tokens matching any provided matcher. By default such
+  // messages are rejected
+  std::shared_ptr<const std::vector<Matchers::StringMatcherPtr>> ignore_upgrade_matchers_;
 
   enum class HeaderKeyFormat {
     // By default no formatting is performed, presenting all headers in lowercase (as Envoy

source/common/http/http1/BUILD

@@ -94,6 +94,7 @@ envoy_cc_library(
     deps = [
         "//envoy/http:codec_interface",
         "//envoy/protobuf:message_validator_interface",
+        "//source/common/common:matchers_lib",
         "//source/common/config:utility_lib",
         "//source/common/runtime:runtime_features_lib",
         "@com_google_absl//absl/types:optional",

source/common/http/http1/codec_impl.cc

@@ -65,7 +65,12 @@ static constexpr uint32_t kMaxOutboundResponses = 2;
 using Http1ResponseCodeDetails = ConstSingleton<Http1ResponseCodeDetailValues>;
 using Http1HeaderTypes = ConstSingleton<Http1HeaderTypesValues>;
 
-const StringUtil::CaseUnorderedSet& caseUnorderdSetContainingUpgradeAndHttp2Settings() {
+const StringUtil::CaseUnorderedSet& caseUnorderedSetContainingUpgrade() {
+  CONSTRUCT_ON_FIRST_USE(StringUtil::CaseUnorderedSet,
+                         Http::Headers::get().ConnectionValues.Upgrade);
+}
+
+const StringUtil::CaseUnorderedSet& caseUnorderedSetContainingUpgradeAndHttp2Settings() {
   CONSTRUCT_ON_FIRST_USE(StringUtil::CaseUnorderedSet,
                          Http::Headers::get().ConnectionValues.Upgrade,
                          Http::Headers::get().ConnectionValues.Http2Settings);
@@ -846,24 +851,33 @@ StatusOr<CallbackResult> ConnectionImpl::onHeadersCompleteImpl() {
   RequestOrResponseHeaderMap& request_or_response_headers = requestOrResponseHeaders();
   const Http::HeaderValues& header_values = Http::Headers::get();
   if (Utility::isUpgrade(request_or_response_headers) && upgradeAllowed()) {
+    auto upgrade_value = request_or_response_headers.getUpgradeValue();
+    const bool is_h2c = absl::EqualsIgnoreCase(upgrade_value, header_values.UpgradeValues.H2c);
+
     // Ignore h2c upgrade requests until we support them.
     // See https://github.com/envoyproxy/envoy/issues/7161 for details.
-    if (absl::EqualsIgnoreCase(request_or_response_headers.getUpgradeValue(),
-                               header_values.UpgradeValues.H2c)) {
+    // Upgrades are rejected unless ignore_http_11_upgrade is configured.
+    // See https://github.com/envoyproxy/envoy/issues/36305 for details.
+    if (is_h2c) {
       ENVOY_CONN_LOG(trace, "removing unsupported h2c upgrade headers.", connection_);
       request_or_response_headers.removeUpgrade();
-      if (request_or_response_headers.Connection()) {
-        const auto& tokens_to_remove = caseUnorderdSetContainingUpgradeAndHttp2Settings();
-        std::string new_value = StringUtil::removeTokens(
-            request_or_response_headers.getConnectionValue(), ",", tokens_to_remove, ",");
-        if (new_value.empty()) {
-          request_or_response_headers.removeConnection();
-        } else {
-          request_or_response_headers.setConnection(new_value);
-        }
-      }
+      Utility::removeConnectionUpgrade(request_or_response_headers,
+                                       caseUnorderedSetContainingUpgradeAndHttp2Settings());
       request_or_response_headers.remove(header_values.Http2Settings);
-    } else {
+    } else if (codec_settings_.ignore_upgrade_matchers_ != nullptr &&
+               !codec_settings_.ignore_upgrade_matchers_->empty()) {
+      ENVOY_CONN_LOG(trace, "removing ignored upgrade headers.", connection_);
+
+      Utility::removeUpgrade(request_or_response_headers,
+                             *codec_settings_.ignore_upgrade_matchers_);
+
+      if (!request_or_response_headers.Upgrade()) {
+        Utility::removeConnectionUpgrade(request_or_response_headers,
+                                         caseUnorderedSetContainingUpgrade());
+      }
+    }
+
+    if (Utility::isUpgrade(request_or_response_headers)) {
       ENVOY_CONN_LOG(trace, "codec entering upgrade mode.", connection_);
       handling_upgrade_ = true;
     }

source/common/http/http1/settings.cc

@@ -2,6 +2,7 @@
 
 #include "envoy/http/header_formatter.h"
 
+#include "source/common/common/matchers.h"
 #include "source/common/config/utility.h"
 #include "source/common/runtime/runtime_features.h"
 
@@ -10,6 +11,7 @@ namespace Http {
 namespace Http1 {
 
 Http1Settings parseHttp1Settings(const envoy::config::core::v3::Http1ProtocolOptions& config,
+                                 Server::Configuration::CommonFactoryContext& context,
                                  ProtobufMessage::ValidationVisitor& validation_visitor) {
   Http1Settings ret;
   ret.allow_absolute_url_ = PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, allow_absolute_url, true);
@@ -19,6 +21,18 @@ Http1Settings parseHttp1Settings(const envoy::config::core::v3::Http1ProtocolOpt
   ret.enable_trailers_ = config.enable_trailers();
   ret.allow_chunked_length_ = config.allow_chunked_length();
 
+  if (!config.ignore_http_11_upgrade().empty()) {
+    std::vector<Matchers::StringMatcherPtr> matchers;
+    for (const auto& matcher : config.ignore_http_11_upgrade()) {
+      matchers.emplace_back(
+          std::make_unique<
+              Envoy::Matchers::StringMatcherImpl<envoy::type::matcher::v3::StringMatcher>>(
+              matcher, context));
+    }
+    ret.ignore_upgrade_matchers_ =
+        std::make_shared<const std::vector<Matchers::StringMatcherPtr>>(std::move(matchers));
+  }
+
   if (config.header_key_format().has_proper_case_words()) {
     ret.header_key_format_ = Http1Settings::HeaderKeyFormat::ProperCase;
   } else if (config.header_key_format().has_stateful_formatter()) {
@@ -45,10 +59,11 @@ Http1Settings parseHttp1Settings(const envoy::config::core::v3::Http1ProtocolOpt
 }
 
 Http1Settings parseHttp1Settings(const envoy::config::core::v3::Http1ProtocolOptions& config,
+                                 Server::Configuration::CommonFactoryContext& context,
                                  ProtobufMessage::ValidationVisitor& validation_visitor,
                                  const ProtobufWkt::BoolValue& hcm_stream_error,
                                  bool validate_scheme) {
-  Http1Settings ret = parseHttp1Settings(config, validation_visitor);
+  Http1Settings ret = parseHttp1Settings(config, context, validation_visitor);
   ret.validate_scheme_ = validate_scheme;
 
   if (config.has_override_stream_error_on_invalid_http_message()) {

source/common/http/http1/settings.h

@@ -3,6 +3,7 @@
 #include "envoy/config/core/v3/protocol.pb.h"
 #include "envoy/http/codec.h"
 #include "envoy/protobuf/message_validator.h"
+#include "envoy/server/factory_context.h"
 
 namespace Envoy {
 namespace Http {
@@ -13,9 +14,11 @@ namespace Http1 {
  * envoy::config::core::v3::Http1ProtocolOptions config.
  */
 Http1Settings parseHttp1Settings(const envoy::config::core::v3::Http1ProtocolOptions& config,
+                                 Server::Configuration::CommonFactoryContext& context,
                                  ProtobufMessage::ValidationVisitor& validation_visitor);
 
 Http1Settings parseHttp1Settings(const envoy::config::core::v3::Http1ProtocolOptions& config,
+                                 Server::Configuration::CommonFactoryContext& context,
                                  ProtobufMessage::ValidationVisitor& validation_visitor,
                                  const ProtobufWkt::BoolValue& hcm_stream_error,
                                  bool validate_scheme);

source/common/http/utility.cc

@@ -649,6 +649,41 @@ bool Utility::isWebSocketUpgradeRequest(const RequestHeaderMap& headers) {
                                  Http::Headers::get().UpgradeValues.WebSocket));
 }
 
+void Utility::removeUpgrade(RequestOrResponseHeaderMap& headers,
+                            const std::vector<Matchers::StringMatcherPtr>& matchers) {
+  if (headers.Upgrade()) {
+    std::vector<absl::string_view> tokens =
+        Envoy::StringUtil::splitToken(headers.getUpgradeValue(), ",", false, true);
+
+    auto end = std::remove_if(tokens.begin(), tokens.end(), [&](absl::string_view token) {
+      return std::any_of(
+          matchers.begin(), matchers.end(),
+          [&token](const Matchers::StringMatcherPtr& matcher) { return matcher->match(token); });
+    });
+
+    const std::string new_value = absl::StrJoin(tokens.begin(), end, ",");
+
+    if (new_value.empty()) {
+      headers.removeUpgrade();
+    } else {
+      headers.setUpgrade(new_value);
+    }
+  }
+}
+
+void Utility::removeConnectionUpgrade(RequestOrResponseHeaderMap& headers,
+                                      StringUtil::CaseUnorderedSet tokens_to_remove) {
+  if (headers.Connection()) {
+    const std::string new_value =
+        StringUtil::removeTokens(headers.getConnectionValue(), ",", tokens_to_remove, ",");
+    if (new_value.empty()) {
+      headers.removeConnection();
+    } else {
+      headers.setConnection(new_value);
+    }
+  }
+}
+
 Utility::PreparedLocalReplyPtr Utility::prepareLocalReply(const EncodeFunctions& encode_functions,
                                                           const LocalReplyData& local_reply_data) {
   Code response_code = local_reply_data.response_code_;

source/common/http/utility.h

@@ -315,6 +315,20 @@ bool isH3UpgradeRequest(const RequestHeaderMap& headers);
  */
 bool isWebSocketUpgradeRequest(const RequestHeaderMap& headers);
 
+/**
+ * Removes tokens from `Upgrade` header matching one of the matchers. Removes the `Upgrade`
+ * header if result is empty.
+ */
+void removeUpgrade(RequestOrResponseHeaderMap& headers,
+                   const std::vector<Matchers::StringMatcherPtr>& matchers);
+
+/**
+ * Removes `tokens_to_remove` from the `Connection` header, if present and part of a comma separated
+ * set of values. Removes the `Connection` header if it only contains `tokens_to_remove`.
+ */
+void removeConnectionUpgrade(RequestOrResponseHeaderMap& headers,
+                             StringUtil::CaseUnorderedSet tokens_to_remove);
+
 struct EncodeFunctions {
   // Function to modify locally generated response headers.
   std::function<void(ResponseHeaderMap& headers)> modify_headers_;

source/common/upstream/upstream_impl.cc

@@ -1032,7 +1032,7 @@ ClusterInfoImpl::generateTimeoutBudgetStats(Stats::Scope& scope,
 absl::StatusOr<std::shared_ptr<const ClusterInfoImpl::HttpProtocolOptionsConfigImpl>>
 createOptions(const envoy::config::cluster::v3::Cluster& config,
               std::shared_ptr<const ClusterInfoImpl::HttpProtocolOptionsConfigImpl>&& options,
-              ProtobufMessage::ValidationVisitor& validation_visitor) {
+              Server::Configuration::ProtocolOptionsFactoryContext& factory_context) {
   if (options) {
     return std::move(options);
   }
@@ -1056,7 +1056,8 @@ createOptions(const envoy::config::cluster::v3::Cluster& config,
                : absl::nullopt),
           config.protocol_selection() ==
               envoy::config::cluster::v3::Cluster::USE_DOWNSTREAM_PROTOCOL,
-          config.has_http2_protocol_options(), validation_visitor);
+          config.has_http2_protocol_options(), factory_context.serverFactoryContext(),
+          factory_context.messageValidationVisitor());
   RETURN_IF_NOT_OK_REF(options_or_error.status());
   return options_or_error.value();
 }
@@ -1170,7 +1171,7 @@ ClusterInfoImpl::ClusterInfoImpl(
           createOptions(config,
                         extensionProtocolOptionsTyped<HttpProtocolOptionsConfigImpl>(
                             "envoy.extensions.upstreams.http.v3.HttpProtocolOptions"),
-                        factory_context.messageValidationVisitor()),
+                        factory_context),
           std::shared_ptr<const ClusterInfoImpl::HttpProtocolOptionsConfigImpl>)),
       tcp_protocol_options_(extensionProtocolOptionsTyped<TcpProtocolOptionsConfigImpl>(
           "envoy.extensions.upstreams.tcp.v3.TcpProtocolOptions")),

source/extensions/filters/network/http_connection_manager/config.cc

@@ -371,8 +371,8 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig(
           config.http3_protocol_options(), config.has_stream_error_on_invalid_http_message(),
           config.stream_error_on_invalid_http_message())),
       http1_settings_(Http::Http1::parseHttp1Settings(
-          config.http_protocol_options(), context.messageValidationVisitor(),
-          config.stream_error_on_invalid_http_message(),
+          config.http_protocol_options(), context.serverFactoryContext(),
+          context.messageValidationVisitor(), config.stream_error_on_invalid_http_message(),
           xff_num_trusted_hops_ == 0 && use_remote_address_)),
       max_request_headers_kb_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(
           config, max_request_headers_kb,

source/extensions/upstreams/http/config.cc

@@ -206,12 +206,13 @@ ProtocolOptionsConfigImpl::createProtocolOptionsConfig(
     const envoy::config::core::v3::HttpProtocolOptions& common_options,
     const absl::optional<envoy::config::core::v3::UpstreamHttpProtocolOptions> upstream_options,
     bool use_downstream_protocol, bool use_http2,
+    Server::Configuration::ServerFactoryContext& server_context,
     ProtobufMessage::ValidationVisitor& validation_visitor) {
   auto options_or_error = Http2::Utility::initializeAndValidateOptions(http2_options);
   RETURN_IF_NOT_OK_REF(options_or_error.status());
   return std::shared_ptr<ProtocolOptionsConfigImpl>(new ProtocolOptionsConfigImpl(
       http1_settings, options_or_error.value(), common_options, upstream_options,
-      use_downstream_protocol, use_http2, validation_visitor));
+      use_downstream_protocol, use_http2, server_context, validation_visitor));
 }
 
 ProtocolOptionsConfigImpl::ProtocolOptionsConfigImpl(
@@ -221,7 +222,7 @@ ProtocolOptionsConfigImpl::ProtocolOptionsConfigImpl(
     absl::optional<const envoy::config::core::v3::AlternateProtocolsCacheOptions> cache_options,
     Server::Configuration::ServerFactoryContext& server_context)
     : http1_settings_(Envoy::Http::Http1::parseHttp1Settings(
-          getHttpOptions(options), server_context.messageValidationVisitor())),
+          getHttpOptions(options), server_context, server_context.messageValidationVisitor())),
       http2_options_(std::move(http2_options)), http3_options_(getHttp3Options(options)),
       common_http_protocol_options_(options.common_http_protocol_options()),
       upstream_http_protocol_options_(
@@ -244,8 +245,10 @@ ProtocolOptionsConfigImpl::ProtocolOptionsConfigImpl(
     const envoy::config::core::v3::HttpProtocolOptions& common_options,
     const absl::optional<envoy::config::core::v3::UpstreamHttpProtocolOptions> upstream_options,
     bool use_downstream_protocol, bool use_http2,
+    Server::Configuration::ServerFactoryContext& server_context,
     ProtobufMessage::ValidationVisitor& validation_visitor)
-    : http1_settings_(Envoy::Http::Http1::parseHttp1Settings(http1_settings, validation_visitor)),
+    : http1_settings_(Envoy::Http::Http1::parseHttp1Settings(http1_settings, server_context,
+                                                             validation_visitor)),
       http2_options_(validated_http2_options), common_http_protocol_options_(common_options),
       upstream_http_protocol_options_(upstream_options),
       use_downstream_protocol_(use_downstream_protocol), use_http2_(use_http2) {

source/extensions/upstreams/http/config.h

@@ -35,6 +35,7 @@ class ProtocolOptionsConfigImpl : public Upstream::ProtocolOptionsConfig {
       const envoy::config::core::v3::HttpProtocolOptions& common_options,
       const absl::optional<envoy::config::core::v3::UpstreamHttpProtocolOptions> upstream_options,
       bool use_downstream_protocol, bool use_http2,
+      Server::Configuration::ServerFactoryContext& server_context,
       ProtobufMessage::ValidationVisitor& validation_visitor);
 
   // Given the supplied cluster config, and protocol options configuration,
@@ -74,6 +75,7 @@ class ProtocolOptionsConfigImpl : public Upstream::ProtocolOptionsConfig {
       const envoy::config::core::v3::HttpProtocolOptions& common_options,
       const absl::optional<envoy::config::core::v3::UpstreamHttpProtocolOptions> upstream_options,
       bool use_downstream_protocol, bool use_http2,
+      Server::Configuration::ServerFactoryContext& server_context,
       ProtobufMessage::ValidationVisitor& validation_visitor);
 };
 

test/common/http/BUILD

@@ -442,6 +442,7 @@ envoy_cc_test(
         "//source/common/http:utility_lib",
         "//source/common/network:address_lib",
         "//test/mocks/http:http_mocks",
+        "//test/mocks/server:factory_context_mocks",
         "//test/mocks/upstream:upstream_mocks",
         "//test/test_common:test_runtime_lib",
         "//test/test_common:utility_lib",

test/common/http/http1/BUILD

@@ -39,6 +39,7 @@ envoy_cc_test(
         "//test/mocks/local_info:local_info_mocks",
         "//test/mocks/network:network_mocks",
         "//test/mocks/protobuf:protobuf_mocks",
+        "//test/mocks/server:factory_context_mocks",
         "//test/mocks/server:overload_manager_mocks",
         "//test/mocks/stream_info:stream_info_mocks",
         "//test/mocks/thread_local:thread_local_mocks",