diff --git a/.bazelversion b/.bazelversion index 0b2eb36f50859..ee74734aa2258 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -3.7.2 +4.1.0 diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 6d5ddefaf2158..119ae1ccf1614 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -75,3 +75,13 @@ updates: directory: "/tools/testing" schedule: interval: "daily" + + - package-ecosystem: "docker" + directory: "/ci" + schedule: + interval: daily + + - package-ecosystem: "docker" + directory: "/.devcontainer" + schedule: + interval: daily diff --git a/CODEOWNERS b/CODEOWNERS index 82b28a9924e74..2fc70c0398b79 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -183,6 +183,10 @@ extensions/filters/http/oauth2 @rgs1 @derekargueta @snowp /*/extensions/filters/common/ext_authz @esmet @gsagula @dio /*/extensions/filters/http/ext_authz @esmet @gsagula @dio /*/extensions/filters/network/ext_authz @esmet @gsagula @dio +# HTTP Bandwidth Limit +/*/extensions/filters/http/bandwidth_limit @nitgoy @mattklein123 @yanavlasov @tonya11en # Original IP detection /*/extensions/http/original_ip_detection/custom_header @rgs1 @alyssawilk @antoniovicente /*/extensions/http/original_ip_detection/xff @rgs1 @alyssawilk @antoniovicente +# set_metadata extension +/*/extensions/filters/http/set_metadata @aguinet @snowp diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0190c14a4eb32..4f73d3353d7af 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -259,6 +259,7 @@ API can be found [here](api/STYLE.md#adding-an-extension-configuration-to-the-ap Other changes will likely include * Editing [source/extensions/extensions_build_config.bzl](source/extensions/extensions_build_config.bzl) to include the new extensions + * Editing [source/extensions/extensions_metadata.yaml](source/extensions/extensions_metadata.yaml) to include metadata for the new extensions * Editing [docs/root/api-v3/config/config.rst](docs/root/api-v3/config/config.rst) to add area/area * Adding `docs/root/api-v3/config/area/area.rst` to add a table of contents for the API docs * Adding `source/extensions/area/well_known_names.h` for registered plugins diff --git a/DEVELOPER.md b/DEVELOPER.md index 7a4ec69f54c6b..1fa2c5076de44 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -19,6 +19,8 @@ Below is a list of additional documentation to aid the development process: - [Guide to Envoy Bazel rules (managing `BUILD` files)](https://github.com/envoyproxy/envoy/blob/main/bazel/DEVELOPER.md) +- [Guide to setup development environment with Visual Studio Code](https://github.com/envoyproxy/envoy/blob/main/tools/vscode/README.md) + - [Using Docker for building and testing](https://github.com/envoyproxy/envoy/tree/main/ci) - [Guide to contributing to Envoy](https://github.com/envoyproxy/envoy/blob/main/CONTRIBUTING.md) diff --git a/SECURITY.md b/SECURITY.md index 8506ae40cfc9e..22e7c92f60624 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -449,22 +449,23 @@ and security team to ensure they still qualify for inclusion on the list. ### Members -| E-mail | Organization | End User | Last Review | -|-------------------------------------------------------|:-------------:|:--------:|:-----------:| -| envoy-security-team@aspenmesh.io | Aspen Mesh | No | 12/19 | -| aws-app-mesh-security@amazon.com | AWS | No | 12/19 | -| security@cilium.io | Cilium | No | 12/19 | -| vulnerabilityreports@cloudfoundry.org | Cloud Foundry | No | 12/19 | -| secalert@datawire.io | Datawire | No | 12/19 | -| google-internal-envoy-security@google.com | Google | No | 12/19 | -| argoprod@us.ibm.com | IBM | No | 12/19 | -| istio-security-vulnerability-reports@googlegroups.com | Istio | No | 12/19 | -| envoy-security@microsoft.com | Microsoft | No | 2/21 | -| secalert@redhat.com | Red Hat | No | 12/19 | -| envoy-security@solo.io | solo.io | No | 12/19 | -| envoy-security@tetrate.io | Tetrate | No | 12/19 | -| security@vmware.com | VMware | No | 12/19 | -| envoy-security@pinterest.com | Pinterest | Yes | 12/19 | -| envoy-security@dropbox.com | Dropbox | Yes | 01/20 | -| envoy-security-predisclosure@stripe.com | Stripe | Yes | 01/20 | -| envoy-security@squareup.com | Square | Yes | 05/21 | +| Organization | End User | Last Review | +|:-------------:|:--------:|:-----------:| +| Aspen Mesh | No | 12/19 | +| AWS | No | 12/19 | +| Cilium | No | 12/19 | +| Cloud Foundry | No | 12/19 | +| Datawire | No | 12/19 | +| Google | No | 12/19 | +| IBM | No | 12/19 | +| Istio | No | 12/19 | +| Microsoft | No | 2/21 | +| Red Hat | No | 12/19 | +| solo.io | No | 12/19 | +| Tetrate | No | 12/19 | +| VMware | No | 12/19 | +| Pinterest | Yes | 12/19 | +| Dropbox | Yes | 01/20 | +| Stripe | Yes | 01/20 | +| Square | Yes | 05/21 | +| Apple | Yes | 05/21 | diff --git a/api/BUILD b/api/BUILD index 04b94ff211fd4..e051250b80753 100644 --- a/api/BUILD +++ b/api/BUILD @@ -177,6 +177,7 @@ proto_library( "//envoy/extensions/filters/http/admission_control/v3alpha:pkg", "//envoy/extensions/filters/http/aws_lambda/v3:pkg", "//envoy/extensions/filters/http/aws_request_signing/v3:pkg", + "//envoy/extensions/filters/http/bandwidth_limit/v3alpha:pkg", "//envoy/extensions/filters/http/buffer/v3:pkg", "//envoy/extensions/filters/http/cache/v3alpha:pkg", "//envoy/extensions/filters/http/cdn_loop/v3alpha:pkg", @@ -209,6 +210,7 @@ proto_library( "//envoy/extensions/filters/http/ratelimit/v3:pkg", "//envoy/extensions/filters/http/rbac/v3:pkg", "//envoy/extensions/filters/http/router/v3:pkg", + "//envoy/extensions/filters/http/set_metadata/v3:pkg", "//envoy/extensions/filters/http/squash/v3:pkg", "//envoy/extensions/filters/http/tap/v3:pkg", "//envoy/extensions/filters/http/wasm/v3:pkg", diff --git a/api/STYLE.md b/api/STYLE.md index 18d96fd4ae47a..d73e17b773b24 100644 --- a/api/STYLE.md +++ b/api/STYLE.md @@ -113,11 +113,11 @@ organization](#package-organization) above. To add an extension config to the API, the steps below should be followed: 1. If this is still WiP and subject to breaking changes, use `vNalpha` instead of `vN` in steps - below. Refer to the [Cache filter config](envoy/extensions/filter/http/cache/v3alpha/cache.proto) + below. Refer to the [Cache filter config](envoy/extensions/filters/http/cache/v3alpha/cache.proto) as an example of `v3alpha`, and the - [Buffer filter config](envoy/extensions/filter/http/buffer/v3/buffer.proto) as an example of `v3`. + [Buffer filter config](envoy/extensions/filters/http/buffer/v3/buffer.proto) as an example of `v3`. 1. Place the v3 extension configuration `.proto` in `api/envoy/extensions`, e.g. - `api/envoy/extensions/filter/http/foobar/v3/foobar.proto` together with an initial BUILD file: + `api/envoy/extensions/filters/http/foobar/v3/foobar.proto` together with an initial BUILD file: ```bazel load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") diff --git a/api/envoy/config/core/v3/resolver.proto b/api/envoy/config/core/v3/resolver.proto new file mode 100644 index 0000000000000..b5e31814f56aa --- /dev/null +++ b/api/envoy/config/core/v3/resolver.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; + +package envoy.config.core.v3; + +import "envoy/config/core/v3/address.proto"; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.core.v3"; +option java_outer_classname = "ResolverProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Resolver] + +// DNS resolver configuration which includes the underlying dns resolver addresses and options. +message DnsResolver { + // A list of dns resolver addresses + // Setting this value causes failure if the + // ``envoy.restart_features.use_apple_api_for_dns_lookups`` runtime value is true during + // server startup. Apple's API only allows overriding DNS resolvers via system settings. + repeated Address resolvers = 1 [(validate.rules).repeated = {min_items: 1}]; +} diff --git a/api/envoy/config/core/v4alpha/resolver.proto b/api/envoy/config/core/v4alpha/resolver.proto new file mode 100644 index 0000000000000..ad597c85549de --- /dev/null +++ b/api/envoy/config/core/v4alpha/resolver.proto @@ -0,0 +1,27 @@ +syntax = "proto3"; + +package envoy.config.core.v4alpha; + +import "envoy/config/core/v4alpha/address.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.core.v4alpha"; +option java_outer_classname = "ResolverProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; + +// [#protodoc-title: Resolver] + +// DNS resolver configuration which includes the underlying dns resolver addresses and options. +message DnsResolver { + option (udpa.annotations.versioning).previous_message_type = "envoy.config.core.v3.DnsResolver"; + + // A list of dns resolver addresses + // Setting this value causes failure if the + // ``envoy.restart_features.use_apple_api_for_dns_lookups`` runtime value is true during + // server startup. Apple's API only allows overriding DNS resolvers via system settings. + repeated Address resolvers = 1 [(validate.rules).repeated = {min_items: 1}]; +} diff --git a/api/envoy/config/listener/v3/quic_config.proto b/api/envoy/config/listener/v3/quic_config.proto index 69df722c6fbb4..d1e62cdaaf15a 100644 --- a/api/envoy/config/listener/v3/quic_config.proto +++ b/api/envoy/config/listener/v3/quic_config.proto @@ -6,9 +6,11 @@ import "envoy/config/core/v3/base.proto"; import "envoy/config/core/v3/protocol.proto"; import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; import "udpa/annotations/status.proto"; import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.config.listener.v3"; option java_outer_classname = "QuicConfigProto"; @@ -18,6 +20,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: QUIC listener config] // Configuration specific to the UDP QUIC listener. +// [#next-free-field: 6] message QuicProtocolOptions { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.listener.QuicProtocolOptions"; @@ -35,4 +38,14 @@ message QuicProtocolOptions { // Runtime flag that controls whether the listener is enabled or not. If not specified, defaults // to enabled. core.v3.RuntimeFeatureFlag enabled = 4; + + // A multiplier to number of connections which is used to determine how many packets to read per + // event loop. A reasonable number should allow the listener to process enough payload but not + // starve TCP and other UDP sockets and also prevent long event loop duration. + // The default value is 32. This means if there are N QUIC connections, the total number of + // packets to read in each read event will be 32 * N. + // The actual number of packets to read in total by the UDP listener is also + // bound by 6000, regardless of this field or how many connections there are. + google.protobuf.UInt32Value packets_to_read_to_connection_count_ratio = 5 + [(validate.rules).uint32 = {gte: 1}]; } diff --git a/api/envoy/config/listener/v4alpha/quic_config.proto b/api/envoy/config/listener/v4alpha/quic_config.proto index c9e218137ae27..6d0f5e51493b6 100644 --- a/api/envoy/config/listener/v4alpha/quic_config.proto +++ b/api/envoy/config/listener/v4alpha/quic_config.proto @@ -6,9 +6,11 @@ import "envoy/config/core/v4alpha/base.proto"; import "envoy/config/core/v4alpha/protocol.proto"; import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; import "udpa/annotations/status.proto"; import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.config.listener.v4alpha"; option java_outer_classname = "QuicConfigProto"; @@ -18,6 +20,7 @@ option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSIO // [#protodoc-title: QUIC listener config] // Configuration specific to the UDP QUIC listener. +// [#next-free-field: 6] message QuicProtocolOptions { option (udpa.annotations.versioning).previous_message_type = "envoy.config.listener.v3.QuicProtocolOptions"; @@ -35,4 +38,14 @@ message QuicProtocolOptions { // Runtime flag that controls whether the listener is enabled or not. If not specified, defaults // to enabled. core.v4alpha.RuntimeFeatureFlag enabled = 4; + + // A multiplier to number of connections which is used to determine how many packets to read per + // event loop. A reasonable number should allow the listener to process enough payload but not + // starve TCP and other UDP sockets and also prevent long event loop duration. + // The default value is 32. This means if there are N QUIC connections, the total number of + // packets to read in each read event will be 32 * N. + // The actual number of packets to read in total by the UDP listener is also + // bound by 6000, regardless of this field or how many connections there are. + google.protobuf.UInt32Value packets_to_read_to_connection_count_ratio = 5 + [(validate.rules).uint32 = {gte: 1}]; } diff --git a/api/envoy/config/route/v3/route_components.proto b/api/envoy/config/route/v3/route_components.proto index 58ac31d41515d..ee82e8f732261 100644 --- a/api/envoy/config/route/v3/route_components.proto +++ b/api/envoy/config/route/v3/route_components.proto @@ -1885,8 +1885,8 @@ message HeaderMatcher { // "-1somestring" type.v3.Int64Range range_match = 6; - // If specified, header match will be performed based on whether the header is in the - // request. + // If specified as true, header match will be performed based on whether the header is in the + // request. If specified as false, header match will be performed based on whether the header is absent. bool present_match = 7; // If specified, header match will be performed based on the prefix of the header value. diff --git a/api/envoy/config/route/v4alpha/route_components.proto b/api/envoy/config/route/v4alpha/route_components.proto index ce3b1c479969a..256a3c742ff3a 100644 --- a/api/envoy/config/route/v4alpha/route_components.proto +++ b/api/envoy/config/route/v4alpha/route_components.proto @@ -1837,8 +1837,8 @@ message HeaderMatcher { // "-1somestring" type.v3.Int64Range range_match = 6; - // If specified, header match will be performed based on whether the header is in the - // request. + // If specified as true, header match will be performed based on whether the header is in the + // request. If specified as false, header match will be performed based on whether the header is absent. bool present_match = 7; // If specified, header match will be performed based on the prefix of the header value. diff --git a/api/envoy/extensions/common/dynamic_forward_proxy/v3/BUILD b/api/envoy/extensions/common/dynamic_forward_proxy/v3/BUILD index b7a0695c214bc..6e74b985c580d 100644 --- a/api/envoy/extensions/common/dynamic_forward_proxy/v3/BUILD +++ b/api/envoy/extensions/common/dynamic_forward_proxy/v3/BUILD @@ -8,6 +8,7 @@ api_proto_package( deps = [ "//envoy/config/cluster/v3:pkg", "//envoy/config/common/dynamic_forward_proxy/v2alpha:pkg", + "//envoy/config/core/v3:pkg", "@com_github_cncf_udpa//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/common/dynamic_forward_proxy/v3/dns_cache.proto b/api/envoy/extensions/common/dynamic_forward_proxy/v3/dns_cache.proto index 4b182a29711bd..1c14739b94c86 100644 --- a/api/envoy/extensions/common/dynamic_forward_proxy/v3/dns_cache.proto +++ b/api/envoy/extensions/common/dynamic_forward_proxy/v3/dns_cache.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package envoy.extensions.common.dynamic_forward_proxy.v3; import "envoy/config/cluster/v3/cluster.proto"; +import "envoy/config/core/v3/resolver.proto"; import "google/protobuf/duration.proto"; import "google/protobuf/wrappers.proto"; @@ -27,7 +28,7 @@ message DnsCacheCircuitBreakers { // Configuration for the dynamic forward proxy DNS cache. See the :ref:`architecture overview // ` for more information. -// [#next-free-field: 9] +// [#next-free-field: 10] message DnsCacheConfig { option (udpa.annotations.versioning).previous_message_type = "envoy.config.common.dynamic_forward_proxy.v2alpha.DnsCacheConfig"; @@ -101,4 +102,9 @@ message DnsCacheConfig { // ``envoy.restart_features.use_apple_api_for_dns_lookups`` runtime value is true during // server startup. Apple' API only uses UDP for DNS resolution. bool use_tcp_for_dns_lookups = 8; + + // DNS resolver configuration + // If specified, DNS cache will perform resolution via the underlying DNS resolvers. + // Otherwise, the default system resolvers (e.g., /etc/resolv.conf) will be used. + config.core.v3.DnsResolver dns_resolver = 9; } diff --git a/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/BUILD b/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/BUILD new file mode 100644 index 0000000000000..1c1a6f6b44235 --- /dev/null +++ b/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/BUILD @@ -0,0 +1,12 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/config/core/v3:pkg", + "@com_github_cncf_udpa//udpa/annotations:pkg", + ], +) diff --git a/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto b/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto new file mode 100644 index 0000000000000..4cd5f8268b704 --- /dev/null +++ b/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto @@ -0,0 +1,70 @@ +syntax = "proto3"; + +package envoy.extensions.filters.http.bandwidth_limit.v3alpha; + +import "envoy/config/core/v3/base.proto"; + +import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.filters.http.bandwidth_limit.v3alpha"; +option java_outer_classname = "BandwidthLimitProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).work_in_progress = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Bandwidth limit] +// Bandwidth limit :ref:`configuration overview `. +// [#extension: envoy.filters.http.bandwidth_limit] + +// [#next-free-field: 6] +message BandwidthLimit { + // Defines the mode for the bandwidth limit filter. + // Values represent bitmask. + enum EnableMode { + // Filter is disabled. + DISABLED = 0; + + // Filter enabled only for incoming traffic. + REQUEST = 1; + + // Filter enabled only for outgoing traffic. + RESPONSE = 2; + + // Filter enabled for both incoming and outgoing traffic. + REQUEST_AND_RESPONSE = 3; + } + + // The human readable prefix to use when emitting stats. + string stat_prefix = 1 [(validate.rules).string = {min_len: 1}]; + + // The enable mode for the bandwidth limit filter. + // Default is Disabled. + EnableMode enable_mode = 2 [(validate.rules).enum = {defined_only: true}]; + + // The limit supplied in KiB/s. + // + // .. note:: + // It's fine for the limit to be unset for the global configuration since the bandwidth limit + // can be applied at a the virtual host or route level. Thus, the limit must be set for the + // per route configuration otherwise the config will be rejected. + // + // .. note:: + // When using per route configuration, the limit becomes unique to that route. + // + google.protobuf.UInt64Value limit_kbps = 3 [(validate.rules).uint64 = {gte: 1}]; + + // Optional Fill interval in milliseconds for the token refills. Defaults to 50ms. + // It must be at least 20ms to avoid too aggressive refills. + google.protobuf.Duration fill_interval = 4 [(validate.rules).duration = { + lte {seconds: 1} + gte {nanos: 20000000} + }]; + + // Runtime flag that controls whether the filter is enabled or not. If not specified, defaults + // to enabled. + config.core.v3.RuntimeFeatureFlag runtime_enabled = 5; +} diff --git a/api/envoy/extensions/filters/http/jwt_authn/v3/config.proto b/api/envoy/extensions/filters/http/jwt_authn/v3/config.proto index 08ef7a09feb20..afc761c07c7e1 100644 --- a/api/envoy/extensions/filters/http/jwt_authn/v3/config.proto +++ b/api/envoy/extensions/filters/http/jwt_authn/v3/config.proto @@ -232,6 +232,35 @@ message RemoteJwks { // Duration after which the cached JWKS should be expired. If not specified, default cache // duration is 5 minutes. google.protobuf.Duration cache_duration = 2; + + // Fetch Jwks asynchronously in the main thread before the listener is activated. + // Fetched Jwks can be used by all worker threads. + // + // If this feature is not enabled: + // + // * The Jwks is fetched on-demand when the requests come. During the fetching, first + // few requests are paused until the Jwks is fetched. + // * Each worker thread fetches its own Jwks since Jwks cache is per worker thread. + // + // If this feature is enabled: + // + // * Fetched Jwks is done in the main thread before the listener is activated. Its fetched + // Jwks can be used by all worker threads. Each worker thread doesn't need to fetch its own. + // * Jwks is ready when the requests come, not need to wait for the Jwks fetching. + // + JwksAsyncFetch async_fetch = 3; +} + +// Fetch Jwks asynchronously in the main thread when the filter config is parsed. +// The listener is activated only after the Jwks is fetched. +// When the Jwks is expired in the cache, it is fetched again in the main thread. +// The fetched Jwks from the main thread can be used by all worker threads. +message JwksAsyncFetch { + // If false, the listener is activated after the initial fetch is completed. + // The initial fetch result can be either successful or failed. + // If true, it is activated without waiting for the initial fetch to complete. + // Default is false. + bool fast_listener = 1; } // This message specifies a header location to extract JWT token. diff --git a/api/envoy/extensions/filters/http/jwt_authn/v4alpha/config.proto b/api/envoy/extensions/filters/http/jwt_authn/v4alpha/config.proto index 7656f09912e9b..442ba7df061ee 100644 --- a/api/envoy/extensions/filters/http/jwt_authn/v4alpha/config.proto +++ b/api/envoy/extensions/filters/http/jwt_authn/v4alpha/config.proto @@ -232,6 +232,38 @@ message RemoteJwks { // Duration after which the cached JWKS should be expired. If not specified, default cache // duration is 5 minutes. google.protobuf.Duration cache_duration = 2; + + // Fetch Jwks asynchronously in the main thread before the listener is activated. + // Fetched Jwks can be used by all worker threads. + // + // If this feature is not enabled: + // + // * The Jwks is fetched on-demand when the requests come. During the fetching, first + // few requests are paused until the Jwks is fetched. + // * Each worker thread fetches its own Jwks since Jwks cache is per worker thread. + // + // If this feature is enabled: + // + // * Fetched Jwks is done in the main thread before the listener is activated. Its fetched + // Jwks can be used by all worker threads. Each worker thread doesn't need to fetch its own. + // * Jwks is ready when the requests come, not need to wait for the Jwks fetching. + // + JwksAsyncFetch async_fetch = 3; +} + +// Fetch Jwks asynchronously in the main thread when the filter config is parsed. +// The listener is activated only after the Jwks is fetched. +// When the Jwks is expired in the cache, it is fetched again in the main thread. +// The fetched Jwks from the main thread can be used by all worker threads. +message JwksAsyncFetch { + option (udpa.annotations.versioning).previous_message_type = + "envoy.extensions.filters.http.jwt_authn.v3.JwksAsyncFetch"; + + // If false, the listener is activated after the initial fetch is completed. + // The initial fetch result can be either successful or failed. + // If true, it is activated without waiting for the initial fetch to complete. + // Default is false. + bool fast_listener = 1; } // This message specifies a header location to extract JWT token. diff --git a/api/envoy/extensions/filters/http/local_ratelimit/v3/local_rate_limit.proto b/api/envoy/extensions/filters/http/local_ratelimit/v3/local_rate_limit.proto index 7766ee2573d00..1cf6c5f2fa52c 100644 --- a/api/envoy/extensions/filters/http/local_ratelimit/v3/local_rate_limit.proto +++ b/api/envoy/extensions/filters/http/local_ratelimit/v3/local_rate_limit.proto @@ -19,7 +19,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // Local Rate limit :ref:`configuration overview `. // [#extension: envoy.filters.http.local_ratelimit] -// [#next-free-field: 11] +// [#next-free-field: 12] message LocalRateLimit { // The human readable prefix to use when emitting stats. string stat_prefix = 1 [(validate.rules).string = {min_len: 1}]; @@ -97,4 +97,13 @@ message LocalRateLimit { // // The filter supports a range of 0 - 10 inclusively for stage numbers. uint32 stage = 9 [(validate.rules).uint32 = {lte: 10}]; + + // Specifies the scope of the rate limiter's token bucket. + // If set to false, the token bucket is shared across all worker threads, + // thus the rate limits are applied per Envoy process. + // If set to true, a token bucket is allocated for each connection. + // Thus the rate limits are applied per connection thereby allowing + // one to rate limit requests on a per connection basis. + // If unspecified, the default value is false. + bool local_rate_limit_per_downstream_connection = 11; } diff --git a/api/envoy/extensions/filters/http/set_metadata/v3/BUILD b/api/envoy/extensions/filters/http/set_metadata/v3/BUILD new file mode 100644 index 0000000000000..ee92fb652582e --- /dev/null +++ b/api/envoy/extensions/filters/http/set_metadata/v3/BUILD @@ -0,0 +1,9 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], +) diff --git a/api/envoy/extensions/filters/http/set_metadata/v3/set_metadata.proto b/api/envoy/extensions/filters/http/set_metadata/v3/set_metadata.proto new file mode 100644 index 0000000000000..7b2cf0e0965d8 --- /dev/null +++ b/api/envoy/extensions/filters/http/set_metadata/v3/set_metadata.proto @@ -0,0 +1,31 @@ +syntax = "proto3"; + +package envoy.extensions.filters.http.set_metadata.v3; + +import "google/protobuf/struct.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.filters.http.set_metadata.v3"; +option java_outer_classname = "SetMetadataProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Set-Metadata Filter] +// +// This filters adds or updates dynamic metadata with static data. +// +// [#extension: envoy.filters.http.set_metadata] + +message Config { + // The metadata namespace. + string metadata_namespace = 1 [(validate.rules).string = {min_len: 1}]; + + // The value to update the namespace with. See + // :ref:`the filter documentation ` for + // more information on how this value is merged with potentially existing + // ones. + google.protobuf.Struct value = 2; +} diff --git a/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto b/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto index df141cb13087b..856249c2a25ac 100644 --- a/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto +++ b/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto @@ -35,7 +35,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // HTTP connection manager :ref:`configuration overview `. // [#extension: envoy.filters.network.http_connection_manager] -// [#next-free-field: 47] +// [#next-free-field: 48] message HttpConnectionManager { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager"; @@ -702,6 +702,16 @@ message HttpConnectionManager { // ` // for details. PathNormalizationOptions path_normalization_options = 43; + + // Determines if trailing dot of the host should be removed from host/authority header before any + // processing of request by HTTP filters or routing. + // This affects the upstream host header. + // Without setting this option, incoming requests with host `example.com.` will not match against + // route with :ref:`domains` match set to `example.com`. Defaults to `false`. + // When the incoming request contains a host/authority header that includes a port number, + // setting this option will strip a trailing dot, if present, from the host section, + // leaving the port as is (e.g. host value `example.com.:443` will be updated to `example.com:443`). + bool strip_trailing_host_dot = 47; } // The configuration to customize local reply returned by Envoy. diff --git a/api/envoy/extensions/filters/network/http_connection_manager/v4alpha/http_connection_manager.proto b/api/envoy/extensions/filters/network/http_connection_manager/v4alpha/http_connection_manager.proto index 37f15c7d49633..c9f4333f3c7cb 100644 --- a/api/envoy/extensions/filters/network/http_connection_manager/v4alpha/http_connection_manager.proto +++ b/api/envoy/extensions/filters/network/http_connection_manager/v4alpha/http_connection_manager.proto @@ -33,7 +33,7 @@ option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSIO // HTTP connection manager :ref:`configuration overview `. // [#extension: envoy.filters.network.http_connection_manager] -// [#next-free-field: 47] +// [#next-free-field: 48] message HttpConnectionManager { option (udpa.annotations.versioning).previous_message_type = "envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager"; @@ -680,6 +680,16 @@ message HttpConnectionManager { // ` // for details. PathNormalizationOptions path_normalization_options = 43; + + // Determines if trailing dot of the host should be removed from host/authority header before any + // processing of request by HTTP filters or routing. + // This affects the upstream host header. + // Without setting this option, incoming requests with host `example.com.` will not match against + // route with :ref:`domains` match set to `example.com`. Defaults to `false`. + // When the incoming request contains a host/authority header that includes a port number, + // setting this option will strip a trailing dot, if present, from the host section, + // leaving the port as is (e.g. host value `example.com.:443` will be updated to `example.com:443`). + bool strip_trailing_host_dot = 47; } // The configuration to customize local reply returned by Envoy. diff --git a/api/envoy/extensions/transport_sockets/tls/v3/tls.proto b/api/envoy/extensions/transport_sockets/tls/v3/tls.proto index 44325bdbee6a4..02287de5875fb 100644 --- a/api/envoy/extensions/transport_sockets/tls/v3/tls.proto +++ b/api/envoy/extensions/transport_sockets/tls/v3/tls.proto @@ -216,8 +216,14 @@ message CommonTlsContext { // Configs for fetching TLS certificates via SDS API. Note SDS API allows certificates to be // fetched/refreshed over the network asynchronously with respect to the TLS handshake. + // + // The same number and types of certificates as :ref:`tls_certificates ` + // are valid in the the certificates fetched through this setting. + // + // If :ref:`tls_certificates ` + // is non-empty, this field is ignored. repeated SdsSecretConfig tls_certificate_sds_secret_configs = 6 - [(validate.rules).repeated = {max_items: 1}]; + [(validate.rules).repeated = {max_items: 2}]; // Certificate provider for fetching TLS certificates. // [#not-implemented-hide:] diff --git a/api/envoy/extensions/transport_sockets/tls/v4alpha/tls.proto b/api/envoy/extensions/transport_sockets/tls/v4alpha/tls.proto index 7ddf55dfa6fab..b92cae619dd9c 100644 --- a/api/envoy/extensions/transport_sockets/tls/v4alpha/tls.proto +++ b/api/envoy/extensions/transport_sockets/tls/v4alpha/tls.proto @@ -221,8 +221,14 @@ message CommonTlsContext { // Configs for fetching TLS certificates via SDS API. Note SDS API allows certificates to be // fetched/refreshed over the network asynchronously with respect to the TLS handshake. + // + // The same number and types of certificates as :ref:`tls_certificates ` + // are valid in the the certificates fetched through this setting. + // + // If :ref:`tls_certificates ` + // is non-empty, this field is ignored. repeated SdsSecretConfig tls_certificate_sds_secret_configs = 6 - [(validate.rules).repeated = {max_items: 1}]; + [(validate.rules).repeated = {max_items: 2}]; // Certificate provider for fetching TLS certificates. // [#not-implemented-hide:] diff --git a/api/versioning/BUILD b/api/versioning/BUILD index eac32a5cdfd38..90f48a6b33fe6 100644 --- a/api/versioning/BUILD +++ b/api/versioning/BUILD @@ -60,6 +60,7 @@ proto_library( "//envoy/extensions/filters/http/admission_control/v3alpha:pkg", "//envoy/extensions/filters/http/aws_lambda/v3:pkg", "//envoy/extensions/filters/http/aws_request_signing/v3:pkg", + "//envoy/extensions/filters/http/bandwidth_limit/v3alpha:pkg", "//envoy/extensions/filters/http/buffer/v3:pkg", "//envoy/extensions/filters/http/cache/v3alpha:pkg", "//envoy/extensions/filters/http/cdn_loop/v3alpha:pkg", @@ -92,6 +93,7 @@ proto_library( "//envoy/extensions/filters/http/ratelimit/v3:pkg", "//envoy/extensions/filters/http/rbac/v3:pkg", "//envoy/extensions/filters/http/router/v3:pkg", + "//envoy/extensions/filters/http/set_metadata/v3:pkg", "//envoy/extensions/filters/http/squash/v3:pkg", "//envoy/extensions/filters/http/tap/v3:pkg", "//envoy/extensions/filters/http/wasm/v3:pkg", diff --git a/bazel/envoy_library.bzl b/bazel/envoy_library.bzl index 0c4ae3a53a361..30d5106edb8e2 100644 --- a/bazel/envoy_library.bzl +++ b/bazel/envoy_library.bzl @@ -44,99 +44,12 @@ def envoy_basic_cc_library(name, deps = [], external_deps = [], **kargs): **kargs ) -# All Envoy extensions must be tagged with their security hardening stance with -# respect to downstream and upstream data plane threats. These are verbose -# labels intended to make clear the trust that operators may place in -# extensions. -EXTENSION_SECURITY_POSTURES = [ - # This extension is hardened against untrusted downstream traffic. It - # assumes that the upstream is trusted. - "robust_to_untrusted_downstream", - # This extension is hardened against both untrusted downstream and upstream - # traffic. - "robust_to_untrusted_downstream_and_upstream", - # This extension is not hardened and should only be used in deployments - # where both the downstream and upstream are trusted. - "requires_trusted_downstream_and_upstream", - # This is functionally equivalent to - # requires_trusted_downstream_and_upstream, but acts as a placeholder to - # allow us to identify extensions that need classifying. - "unknown", - # Not relevant to data plane threats, e.g. stats sinks. - "data_plane_agnostic", -] - -# Extension categories as defined by factories -EXTENSION_CATEGORIES = [ - "envoy.access_loggers", - "envoy.bootstrap", - "envoy.clusters", - "envoy.compression.compressor", - "envoy.compression.decompressor", - "envoy.filters.http", - "envoy.filters.http.cache", - "envoy.filters.listener", - "envoy.filters.network", - "envoy.filters.udp_listener", - "envoy.grpc_credentials", - "envoy.guarddog_actions", - "envoy.health_checkers", - "envoy.http.stateful_header_formatters", - "envoy.internal_redirect_predicates", - "envoy.io_socket", - "envoy.http.original_ip_detection", - "envoy.matching.common_inputs", - "envoy.matching.input_matchers", - "envoy.rate_limit_descriptors", - "envoy.request_id", - "envoy.resource_monitors", - "envoy.retry_host_predicates", - "envoy.retry_priorities", - "envoy.stats_sinks", - "envoy.thrift_proxy.filters", - "envoy.tracers", - "envoy.transport_sockets.downstream", - "envoy.transport_sockets.upstream", - "envoy.tls.cert_validator", - "envoy.upstreams", - "envoy.wasm.runtime", - "DELIBERATELY_OMITTED", -] - -EXTENSION_STATUS_VALUES = [ - # This extension is stable and is expected to be production usable. - "stable", - # This extension is functional but has not had substantial production burn - # time, use only with this caveat. - "alpha", - # This extension is work-in-progress. Functionality is incomplete and it is - # not intended for production use. - "wip", -] - def envoy_cc_extension( name, - security_posture, - category = None, - # Only set this for internal, undocumented extensions. - undocumented = False, - status = "stable", tags = [], extra_visibility = [], visibility = EXTENSION_CONFIG_VISIBILITY, **kwargs): - if not category: - fail("Category not set for %s" % name) - if type(category) == "string": - category = (category,) - for cat in category: - if cat not in EXTENSION_CATEGORIES: - fail("Unknown extension category for %s: %s" % - (name, cat)) - if security_posture not in EXTENSION_SECURITY_POSTURES: - fail("Unknown extension security posture: " + security_posture) - if status not in EXTENSION_STATUS_VALUES: - fail("Unknown extension status: " + status) if "//visibility:public" not in visibility: visibility = visibility + extra_visibility diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 106d7a1799184..e37ee8ea607e4 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -25,14 +25,14 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "bazel-toolchains", project_desc = "Bazel toolchain configs for RBE", project_url = "https://github.com/bazelbuild/bazel-toolchains", - version = "3.7.2", - sha256 = "1caf8584434d3e31be674067996be787cfa511fda2a0f05811131b588886477f", + version = "4.1.0", + sha256 = "179ec02f809e86abf56356d8898c8bd74069f1bd7c56044050c2cd3d79d0e024", strip_prefix = "bazel-toolchains-{version}", urls = [ "https://github.com/bazelbuild/bazel-toolchains/releases/download/{version}/bazel-toolchains-{version}.tar.gz", "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/{version}.tar.gz", ], - release_date = "2021-01-07", + release_date = "2021-05-21", use_category = ["build"], ), build_bazel_rules_apple = dict( @@ -65,11 +65,11 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "envoy-build-tools", project_desc = "Common build tools shared by the Envoy/UDPA ecosystem", project_url = "https://github.com/envoyproxy/envoy-build-tools", - version = "2d4bdba38113cd9bf758c2609f40ce90014e52af", - sha256 = "8e872990609d67f9b635790020672d1972b906bed30e4d08d97f964be1ced483", + version = "a955a00bed5f35777a83899ee680f8530eee4718", + sha256 = "b0830dc6fc1e3a095c5d817ca768c89c407bdd71894e1641daf500d28cb269da", strip_prefix = "envoy-build-tools-{version}", urls = ["https://github.com/envoyproxy/envoy-build-tools/archive/{version}.tar.gz"], - release_date = "2021-05-06", + release_date = "2021-05-25", use_category = ["build"], ), boringssl = dict( @@ -750,8 +750,8 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "curl", project_desc = "Library for transferring data with URLs", project_url = "https://curl.haxx.se", - version = "7.76.0", - sha256 = "3b4378156ba09e224008e81dcce854b7ce4d182b1f9cfb97fe5ed9e9c18c6bd3", + version = "7.77.0", + sha256 = "b0a3428acb60fa59044c4d0baae4e4fc09ae9af1d8a3aa84b2e3fbcd99841f77", strip_prefix = "curl-{version}", urls = ["https://github.com/curl/curl/releases/download/curl-{underscore_version}/curl-{version}.tar.gz"], use_category = ["dataplane_ext", "observability_ext"], @@ -761,7 +761,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( "envoy.grpc_credentials.aws_iam", "envoy.tracers.opencensus", ], - release_date = "2021-03-30", + release_date = "2021-05-26", cpe = "cpe:2.3:a:haxx:libcurl:*", ), com_googlesource_chromium_v8 = dict( @@ -925,8 +925,8 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "WebAssembly for Proxies (C++ SDK)", project_desc = "WebAssembly for Proxies (C++ SDK)", project_url = "https://github.com/proxy-wasm/proxy-wasm-cpp-sdk", - version = "258b4c6974dba5255a9c433450971a56b29228ff", - sha256 = "8ff6231a5f0cc07bc865293e56eb37f60f5fd8d5a3889455c4e4ad9dbe54a5f7", + version = "d9baeb21d46ab07d4eb9295a5d53a1803b7b80af", + sha256 = "b517ac487e0ac4b5d4f951ec805f2e54d5aecece34159b053c5fb781fac5e0f5", strip_prefix = "proxy-wasm-cpp-sdk-{version}", urls = ["https://github.com/proxy-wasm/proxy-wasm-cpp-sdk/archive/{version}.tar.gz"], use_category = ["dataplane_ext"], @@ -941,7 +941,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( "envoy.wasm.runtime.wavm", "envoy.wasm.runtime.wasmtime", ], - release_date = "2021-03-10", + release_date = "2021-05-15", cpe = "N/A", ), proxy_wasm_cpp_host = dict( diff --git a/ci/format_pre.sh b/ci/format_pre.sh index 92df445170971..518294d648bdd 100755 --- a/ci/format_pre.sh +++ b/ci/format_pre.sh @@ -46,6 +46,9 @@ bazel run "${BAZEL_BUILD_OPTIONS[@]}" //configs:example_configs_validation CURRENT=python bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/code_format:python_check -- --diff-file="$DIFF_OUTPUT" --fix "$(pwd)" +CURRENT=extensions +bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/extensions:validate_extensions + if [[ "${#FAILED[@]}" -ne "0" ]]; then echo "TESTS FAILED:" >&2 for failed in "${FAILED[@]}"; do diff --git a/configs/Dockerfile b/configs/Dockerfile deleted file mode 100644 index 8167a41d4dd4b..0000000000000 --- a/configs/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -# This configuration will build a Docker container containing -# an Envoy proxy that routes to envoyproxy.io. - -FROM envoyproxy/envoy-dev:latest -RUN apt-get update -COPY envoy-demo.yaml /etc/envoy.yaml -CMD /usr/local/bin/envoy -c /etc/envoy.yaml diff --git a/configs/envoy-demo.yaml b/configs/envoy-demo.yaml index f84b00c5eab46..39bfae994947c 100644 --- a/configs/envoy-demo.yaml +++ b/configs/envoy-demo.yaml @@ -37,7 +37,6 @@ static_resources: - name: envoy.filters.http.router clusters: - name: service_envoyproxy_io - connect_timeout: 30s type: LOGICAL_DNS # Comment out the following line to test on v6 networks dns_lookup_family: V4_ONLY diff --git a/configs/envoy-tap-config.yaml b/configs/envoy-tap-config.yaml index f8d4ef0836484..655580c77210b 100644 --- a/configs/envoy-tap-config.yaml +++ b/configs/envoy-tap-config.yaml @@ -37,7 +37,6 @@ static_resources: - name: envoy.filters.http.router clusters: - name: service_envoyproxy_io - connect_timeout: 30s type: LOGICAL_DNS # Comment out the following line to test on v6 networks dns_lookup_family: V4_ONLY diff --git a/configs/envoy_double_proxy.template.yaml b/configs/envoy_double_proxy.template.yaml index 806ba9d03a418..223a0d5ccf86b 100644 --- a/configs/envoy_double_proxy.template.yaml +++ b/configs/envoy_double_proxy.template.yaml @@ -112,7 +112,6 @@ static_resources: clusters: - name: statsd type: STATIC - connect_timeout: 0.25s lb_policy: ROUND_ROBIN load_assignment: cluster_name: statsd @@ -126,7 +125,6 @@ static_resources: protocol: TCP - name: backhaul type: STRICT_DNS - connect_timeout: 1s lb_policy: ROUND_ROBIN load_assignment: cluster_name: backhaul @@ -165,7 +163,6 @@ static_resources: http2_protocol_options: {} - name: lightstep_saas type: LOGICAL_DNS - connect_timeout: 1s lb_policy: ROUND_ROBIN load_assignment: cluster_name: lightstep_saas diff --git a/configs/example_configs_validation.py b/configs/example_configs_validation.py index ce8095ee84e28..e05f52b43a448 100644 --- a/configs/example_configs_validation.py +++ b/configs/example_configs_validation.py @@ -1,22 +1,18 @@ import pathlib import sys -import yaml - from google.protobuf.json_format import ParseError sys.path = [p for p in sys.path if not p.endswith('bazel_tools')] -from tools.config_validation.validate_fragment import validate_fragment +from tools.config_validation.validate_fragment import validate_yaml def main(): errors = [] for arg in sys.argv[1:]: try: - validate_fragment( - "envoy.config.bootstrap.v3.Bootstrap", - yaml.safe_load(pathlib.Path(arg).read_text())) + validate_yaml("envoy.config.bootstrap.v3.Bootstrap", pathlib.Path(arg).read_text()) except (ParseError, KeyError) as e: errors.append(arg) print(f"\nERROR (validation failed): {arg}\n{e}\n\n") diff --git a/docs/build.sh b/docs/build.sh index 6e0b83731ba28..bca3935841fe2 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -63,10 +63,7 @@ pip3 install --require-hashes -r "${SCRIPT_DIR}"/requirements.txt # files still. rm -rf bazel-bin/external/envoy_api_canonical -EXTENSION_DB_PATH="$(realpath "${BUILD_DIR}/extension_db.json")" -rm -rf "${EXTENSION_DB_PATH}" GENERATED_RST_DIR="$(realpath "${GENERATED_RST_DIR}")" -export EXTENSION_DB_PATH export GENERATED_RST_DIR # This is for local RBE setup, should be no-op for builds without RBE setting in bazelrc files. @@ -74,8 +71,7 @@ IFS=" " read -ra BAZEL_BUILD_OPTIONS <<< "${BAZEL_BUILD_OPTIONS:-}" BAZEL_BUILD_OPTIONS+=( "--remote_download_outputs=all" "--strategy=protodoc=sandboxed,local" - "--action_env=ENVOY_BLOB_SHA" - "--action_env=EXTENSION_DB_PATH") + "--action_env=ENVOY_BLOB_SHA") # Generate RST for the lists of trusted/untrusted extensions in # intro/arch_overview/security docs. diff --git a/docs/requirements.txt b/docs/requirements.txt index 6a3f7c3cc01e7..5c028bb2d4639 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -35,6 +35,7 @@ docutils==0.16 \ # -r docs/requirements.txt # sphinx # sphinx-rtd-theme + # sphinx-tabs gitdb==4.0.7 \ --hash=sha256:6c4cc71933456991da20917998acbe6cf4fb41eeaab7d6d67fbc05ecd4c865b0 \ --hash=sha256:96bf5c08b157a666fec41129e6d327235284cca4c81e92109260f353ba138005 @@ -101,6 +102,7 @@ markupsafe==2.0.1 \ # via # -r docs/requirements.txt # jinja2 + # sphinx packaging==20.9 \ --hash=sha256:5b327ac1320dc863dca72f4514ecc086f31186744b84a230374cc1fd776feae5 \ --hash=sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a @@ -108,8 +110,8 @@ packaging==20.9 \ # -r docs/requirements.txt # sphinx pygments==2.9.0 \ - --hash=sha256:d66e804411278594d764fc69ec36ec13d9ae9147193a1740cd34d272ca383b8e \ - --hash=sha256:a18f47b506a429f6f4b9df81bb02beab9ca21d0a5fee38ed15aef65f0545519f + --hash=sha256:a18f47b506a429f6f4b9df81bb02beab9ca21d0a5fee38ed15aef65f0545519f \ + --hash=sha256:d66e804411278594d764fc69ec36ec13d9ae9147193a1740cd34d272ca383b8e # via # -r docs/requirements.txt # sphinx @@ -126,6 +128,37 @@ pytz==2021.1 \ # via # -r docs/requirements.txt # babel +pyyaml==5.4.1 \ + --hash=sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf \ + --hash=sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696 \ + --hash=sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393 \ + --hash=sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77 \ + --hash=sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922 \ + --hash=sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5 \ + --hash=sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8 \ + --hash=sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10 \ + --hash=sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc \ + --hash=sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018 \ + --hash=sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e \ + --hash=sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253 \ + --hash=sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347 \ + --hash=sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183 \ + --hash=sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541 \ + --hash=sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb \ + --hash=sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185 \ + --hash=sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc \ + --hash=sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db \ + --hash=sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa \ + --hash=sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46 \ + --hash=sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122 \ + --hash=sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b \ + --hash=sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63 \ + --hash=sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df \ + --hash=sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc \ + --hash=sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247 \ + --hash=sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6 \ + --hash=sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0 + # via -r docs/requirements.txt requests==2.25.1 \ --hash=sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804 \ --hash=sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e @@ -133,8 +166,8 @@ requests==2.25.1 \ # -r docs/requirements.txt # sphinx six==1.16.0 \ - --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 \ - --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 + --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ + --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 # via # -r docs/requirements.txt # sphinxcontrib-httpdomain @@ -184,9 +217,9 @@ sphinxcontrib-devhelp==1.0.2 \ # via # -r docs/requirements.txt # sphinx -sphinxcontrib-htmlhelp==1.0.3 \ - --hash=sha256:3c0bc24a2c41e340ac37c85ced6dafc879ab485c095b1d65d2461ac2f7cca86f \ - --hash=sha256:e8f5bb7e31b2dbb25b9cc435c8ab7a79787ebf7f906155729338f3156d93659b +sphinxcontrib-htmlhelp==2.0.0 \ + --hash=sha256:d412243dfb797ae3ec2b59eca0e52dac12e75a241bf0e4eb861e450d06c6ed07 \ + --hash=sha256:f5f8bb2d0d629f398bf47d0d69c07bc13b65f75a81ad9e2f71a63d4b7a2f6db2 # via # -r docs/requirements.txt # sphinx @@ -206,9 +239,9 @@ sphinxcontrib-qthelp==1.0.3 \ # via # -r docs/requirements.txt # sphinx -sphinxcontrib-serializinghtml==1.1.4 \ - --hash=sha256:eaa0eccc86e982a9b939b2b82d12cc5d013385ba5eadcc7e4fed23f4405f77bc \ - --hash=sha256:f242a81d423f59617a8e5cf16f5d4d74e28ee9a66f9e5b637a18082991db5a9a +sphinxcontrib-serializinghtml==1.1.5 \ + --hash=sha256:352a9a00ae864471d3a7ead8d7d79f5fc0b57e8b3f95e9867eb9eb28999b92fd \ + --hash=sha256:aa5f6de5dfdf809ef505c4895e51ef5c9eac17d0f287933eb49ec495280b6952 # via # -r docs/requirements.txt # sphinx @@ -216,9 +249,9 @@ sphinxext-rediraffe==0.2.7 \ --hash=sha256:651dcbfae5ffda9ffd534dfb8025f36120e5efb6ea1a33f5420023862b9f725d \ --hash=sha256:9e430a52d4403847f4ffb3a8dd6dfc34a9fe43525305131f52ed899743a5fd8c # via -r docs/requirements.txt -urllib3==1.26.4 \ - --hash=sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df \ - --hash=sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937 +urllib3==1.26.5 \ + --hash=sha256:753a0374df26658f99d826cfe40394a686d05985786d946fbe4165b5148f5a7c \ + --hash=sha256:a7acd0977125325f516bda9735fa7142b909a8d01e8b2e4c8108d0984e6e0098 # via # -r docs/requirements.txt # requests diff --git a/docs/root/_include/ads.yaml b/docs/root/_include/ads.yaml index 81d30842e113c..962f1eef9b8b5 100644 --- a/docs/root/_include/ads.yaml +++ b/docs/root/_include/ads.yaml @@ -21,7 +21,6 @@ dynamic_resources: static_resources: clusters: - name: ads_cluster - connect_timeout: 5s type: STRICT_DNS load_assignment: cluster_name: ads_cluster diff --git a/docs/root/api-v3/common_messages/common_messages.rst b/docs/root/api-v3/common_messages/common_messages.rst index 843154a3bb9e1..2826b6c67ecdb 100644 --- a/docs/root/api-v3/common_messages/common_messages.rst +++ b/docs/root/api-v3/common_messages/common_messages.rst @@ -16,6 +16,7 @@ Common messages ../config/core/v3/grpc_service.proto ../config/core/v3/grpc_method_list.proto ../config/core/v3/http_uri.proto + ../config/core/v3/resolver.proto ../config/core/v3/socket_option.proto ../config/core/v3/udp_socket_config.proto ../config/core/v3/substitution_format_string.proto diff --git a/docs/root/api-v3/config/filter/http/http.rst b/docs/root/api-v3/config/filter/http/http.rst index 20f2c75664dbc..861a920b5a8ec 100644 --- a/docs/root/api-v3/config/filter/http/http.rst +++ b/docs/root/api-v3/config/filter/http/http.rst @@ -6,4 +6,4 @@ HTTP filters :maxdepth: 2 */empty/* - ../../../extensions/filters/http/*/v3/* + ../../../extensions/filters/http/*/v3*/* diff --git a/docs/root/configuration/best_practices/_include/edge.yaml b/docs/root/configuration/best_practices/_include/edge.yaml index d74f572c3625d..cfcb3c4bd5980 100644 --- a/docs/root/configuration/best_practices/_include/edge.yaml +++ b/docs/root/configuration/best_practices/_include/edge.yaml @@ -78,7 +78,6 @@ static_resources: idle_timeout: 15s # must be disabled for long-lived and streaming requests clusters: - name: service_foo - connect_timeout: 15s per_connection_buffer_limit_bytes: 32768 # 32 KiB load_assignment: cluster_name: some_service diff --git a/docs/root/configuration/http/http_conn_man/_include/preserve-case.yaml b/docs/root/configuration/http/http_conn_man/_include/preserve-case.yaml index cf637b1eeb7a1..a4ea11ac4af5e 100644 --- a/docs/root/configuration/http/http_conn_man/_include/preserve-case.yaml +++ b/docs/root/configuration/http/http_conn_man/_include/preserve-case.yaml @@ -28,7 +28,6 @@ static_resources: cluster: service_foo clusters: - name: service_foo - connect_timeout: 15s typed_extension_protocol_options: envoy.extensions.upstreams.http.v3.HttpProtocolOptions: "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions diff --git a/docs/root/configuration/http/http_conn_man/stats.rst b/docs/root/configuration/http/http_conn_man/stats.rst index 7fd66553222f9..dd6891ce2d220 100644 --- a/docs/root/configuration/http/http_conn_man/stats.rst +++ b/docs/root/configuration/http/http_conn_man/stats.rst @@ -160,6 +160,24 @@ On the upstream side all http2 statistics are rooted at *cluster..http2.* `downstream_rq_active` gauge due to differences in stream accounting between the codec and the HTTP connection manager. +Http3 codec statistics +~~~~~~~~~~~~~~~~~~~~~~ + +On the downstream side all http3 statistics are rooted at *http3.* + +On the upstream side all http3 statistics are rooted at *cluster..http3.* + +.. csv-table:: + :header: Name, Type, Description + :widths: 1, 1, 2 + + dropped_headers_with_underscores, Counter, Total number of dropped headers with names containing underscores. This action is configured by setting the :ref:`headers_with_underscores_action config setting `. + requests_rejected_with_underscores_in_headers, Counter, Total numbers of rejected requests due to header names containing underscores. This action is configured by setting the :ref:`headers_with_underscores_action config setting `. + rx_reset, Counter, Total number of reset stream frames received by Envoy + tx_reset, Counter, Total number of reset stream frames transmitted by Envoy + metadata_not_supported_error, Counter, Total number of metadata dropped during HTTP/3 encoding + + Tracing statistics ------------------ diff --git a/docs/root/configuration/http/http_filters/_include/bandwidth-limit-filter.yaml b/docs/root/configuration/http/http_filters/_include/bandwidth-limit-filter.yaml new file mode 100644 index 0000000000000..2f63fb2147bc1 --- /dev/null +++ b/docs/root/configuration/http/http_filters/_include/bandwidth-limit-filter.yaml @@ -0,0 +1,63 @@ +static_resources: + listeners: + - address: + socket_address: + address: 0.0.0.0 + port_value: 8000 + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + codec_type: AUTO + stat_prefix: ingress_http + route_config: + name: local_route + virtual_hosts: + - name: local_service + domains: ["*"] + routes: + - match: { prefix: "/path/with/bandwidth/limit" } + route: { cluster: service_protected_by_bandwidth_limit } + typed_per_filter_config: + envoy.filters.http.bandwidth_limit: + "@type": type.googleapis.com/envoy.extensions.filters.http.bandwidth_limit.v3alpha.BandwidthLimit + stat_prefix: bandwidth_limiter_custom_route + enable_mode: REQUEST_AND_RESPONSE + limit_kbps: 500 + fill_interval: 0.1s + - match: { prefix: "/" } + route: { cluster: web_service } + http_filters: + - name: envoy.filters.http.bandwidth_limit + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.bandwidth_limit.v3alpha.BandwidthLimit + stat_prefix: bandwidth_limiter_default + - name: envoy.filters.http.router + typed_config: {} + + clusters: + - name: service_protected_by_bandwidth_limit + type: STRICT_DNS + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: service1 + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: web_service + port_value: 9000 + - name: web_service + type: STRICT_DNS + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: service1 + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: web_service + port_value: 9000 diff --git a/docs/root/configuration/http/http_filters/_include/composite.yaml b/docs/root/configuration/http/http_filters/_include/composite.yaml index 48109818710bf..72d11abde1114 100644 --- a/docs/root/configuration/http/http_filters/_include/composite.yaml +++ b/docs/root/configuration/http/http_filters/_include/composite.yaml @@ -73,7 +73,6 @@ static_resources: clusters: - name: grpc - connect_timeout: 1.25s type: LOGICAL_DNS lb_policy: ROUND_ROBIN dns_lookup_family: V4_ONLY diff --git a/docs/root/configuration/http/http_filters/_include/dns-cache-circuit-breaker.yaml b/docs/root/configuration/http/http_filters/_include/dns-cache-circuit-breaker.yaml index 1efb16f48c768..8742d6954f5a6 100644 --- a/docs/root/configuration/http/http_filters/_include/dns-cache-circuit-breaker.yaml +++ b/docs/root/configuration/http/http_filters/_include/dns-cache-circuit-breaker.yaml @@ -48,7 +48,6 @@ static_resources: "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router clusters: - name: dynamic_forward_proxy_cluster - connect_timeout: 1s lb_policy: CLUSTER_PROVIDED cluster_type: name: envoy.clusters.dynamic_forward_proxy diff --git a/docs/root/configuration/http/http_filters/_include/grpc-reverse-bridge-filter.yaml b/docs/root/configuration/http/http_filters/_include/grpc-reverse-bridge-filter.yaml index 1cca540f59bdd..4ffc8af0e483e 100644 --- a/docs/root/configuration/http/http_filters/_include/grpc-reverse-bridge-filter.yaml +++ b/docs/root/configuration/http/http_filters/_include/grpc-reverse-bridge-filter.yaml @@ -53,7 +53,6 @@ static_resources: typed_config: {} clusters: - name: other - connect_timeout: 5.00s type: LOGICAL_DNS dns_lookup_family: V4_ONLY lb_policy: ROUND_ROBIN @@ -67,7 +66,6 @@ static_resources: address: localhost port_value: 4630 - name: grpc - connect_timeout: 5.00s type: STRICT_DNS lb_policy: ROUND_ROBIN typed_extension_protocol_options: diff --git a/docs/root/configuration/http/http_filters/_include/grpc-transcoder-filter.yaml b/docs/root/configuration/http/http_filters/_include/grpc-transcoder-filter.yaml index 91802786b5101..704b58d4f9b29 100644 --- a/docs/root/configuration/http/http_filters/_include/grpc-transcoder-filter.yaml +++ b/docs/root/configuration/http/http_filters/_include/grpc-transcoder-filter.yaml @@ -39,7 +39,6 @@ static_resources: clusters: - name: grpc - connect_timeout: 1.25s type: LOGICAL_DNS lb_policy: ROUND_ROBIN dns_lookup_family: V4_ONLY diff --git a/docs/root/configuration/http/http_filters/bandwidth_limit_filter.rst b/docs/root/configuration/http/http_filters/bandwidth_limit_filter.rst new file mode 100644 index 0000000000000..4576e9d3ac36a --- /dev/null +++ b/docs/root/configuration/http/http_filters/bandwidth_limit_filter.rst @@ -0,0 +1,64 @@ +.. _config_http_filters_bandwidth_limit: + +Bandwidth limit +==================== + +* Bandwidth limiting :ref:`architecture overview ` +* :ref:`v3 API reference ` +* This filter should be configured with the name ``envoy.filters.http.bandwidth_limit``. + +The HTTP Bandwidth limit filter limits the size of data flow to the max bandwidth set in the ``limit_kbps`` +when the request's route, virtual host or filter chain has a +:ref:`bandwidth limit configuration `. + +If the bandwidth limit has been exhausted the filter stops further transfer until more bandwidth gets allocated +according to the ``fill_interval`` (default is 50 milliseconds). If the connection buffer fills up with accumulated +data then the source of data will have ``readDisable(true)`` set as described in the :repo:`flow control doc`. + +.. note:: + The token bucket is shared across all workers, thus the limits are applied per Envoy process. + +Example configuration +--------------------- + +Example filter configuration for a globally disabled bandwidth limiter but enabled for a specific route: + +.. literalinclude:: _include/bandwidth-limit-filter.yaml + :language: yaml + :lines: 11-53 + :emphasize-lines: 9-25 + :caption: :download:`bandwidth-limit-filter.yaml <_include/bandwidth-limit-filter.yaml>` + +Note that if this filter is configured as globally disabled and there are no virtual host or route level +token buckets, no bandwidth limiting will be applied. + +Statistics +---------- + +The HTTP bandwidth limit filter outputs statistics in the ``.http_bandwidth_limit.`` namespace. + +.. csv-table:: + :header: Name, Type, Description + :widths: 1, 1, 2 + + request_enabled, Counter, Total number of request streams for which the bandwidth limiter was consulted + request_pending, GAUGE, Number of request streams which are currently pending transfer in bandwidth limiter + request_incoming_size, GAUGE, Size in bytes of incoming request data to bandwidth limiter + request_allowed_size, GAUGE, Size in bytes of outgoing request data from bandwidth limiter + request_transfer_duration, HISTOGRAM, Total time (including added delay) it took for the request stream transfer + response_enabled, Counter, Total number of response streams for which the bandwidth limiter was consulted + response_pending, GAUGE, Number of response streams which are currently pending transfer in bandwidth limiter + response_incoming_size, GAUGE, Size in bytes of incoming response data to bandwidth limiter + response_allowed_size, GAUGE, Size in bytes of outgoing response data from bandwidth limiter + response_transfer_duration, HISTOGRAM, Total time (including added delay) it took for the response stream transfer + +.. _config_http_filters_bandwidth_limit_runtime: + +Runtime +------- + +The HTTP bandwidth limit filter supports the following runtime settings: + +The bandwidth limit filter can be runtime feature flagged via the :ref:`enabled +` +configuration field. diff --git a/docs/root/configuration/http/http_filters/http_filters.rst b/docs/root/configuration/http/http_filters/http_filters.rst index 5bd61f14dc44b..b983afd3fd17d 100644 --- a/docs/root/configuration/http/http_filters/http_filters.rst +++ b/docs/root/configuration/http/http_filters/http_filters.rst @@ -10,6 +10,7 @@ HTTP filters admission_control_filter aws_lambda_filter aws_request_signing_filter + bandwidth_limit_filter buffer_filter cdn_loop_filter compressor_filter @@ -41,6 +42,7 @@ HTTP filters rate_limit_filter rbac_filter router_filter + set_metadata_filter squash_filter tap_filter wasm_filter diff --git a/docs/root/configuration/http/http_filters/local_rate_limit_filter.rst b/docs/root/configuration/http/http_filters/local_rate_limit_filter.rst index 4467ba080a41f..0b890cce0bf5e 100644 --- a/docs/root/configuration/http/http_filters/local_rate_limit_filter.rst +++ b/docs/root/configuration/http/http_filters/local_rate_limit_filter.rst @@ -22,8 +22,9 @@ configured to be returned. ` can be configured to be added to forwarded requests to the upstream when the local rate limit filter is enabled but not enforced. -.. note:: - The token bucket is shared across all workers, thus the rate limits are applied per Envoy process. +Depending on the value of the config :ref:`local_rate_limit_per_downstream_connection `, +the token bucket is either shared across all workers or on a per connection basis. This results in the local rate limits being applied either per Envoy process or per downstream connection. +By default the rate limits are applied per Envoy process. Example configuration --------------------- @@ -55,6 +56,7 @@ Example filter configuration for a globally set rate limiter (e.g.: all vhosts/r header: key: x-local-rate-limit value: 'true' + local_rate_limit_per_downstream_connection: false Example filter configuration for a globally disabled rate limiter but enabled for a specific route: diff --git a/docs/root/configuration/http/http_filters/set_metadata_filter.rst b/docs/root/configuration/http/http_filters/set_metadata_filter.rst new file mode 100644 index 0000000000000..b56367e0105f0 --- /dev/null +++ b/docs/root/configuration/http/http_filters/set_metadata_filter.rst @@ -0,0 +1,52 @@ +.. _config_http_filters_set_metadata: + +Set Metadata +============ +* :ref:`v3 API reference ` +* This filter should be configured with the name *envoy.filters.http.set_metadata*. + +This filters adds or updates dynamic metadata with static data. + +Dynamic metadata values are updated with the following scheme. If a key +does not exists, it's just copied into the current metadata. If the key exists +but has a different type, it is replaced by the new value. Otherwise: + + * for scalar values (null, string, number, boolean) are replaced with the new value + * for lists: new values are added to the current list + * for structures: recursively apply this scheme + +For instance, if the namespace already contains this structure: + +.. code-block:: yaml + + myint: 1 + mylist: ["a"] + mykey: ["val"] + mytags: + tag0: 1 + +and the value to set is: + +.. code-block:: yaml + + myint: 2 + mylist: ["b","c"] + mykey: 1 + mytags: + tag1: 1 + +After applying this filter, the namespace will contain: + +.. code-block:: yaml + + myint: 2 + mylist: ["a","b","c"] + mykey: 1 + mytags: + tag0: 1 + tag1: 1 + +Statistics +---------- + +Currently, this filter generates no statistics. diff --git a/docs/root/configuration/listeners/network_filters/_include/sni-dynamic-forward-proxy-filter.yaml b/docs/root/configuration/listeners/network_filters/_include/sni-dynamic-forward-proxy-filter.yaml index 5150a03aba7a3..c78ad55d80324 100644 --- a/docs/root/configuration/listeners/network_filters/_include/sni-dynamic-forward-proxy-filter.yaml +++ b/docs/root/configuration/listeners/network_filters/_include/sni-dynamic-forward-proxy-filter.yaml @@ -30,7 +30,6 @@ static_resources: cluster: dynamic_forward_proxy_cluster clusters: - name: dynamic_forward_proxy_cluster - connect_timeout: 1s lb_policy: CLUSTER_PROVIDED cluster_type: name: envoy.clusters.dynamic_forward_proxy diff --git a/docs/root/configuration/listeners/network_filters/_include/zookeeper-filter-proxy.yaml b/docs/root/configuration/listeners/network_filters/_include/zookeeper-filter-proxy.yaml index aafca89351bf5..a9850d4672243 100644 --- a/docs/root/configuration/listeners/network_filters/_include/zookeeper-filter-proxy.yaml +++ b/docs/root/configuration/listeners/network_filters/_include/zookeeper-filter-proxy.yaml @@ -18,7 +18,6 @@ static_resources: cluster: local_zk_server clusters: - name: local_zk_server - connect_timeout: 120s type: LOGICAL_DNS lb_policy: ROUND_ROBIN typed_extension_protocol_options: diff --git a/docs/root/configuration/listeners/udp_filters/_include/udp-proxy.yaml b/docs/root/configuration/listeners/udp_filters/_include/udp-proxy.yaml index 816a7657fd484..bcffbb1404df5 100644 --- a/docs/root/configuration/listeners/udp_filters/_include/udp-proxy.yaml +++ b/docs/root/configuration/listeners/udp_filters/_include/udp-proxy.yaml @@ -26,7 +26,6 @@ static_resources: max_rx_datagram_size: 9000 clusters: - name: service_udp - connect_timeout: 0.25s type: STATIC lb_policy: ROUND_ROBIN load_assignment: diff --git a/docs/root/configuration/other_protocols/thrift_filters/router_filter.rst b/docs/root/configuration/other_protocols/thrift_filters/router_filter.rst index a6d5c50ad4d2e..dae0be24d7429 100644 --- a/docs/root/configuration/other_protocols/thrift_filters/router_filter.rst +++ b/docs/root/configuration/other_protocols/thrift_filters/router_filter.rst @@ -41,3 +41,10 @@ Since these stats utilize the underlying cluster scope, we prefix with the ``thr thrift.upstream_resp_exception, Counter, Total responses with the "Exception" message type. thrift.upstream_resp_invalid_type, Counter, Total responses with an unsupported message type. thrift.upstream_rq_time, Histogram, total rq time from rq complete to resp complete; includes oneway messages. + thrift.upstream_rq_size, Histogram, Request message size in bytes per upstream + thrift.upstream_resp_size, Histogram, Response message size in bytes per upstream + +.. note:: + + The request and response size histograms include what's sent and received during protocol upgrade. + However, invalid responses are not included in the response size histogram. diff --git a/docs/root/configuration/overview/_include/tagged.yaml b/docs/root/configuration/overview/_include/tagged.yaml new file mode 100644 index 0000000000000..a333ec7ffda1a --- /dev/null +++ b/docs/root/configuration/overview/_include/tagged.yaml @@ -0,0 +1,43 @@ +!ignore dynamic_sockets: + - &admin_address { address: 127.0.0.1, port_value: 9901 } + - &listener_address { address: 127.0.0.1, port_value: 10000 } + - &lb_address { address: 127.0.0.1, port_value: 1234 } + +admin: + address: + socket_address: *admin_address + +static_resources: + listeners: + - name: listener_0 + address: + socket_address: *listener_address + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: ingress_http + codec_type: AUTO + route_config: + name: local_route + virtual_hosts: + - name: local_service + domains: ["*"] + routes: + - match: { prefix: "/" } + route: { cluster: some_service } + http_filters: + - name: envoy.filters.http.router + clusters: + - name: some_service + connect_timeout: 0.25s + type: STATIC + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: some_service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: *lb_address diff --git a/docs/root/configuration/overview/examples.rst b/docs/root/configuration/overview/examples.rst index f6d14cf5df4b2..03811aa1952ab 100644 --- a/docs/root/configuration/overview/examples.rst +++ b/docs/root/configuration/overview/examples.rst @@ -302,3 +302,36 @@ The management server could respond to EDS requests with: socket_address: address: 127.0.0.2 port_value: 1234 + +Special YAML usage +~~~~~~~~~~~~~~~~~~ + +When loading YAML configuration, the Envoy loader will interpret map keys tagged with !ignore +specially, and omit them entirely from the native configuration tree. Ordinarily, the YAML stream +must adhere strictly to the proto schemas defined for Envoy configuration. This allows content to +be declared that is explicitly handled as a non-represented type. + +This lets you split your file into two parts: one in which we have YAML content not subject to +parsing according to the schema and another part that is parsed. YAML anchors in the first part +may be referenced by aliases in the second part. This mechanism can simplify setups that need to +re-use or dynamically generate configuration fragments. + +See the following example: + +.. literalinclude:: _include/tagged.yaml + :language: yaml + +.. warning:: + If you parse Envoy YAML configuration using external loaders, you may need to inform these + loaders about the !ignore tag. Compliant YAML loaders will typically expose an interface to + allow you to choose how to handle a custom tag. + +For example, this will instruct `PyYAML ` to treat an ignored +node as a simple scalar when loading: + +.. code-block:: python3 + + yaml.SafeLoader.add_constructor('!ignore', yaml.loader.SafeConstructor.construct_scalar) + +Alternatively, :repo:`this is how ` +Envoy registers the !ignore tag in config validation. diff --git a/docs/root/faq/configuration/timeouts.rst b/docs/root/faq/configuration/timeouts.rst index 43a228e0c27a5..72c090aac266b 100644 --- a/docs/root/faq/configuration/timeouts.rst +++ b/docs/root/faq/configuration/timeouts.rst @@ -100,6 +100,20 @@ stream timeouts already introduced above. :ref:`max_stream_duration ` for individual routes as well as setting both limits and a fixed time offset on grpc-timeout headers. +Scaled timeouts +^^^^^^^^^^^^^^^ + +In situations where envoy is under high load, Envoy can dynamically configure timeouts using scaled timeouts. +Envoy supports scaled timeouts through the :ref:`Overload Manager `, configured +in envoy :ref:`bootstrap configuration `. +Using a :ref:`reduce timeouts ` overload action, +the Overload Manager can be configured to monitor :ref:`resources ` +and scale timeouts accordingly. For example, a common use case may be to monitor the Envoy :ref:`heap size ` +and set the scaled TimerType to :ref:`HTTP_DOWNSTREAM_CONNECTION_IDLE `. +The overload manager will scale down the :ref:`idle timeout ` once the :ref:`scaling_threshold ` has been met +and will set the timeout to the :ref:`min timeout ` once the :ref:`scaling_threshold ` is met. +The full list of supported timers that can be scaled is available in the overload manager :ref:`docs `. + TCP --- diff --git a/docs/root/intro/_include/life-of-a-request.yaml b/docs/root/intro/_include/life-of-a-request.yaml index 6875f8861a655..9dbe644dcaaeb 100644 --- a/docs/root/intro/_include/life-of-a-request.yaml +++ b/docs/root/intro/_include/life-of-a-request.yaml @@ -58,7 +58,6 @@ static_resources: - name: envoy.filters.http.router clusters: - name: some_service - connect_timeout: 5s # Upstream TLS configuration. transport_socket: name: envoy.transport_sockets.tls @@ -86,7 +85,6 @@ static_resources: http2_protocol_options: max_concurrent_streams: 100 - name: some_statsd_sink - connect_timeout: 5s # The rest of the configuration for statsd sink cluster. # statsd sink. stats_sinks: diff --git a/docs/root/intro/arch_overview/advanced/matching/_include/complicated.yaml b/docs/root/intro/arch_overview/advanced/matching/_include/complicated.yaml index 0c385de903e42..a9dd901d638b2 100644 --- a/docs/root/intro/arch_overview/advanced/matching/_include/complicated.yaml +++ b/docs/root/intro/arch_overview/advanced/matching/_include/complicated.yaml @@ -81,7 +81,6 @@ static_resources: cluster: service_foo clusters: - name: service_foo - connect_timeout: 15s load_assignment: cluster_name: some_service endpoints: diff --git a/docs/root/intro/arch_overview/advanced/matching/_include/request_response.yaml b/docs/root/intro/arch_overview/advanced/matching/_include/request_response.yaml index 78ae0b18b1b79..ff48ee5e13f52 100644 --- a/docs/root/intro/arch_overview/advanced/matching/_include/request_response.yaml +++ b/docs/root/intro/arch_overview/advanced/matching/_include/request_response.yaml @@ -67,7 +67,6 @@ static_resources: cluster: service_foo clusters: - name: service_foo - connect_timeout: 15s load_assignment: cluster_name: some_service endpoints: diff --git a/docs/root/intro/arch_overview/advanced/matching/_include/simple.yaml b/docs/root/intro/arch_overview/advanced/matching/_include/simple.yaml index 9e00c6dba57d9..fdf2f1c8c183a 100644 --- a/docs/root/intro/arch_overview/advanced/matching/_include/simple.yaml +++ b/docs/root/intro/arch_overview/advanced/matching/_include/simple.yaml @@ -55,7 +55,6 @@ static_resources: cluster: service_foo clusters: - name: service_foo - connect_timeout: 15s load_assignment: cluster_name: some_service endpoints: diff --git a/docs/root/intro/arch_overview/other_features/bandwidth_limiting.rst b/docs/root/intro/arch_overview/other_features/bandwidth_limiting.rst new file mode 100644 index 0000000000000..146ce50cc8630 --- /dev/null +++ b/docs/root/intro/arch_overview/other_features/bandwidth_limiting.rst @@ -0,0 +1,9 @@ +.. _arch_overview_bandwidth_limit: + +Bandwidth limiting +=================== + +Envoy supports local (non-distributed) bandwidth limiting of HTTP requests and response via the +:ref:`HTTP bandwidth limit filter `. This can be activated +globally at the listener level or at a more specific level (e.g.: the virtual host or route level). + diff --git a/docs/root/intro/arch_overview/other_features/other_features.rst b/docs/root/intro/arch_overview/other_features/other_features.rst index 2e2a2c054b74b..be43344957d6d 100644 --- a/docs/root/intro/arch_overview/other_features/other_features.rst +++ b/docs/root/intro/arch_overview/other_features/other_features.rst @@ -6,6 +6,7 @@ Other features local_rate_limiting global_rate_limiting + bandwidth_limiting scripting ip_transparency compression/libraries diff --git a/docs/root/intro/arch_overview/security/_include/ssl.yaml b/docs/root/intro/arch_overview/security/_include/ssl.yaml index 9c6f6224bf287..e23727f0d1473 100644 --- a/docs/root/intro/arch_overview/security/_include/ssl.yaml +++ b/docs/root/intro/arch_overview/security/_include/ssl.yaml @@ -29,7 +29,6 @@ static_resources: filename: certs/cacert.pem clusters: - name: some_service - connect_timeout: 0.25s type: STATIC lb_policy: ROUND_ROBIN load_assignment: diff --git a/docs/root/start/install.rst b/docs/root/start/install.rst index 599883fe54aba..01674263ac874 100644 --- a/docs/root/start/install.rst +++ b/docs/root/start/install.rst @@ -57,38 +57,16 @@ using `Get Envoy `__. To add the nightly repository instead, replace the word ``stable`` with ``nightly``, when adding the ``apt`` repository. -Install Envoy on CentOS Linux -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -You can `install Envoy on CentOS `_ -using `Get Envoy `__. - -.. code-block:: console - - $ sudo yum install yum-utils - $ sudo yum-config-manager --add-repo https://getenvoy.io/linux/centos/tetrate-getenvoy.repo - $ sudo yum install getenvoy-envoy - -.. tip:: - - You can enable/disable ``nightly`` using ``yum-config-manager``: - - .. code-block:: console - - $ sudo yum-config-manager --enable tetrate-getenvoy-nightly - $ sudo yum-config-manager --disable tetrate-getenvoy-nightly - -Install Envoy on Redhat Enterprise Linux (RHEL) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Install Envoy on RPM-based distros +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -You can -`install Envoy on Redhat Enterprise Linux (RHEL) `_ +You can `install Envoy on Centos/Redhat Enterprise Linux (RHEL) `_ using `Get Envoy `__. .. code-block:: console $ sudo yum install yum-utils - $ sudo yum-config-manager --add-repo https://getenvoy.io/linux/rhel/tetrate-getenvoy.repo + $ sudo yum-config-manager --add-repo https://getenvoy.io/linux/rpm/tetrate-getenvoy.repo $ sudo yum install getenvoy-envoy .. tip:: diff --git a/docs/root/start/quick-start/_include/envoy-demo-tls-client-auth.yaml b/docs/root/start/quick-start/_include/envoy-demo-tls-client-auth.yaml index d921322280c00..84367c637f686 100644 --- a/docs/root/start/quick-start/_include/envoy-demo-tls-client-auth.yaml +++ b/docs/root/start/quick-start/_include/envoy-demo-tls-client-auth.yaml @@ -44,7 +44,6 @@ static_resources: clusters: - name: service_envoyproxy_io - connect_timeout: 30s type: LOGICAL_DNS # Comment out the following line to test on v6 networks dns_lookup_family: V4_ONLY diff --git a/docs/root/start/quick-start/_include/envoy-demo-tls-sni.yaml b/docs/root/start/quick-start/_include/envoy-demo-tls-sni.yaml index ae2936cabcffa..40f8533e1ce2d 100644 --- a/docs/root/start/quick-start/_include/envoy-demo-tls-sni.yaml +++ b/docs/root/start/quick-start/_include/envoy-demo-tls-sni.yaml @@ -41,7 +41,6 @@ static_resources: clusters: - name: service_envoyproxy_io - connect_timeout: 30s type: LOGICAL_DNS # Comment out the following line to test on v6 networks dns_lookup_family: V4_ONLY diff --git a/docs/root/start/quick-start/_include/envoy-demo-tls-validation.yaml b/docs/root/start/quick-start/_include/envoy-demo-tls-validation.yaml index 4260b1fca986e..b9ad6cc0635e9 100644 --- a/docs/root/start/quick-start/_include/envoy-demo-tls-validation.yaml +++ b/docs/root/start/quick-start/_include/envoy-demo-tls-validation.yaml @@ -28,7 +28,6 @@ static_resources: clusters: - name: service_envoyproxy_io - connect_timeout: 30s type: LOGICAL_DNS # Comment out the following line to test on v6 networks dns_lookup_family: V4_ONLY diff --git a/docs/root/start/quick-start/_include/envoy-demo-tls.yaml b/docs/root/start/quick-start/_include/envoy-demo-tls.yaml index 6e0830d873c0c..960b9e9bca123 100644 --- a/docs/root/start/quick-start/_include/envoy-demo-tls.yaml +++ b/docs/root/start/quick-start/_include/envoy-demo-tls.yaml @@ -38,7 +38,6 @@ static_resources: clusters: - name: service_envoyproxy_io - connect_timeout: 30s type: LOGICAL_DNS # Comment out the following line to test on v6 networks dns_lookup_family: V4_ONLY diff --git a/docs/root/start/quick-start/_include/envoy-demo.yaml b/docs/root/start/quick-start/_include/envoy-demo.yaml index 3d3b025a8b5e1..eaf9469b5aa76 100644 --- a/docs/root/start/quick-start/_include/envoy-demo.yaml +++ b/docs/root/start/quick-start/_include/envoy-demo.yaml @@ -32,7 +32,6 @@ static_resources: clusters: - name: service_envoyproxy_io - connect_timeout: 30s type: LOGICAL_DNS # Comment out the following line to test on v6 networks dns_lookup_family: V4_ONLY diff --git a/docs/root/start/quick-start/_include/envoy-dynamic-cds-demo.yaml b/docs/root/start/quick-start/_include/envoy-dynamic-cds-demo.yaml index 65e41b87b30ac..fc564cbc9ad02 100644 --- a/docs/root/start/quick-start/_include/envoy-dynamic-cds-demo.yaml +++ b/docs/root/start/quick-start/_include/envoy-dynamic-cds-demo.yaml @@ -1,7 +1,6 @@ resources: - "@type": type.googleapis.com/envoy.config.cluster.v3.Cluster name: example_proxy_cluster - connect_timeout: 1s type: STRICT_DNS typed_extension_protocol_options: envoy.extensions.upstreams.http.v3.HttpProtocolOptions: diff --git a/docs/root/start/quick-start/_include/envoy-dynamic-control-plane-demo.yaml b/docs/root/start/quick-start/_include/envoy-dynamic-control-plane-demo.yaml index e80f175d94f50..e0b300e98bcea 100644 --- a/docs/root/start/quick-start/_include/envoy-dynamic-control-plane-demo.yaml +++ b/docs/root/start/quick-start/_include/envoy-dynamic-control-plane-demo.yaml @@ -18,8 +18,7 @@ dynamic_resources: static_resources: clusters: - - connect_timeout: 1s - type: STRICT_DNS + - type: STRICT_DNS typed_extension_protocol_options: envoy.extensions.upstreams.http.v3.HttpProtocolOptions: "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions diff --git a/docs/root/start/quick-start/configuration-dynamic-control-plane.rst b/docs/root/start/quick-start/configuration-dynamic-control-plane.rst index 1ccdcfc56a745..eb2bbe395b801 100644 --- a/docs/root/start/quick-start/configuration-dynamic-control-plane.rst +++ b/docs/root/start/quick-start/configuration-dynamic-control-plane.rst @@ -69,6 +69,6 @@ The ``xds_cluster`` is configured to query a control plane at http://my-control- .. literalinclude:: _include/envoy-dynamic-control-plane-demo.yaml :language: yaml :linenos: - :lines: 17-39 + :lines: 17-38 :lineno-start: 17 - :emphasize-lines: 3-21 + :emphasize-lines: 3-20 diff --git a/docs/root/start/quick-start/configuration-dynamic-filesystem.rst b/docs/root/start/quick-start/configuration-dynamic-filesystem.rst index a251aa3886054..3fd80238a8d3a 100644 --- a/docs/root/start/quick-start/configuration-dynamic-filesystem.rst +++ b/docs/root/start/quick-start/configuration-dynamic-filesystem.rst @@ -89,4 +89,4 @@ proxies over ``TLS`` to https://www.envoyproxy.io. .. literalinclude:: _include/envoy-dynamic-cds-demo.yaml :language: yaml :linenos: - :emphasize-lines: 12, 18-19, 23-24 + :emphasize-lines: 11, 17-18, 22-23 diff --git a/docs/root/start/quick-start/configuration-static.rst b/docs/root/start/quick-start/configuration-static.rst index c269e7fd293e7..1e6042483f0f0 100644 --- a/docs/root/start/quick-start/configuration-static.rst +++ b/docs/root/start/quick-start/configuration-static.rst @@ -55,5 +55,5 @@ proxies over ``TLS`` to https://www.envoyproxy.io. .. literalinclude:: _include/envoy-demo.yaml :language: yaml :lineno-start: 29 - :lines: 29-52 - :emphasize-lines: 5-24 + :lines: 29-51 + :emphasize-lines: 5-23 diff --git a/docs/root/start/quick-start/securing.rst b/docs/root/start/quick-start/securing.rst index e4e74e17f54d1..cf9f0b558c3c3 100644 --- a/docs/root/start/quick-start/securing.rst +++ b/docs/root/start/quick-start/securing.rst @@ -59,8 +59,8 @@ to the :ref:`transport_socket ` of a :language: yaml :linenos: :lineno-start: 39 - :lines: 39-57 - :emphasize-lines: 16-19 + :lines: 39-56 + :emphasize-lines: 15-18 :caption: :download:`envoy-demo-tls.yaml <_include/envoy-demo-tls.yaml>` .. _start_quick_start_securing_validation: @@ -79,8 +79,8 @@ Firstly, you can ensure that the certificates are from a mutually trusted certif .. literalinclude:: _include/envoy-demo-tls-validation.yaml :language: yaml :linenos: - :lineno-start: 43 - :lines: 43-53 + :lineno-start: 42 + :lines: 42-52 :emphasize-lines: 6-9 :caption: :download:`envoy-demo-tls-validation.yaml <_include/envoy-demo-tls-validation.yaml>` @@ -92,8 +92,8 @@ certificate is valid for. .. literalinclude:: _include/envoy-demo-tls-validation.yaml :language: yaml :linenos: - :lineno-start: 43 - :lines: 43-53 + :lineno-start: 42 + :lines: 42-52 :emphasize-lines: 6-7, 10-11 :caption: :download:`envoy-demo-tls-validation.yaml <_include/envoy-demo-tls-validation.yaml>` @@ -154,9 +154,9 @@ When connecting to an upstream with client certificates you can set them as foll .. literalinclude:: _include/envoy-demo-tls-client-auth.yaml :language: yaml :linenos: - :lineno-start: 45 - :lines: 45-69 - :emphasize-lines: 21-25 + :lineno-start: 44 + :lines: 44-68 + :emphasize-lines: 20-25 :caption: :download:`envoy-demo-tls-client-auth.yaml <_include/envoy-demo-tls-client-auth.yaml>` .. _start_quick_start_securing_sni: @@ -195,8 +195,8 @@ This will usually be the DNS name of the service you are connecting to. .. literalinclude:: _include/envoy-demo-tls-sni.yaml :language: yaml :linenos: - :lineno-start: 56 - :lines: 56-61 + :lineno-start: 55 + :lines: 55-60 :emphasize-lines: 6 :caption: :download:`envoy-demo-tls-sni.yaml <_include/envoy-demo-tls-sni.yaml>` diff --git a/docs/root/start/sandboxes/dynamic-configuration-filesystem.rst b/docs/root/start/sandboxes/dynamic-configuration-filesystem.rst index f3ab8d01051f2..eb52a8f9cf2dc 100644 --- a/docs/root/start/sandboxes/dynamic-configuration-filesystem.rst +++ b/docs/root/start/sandboxes/dynamic-configuration-filesystem.rst @@ -90,7 +90,7 @@ from ``service1`` to ``service2``: .. literalinclude:: _include/dynamic-config-fs/configs/cds.yaml :language: yaml :linenos: - :lines: 6-14 + :lines: 6-13 :lineno-start: 6 :emphasize-lines: 8 diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 449d76c1bd2c0..27a50d16838cc 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -30,18 +30,22 @@ Minor Behavior Changes ``envoy.reloadable_features.send_strict_1xx_and_204_response_headers`` (do not send 1xx or 204 responses with these headers). Both are true by default. * http: serve HEAD requests from cache. +* http: the behavior of the *present_match* in route header matcher changed. The value of *present_match* is ignored in the past. The new behavior is *present_match* performed when value is true. absent match performed when the value is false. Please reference :ref:`present_match + `. * listener: respect the :ref:`connection balance config ` defined within the listener where the sockets are redirected to. Clear that field to restore the previous behavior. * tcp: switched to the new connection pool by default. Any unexpected behavioral changes can be reverted by setting runtime guard ``envoy.reloadable_features.new_tcp_connection_pool`` to false. - Bug Fixes --------- *Changes expected to improve the state of the world and are unlikely to have negative effects* +* aws_lambda: if `payload_passthrough` is set to ``false``, the downstream response content-type header will now be set from the content-type entry in the JSON response's headers map, if present. +* hot_restart: fix double counting of `server.seconds_until_first_ocsp_response_expiring` and `server.days_until_first_cert_expiring` during hot-restart. This stat was only incorrect until the parent process terminated. * http: port stripping now works for CONNECT requests, though the port will be restored if the CONNECT request is sent upstream. This behavior can be temporarily reverted by setting ``envoy.reloadable_features.strip_port_from_connect`` to false. * http: raise max configurable max_request_headers_kb limit to 8192 KiB (8MiB) from 96 KiB in http connection manager. * listener: fix the crash which could happen when the ongoing filter chain only listener update is followed by the listener removal or full listener update. +* udp: limit each UDP listener to read maxmium 6000 packets per event loop. This behavior can be temporarily reverted by setting ``envoy.reloadable_features.udp_per_event_loop_read_limit`` to false. * validation: fix an issue that causes TAP sockets to panic during config validation mode. * xray: fix the default sampling 'rate' for AWS X-Ray tracer extension to be 5% as opposed to 50%. * zipkin: fix timestamp serializaiton in annotations. A prior bug fix exposed an issue with timestamps being serialized as strings. @@ -62,19 +66,27 @@ Removed Config or Runtime New Features ------------ -* crash support: restore crash context when continuing to processing requests or responses as a result of an asynchronous callback that invokes a filter directly. This is unlike the call stacks that go through the various network layers, to eventually reach the filter. For a concrete example see: ``Envoy::Extensions::HttpFilters::Cache::CacheFilter::getHeaders`` which posts a callback on the dispatcher that will invoke the filter directly. + +* bandwidth_limit: added new :ref:`HTTP bandwidth limit filter `. +* crash support: restore crash context when continuing to processing requests or responses as a result of an asynchronous callback that invokes a filter directly. This is unlike the call stacks that go through the various network layers, to eventually reach the filter. For a concrete example see: ``Envoy::Extensions::HttpFilters::Cache::CacheFilter::getHeaders`` which posts a callback on the dispatcher that will invoke the filter directly. +* dynamic_forward_proxy: added :ref:`dns_resolver` option to the DNS cache config in order use custom DNS resolvers instead of the system default resolvers. * http: a new field `is_optional` is added to `extensions.filters.network.http_connection_manager.v3.HttpFilter`. When value is `true`, the unsupported http filter will be ignored by envoy. This is also same with unsupported http filter in the typed per filter config. For more information, please reference :ref:`HttpFilter `. +* http: added :ref:`stripping trailing host dot from host header` support. * http: added support for :ref:`original IP detection extensions`. Two initial extensions were added, the :ref:`custom header ` extension and the :ref:`xff ` extension. * http: added the ability to :ref:`unescape slash sequences` in the path. Requests with unescaped slashes can be proxied, rejected or redirected to the new unescaped path. By default this feature is disabled. The default behavior can be overridden through :ref:`http_connection_manager.path_with_escaped_slashes_action` runtime variable. This action can be selectively enabled for a portion of requests by setting the :ref:`http_connection_manager.path_with_escaped_slashes_action_sampling` runtime variable. * http: added upstream and downstream alpha HTTP/3 support! See :ref:`quic_options ` for downstream and the new http3_protocol_options in :ref:`http_protocol_options ` for upstream HTTP/3. +* jwt_authn: added support to fetch remote jwks asynchronously specified by :ref:`async_fetch `. * listener: added ability to change an existing listener's address. +* local_rate_limit_filter: added suppoort for locally rate limiting http requests on a per connection basis. This can be enabled by setting the :ref:`local_rate_limit_per_downstream_connection ` field to true. * metric service: added support for sending metric tags as labels. This can be enabled by setting the :ref:`emit_tags_as_labels ` field to true. * tcp: added support for :ref:`preconnecting `. Preconnecting is off by default, but recommended for clusters serving latency-sensitive traffic. +* thrift_proxy: added per upstream metrics within the :ref:`thrift router ` for request and response size histograms. +* tls: allow dual ECDSA/RSA certs via SDS. Previously, SDS only supported a single certificate per context, and dual cert was only supported via non-SDS. * udp_proxy: added :ref:`key ` as another hash policy to support hash based routing on any given key. Deprecated diff --git a/examples/cache/front-envoy.yaml b/examples/cache/front-envoy.yaml index 345e96f4f7346..00091a052ca86 100644 --- a/examples/cache/front-envoy.yaml +++ b/examples/cache/front-envoy.yaml @@ -37,7 +37,6 @@ static_resources: clusters: - name: service1 - connect_timeout: 0.25s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: @@ -50,7 +49,6 @@ static_resources: address: service1 port_value: 8000 - name: service2 - connect_timeout: 0.25s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: diff --git a/examples/cache/service-envoy.yaml b/examples/cache/service-envoy.yaml index b3ea728cc5f60..eaa849dd40797 100644 --- a/examples/cache/service-envoy.yaml +++ b/examples/cache/service-envoy.yaml @@ -27,7 +27,6 @@ static_resources: typed_config: {} clusters: - name: local_service - connect_timeout: 0.25s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: diff --git a/examples/cors/backend/front-envoy.yaml b/examples/cors/backend/front-envoy.yaml index b6b4f9d63e0b3..cd95621758d0f 100644 --- a/examples/cors/backend/front-envoy.yaml +++ b/examples/cors/backend/front-envoy.yaml @@ -72,7 +72,6 @@ static_resources: typed_config: {} clusters: - name: backend_service - connect_timeout: 0.25s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: diff --git a/examples/cors/backend/service-envoy.yaml b/examples/cors/backend/service-envoy.yaml index c596eb508ff87..d5943d46ea7ba 100644 --- a/examples/cors/backend/service-envoy.yaml +++ b/examples/cors/backend/service-envoy.yaml @@ -27,7 +27,6 @@ static_resources: typed_config: {} clusters: - name: local_service - connect_timeout: 0.25s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: diff --git a/examples/cors/frontend/front-envoy.yaml b/examples/cors/frontend/front-envoy.yaml index 604ada75465b6..51411fafa9234 100644 --- a/examples/cors/frontend/front-envoy.yaml +++ b/examples/cors/frontend/front-envoy.yaml @@ -33,7 +33,6 @@ static_resources: typed_config: {} clusters: - name: frontend_service - connect_timeout: 0.25s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: diff --git a/examples/cors/frontend/service-envoy.yaml b/examples/cors/frontend/service-envoy.yaml index c596eb508ff87..d5943d46ea7ba 100644 --- a/examples/cors/frontend/service-envoy.yaml +++ b/examples/cors/frontend/service-envoy.yaml @@ -27,7 +27,6 @@ static_resources: typed_config: {} clusters: - name: local_service - connect_timeout: 0.25s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: diff --git a/examples/csrf/crosssite/front-envoy.yaml b/examples/csrf/crosssite/front-envoy.yaml index 114b0f06ddd03..18f8be37fa47d 100644 --- a/examples/csrf/crosssite/front-envoy.yaml +++ b/examples/csrf/crosssite/front-envoy.yaml @@ -31,7 +31,6 @@ static_resources: typed_config: {} clusters: - name: generic_service - connect_timeout: 0.25s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: diff --git a/examples/csrf/samesite/front-envoy.yaml b/examples/csrf/samesite/front-envoy.yaml index c8fc0c77eb3de..de99d4a8c4f13 100644 --- a/examples/csrf/samesite/front-envoy.yaml +++ b/examples/csrf/samesite/front-envoy.yaml @@ -103,7 +103,6 @@ static_resources: typed_config: {} clusters: - name: generic_service - connect_timeout: 0.25s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: diff --git a/examples/csrf/service-envoy.yaml b/examples/csrf/service-envoy.yaml index c596eb508ff87..d5943d46ea7ba 100644 --- a/examples/csrf/service-envoy.yaml +++ b/examples/csrf/service-envoy.yaml @@ -27,7 +27,6 @@ static_resources: typed_config: {} clusters: - name: local_service - connect_timeout: 0.25s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: diff --git a/examples/double-proxy/envoy-backend.yaml b/examples/double-proxy/envoy-backend.yaml index 598749325af49..d04f33f2996f5 100644 --- a/examples/double-proxy/envoy-backend.yaml +++ b/examples/double-proxy/envoy-backend.yaml @@ -38,7 +38,6 @@ static_resources: clusters: - name: postgres_cluster - connect_timeout: 1s type: STRICT_DNS load_assignment: cluster_name: postgres_cluster diff --git a/examples/double-proxy/envoy-frontend.yaml b/examples/double-proxy/envoy-frontend.yaml index 49cee39333250..af836ba8fc30d 100644 --- a/examples/double-proxy/envoy-frontend.yaml +++ b/examples/double-proxy/envoy-frontend.yaml @@ -19,7 +19,6 @@ static_resources: clusters: - name: postgres_cluster - connect_timeout: 1s type: STRICT_DNS load_assignment: cluster_name: postgres_cluster diff --git a/examples/double-proxy/envoy.yaml b/examples/double-proxy/envoy.yaml index 97fb2c039d2d3..a962715924720 100644 --- a/examples/double-proxy/envoy.yaml +++ b/examples/double-proxy/envoy.yaml @@ -27,7 +27,6 @@ static_resources: clusters: - name: service1 - connect_timeout: 0.25s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: diff --git a/examples/dynamic-config-cp/envoy.yaml b/examples/dynamic-config-cp/envoy.yaml index 9b7eb57dd1b90..aafd750656cfb 100644 --- a/examples/dynamic-config-cp/envoy.yaml +++ b/examples/dynamic-config-cp/envoy.yaml @@ -18,8 +18,7 @@ dynamic_resources: static_resources: clusters: - - connect_timeout: 1s - type: STRICT_DNS + - type: STRICT_DNS typed_extension_protocol_options: envoy.extensions.upstreams.http.v3.HttpProtocolOptions: "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions diff --git a/examples/dynamic-config-fs/configs/cds.yaml b/examples/dynamic-config-fs/configs/cds.yaml index f22089e0c7f0a..8cc9e3c0b26cd 100644 --- a/examples/dynamic-config-fs/configs/cds.yaml +++ b/examples/dynamic-config-fs/configs/cds.yaml @@ -1,7 +1,6 @@ resources: - "@type": type.googleapis.com/envoy.config.cluster.v3.Cluster name: example_proxy_cluster - connect_timeout: 1s type: STRICT_DNS load_assignment: cluster_name: example_proxy_cluster diff --git a/examples/ext_authz/config/grpc-service/v3.yaml b/examples/ext_authz/config/grpc-service/v3.yaml index 8ad48325eb410..f42f332137957 100644 --- a/examples/ext_authz/config/grpc-service/v3.yaml +++ b/examples/ext_authz/config/grpc-service/v3.yaml @@ -36,7 +36,6 @@ static_resources: clusters: - name: upstream-service - connect_timeout: 0.250s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: @@ -50,7 +49,6 @@ static_resources: port_value: 8080 - name: ext_authz-grpc-service - connect_timeout: 0.250s type: STRICT_DNS lb_policy: ROUND_ROBIN typed_extension_protocol_options: diff --git a/examples/ext_authz/config/http-service.yaml b/examples/ext_authz/config/http-service.yaml index 1bfb7df4ff96b..581f48961d131 100644 --- a/examples/ext_authz/config/http-service.yaml +++ b/examples/ext_authz/config/http-service.yaml @@ -41,7 +41,6 @@ static_resources: clusters: - name: upstream-service - connect_timeout: 0.250s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: @@ -55,7 +54,6 @@ static_resources: port_value: 8080 - name: ext_authz-http-service - connect_timeout: 0.250s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: diff --git a/examples/ext_authz/config/opa-service/v3.yaml b/examples/ext_authz/config/opa-service/v3.yaml index 72dd285de3ea6..f5a6697cb4661 100644 --- a/examples/ext_authz/config/opa-service/v3.yaml +++ b/examples/ext_authz/config/opa-service/v3.yaml @@ -36,7 +36,6 @@ static_resources: clusters: - name: upstream-service - connect_timeout: 0.250s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: @@ -50,7 +49,6 @@ static_resources: port_value: 8080 - name: ext_authz-opa-service - connect_timeout: 0.250s type: STRICT_DNS lb_policy: ROUND_ROBIN typed_extension_protocol_options: diff --git a/examples/fault-injection/envoy.yaml b/examples/fault-injection/envoy.yaml index 1cce4ec52aea8..a526edb6fa016 100644 --- a/examples/fault-injection/envoy.yaml +++ b/examples/fault-injection/envoy.yaml @@ -44,7 +44,6 @@ static_resources: typed_config: {} clusters: - name: local_service - connect_timeout: 1s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: diff --git a/examples/front-proxy/front-envoy.yaml b/examples/front-proxy/front-envoy.yaml index d96a5d31538c3..b97ac7b278fa9 100644 --- a/examples/front-proxy/front-envoy.yaml +++ b/examples/front-proxy/front-envoy.yaml @@ -126,7 +126,6 @@ static_resources: clusters: - name: service1 - connect_timeout: 0.25s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: @@ -139,7 +138,6 @@ static_resources: address: service1 port_value: 8000 - name: service2 - connect_timeout: 0.25s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: diff --git a/examples/front-proxy/service-envoy.yaml b/examples/front-proxy/service-envoy.yaml index b3ea728cc5f60..eaa849dd40797 100644 --- a/examples/front-proxy/service-envoy.yaml +++ b/examples/front-proxy/service-envoy.yaml @@ -27,7 +27,6 @@ static_resources: typed_config: {} clusters: - name: local_service - connect_timeout: 0.25s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: diff --git a/examples/grpc-bridge/client/envoy-proxy.yaml b/examples/grpc-bridge/client/envoy-proxy.yaml index 91ea427889885..c96d32a6dfc43 100644 --- a/examples/grpc-bridge/client/envoy-proxy.yaml +++ b/examples/grpc-bridge/client/envoy-proxy.yaml @@ -40,7 +40,6 @@ static_resources: type: LOGICAL_DNS dns_lookup_family: V4_ONLY lb_policy: ROUND_ROBIN - connect_timeout: 0.250s http_protocol_options: {} load_assignment: cluster_name: backend-proxy diff --git a/examples/grpc-bridge/client/requirements.txt b/examples/grpc-bridge/client/requirements.txt index bda4a54f6297d..43e9983674907 100644 --- a/examples/grpc-bridge/client/requirements.txt +++ b/examples/grpc-bridge/client/requirements.txt @@ -1,4 +1,4 @@ requests>=2.22.0 grpcio grpcio-tools -protobuf==3.17.0 +protobuf==3.17.1 diff --git a/examples/grpc-bridge/server/envoy-proxy.yaml b/examples/grpc-bridge/server/envoy-proxy.yaml index d03dd25c53d73..8e63706b5aaa6 100644 --- a/examples/grpc-bridge/server/envoy-proxy.yaml +++ b/examples/grpc-bridge/server/envoy-proxy.yaml @@ -32,7 +32,6 @@ static_resources: typed_config: {} clusters: - name: backend_grpc_service - connect_timeout: 0.250s type: STRICT_DNS lb_policy: ROUND_ROBIN typed_extension_protocol_options: diff --git a/examples/jaeger-native-tracing/front-envoy-jaeger.yaml b/examples/jaeger-native-tracing/front-envoy-jaeger.yaml index 9569202399179..a32051d5dc7bb 100644 --- a/examples/jaeger-native-tracing/front-envoy-jaeger.yaml +++ b/examples/jaeger-native-tracing/front-envoy-jaeger.yaml @@ -52,7 +52,6 @@ static_resources: use_remote_address: true clusters: - name: service1 - connect_timeout: 0.250s type: strict_dns lb_policy: round_robin load_assignment: diff --git a/examples/jaeger-native-tracing/service1-envoy-jaeger.yaml b/examples/jaeger-native-tracing/service1-envoy-jaeger.yaml index cfb10f3259431..7614f6354c883 100644 --- a/examples/jaeger-native-tracing/service1-envoy-jaeger.yaml +++ b/examples/jaeger-native-tracing/service1-envoy-jaeger.yaml @@ -78,7 +78,6 @@ static_resources: typed_config: {} clusters: - name: local_service - connect_timeout: 0.250s type: strict_dns lb_policy: round_robin load_assignment: @@ -91,7 +90,6 @@ static_resources: address: 127.0.0.1 port_value: 8080 - name: service2 - connect_timeout: 0.250s type: strict_dns lb_policy: round_robin load_assignment: diff --git a/examples/jaeger-native-tracing/service2-envoy-jaeger.yaml b/examples/jaeger-native-tracing/service2-envoy-jaeger.yaml index b2349b7fac9a6..a56ba608c3120 100644 --- a/examples/jaeger-native-tracing/service2-envoy-jaeger.yaml +++ b/examples/jaeger-native-tracing/service2-envoy-jaeger.yaml @@ -50,7 +50,6 @@ static_resources: typed_config: {} clusters: - name: local_service - connect_timeout: 0.250s type: strict_dns lb_policy: round_robin load_assignment: diff --git a/examples/jaeger-tracing/front-envoy-jaeger.yaml b/examples/jaeger-tracing/front-envoy-jaeger.yaml index 831175b810b29..b5ac02f727e66 100644 --- a/examples/jaeger-tracing/front-envoy-jaeger.yaml +++ b/examples/jaeger-tracing/front-envoy-jaeger.yaml @@ -41,7 +41,6 @@ static_resources: use_remote_address: true clusters: - name: service1 - connect_timeout: 0.250s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: @@ -54,7 +53,6 @@ static_resources: address: service1 port_value: 8000 - name: jaeger - connect_timeout: 1s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: diff --git a/examples/jaeger-tracing/service1-envoy-jaeger.yaml b/examples/jaeger-tracing/service1-envoy-jaeger.yaml index a33970821c97d..7ef911a69ea80 100644 --- a/examples/jaeger-tracing/service1-envoy-jaeger.yaml +++ b/examples/jaeger-tracing/service1-envoy-jaeger.yaml @@ -76,7 +76,6 @@ static_resources: typed_config: {} clusters: - name: local_service - connect_timeout: 0.250s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: @@ -89,7 +88,6 @@ static_resources: address: 127.0.0.1 port_value: 8080 - name: service2 - connect_timeout: 0.250s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: @@ -102,7 +100,6 @@ static_resources: address: service2 port_value: 8000 - name: jaeger - connect_timeout: 1s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: diff --git a/examples/jaeger-tracing/service2-envoy-jaeger.yaml b/examples/jaeger-tracing/service2-envoy-jaeger.yaml index c48878e9c1891..afb0a2d70df5f 100644 --- a/examples/jaeger-tracing/service2-envoy-jaeger.yaml +++ b/examples/jaeger-tracing/service2-envoy-jaeger.yaml @@ -39,7 +39,6 @@ static_resources: typed_config: {} clusters: - name: local_service - connect_timeout: 0.250s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: @@ -52,7 +51,6 @@ static_resources: address: 127.0.0.1 port_value: 8080 - name: jaeger - connect_timeout: 1s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: diff --git a/examples/load-reporting-service/service-envoy-w-lrs.yaml b/examples/load-reporting-service/service-envoy-w-lrs.yaml index e908f8ff15826..c5da0d64a1651 100644 --- a/examples/load-reporting-service/service-envoy-w-lrs.yaml +++ b/examples/load-reporting-service/service-envoy-w-lrs.yaml @@ -27,7 +27,6 @@ static_resources: typed_config: {} clusters: - name: local_service - connect_timeout: 0.25s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: @@ -40,7 +39,6 @@ static_resources: address: 127.0.0.1 port_value: 8082 - name: load_reporting_cluster - connect_timeout: 0.25s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: diff --git a/examples/lua/envoy.yaml b/examples/lua/envoy.yaml index f469888229908..a9cc2555282e1 100644 --- a/examples/lua/envoy.yaml +++ b/examples/lua/envoy.yaml @@ -42,7 +42,6 @@ static_resources: clusters: - name: web_service - connect_timeout: 0.25s type: STRICT_DNS # static lb_policy: ROUND_ROBIN load_assignment: diff --git a/examples/mysql/envoy.yaml b/examples/mysql/envoy.yaml index b530e0e2f6942..4760b2c3eaca4 100644 --- a/examples/mysql/envoy.yaml +++ b/examples/mysql/envoy.yaml @@ -19,7 +19,6 @@ static_resources: clusters: - name: mysql_cluster - connect_timeout: 1s type: STRICT_DNS load_assignment: cluster_name: mysql_cluster diff --git a/examples/postgres/envoy.yaml b/examples/postgres/envoy.yaml index 5433bc54b6ebf..bc3bbe7b373e1 100644 --- a/examples/postgres/envoy.yaml +++ b/examples/postgres/envoy.yaml @@ -19,7 +19,6 @@ static_resources: clusters: - name: postgres_cluster - connect_timeout: 1s type: STRICT_DNS load_assignment: cluster_name: postgres_cluster diff --git a/examples/redis/envoy.yaml b/examples/redis/envoy.yaml index d9f4bd9450b41..1960f7925daf9 100644 --- a/examples/redis/envoy.yaml +++ b/examples/redis/envoy.yaml @@ -18,7 +18,6 @@ static_resources: cluster: redis_cluster clusters: - name: redis_cluster - connect_timeout: 1s type: STRICT_DNS # static lb_policy: MAGLEV load_assignment: diff --git a/examples/skywalking-tracing/front-envoy-skywalking.yaml b/examples/skywalking-tracing/front-envoy-skywalking.yaml index 61baeec4add10..eddedbb407873 100644 --- a/examples/skywalking-tracing/front-envoy-skywalking.yaml +++ b/examples/skywalking-tracing/front-envoy-skywalking.yaml @@ -45,7 +45,6 @@ static_resources: start_child_span: true clusters: - name: service1 - connect_timeout: 0.250s type: STRICT_DNS lb_policy: ROUND_ROBIN typed_extension_protocol_options: @@ -63,7 +62,6 @@ static_resources: address: service1 port_value: 8000 - name: skywalking - connect_timeout: 1s type: STRICT_DNS lb_policy: ROUND_ROBIN typed_extension_protocol_options: diff --git a/examples/skywalking-tracing/service1-envoy-skywalking.yaml b/examples/skywalking-tracing/service1-envoy-skywalking.yaml index fc3220bb5a000..96937a17b4fcd 100644 --- a/examples/skywalking-tracing/service1-envoy-skywalking.yaml +++ b/examples/skywalking-tracing/service1-envoy-skywalking.yaml @@ -86,7 +86,6 @@ static_resources: start_child_span: true clusters: - name: local_service - connect_timeout: 0.250s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: @@ -99,7 +98,6 @@ static_resources: address: 127.0.0.1 port_value: 8080 - name: service2 - connect_timeout: 0.250s type: STRICT_DNS lb_policy: ROUND_ROBIN typed_extension_protocol_options: @@ -117,7 +115,6 @@ static_resources: address: service2 port_value: 8000 - name: skywalking - connect_timeout: 1s type: STRICT_DNS lb_policy: ROUND_ROBIN typed_extension_protocol_options: diff --git a/examples/skywalking-tracing/service2-envoy-skywalking.yaml b/examples/skywalking-tracing/service2-envoy-skywalking.yaml index 0cdb6ded67a5a..5a9eb92e9dd7d 100644 --- a/examples/skywalking-tracing/service2-envoy-skywalking.yaml +++ b/examples/skywalking-tracing/service2-envoy-skywalking.yaml @@ -44,7 +44,6 @@ static_resources: start_child_span: true clusters: - name: local_service - connect_timeout: 0.250s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: @@ -57,7 +56,6 @@ static_resources: address: 127.0.0.1 port_value: 8080 - name: skywalking - connect_timeout: 1s type: STRICT_DNS lb_policy: ROUND_ROBIN typed_extension_protocol_options: diff --git a/examples/tls-inspector/envoy.yaml b/examples/tls-inspector/envoy.yaml index d4462920d8916..634f82075de0c 100644 --- a/examples/tls-inspector/envoy.yaml +++ b/examples/tls-inspector/envoy.yaml @@ -43,7 +43,6 @@ static_resources: clusters: - name: service-https-http2 - connect_timeout: 0.25s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: @@ -56,7 +55,6 @@ static_resources: address: service-https-http2 port_value: 443 - name: service-https-http1.1 - connect_timeout: 0.25s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: @@ -69,7 +67,6 @@ static_resources: address: service-https-http1.1 port_value: 443 - name: service-http - connect_timeout: 0.25s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: diff --git a/examples/tls-sni/envoy-client.yaml b/examples/tls-sni/envoy-client.yaml index e64aae12dc2a6..b99bbeead452c 100644 --- a/examples/tls-sni/envoy-client.yaml +++ b/examples/tls-sni/envoy-client.yaml @@ -35,7 +35,6 @@ static_resources: clusters: - name: proxy-client-domain1 - connect_timeout: 0.25s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: @@ -54,7 +53,6 @@ static_resources: sni: domain1.example.com - name: proxy-client-domain2 - connect_timeout: 0.25s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: @@ -73,7 +71,6 @@ static_resources: sni: domain2.example.com - name: proxy-client-domain3 - connect_timeout: 0.25s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: diff --git a/examples/tls-sni/envoy.yaml b/examples/tls-sni/envoy.yaml index 7ad9843c6d8c4..b6248f9d748ae 100644 --- a/examples/tls-sni/envoy.yaml +++ b/examples/tls-sni/envoy.yaml @@ -85,7 +85,6 @@ static_resources: clusters: - name: proxy-domain1 - connect_timeout: 0.25s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: @@ -99,7 +98,6 @@ static_resources: port_value: 80 - name: proxy-domain2 - connect_timeout: 0.25s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: @@ -113,7 +111,6 @@ static_resources: port_value: 80 - name: proxy-domain3 - connect_timeout: 0.25s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: diff --git a/examples/tls/envoy-http-https.yaml b/examples/tls/envoy-http-https.yaml index 0695099e89814..ae532dd7c355e 100644 --- a/examples/tls/envoy-http-https.yaml +++ b/examples/tls/envoy-http-https.yaml @@ -27,7 +27,6 @@ static_resources: clusters: - name: service-https - connect_timeout: 0.25s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: diff --git a/examples/tls/envoy-https-http.yaml b/examples/tls/envoy-https-http.yaml index 79496d520e00d..640b0ca8429d1 100644 --- a/examples/tls/envoy-https-http.yaml +++ b/examples/tls/envoy-https-http.yaml @@ -90,7 +90,6 @@ static_resources: clusters: - name: service-http - connect_timeout: 0.25s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: diff --git a/examples/tls/envoy-https-https.yaml b/examples/tls/envoy-https-https.yaml index 2702adb6b8e91..13a60d4d9c361 100644 --- a/examples/tls/envoy-https-https.yaml +++ b/examples/tls/envoy-https-https.yaml @@ -90,7 +90,6 @@ static_resources: clusters: - name: service-https - connect_timeout: 0.25s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: diff --git a/examples/tls/envoy-https-passthrough.yaml b/examples/tls/envoy-https-passthrough.yaml index 3c7a221ff0604..8707204a5c416 100644 --- a/examples/tls/envoy-https-passthrough.yaml +++ b/examples/tls/envoy-https-passthrough.yaml @@ -14,7 +14,6 @@ static_resources: clusters: - name: service-https - connect_timeout: 0.25s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: diff --git a/examples/udp/envoy.yaml b/examples/udp/envoy.yaml index 6ae56223a2fca..9937bba45f1f9 100644 --- a/examples/udp/envoy.yaml +++ b/examples/udp/envoy.yaml @@ -16,7 +16,6 @@ static_resources: clusters: - name: service_udp - connect_timeout: 0.25s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: diff --git a/examples/wasm-cc/envoy.yaml b/examples/wasm-cc/envoy.yaml index 1d04c7a26f62c..e65d4e6ce8265 100644 --- a/examples/wasm-cc/envoy.yaml +++ b/examples/wasm-cc/envoy.yaml @@ -47,7 +47,6 @@ static_resources: clusters: - name: web_service - connect_timeout: 0.25s type: strict_dns lb_policy: round_robin load_assignment: diff --git a/examples/websocket/envoy-ws.yaml b/examples/websocket/envoy-ws.yaml index aa670d2bd412e..5db9005f3d531 100644 --- a/examples/websocket/envoy-ws.yaml +++ b/examples/websocket/envoy-ws.yaml @@ -28,7 +28,6 @@ static_resources: clusters: - name: service_ws - connect_timeout: 0.25s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: diff --git a/examples/websocket/envoy-wss-passthrough.yaml b/examples/websocket/envoy-wss-passthrough.yaml index 5d546329c3e18..6634d483def68 100644 --- a/examples/websocket/envoy-wss-passthrough.yaml +++ b/examples/websocket/envoy-wss-passthrough.yaml @@ -14,7 +14,6 @@ static_resources: clusters: - name: service_wss_passthrough - connect_timeout: 0.25s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: diff --git a/examples/websocket/envoy-wss.yaml b/examples/websocket/envoy-wss.yaml index 8e94de35f5915..f4dee8fd0bd7b 100644 --- a/examples/websocket/envoy-wss.yaml +++ b/examples/websocket/envoy-wss.yaml @@ -91,7 +91,6 @@ static_resources: clusters: - name: service_wss - connect_timeout: 0.25s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: diff --git a/examples/zipkin-tracing/front-envoy-zipkin.yaml b/examples/zipkin-tracing/front-envoy-zipkin.yaml index 599b28d009b56..7fc5cfb61f70e 100644 --- a/examples/zipkin-tracing/front-envoy-zipkin.yaml +++ b/examples/zipkin-tracing/front-envoy-zipkin.yaml @@ -46,7 +46,6 @@ static_resources: typed_config: {} clusters: - name: service1 - connect_timeout: 0.250s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: @@ -59,7 +58,6 @@ static_resources: address: service1 port_value: 8000 - name: zipkin - connect_timeout: 1s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: diff --git a/examples/zipkin-tracing/service1-envoy-zipkin.yaml b/examples/zipkin-tracing/service1-envoy-zipkin.yaml index b006bed4b4195..fd790aa3c1e94 100644 --- a/examples/zipkin-tracing/service1-envoy-zipkin.yaml +++ b/examples/zipkin-tracing/service1-envoy-zipkin.yaml @@ -74,7 +74,6 @@ static_resources: typed_config: {} clusters: - name: local_service - connect_timeout: 0.250s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: @@ -87,7 +86,6 @@ static_resources: address: 127.0.0.1 port_value: 8080 - name: service2 - connect_timeout: 0.250s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: @@ -100,7 +98,6 @@ static_resources: address: service2 port_value: 8000 - name: zipkin - connect_timeout: 1s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: diff --git a/examples/zipkin-tracing/service2-envoy-zipkin.yaml b/examples/zipkin-tracing/service2-envoy-zipkin.yaml index be152059d2348..2078d93d00777 100644 --- a/examples/zipkin-tracing/service2-envoy-zipkin.yaml +++ b/examples/zipkin-tracing/service2-envoy-zipkin.yaml @@ -38,7 +38,6 @@ static_resources: typed_config: {} clusters: - name: local_service - connect_timeout: 0.250s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: @@ -51,7 +50,6 @@ static_resources: address: 127.0.0.1 port_value: 8080 - name: zipkin - connect_timeout: 1s type: STRICT_DNS lb_policy: ROUND_ROBIN load_assignment: diff --git a/generated_api_shadow/BUILD b/generated_api_shadow/BUILD index 04b94ff211fd4..e051250b80753 100644 --- a/generated_api_shadow/BUILD +++ b/generated_api_shadow/BUILD @@ -177,6 +177,7 @@ proto_library( "//envoy/extensions/filters/http/admission_control/v3alpha:pkg", "//envoy/extensions/filters/http/aws_lambda/v3:pkg", "//envoy/extensions/filters/http/aws_request_signing/v3:pkg", + "//envoy/extensions/filters/http/bandwidth_limit/v3alpha:pkg", "//envoy/extensions/filters/http/buffer/v3:pkg", "//envoy/extensions/filters/http/cache/v3alpha:pkg", "//envoy/extensions/filters/http/cdn_loop/v3alpha:pkg", @@ -209,6 +210,7 @@ proto_library( "//envoy/extensions/filters/http/ratelimit/v3:pkg", "//envoy/extensions/filters/http/rbac/v3:pkg", "//envoy/extensions/filters/http/router/v3:pkg", + "//envoy/extensions/filters/http/set_metadata/v3:pkg", "//envoy/extensions/filters/http/squash/v3:pkg", "//envoy/extensions/filters/http/tap/v3:pkg", "//envoy/extensions/filters/http/wasm/v3:pkg", diff --git a/generated_api_shadow/envoy/config/core/v3/resolver.proto b/generated_api_shadow/envoy/config/core/v3/resolver.proto new file mode 100644 index 0000000000000..b5e31814f56aa --- /dev/null +++ b/generated_api_shadow/envoy/config/core/v3/resolver.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; + +package envoy.config.core.v3; + +import "envoy/config/core/v3/address.proto"; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.core.v3"; +option java_outer_classname = "ResolverProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Resolver] + +// DNS resolver configuration which includes the underlying dns resolver addresses and options. +message DnsResolver { + // A list of dns resolver addresses + // Setting this value causes failure if the + // ``envoy.restart_features.use_apple_api_for_dns_lookups`` runtime value is true during + // server startup. Apple's API only allows overriding DNS resolvers via system settings. + repeated Address resolvers = 1 [(validate.rules).repeated = {min_items: 1}]; +} diff --git a/generated_api_shadow/envoy/config/core/v4alpha/resolver.proto b/generated_api_shadow/envoy/config/core/v4alpha/resolver.proto new file mode 100644 index 0000000000000..ad597c85549de --- /dev/null +++ b/generated_api_shadow/envoy/config/core/v4alpha/resolver.proto @@ -0,0 +1,27 @@ +syntax = "proto3"; + +package envoy.config.core.v4alpha; + +import "envoy/config/core/v4alpha/address.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.core.v4alpha"; +option java_outer_classname = "ResolverProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; + +// [#protodoc-title: Resolver] + +// DNS resolver configuration which includes the underlying dns resolver addresses and options. +message DnsResolver { + option (udpa.annotations.versioning).previous_message_type = "envoy.config.core.v3.DnsResolver"; + + // A list of dns resolver addresses + // Setting this value causes failure if the + // ``envoy.restart_features.use_apple_api_for_dns_lookups`` runtime value is true during + // server startup. Apple's API only allows overriding DNS resolvers via system settings. + repeated Address resolvers = 1 [(validate.rules).repeated = {min_items: 1}]; +} diff --git a/generated_api_shadow/envoy/config/listener/v3/quic_config.proto b/generated_api_shadow/envoy/config/listener/v3/quic_config.proto index 69df722c6fbb4..d1e62cdaaf15a 100644 --- a/generated_api_shadow/envoy/config/listener/v3/quic_config.proto +++ b/generated_api_shadow/envoy/config/listener/v3/quic_config.proto @@ -6,9 +6,11 @@ import "envoy/config/core/v3/base.proto"; import "envoy/config/core/v3/protocol.proto"; import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; import "udpa/annotations/status.proto"; import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.config.listener.v3"; option java_outer_classname = "QuicConfigProto"; @@ -18,6 +20,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: QUIC listener config] // Configuration specific to the UDP QUIC listener. +// [#next-free-field: 6] message QuicProtocolOptions { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.listener.QuicProtocolOptions"; @@ -35,4 +38,14 @@ message QuicProtocolOptions { // Runtime flag that controls whether the listener is enabled or not. If not specified, defaults // to enabled. core.v3.RuntimeFeatureFlag enabled = 4; + + // A multiplier to number of connections which is used to determine how many packets to read per + // event loop. A reasonable number should allow the listener to process enough payload but not + // starve TCP and other UDP sockets and also prevent long event loop duration. + // The default value is 32. This means if there are N QUIC connections, the total number of + // packets to read in each read event will be 32 * N. + // The actual number of packets to read in total by the UDP listener is also + // bound by 6000, regardless of this field or how many connections there are. + google.protobuf.UInt32Value packets_to_read_to_connection_count_ratio = 5 + [(validate.rules).uint32 = {gte: 1}]; } diff --git a/generated_api_shadow/envoy/config/listener/v4alpha/quic_config.proto b/generated_api_shadow/envoy/config/listener/v4alpha/quic_config.proto index c9e218137ae27..6d0f5e51493b6 100644 --- a/generated_api_shadow/envoy/config/listener/v4alpha/quic_config.proto +++ b/generated_api_shadow/envoy/config/listener/v4alpha/quic_config.proto @@ -6,9 +6,11 @@ import "envoy/config/core/v4alpha/base.proto"; import "envoy/config/core/v4alpha/protocol.proto"; import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; import "udpa/annotations/status.proto"; import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.config.listener.v4alpha"; option java_outer_classname = "QuicConfigProto"; @@ -18,6 +20,7 @@ option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSIO // [#protodoc-title: QUIC listener config] // Configuration specific to the UDP QUIC listener. +// [#next-free-field: 6] message QuicProtocolOptions { option (udpa.annotations.versioning).previous_message_type = "envoy.config.listener.v3.QuicProtocolOptions"; @@ -35,4 +38,14 @@ message QuicProtocolOptions { // Runtime flag that controls whether the listener is enabled or not. If not specified, defaults // to enabled. core.v4alpha.RuntimeFeatureFlag enabled = 4; + + // A multiplier to number of connections which is used to determine how many packets to read per + // event loop. A reasonable number should allow the listener to process enough payload but not + // starve TCP and other UDP sockets and also prevent long event loop duration. + // The default value is 32. This means if there are N QUIC connections, the total number of + // packets to read in each read event will be 32 * N. + // The actual number of packets to read in total by the UDP listener is also + // bound by 6000, regardless of this field or how many connections there are. + google.protobuf.UInt32Value packets_to_read_to_connection_count_ratio = 5 + [(validate.rules).uint32 = {gte: 1}]; } diff --git a/generated_api_shadow/envoy/config/route/v3/route_components.proto b/generated_api_shadow/envoy/config/route/v3/route_components.proto index 1b70f539c2c0a..b8f03cde3a9dd 100644 --- a/generated_api_shadow/envoy/config/route/v3/route_components.proto +++ b/generated_api_shadow/envoy/config/route/v3/route_components.proto @@ -1916,8 +1916,8 @@ message HeaderMatcher { // "-1somestring" type.v3.Int64Range range_match = 6; - // If specified, header match will be performed based on whether the header is in the - // request. + // If specified as true, header match will be performed based on whether the header is in the + // request. If specified as false, header match will be performed based on whether the header is absent. bool present_match = 7; // If specified, header match will be performed based on the prefix of the header value. diff --git a/generated_api_shadow/envoy/config/route/v4alpha/route_components.proto b/generated_api_shadow/envoy/config/route/v4alpha/route_components.proto index 34d3762bfdde9..6b8c146582a33 100644 --- a/generated_api_shadow/envoy/config/route/v4alpha/route_components.proto +++ b/generated_api_shadow/envoy/config/route/v4alpha/route_components.proto @@ -1915,8 +1915,8 @@ message HeaderMatcher { // "-1somestring" type.v3.Int64Range range_match = 6; - // If specified, header match will be performed based on whether the header is in the - // request. + // If specified as true, header match will be performed based on whether the header is in the + // request. If specified as false, header match will be performed based on whether the header is absent. bool present_match = 7; // If specified, header match will be performed based on the prefix of the header value. diff --git a/generated_api_shadow/envoy/extensions/common/dynamic_forward_proxy/v3/BUILD b/generated_api_shadow/envoy/extensions/common/dynamic_forward_proxy/v3/BUILD index b7a0695c214bc..6e74b985c580d 100644 --- a/generated_api_shadow/envoy/extensions/common/dynamic_forward_proxy/v3/BUILD +++ b/generated_api_shadow/envoy/extensions/common/dynamic_forward_proxy/v3/BUILD @@ -8,6 +8,7 @@ api_proto_package( deps = [ "//envoy/config/cluster/v3:pkg", "//envoy/config/common/dynamic_forward_proxy/v2alpha:pkg", + "//envoy/config/core/v3:pkg", "@com_github_cncf_udpa//udpa/annotations:pkg", ], ) diff --git a/generated_api_shadow/envoy/extensions/common/dynamic_forward_proxy/v3/dns_cache.proto b/generated_api_shadow/envoy/extensions/common/dynamic_forward_proxy/v3/dns_cache.proto index 4b182a29711bd..1c14739b94c86 100644 --- a/generated_api_shadow/envoy/extensions/common/dynamic_forward_proxy/v3/dns_cache.proto +++ b/generated_api_shadow/envoy/extensions/common/dynamic_forward_proxy/v3/dns_cache.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package envoy.extensions.common.dynamic_forward_proxy.v3; import "envoy/config/cluster/v3/cluster.proto"; +import "envoy/config/core/v3/resolver.proto"; import "google/protobuf/duration.proto"; import "google/protobuf/wrappers.proto"; @@ -27,7 +28,7 @@ message DnsCacheCircuitBreakers { // Configuration for the dynamic forward proxy DNS cache. See the :ref:`architecture overview // ` for more information. -// [#next-free-field: 9] +// [#next-free-field: 10] message DnsCacheConfig { option (udpa.annotations.versioning).previous_message_type = "envoy.config.common.dynamic_forward_proxy.v2alpha.DnsCacheConfig"; @@ -101,4 +102,9 @@ message DnsCacheConfig { // ``envoy.restart_features.use_apple_api_for_dns_lookups`` runtime value is true during // server startup. Apple' API only uses UDP for DNS resolution. bool use_tcp_for_dns_lookups = 8; + + // DNS resolver configuration + // If specified, DNS cache will perform resolution via the underlying DNS resolvers. + // Otherwise, the default system resolvers (e.g., /etc/resolv.conf) will be used. + config.core.v3.DnsResolver dns_resolver = 9; } diff --git a/generated_api_shadow/envoy/extensions/filters/http/bandwidth_limit/v3alpha/BUILD b/generated_api_shadow/envoy/extensions/filters/http/bandwidth_limit/v3alpha/BUILD new file mode 100644 index 0000000000000..1c1a6f6b44235 --- /dev/null +++ b/generated_api_shadow/envoy/extensions/filters/http/bandwidth_limit/v3alpha/BUILD @@ -0,0 +1,12 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/config/core/v3:pkg", + "@com_github_cncf_udpa//udpa/annotations:pkg", + ], +) diff --git a/generated_api_shadow/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto b/generated_api_shadow/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto new file mode 100644 index 0000000000000..4cd5f8268b704 --- /dev/null +++ b/generated_api_shadow/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto @@ -0,0 +1,70 @@ +syntax = "proto3"; + +package envoy.extensions.filters.http.bandwidth_limit.v3alpha; + +import "envoy/config/core/v3/base.proto"; + +import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.filters.http.bandwidth_limit.v3alpha"; +option java_outer_classname = "BandwidthLimitProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).work_in_progress = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Bandwidth limit] +// Bandwidth limit :ref:`configuration overview `. +// [#extension: envoy.filters.http.bandwidth_limit] + +// [#next-free-field: 6] +message BandwidthLimit { + // Defines the mode for the bandwidth limit filter. + // Values represent bitmask. + enum EnableMode { + // Filter is disabled. + DISABLED = 0; + + // Filter enabled only for incoming traffic. + REQUEST = 1; + + // Filter enabled only for outgoing traffic. + RESPONSE = 2; + + // Filter enabled for both incoming and outgoing traffic. + REQUEST_AND_RESPONSE = 3; + } + + // The human readable prefix to use when emitting stats. + string stat_prefix = 1 [(validate.rules).string = {min_len: 1}]; + + // The enable mode for the bandwidth limit filter. + // Default is Disabled. + EnableMode enable_mode = 2 [(validate.rules).enum = {defined_only: true}]; + + // The limit supplied in KiB/s. + // + // .. note:: + // It's fine for the limit to be unset for the global configuration since the bandwidth limit + // can be applied at a the virtual host or route level. Thus, the limit must be set for the + // per route configuration otherwise the config will be rejected. + // + // .. note:: + // When using per route configuration, the limit becomes unique to that route. + // + google.protobuf.UInt64Value limit_kbps = 3 [(validate.rules).uint64 = {gte: 1}]; + + // Optional Fill interval in milliseconds for the token refills. Defaults to 50ms. + // It must be at least 20ms to avoid too aggressive refills. + google.protobuf.Duration fill_interval = 4 [(validate.rules).duration = { + lte {seconds: 1} + gte {nanos: 20000000} + }]; + + // Runtime flag that controls whether the filter is enabled or not. If not specified, defaults + // to enabled. + config.core.v3.RuntimeFeatureFlag runtime_enabled = 5; +} diff --git a/generated_api_shadow/envoy/extensions/filters/http/jwt_authn/v3/config.proto b/generated_api_shadow/envoy/extensions/filters/http/jwt_authn/v3/config.proto index 08ef7a09feb20..afc761c07c7e1 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/jwt_authn/v3/config.proto +++ b/generated_api_shadow/envoy/extensions/filters/http/jwt_authn/v3/config.proto @@ -232,6 +232,35 @@ message RemoteJwks { // Duration after which the cached JWKS should be expired. If not specified, default cache // duration is 5 minutes. google.protobuf.Duration cache_duration = 2; + + // Fetch Jwks asynchronously in the main thread before the listener is activated. + // Fetched Jwks can be used by all worker threads. + // + // If this feature is not enabled: + // + // * The Jwks is fetched on-demand when the requests come. During the fetching, first + // few requests are paused until the Jwks is fetched. + // * Each worker thread fetches its own Jwks since Jwks cache is per worker thread. + // + // If this feature is enabled: + // + // * Fetched Jwks is done in the main thread before the listener is activated. Its fetched + // Jwks can be used by all worker threads. Each worker thread doesn't need to fetch its own. + // * Jwks is ready when the requests come, not need to wait for the Jwks fetching. + // + JwksAsyncFetch async_fetch = 3; +} + +// Fetch Jwks asynchronously in the main thread when the filter config is parsed. +// The listener is activated only after the Jwks is fetched. +// When the Jwks is expired in the cache, it is fetched again in the main thread. +// The fetched Jwks from the main thread can be used by all worker threads. +message JwksAsyncFetch { + // If false, the listener is activated after the initial fetch is completed. + // The initial fetch result can be either successful or failed. + // If true, it is activated without waiting for the initial fetch to complete. + // Default is false. + bool fast_listener = 1; } // This message specifies a header location to extract JWT token. diff --git a/generated_api_shadow/envoy/extensions/filters/http/jwt_authn/v4alpha/config.proto b/generated_api_shadow/envoy/extensions/filters/http/jwt_authn/v4alpha/config.proto index 7656f09912e9b..442ba7df061ee 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/jwt_authn/v4alpha/config.proto +++ b/generated_api_shadow/envoy/extensions/filters/http/jwt_authn/v4alpha/config.proto @@ -232,6 +232,38 @@ message RemoteJwks { // Duration after which the cached JWKS should be expired. If not specified, default cache // duration is 5 minutes. google.protobuf.Duration cache_duration = 2; + + // Fetch Jwks asynchronously in the main thread before the listener is activated. + // Fetched Jwks can be used by all worker threads. + // + // If this feature is not enabled: + // + // * The Jwks is fetched on-demand when the requests come. During the fetching, first + // few requests are paused until the Jwks is fetched. + // * Each worker thread fetches its own Jwks since Jwks cache is per worker thread. + // + // If this feature is enabled: + // + // * Fetched Jwks is done in the main thread before the listener is activated. Its fetched + // Jwks can be used by all worker threads. Each worker thread doesn't need to fetch its own. + // * Jwks is ready when the requests come, not need to wait for the Jwks fetching. + // + JwksAsyncFetch async_fetch = 3; +} + +// Fetch Jwks asynchronously in the main thread when the filter config is parsed. +// The listener is activated only after the Jwks is fetched. +// When the Jwks is expired in the cache, it is fetched again in the main thread. +// The fetched Jwks from the main thread can be used by all worker threads. +message JwksAsyncFetch { + option (udpa.annotations.versioning).previous_message_type = + "envoy.extensions.filters.http.jwt_authn.v3.JwksAsyncFetch"; + + // If false, the listener is activated after the initial fetch is completed. + // The initial fetch result can be either successful or failed. + // If true, it is activated without waiting for the initial fetch to complete. + // Default is false. + bool fast_listener = 1; } // This message specifies a header location to extract JWT token. diff --git a/generated_api_shadow/envoy/extensions/filters/http/local_ratelimit/v3/local_rate_limit.proto b/generated_api_shadow/envoy/extensions/filters/http/local_ratelimit/v3/local_rate_limit.proto index 7766ee2573d00..1cf6c5f2fa52c 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/local_ratelimit/v3/local_rate_limit.proto +++ b/generated_api_shadow/envoy/extensions/filters/http/local_ratelimit/v3/local_rate_limit.proto @@ -19,7 +19,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // Local Rate limit :ref:`configuration overview `. // [#extension: envoy.filters.http.local_ratelimit] -// [#next-free-field: 11] +// [#next-free-field: 12] message LocalRateLimit { // The human readable prefix to use when emitting stats. string stat_prefix = 1 [(validate.rules).string = {min_len: 1}]; @@ -97,4 +97,13 @@ message LocalRateLimit { // // The filter supports a range of 0 - 10 inclusively for stage numbers. uint32 stage = 9 [(validate.rules).uint32 = {lte: 10}]; + + // Specifies the scope of the rate limiter's token bucket. + // If set to false, the token bucket is shared across all worker threads, + // thus the rate limits are applied per Envoy process. + // If set to true, a token bucket is allocated for each connection. + // Thus the rate limits are applied per connection thereby allowing + // one to rate limit requests on a per connection basis. + // If unspecified, the default value is false. + bool local_rate_limit_per_downstream_connection = 11; } diff --git a/generated_api_shadow/envoy/extensions/filters/http/set_metadata/v3/BUILD b/generated_api_shadow/envoy/extensions/filters/http/set_metadata/v3/BUILD new file mode 100644 index 0000000000000..ee92fb652582e --- /dev/null +++ b/generated_api_shadow/envoy/extensions/filters/http/set_metadata/v3/BUILD @@ -0,0 +1,9 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], +) diff --git a/generated_api_shadow/envoy/extensions/filters/http/set_metadata/v3/set_metadata.proto b/generated_api_shadow/envoy/extensions/filters/http/set_metadata/v3/set_metadata.proto new file mode 100644 index 0000000000000..7b2cf0e0965d8 --- /dev/null +++ b/generated_api_shadow/envoy/extensions/filters/http/set_metadata/v3/set_metadata.proto @@ -0,0 +1,31 @@ +syntax = "proto3"; + +package envoy.extensions.filters.http.set_metadata.v3; + +import "google/protobuf/struct.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.filters.http.set_metadata.v3"; +option java_outer_classname = "SetMetadataProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Set-Metadata Filter] +// +// This filters adds or updates dynamic metadata with static data. +// +// [#extension: envoy.filters.http.set_metadata] + +message Config { + // The metadata namespace. + string metadata_namespace = 1 [(validate.rules).string = {min_len: 1}]; + + // The value to update the namespace with. See + // :ref:`the filter documentation ` for + // more information on how this value is merged with potentially existing + // ones. + google.protobuf.Struct value = 2; +} diff --git a/generated_api_shadow/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto b/generated_api_shadow/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto index 76c8488c32be6..095f3a28d7124 100644 --- a/generated_api_shadow/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto +++ b/generated_api_shadow/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto @@ -36,7 +36,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // HTTP connection manager :ref:`configuration overview `. // [#extension: envoy.filters.network.http_connection_manager] -// [#next-free-field: 47] +// [#next-free-field: 48] message HttpConnectionManager { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager"; @@ -708,6 +708,16 @@ message HttpConnectionManager { // for details. PathNormalizationOptions path_normalization_options = 43; + // Determines if trailing dot of the host should be removed from host/authority header before any + // processing of request by HTTP filters or routing. + // This affects the upstream host header. + // Without setting this option, incoming requests with host `example.com.` will not match against + // route with :ref:`domains` match set to `example.com`. Defaults to `false`. + // When the incoming request contains a host/authority header that includes a port number, + // setting this option will strip a trailing dot, if present, from the host section, + // leaving the port as is (e.g. host value `example.com.:443` will be updated to `example.com:443`). + bool strip_trailing_host_dot = 47; + google.protobuf.Duration hidden_envoy_deprecated_idle_timeout = 11 [ deprecated = true, (envoy.annotations.deprecated_at_minor_version) = "3.0", diff --git a/generated_api_shadow/envoy/extensions/filters/network/http_connection_manager/v4alpha/http_connection_manager.proto b/generated_api_shadow/envoy/extensions/filters/network/http_connection_manager/v4alpha/http_connection_manager.proto index e2f85d4b3e15d..db628cc585ec1 100644 --- a/generated_api_shadow/envoy/extensions/filters/network/http_connection_manager/v4alpha/http_connection_manager.proto +++ b/generated_api_shadow/envoy/extensions/filters/network/http_connection_manager/v4alpha/http_connection_manager.proto @@ -34,7 +34,7 @@ option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSIO // HTTP connection manager :ref:`configuration overview `. // [#extension: envoy.filters.network.http_connection_manager] -// [#next-free-field: 47] +// [#next-free-field: 48] message HttpConnectionManager { option (udpa.annotations.versioning).previous_message_type = "envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager"; @@ -704,6 +704,16 @@ message HttpConnectionManager { // ` // for details. PathNormalizationOptions path_normalization_options = 43; + + // Determines if trailing dot of the host should be removed from host/authority header before any + // processing of request by HTTP filters or routing. + // This affects the upstream host header. + // Without setting this option, incoming requests with host `example.com.` will not match against + // route with :ref:`domains` match set to `example.com`. Defaults to `false`. + // When the incoming request contains a host/authority header that includes a port number, + // setting this option will strip a trailing dot, if present, from the host section, + // leaving the port as is (e.g. host value `example.com.:443` will be updated to `example.com:443`). + bool strip_trailing_host_dot = 47; } // The configuration to customize local reply returned by Envoy. diff --git a/generated_api_shadow/envoy/extensions/transport_sockets/tls/v3/tls.proto b/generated_api_shadow/envoy/extensions/transport_sockets/tls/v3/tls.proto index 44325bdbee6a4..02287de5875fb 100644 --- a/generated_api_shadow/envoy/extensions/transport_sockets/tls/v3/tls.proto +++ b/generated_api_shadow/envoy/extensions/transport_sockets/tls/v3/tls.proto @@ -216,8 +216,14 @@ message CommonTlsContext { // Configs for fetching TLS certificates via SDS API. Note SDS API allows certificates to be // fetched/refreshed over the network asynchronously with respect to the TLS handshake. + // + // The same number and types of certificates as :ref:`tls_certificates ` + // are valid in the the certificates fetched through this setting. + // + // If :ref:`tls_certificates ` + // is non-empty, this field is ignored. repeated SdsSecretConfig tls_certificate_sds_secret_configs = 6 - [(validate.rules).repeated = {max_items: 1}]; + [(validate.rules).repeated = {max_items: 2}]; // Certificate provider for fetching TLS certificates. // [#not-implemented-hide:] diff --git a/generated_api_shadow/envoy/extensions/transport_sockets/tls/v4alpha/tls.proto b/generated_api_shadow/envoy/extensions/transport_sockets/tls/v4alpha/tls.proto index 7ddf55dfa6fab..b92cae619dd9c 100644 --- a/generated_api_shadow/envoy/extensions/transport_sockets/tls/v4alpha/tls.proto +++ b/generated_api_shadow/envoy/extensions/transport_sockets/tls/v4alpha/tls.proto @@ -221,8 +221,14 @@ message CommonTlsContext { // Configs for fetching TLS certificates via SDS API. Note SDS API allows certificates to be // fetched/refreshed over the network asynchronously with respect to the TLS handshake. + // + // The same number and types of certificates as :ref:`tls_certificates ` + // are valid in the the certificates fetched through this setting. + // + // If :ref:`tls_certificates ` + // is non-empty, this field is ignored. repeated SdsSecretConfig tls_certificate_sds_secret_configs = 6 - [(validate.rules).repeated = {max_items: 1}]; + [(validate.rules).repeated = {max_items: 2}]; // Certificate provider for fetching TLS certificates. // [#not-implemented-hide:] diff --git a/include/envoy/buffer/buffer.h b/include/envoy/buffer/buffer.h index d00130d48d901..2ac7674e37668 100644 --- a/include/envoy/buffer/buffer.h +++ b/include/envoy/buffer/buffer.h @@ -131,7 +131,8 @@ class Instance { /** * Binds the account to be charged for resources used by the buffer. This - * should only be called once. + * should only be called when the buffer is empty as existing slices + * within the buffer won't retroactively get tagged. * * @param account a shared_ptr to the account to charge. */ @@ -449,6 +450,7 @@ class Instance { * @param watermark supplies the buffer high watermark size threshold, in bytes. */ virtual void setWatermarks(uint32_t watermark) PURE; + /** * Returns the configured high watermark. A return value of 0 indicates that watermark * functionality is disabled. diff --git a/include/envoy/http/alternate_protocols_cache.h b/include/envoy/http/alternate_protocols_cache.h index 6d94f32eafcb6..2d02cda9bd48e 100644 --- a/include/envoy/http/alternate_protocols_cache.h +++ b/include/envoy/http/alternate_protocols_cache.h @@ -64,16 +64,18 @@ class AlternateProtocolsCache { }; /** - * Represents an alternative protocol that can be used to connect to an origin. + * Represents an alternative protocol that can be used to connect to an origin + * with a specified expiration time. */ struct AlternateProtocol { public: - AlternateProtocol(absl::string_view alpn, absl::string_view hostname, uint32_t port) - : alpn_(alpn), hostname_(hostname), port_(port) {} + AlternateProtocol(absl::string_view alpn, absl::string_view hostname, uint32_t port, + MonotonicTime expiration) + : alpn_(alpn), hostname_(hostname), port_(port), expiration_(expiration) {} bool operator==(const AlternateProtocol& other) const { - return std::tie(alpn_, hostname_, port_) == - std::tie(other.alpn_, other.hostname_, other.port_); + return std::tie(alpn_, hostname_, port_, expiration_) == + std::tie(other.alpn_, other.hostname_, other.port_, other.expiration_); } bool operator!=(const AlternateProtocol& other) const { return !this->operator==(other); } @@ -81,6 +83,7 @@ class AlternateProtocolsCache { std::string alpn_; std::string hostname_; uint32_t port_; + MonotonicTime expiration_; }; virtual ~AlternateProtocolsCache() = default; @@ -90,11 +93,9 @@ class AlternateProtocolsCache { * specified origin. Expires after the specified expiration time. * @param origin The origin to set alternate protocols for. * @param protocols A list of alternate protocols. - * @param expiration The time after which the alternatives are no longer valid. */ virtual void setAlternatives(const Origin& origin, - const std::vector& protocols, - const MonotonicTime& expiration) PURE; + const std::vector& protocols) PURE; /** * Returns the possible alternative protocols which can be used to connect to the diff --git a/include/envoy/http/codec.h b/include/envoy/http/codec.h index 9e91a4d9df4a0..67b1bc95ce813 100644 --- a/include/envoy/http/codec.h +++ b/include/envoy/http/codec.h @@ -19,6 +19,8 @@ namespace Envoy { namespace Http { +enum class CodecType { HTTP1, HTTP2, HTTP3 }; + namespace Http1 { struct CodecStats; } @@ -379,6 +381,12 @@ class Stream { * small window updates as satisfying the idle timeout as this is a potential DoS vector. */ virtual void setFlushTimeout(std::chrono::milliseconds timeout) PURE; + + /** + * Sets the account for this stream, propagating it to all of its buffers. + * @param the account to assign this stream. + */ + virtual void setAccount(Buffer::BufferMemoryAccountSharedPtr account) PURE; }; /** diff --git a/include/envoy/http/filter.h b/include/envoy/http/filter.h index 48a23a7e8e7b0..3d15d9735da19 100644 --- a/include/envoy/http/filter.h +++ b/include/envoy/http/filter.h @@ -6,6 +6,7 @@ #include #include "envoy/access_log/access_log.h" +#include "envoy/buffer/buffer.h" #include "envoy/common/scope_tracker.h" #include "envoy/event/dispatcher.h" #include "envoy/grpc/status.h" @@ -556,6 +557,11 @@ class StreamDecoderFilterCallbacks : public virtual StreamFilterCallbacks { */ virtual uint32_t decoderBufferLimit() PURE; + /** + * @return the account, if any, used by this stream. + */ + virtual Buffer::BufferMemoryAccountSharedPtr account() const PURE; + /** * Takes a stream, and acts as if the headers are newly arrived. * On success, this will result in a creating a new filter chain and likely diff --git a/include/envoy/network/listener.h b/include/envoy/network/listener.h index 7c6eabe3e3c6c..73dbced7fa93a 100644 --- a/include/envoy/network/listener.h +++ b/include/envoy/network/listener.h @@ -330,6 +330,11 @@ class UdpListenerCallbacks { * Posts ``data`` to be delivered on this worker. */ virtual void post(Network::UdpRecvData&& data) PURE; + + /** + * An estimated number of UDP packets this callback expects to process in current read event. + */ + virtual size_t numPacketsExpectedPerEventLoop() const PURE; }; using UdpListenerCallbacksOptRef = absl::optional>; diff --git a/include/envoy/router/router.h b/include/envoy/router/router.h index 95944beddb2b0..7cf920f6f68b2 100644 --- a/include/envoy/router/router.h +++ b/include/envoy/router/router.h @@ -64,9 +64,11 @@ class ResponseEntry { * process them later. Note: do not use unless you are sure that there will be no route * modifications later in the filter chain. * @param stream_info holds additional information about the request. + * @param do_formatting whether or not to evaluate configured transformations; if false, returns + * original values instead. */ - virtual Http::HeaderTransforms - responseHeaderTransforms(const StreamInfo::StreamInfo& stream_info) const PURE; + virtual Http::HeaderTransforms responseHeaderTransforms(const StreamInfo::StreamInfo& stream_info, + bool do_formatting = true) const PURE; }; /** @@ -1327,6 +1329,12 @@ class GenericUpstream { * @param reason supplies the reset reason. */ virtual void resetStream() PURE; + + /** + * Sets the upstream to use the following account. + * @param the account to assign the generic upstream. + */ + virtual void setAccount(Buffer::BufferMemoryAccountSharedPtr account) PURE; }; using GenericConnPoolPtr = std::unique_ptr; diff --git a/security/email-templates.md b/security/email-templates.md index 3e6c178978661..550fa02955b58 100644 --- a/security/email-templates.md +++ b/security/email-templates.md @@ -206,3 +206,32 @@ Thanks, $PERSON (on behalf of the Envoy security team and maintainers) ``` + +## Security Advisory (for feature accidentally marked as production) +``` +Subject: Security advisory +To: envoy-security-announce@googlegroups.com +Cc: envoy-announce@googlegroups.com, envoy-security@googlegroups.com, envoy-maintainers@googlegroups.com + +Hello Envoy Community, + +The Envoy securitty team would like to announce a security advisory for a feature introduced in +$ENVOYRELEASE. As this is a security advisory for a feature not considered production ready that may +have been labeled as such, no fix is provided and the advice is to not make use of this feature in +a production capacity until future hardening has been done. + +$DEFECTSSUMMARY + + + +The CVSS score for this is [$CVSSSTRING]($CVSSURL). + +**Thank you** + +Thank you to $REPORTER, $DEVELOPERS for the coordination in making this release. + +Thanks, + +$PERSON (on behalf of the Envoy security team and maintainers) +``` diff --git a/source/common/buffer/buffer_impl.cc b/source/common/buffer/buffer_impl.cc index 6d2241171a135..3187cae8abe52 100644 --- a/source/common/buffer/buffer_impl.cc +++ b/source/common/buffer/buffer_impl.cc @@ -43,11 +43,11 @@ void OwnedImpl::addDrainTracker(std::function drain_tracker) { void OwnedImpl::bindAccount(BufferMemoryAccountSharedPtr account) { ASSERT(slices_.empty()); - // We don't yet have an account bound. - ASSERT(!account_); account_ = std::move(account); } +BufferMemoryAccountSharedPtr OwnedImpl::getAccountForTest() { return account_; } + void OwnedImpl::add(const void* data, uint64_t size) { addImpl(data, size); } void OwnedImpl::addBufferFragment(BufferFragment& fragment) { diff --git a/source/common/buffer/buffer_impl.h b/source/common/buffer/buffer_impl.h index a794b0e020514..35d2c16b8079c 100644 --- a/source/common/buffer/buffer_impl.h +++ b/source/common/buffer/buffer_impl.h @@ -710,6 +710,11 @@ class OwnedImpl : public LibEventInstance { */ virtual void appendSliceForTest(absl::string_view data); + /** + * @return the BufferMemoryAccount bound to this buffer, if any. + */ + BufferMemoryAccountSharedPtr getAccountForTest(); + // Does not implement watermarking. // TODO(antoniovicente) Implement watermarks by merging the OwnedImpl and WatermarkBuffer // implementations. Also, make high-watermark config a constructor argument. diff --git a/source/common/buffer/watermark_buffer.h b/source/common/buffer/watermark_buffer.h index 68d260b155320..c8700cc0ca151 100644 --- a/source/common/buffer/watermark_buffer.h +++ b/source/common/buffer/watermark_buffer.h @@ -47,7 +47,7 @@ class WatermarkBuffer : public OwnedImpl { protected: virtual void checkHighAndOverflowWatermarks(); - void checkLowWatermark(); + virtual void checkLowWatermark(); private: void commit(uint64_t length, absl::Span slices, diff --git a/source/common/common/base64.cc b/source/common/common/base64.cc index ea215b85b9ee9..fc4cd599c20d3 100644 --- a/source/common/common/base64.cc +++ b/source/common/common/base64.cc @@ -234,13 +234,6 @@ std::string Base64::encode(const char* input, uint64_t length, bool add_padding) return ret; } -void Base64::completePadding(std::string& encoded) { - if (encoded.length() % 4 != 0) { - std::string trailing_padding(4 - encoded.length() % 4, '='); - encoded.append(trailing_padding); - } -} - std::string Base64Url::decode(const std::string& input) { if (input.empty()) { return EMPTY_STRING; diff --git a/source/common/common/base64.h b/source/common/common/base64.h index a69ffbf910a3b..13beff40b64ab 100644 --- a/source/common/common/base64.h +++ b/source/common/common/base64.h @@ -54,12 +54,6 @@ class Base64 { * bytes. */ static std::string decodeWithoutPadding(absl::string_view input); - - /** - * Add the padding in the base64 encoded binary if the padding is missing. - * @param encoded is the target to complete the padding. - */ - static void completePadding(std::string& encoded); }; /** diff --git a/source/common/event/file_event_impl.cc b/source/common/event/file_event_impl.cc index 252bb53650f9e..45f01dffa467c 100644 --- a/source/common/event/file_event_impl.cc +++ b/source/common/event/file_event_impl.cc @@ -85,7 +85,16 @@ void FileEventImpl::assignEvents(uint32_t events, event_base* base) { void FileEventImpl::updateEvents(uint32_t events) { ASSERT(dispatcher_.isThreadSafe()); - if (events == enabled_events_) { + // The update can be skipped in cases where the old and new event mask are the same if the fd is + // using Level or EmulatedEdge trigger modes, but not Edge trigger mode. When the fd is registered + // in edge trigger mode, re-registering the fd will force re-computation of the readable/writable + // state even in cases where the event mask is not changing. See + // https://github.com/envoyproxy/envoy/pull/16389 for more details. + // TODO(antoniovicente) Consider ways to optimize away event registration updates in edge trigger + // mode once setEnabled stops clearing injected_activation_events_ before calling updateEvents + // and/or implement optimizations at the Network::ConnectionImpl level to reduce the number of + // calls to setEnabled. + if (events == enabled_events_ && trigger_ != FileTriggerType::Edge) { return; } auto* base = event_get_base(&raw_event_); diff --git a/source/common/http/BUILD b/source/common/http/BUILD index bc2b727a1b3fb..a581bc5d8d54e 100644 --- a/source/common/http/BUILD +++ b/source/common/http/BUILD @@ -176,6 +176,7 @@ envoy_cc_library( "//include/envoy/singleton:manager_interface", "//include/envoy/thread_local:thread_local_interface", "//include/envoy/upstream:resource_manager_interface", + "//source/common/common:logger_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], ) diff --git a/source/common/http/alternate_protocols_cache_impl.cc b/source/common/http/alternate_protocols_cache_impl.cc index 89d31df615aaf..4368e9a778599 100644 --- a/source/common/http/alternate_protocols_cache_impl.cc +++ b/source/common/http/alternate_protocols_cache_impl.cc @@ -1,5 +1,7 @@ #include "common/http/alternate_protocols_cache_impl.h" +#include "common/common/logger.h" + namespace Envoy { namespace Http { @@ -9,14 +11,13 @@ AlternateProtocolsCacheImpl::AlternateProtocolsCacheImpl(TimeSource& time_source AlternateProtocolsCacheImpl::~AlternateProtocolsCacheImpl() = default; void AlternateProtocolsCacheImpl::setAlternatives(const Origin& origin, - const std::vector& protocols, - const MonotonicTime& expiration) { - Entry& entry = protocols_[origin]; - if (entry.protocols_ != protocols) { - entry.protocols_ = protocols; - } - if (entry.expiration_ != expiration) { - entry.expiration_ = expiration; + const std::vector& protocols) { + protocols_[origin] = protocols; + static const size_t max_protocols = 10; + if (protocols.size() > max_protocols) { + ENVOY_LOG_MISC(trace, "Too many alternate protocols: {}, truncating", protocols.size()); + std::vector& p = protocols_[origin]; + p.erase(p.begin() + max_protocols, p.end()); } } @@ -24,19 +25,24 @@ OptRef> AlternateProtocolsCacheImpl::findAlternatives(const Origin& origin) { auto entry_it = protocols_.find(origin); if (entry_it == protocols_.end()) { - return makeOptRefFromPtr>( - nullptr); + return makeOptRefFromPtr>(nullptr); } - const Entry& entry = entry_it->second; - if (time_source_.monotonicTime() > entry.expiration_) { - // Expire the entry. - // TODO(RyanTheOptimist): expire entries based on a timer. + std::vector& protocols = entry_it->second; + + const MonotonicTime now = time_source_.monotonicTime(); + protocols.erase(std::remove_if(protocols.begin(), protocols.end(), + [now](const AlternateProtocol& protocol) { + return (now > protocol.expiration_); + }), + protocols.end()); + + if (protocols.empty()) { protocols_.erase(entry_it); - return makeOptRefFromPtr>( - nullptr); + return makeOptRefFromPtr>(nullptr); } - return makeOptRef(entry.protocols_); + + return makeOptRef(const_cast&>(protocols)); } size_t AlternateProtocolsCacheImpl::size() const { return protocols_.size(); } diff --git a/source/common/http/alternate_protocols_cache_impl.h b/source/common/http/alternate_protocols_cache_impl.h index f72ec506dc2d4..a029a970c763f 100644 --- a/source/common/http/alternate_protocols_cache_impl.h +++ b/source/common/http/alternate_protocols_cache_impl.h @@ -22,23 +22,18 @@ class AlternateProtocolsCacheImpl : public AlternateProtocolsCache { ~AlternateProtocolsCacheImpl() override; // AlternateProtocolsCache - void setAlternatives(const Origin& origin, const std::vector& protocols, - const MonotonicTime& expiration) override; + void setAlternatives(const Origin& origin, + const std::vector& protocols) override; OptRef> findAlternatives(const Origin& origin) override; size_t size() const override; private: - struct Entry { - std::vector protocols_; - MonotonicTime expiration_; - }; - // Time source used to check expiration of entries. TimeSource& time_source_; // Map from hostname to list of alternate protocols. // TODO(RyanTheOptimist): Add a limit to the size of this map and evict based on usage. - std::map protocols_; + std::map> protocols_; }; } // namespace Http diff --git a/source/common/http/async_client_impl.h b/source/common/http/async_client_impl.h index 798907b0763be..c4e7455bda687 100644 --- a/source/common/http/async_client_impl.h +++ b/source/common/http/async_client_impl.h @@ -237,7 +237,8 @@ class AsyncStreamImpl : public AsyncClient::Stream, bool) const override {} void finalizeResponseHeaders(Http::ResponseHeaderMap&, const StreamInfo::StreamInfo&) const override {} - Http::HeaderTransforms responseHeaderTransforms(const StreamInfo::StreamInfo&) const override { + Http::HeaderTransforms responseHeaderTransforms(const StreamInfo::StreamInfo&, + bool) const override { return {}; } const HashPolicy* hashPolicy() const override { return hash_policy_.get(); } @@ -365,6 +366,8 @@ class AsyncStreamImpl : public AsyncClient::Stream, Upstream::ClusterInfoConstSharedPtr clusterInfo() override { return parent_.cluster_; } void clearRouteCache() override {} uint64_t streamId() const override { return stream_id_; } + // TODO(kbaichoo): Plumb account from owning request filter. + Buffer::BufferMemoryAccountSharedPtr account() const override { return nullptr; } Tracing::Span& activeSpan() override { return active_span_; } const Tracing::Config& tracingConfig() override { return tracing_config_; } void continueDecoding() override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } diff --git a/source/common/http/codec_client.cc b/source/common/http/codec_client.cc index 3353ff81d0904..0b21cf4bc6939 100644 --- a/source/common/http/codec_client.cc +++ b/source/common/http/codec_client.cc @@ -20,12 +20,12 @@ namespace Envoy { namespace Http { -CodecClient::CodecClient(Type type, Network::ClientConnectionPtr&& connection, +CodecClient::CodecClient(CodecType type, Network::ClientConnectionPtr&& connection, Upstream::HostDescriptionConstSharedPtr host, Event::Dispatcher& dispatcher) : type_(type), host_(host), connection_(std::move(connection)), idle_timeout_(host_->cluster().idleTimeout()) { - if (type_ != Type::HTTP3) { + if (type_ != CodecType::HTTP3) { // Make sure upstream connections process data and then the FIN, rather than processing // TCP disconnects immediately. (see https://github.com/envoyproxy/envoy/issues/1679 for // details) @@ -95,7 +95,7 @@ void CodecClient::onEvent(Network::ConnectionEvent event) { } // HTTP/1 can signal end of response by disconnecting. We need to handle that case. - if (type_ == Type::HTTP1 && event == Network::ConnectionEvent::RemoteClose && + if (type_ == CodecType::HTTP1 && event == Network::ConnectionEvent::RemoteClose && !active_requests_.empty()) { Buffer::OwnedImpl empty; onData(empty); @@ -169,26 +169,26 @@ void CodecClient::onData(Buffer::Instance& data) { absl::StrCat("extraneous bytes after response complete: ", data.length())); } -CodecClientProd::CodecClientProd(Type type, Network::ClientConnectionPtr&& connection, +CodecClientProd::CodecClientProd(CodecType type, Network::ClientConnectionPtr&& connection, Upstream::HostDescriptionConstSharedPtr host, Event::Dispatcher& dispatcher, Random::RandomGenerator& random_generator) : CodecClient(type, std::move(connection), host, dispatcher) { switch (type) { - case Type::HTTP1: { + case CodecType::HTTP1: { codec_ = std::make_unique( *connection_, host->cluster().http1CodecStats(), *this, host->cluster().http1Settings(), host->cluster().maxResponseHeadersCount()); break; } - case Type::HTTP2: { + case CodecType::HTTP2: { codec_ = std::make_unique( *connection_, *this, host->cluster().http2CodecStats(), random_generator, host->cluster().http2Options(), Http::DEFAULT_MAX_REQUEST_HEADERS_KB, host->cluster().maxResponseHeadersCount(), Http2::ProdNghttp2SessionFactory::get()); break; } - case Type::HTTP3: { + case CodecType::HTTP3: { #ifdef ENVOY_ENABLE_QUIC auto& quic_session = dynamic_cast(*connection_); codec_ = std::make_unique( diff --git a/source/common/http/codec_client.h b/source/common/http/codec_client.h index 07eb9724d607a..b2cb841010d65 100644 --- a/source/common/http/codec_client.h +++ b/source/common/http/codec_client.h @@ -55,7 +55,8 @@ class CodecClient : protected Logger::Loggable, /** * Type of HTTP codec to use. */ - enum class Type { HTTP1, HTTP2, HTTP3 }; + // This is a legacy alias. + using Type = Envoy::Http::CodecType; ~CodecClient() override; @@ -125,7 +126,7 @@ class CodecClient : protected Logger::Loggable, bool remoteClosed() const { return remote_closed_; } - Type type() const { return type_; } + CodecType type() const { return type_; } // Note this is the L4 stream info, not L7. const StreamInfo::StreamInfo& streamInfo() { return connection_->streamInfo(); } @@ -137,7 +138,7 @@ class CodecClient : protected Logger::Loggable, * @param connection supplies the connection to communicate on. * @param host supplies the owning host. */ - CodecClient(Type type, Network::ClientConnectionPtr&& connection, + CodecClient(CodecType type, Network::ClientConnectionPtr&& connection, Upstream::HostDescriptionConstSharedPtr host, Event::Dispatcher& dispatcher); /** @@ -175,7 +176,7 @@ class CodecClient : protected Logger::Loggable, } } - const Type type_; + const CodecType type_; // The order of host_, connection_, and codec_ matter as during destruction each can refer to // the previous, at least in tests. Upstream::HostDescriptionConstSharedPtr host_; @@ -273,7 +274,7 @@ using CodecClientPtr = std::unique_ptr; */ class CodecClientProd : public CodecClient { public: - CodecClientProd(Type type, Network::ClientConnectionPtr&& connection, + CodecClientProd(CodecType type, Network::ClientConnectionPtr&& connection, Upstream::HostDescriptionConstSharedPtr host, Event::Dispatcher& dispatcher, Random::RandomGenerator& random_generator); }; diff --git a/source/common/http/conn_manager_config.h b/source/common/http/conn_manager_config.h index 8b05b4bbff49b..0569921eb9287 100644 --- a/source/common/http/conn_manager_config.h +++ b/source/common/http/conn_manager_config.h @@ -484,6 +484,12 @@ class ConnectionManagerConfig { */ virtual const std::vector& originalIpDetectionExtensions() const PURE; + + /** + * @return if the HttpConnectionManager should remove trailing host dot from host/authority + * header. + */ + virtual bool shouldStripTrailingHostDot() const PURE; }; } // namespace Http } // namespace Envoy diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index 87fd8073de1fb..b18102f31b5b7 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -259,7 +259,18 @@ RequestDecoder& ConnectionManagerImpl::newStream(ResponseEncoder& response_encod } ENVOY_CONN_LOG(debug, "new stream", read_callbacks_->connection()); - ActiveStreamPtr new_stream(new ActiveStream(*this, response_encoder.getStream().bufferLimit())); + + // Set the account to start accounting if enabled. This is still a + // work-in-progress, and will be removed when other features using the + // accounting are implemented. + Buffer::BufferMemoryAccountSharedPtr downstream_request_account; + if (Runtime::runtimeFeatureEnabled("envoy.test_only.per_stream_buffer_accounting")) { + downstream_request_account = std::make_shared(); + response_encoder.getStream().setAccount(downstream_request_account); + } + ActiveStreamPtr new_stream(new ActiveStream(*this, response_encoder.getStream().bufferLimit(), + std::move(downstream_request_account))); + new_stream->state_.is_internally_created_ = is_internally_created; new_stream->response_encoder_ = &response_encoder; new_stream->response_encoder_->getStream().addCallbacks(*new_stream); @@ -571,13 +582,14 @@ void ConnectionManagerImpl::RdsRouteConfigUpdateRequester::requestSrdsUpdate( } ConnectionManagerImpl::ActiveStream::ActiveStream(ConnectionManagerImpl& connection_manager, - uint32_t buffer_limit) + uint32_t buffer_limit, + Buffer::BufferMemoryAccountSharedPtr account) : connection_manager_(connection_manager), stream_id_(connection_manager.random_generator_.random()), filter_manager_(*this, connection_manager_.read_callbacks_->connection().dispatcher(), connection_manager_.read_callbacks_->connection(), stream_id_, - connection_manager_.config_.proxy100Continue(), buffer_limit, - connection_manager_.config_.filterFactory(), + std::move(account), connection_manager_.config_.proxy100Continue(), + buffer_limit, connection_manager_.config_.filterFactory(), connection_manager_.config_.localReply(), connection_manager_.codec_->protocol(), connection_manager_.timeSource(), connection_manager_.read_callbacks_->connection().streamInfo().filterState(), diff --git a/source/common/http/conn_manager_impl.h b/source/common/http/conn_manager_impl.h index ce8c6648a84e7..7a9156c787101 100644 --- a/source/common/http/conn_manager_impl.h +++ b/source/common/http/conn_manager_impl.h @@ -156,7 +156,8 @@ class ConnectionManagerImpl : Logger::Loggable, public Tracing::Config, public ScopeTrackedObject, public FilterManagerCallbacks { - ActiveStream(ConnectionManagerImpl& connection_manager, uint32_t buffer_limit); + ActiveStream(ConnectionManagerImpl& connection_manager, uint32_t buffer_limit, + Buffer::BufferMemoryAccountSharedPtr account); void completeRequest(); const Network::Connection* connection(); diff --git a/source/common/http/conn_manager_utility.cc b/source/common/http/conn_manager_utility.cc index 1131f40113bb6..887fe09bcd095 100644 --- a/source/common/http/conn_manager_utility.cc +++ b/source/common/http/conn_manager_utility.cc @@ -499,6 +499,9 @@ ConnectionManagerUtility::maybeNormalizePath(RequestHeaderMap& request_headers, absl::optional ConnectionManagerUtility::maybeNormalizeHost(RequestHeaderMap& request_headers, const ConnectionManagerConfig& config, uint32_t port) { + if (config.shouldStripTrailingHostDot()) { + HeaderUtility::stripTrailingHostDot(request_headers); + } if (config.stripPortType() == Http::StripPortType::Any) { return HeaderUtility::stripPortFromHost(request_headers, absl::nullopt); } else if (config.stripPortType() == Http::StripPortType::MatchingHost) { diff --git a/source/common/http/filter_manager.cc b/source/common/http/filter_manager.cc index 982dd80a864cf..a962365e10f36 100644 --- a/source/common/http/filter_manager.cc +++ b/source/common/http/filter_manager.cc @@ -1578,5 +1578,9 @@ void ActiveStreamFilterBase::resetStream() { parent_.filter_manager_callbacks_.r uint64_t ActiveStreamFilterBase::streamId() const { return parent_.streamId(); } +Buffer::BufferMemoryAccountSharedPtr ActiveStreamDecoderFilter::account() const { + return parent_.account(); +} + } // namespace Http } // namespace Envoy diff --git a/source/common/http/filter_manager.h b/source/common/http/filter_manager.h index c861d08a228fc..2f65055c77839 100644 --- a/source/common/http/filter_manager.h +++ b/source/common/http/filter_manager.h @@ -3,6 +3,7 @@ #include #include +#include "envoy/buffer/buffer.h" #include "envoy/common/optref.h" #include "envoy/extensions/filters/common/matcher/action/v3/skip_action.pb.h" #include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.h" @@ -277,6 +278,7 @@ struct ActiveStreamDecoderFilter : public ActiveStreamFilterBase, void addUpstreamSocketOptions(const Network::Socket::OptionsSharedPtr& options) override; Network::Socket::OptionsSharedPtr getUpstreamSocketOptions() const override; + Buffer::BufferMemoryAccountSharedPtr account() const override; // Each decoder filter instance checks if the request passed to the filter is gRPC // so that we can issue gRPC local responses to gRPC requests. Filter's decodeHeaders() @@ -640,15 +642,16 @@ class FilterManager : public ScopeTrackedObject, Logger::Loggable { public: FilterManager(FilterManagerCallbacks& filter_manager_callbacks, Event::Dispatcher& dispatcher, - const Network::Connection& connection, uint64_t stream_id, bool proxy_100_continue, + const Network::Connection& connection, uint64_t stream_id, + Buffer::BufferMemoryAccountSharedPtr account, bool proxy_100_continue, uint32_t buffer_limit, FilterChainFactory& filter_chain_factory, const LocalReply::LocalReply& local_reply, Http::Protocol protocol, TimeSource& time_source, StreamInfo::FilterStateSharedPtr parent_filter_state, StreamInfo::FilterState::LifeSpan filter_state_life_span) : filter_manager_callbacks_(filter_manager_callbacks), dispatcher_(dispatcher), - connection_(connection), stream_id_(stream_id), proxy_100_continue_(proxy_100_continue), - buffer_limit_(buffer_limit), filter_chain_factory_(filter_chain_factory), - local_reply_(local_reply), + connection_(connection), stream_id_(stream_id), account_(std::move(account)), + proxy_100_continue_(proxy_100_continue), buffer_limit_(buffer_limit), + filter_chain_factory_(filter_chain_factory), local_reply_(local_reply), stream_info_(protocol, time_source, connection.addressProviderSharedPtr(), parent_filter_state, filter_state_life_span) {} ~FilterManager() override { @@ -913,6 +916,7 @@ class FilterManager : public ScopeTrackedObject, const Network::Connection* connection() const { return &connection_; } uint64_t streamId() const { return stream_id_; } + Buffer::BufferMemoryAccountSharedPtr account() const { return account_; } Buffer::InstancePtr& bufferedRequestData() { return buffered_request_data_; } @@ -986,6 +990,7 @@ class FilterManager : public ScopeTrackedObject, Event::Dispatcher& dispatcher_; const Network::Connection& connection_; const uint64_t stream_id_; + Buffer::BufferMemoryAccountSharedPtr account_; const bool proxy_100_continue_; std::list decoder_filters_; diff --git a/source/common/http/header_utility.cc b/source/common/http/header_utility.cc index e504a21ad8a23..d019a1799175c 100644 --- a/source/common/http/header_utility.cc +++ b/source/common/http/header_utility.cc @@ -58,6 +58,7 @@ HeaderUtility::HeaderData::HeaderData(const envoy::config::route::v3::HeaderMatc break; case envoy::config::route::v3::HeaderMatcher::HeaderMatchSpecifierCase::kPresentMatch: header_match_type_ = HeaderMatchType::Present; + present_ = config.present_match(); break; case envoy::config::route::v3::HeaderMatcher::HeaderMatchSpecifierCase::kPrefixMatch: header_match_type_ = HeaderMatchType::Prefix; @@ -76,6 +77,7 @@ HeaderUtility::HeaderData::HeaderData(const envoy::config::route::v3::HeaderMatc FALLTHRU; default: header_match_type_ = HeaderMatchType::Present; + present_ = true; break; } } @@ -135,7 +137,11 @@ bool HeaderUtility::matchHeaders(const HeaderMap& request_headers, const HeaderD const auto header_value = getAllOfHeaderAsString(request_headers, header_data.name_); if (!header_value.result().has_value()) { - return header_data.invert_match_ && header_data.header_match_type_ == HeaderMatchType::Present; + if (header_data.invert_match_) { + return header_data.header_match_type_ == HeaderMatchType::Present && header_data.present_; + } else { + return header_data.header_match_type_ == HeaderMatchType::Present && !header_data.present_; + } } bool match; @@ -154,7 +160,7 @@ bool HeaderUtility::matchHeaders(const HeaderMap& request_headers, const HeaderD break; } case HeaderMatchType::Present: - match = true; + match = header_data.present_; break; case HeaderMatchType::Prefix: match = absl::StartsWith(header_value.result().value(), header_data.value_); @@ -216,6 +222,25 @@ bool HeaderUtility::isEnvoyInternalRequest(const RequestHeaderMap& headers) { internal_request_header->value() == Headers::get().EnvoyInternalRequestValues.True; } +void HeaderUtility::stripTrailingHostDot(RequestHeaderMap& headers) { + auto host = headers.getHostValue(); + // If the host ends in a period, remove it. + auto dot_index = host.rfind('.'); + if (dot_index == std::string::npos) { + return; + } else if (dot_index == (host.size() - 1)) { + host.remove_suffix(1); + headers.setHost(host); + return; + } + // If the dot is just before a colon, it must be preceding the port number. + // IPv6 addresses may contain colons or dots, but the dot will never directly + // precede the colon, so this check should be sufficient to detect a trailing port number. + if (host[dot_index + 1] == ':') { + headers.setHost(absl::StrCat(host.substr(0, dot_index), host.substr(dot_index + 1))); + } +} + absl::optional HeaderUtility::stripPortFromHost(RequestHeaderMap& headers, absl::optional listener_port) { if (headers.getMethodValue() == Http::Headers::get().MethodValues.Connect && diff --git a/source/common/http/header_utility.h b/source/common/http/header_utility.h index 20719003733c2..04d1201d61ecf 100644 --- a/source/common/http/header_utility.h +++ b/source/common/http/header_utility.h @@ -66,6 +66,7 @@ class HeaderUtility { Regex::CompiledMatcherPtr regex_; envoy::type::v3::Int64Range range_; const bool invert_match_; + bool present_; // HeaderMatcher bool matchesHeaders(const HeaderMap& headers) const override { @@ -176,6 +177,11 @@ class HeaderUtility { static bool shouldCloseConnection(Http::Protocol protocol, const RequestOrResponseHeaderMap& headers); + /** + * @brief Remove the trailing host dot from host/authority header. + */ + static void stripTrailingHostDot(RequestHeaderMap& headers); + /** * @brief Remove the port part from host/authority header if it is equal to provided port. * @return absl::optional containing the port, if removed, else absl::nullopt. diff --git a/source/common/http/http1/codec_impl.h b/source/common/http/http1/codec_impl.h index 031d972a5a08b..4cfd38b5b1ec9 100644 --- a/source/common/http/http1/codec_impl.h +++ b/source/common/http/http1/codec_impl.h @@ -69,6 +69,10 @@ class StreamEncoderImpl : public virtual StreamEncoder, // require a flush timeout not already covered by other timeouts. } + void setAccount(Buffer::BufferMemoryAccountSharedPtr) override { + // TODO(kbaichoo): implement account tracking for H1. + } + void setIsResponseToHeadRequest(bool value) { is_response_to_head_request_ = value; } void setIsResponseToConnectRequest(bool value) { is_response_to_connect_request_ = value; } void setDetails(absl::string_view details) { details_ = details; } diff --git a/source/common/http/http1/conn_pool.cc b/source/common/http/http1/conn_pool.cc index 8a59c01005572..db90cfc87a48b 100644 --- a/source/common/http/http1/conn_pool.cc +++ b/source/common/http/http1/conn_pool.cc @@ -108,9 +108,9 @@ allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_ random_generator, state, [](HttpConnPoolImplBase* pool) { return std::make_unique(*pool); }, [](Upstream::Host::CreateConnectionData& data, HttpConnPoolImplBase* pool) { - CodecClientPtr codec{new CodecClientProd( - CodecClient::Type::HTTP1, std::move(data.connection_), data.host_description_, - pool->dispatcher(), pool->randomGenerator())}; + CodecClientPtr codec{new CodecClientProd(CodecType::HTTP1, std::move(data.connection_), + data.host_description_, pool->dispatcher(), + pool->randomGenerator())}; return codec; }, std::vector{Protocol::Http11}); diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index 758dea9d326fc..5f3cc901a7684 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -130,8 +130,17 @@ template static T* removeConst(const void* object) { } ConnectionImpl::StreamImpl::StreamImpl(ConnectionImpl& parent, uint32_t buffer_limit) - : parent_(parent), local_end_stream_sent_(false), remote_end_stream_(false), - data_deferred_(false), received_noninformational_headers_(false), + : parent_(parent), + pending_recv_data_(parent_.connection_.dispatcher().getWatermarkFactory().create( + [this]() -> void { this->pendingRecvBufferLowWatermark(); }, + [this]() -> void { this->pendingRecvBufferHighWatermark(); }, + []() -> void { /* TODO(adisuissa): Handle overflow watermark */ })), + pending_send_data_(parent_.connection_.dispatcher().getWatermarkFactory().create( + [this]() -> void { this->pendingSendBufferLowWatermark(); }, + [this]() -> void { this->pendingSendBufferHighWatermark(); }, + []() -> void { /* TODO(adisuissa): Handle overflow watermark */ })), + local_end_stream_sent_(false), remote_end_stream_(false), data_deferred_(false), + received_noninformational_headers_(false), pending_receive_buffer_high_watermark_called_(false), pending_send_buffer_high_watermark_called_(false), reset_due_to_messaging_error_(false) { parent_.stats_.streams_active_.inc(); @@ -145,7 +154,7 @@ ConnectionImpl::StreamImpl::~StreamImpl() { ASSERT(stream_idle_timer_ == nullptr void ConnectionImpl::StreamImpl::destroy() { disarmStreamIdleTimer(); parent_.stats_.streams_active_.dec(); - parent_.stats_.pending_send_bytes_.sub(pending_send_data_.length()); + parent_.stats_.pending_send_bytes_.sub(pending_send_data_->length()); } static void insertHeader(std::vector& headers, const HeaderEntry& header) { @@ -253,7 +262,7 @@ void ConnectionImpl::ServerStreamImpl::encodeHeaders(const ResponseHeaderMap& he void ConnectionImpl::StreamImpl::encodeTrailersBase(const HeaderMap& trailers) { ASSERT(!local_end_stream_); local_end_stream_ = true; - if (pending_send_data_.length() > 0) { + if (pending_send_data_->length() > 0) { // In this case we want trailers to come after we release all pending body data that is // waiting on window updates. We need to save the trailers so that we can emit them later. // However, for empty trailers, we don't need to to save the trailers. @@ -409,13 +418,13 @@ void ConnectionImpl::StreamImpl::submitMetadata(uint8_t flags) { } ssize_t ConnectionImpl::StreamImpl::onDataSourceRead(uint64_t length, uint32_t* data_flags) { - if (pending_send_data_.length() == 0 && !local_end_stream_) { + if (pending_send_data_->length() == 0 && !local_end_stream_) { ASSERT(!data_deferred_); data_deferred_ = true; return NGHTTP2_ERR_DEFERRED; } else { *data_flags |= NGHTTP2_DATA_FLAG_NO_COPY; - if (local_end_stream_ && pending_send_data_.length() <= length) { + if (local_end_stream_ && pending_send_data_->length() <= length) { *data_flags |= NGHTTP2_DATA_FLAG_EOF; if (pending_trailers_to_encode_) { // We need to tell the library to not set end stream so that we can emit the trailers. @@ -425,7 +434,7 @@ ssize_t ConnectionImpl::StreamImpl::onDataSourceRead(uint64_t length, uint32_t* } } - return std::min(length, pending_send_data_.length()); + return std::min(length, pending_send_data_->length()); } } @@ -446,7 +455,7 @@ void ConnectionImpl::StreamImpl::onDataSourceSend(const uint8_t* framehd, size_t } parent_.stats_.pending_send_bytes_.sub(length); - output.move(pending_send_data_, length); + output.move(*pending_send_data_, length); parent_.connection_.write(output, false); } @@ -491,7 +500,9 @@ void ConnectionImpl::StreamImpl::onPendingFlushTimer() { void ConnectionImpl::StreamImpl::encodeData(Buffer::Instance& data, bool end_stream) { ASSERT(!local_end_stream_); - encodeDataHelper(data, end_stream, /*skip_encoding_empty_trailers=*/false); + encodeDataHelper(data, end_stream, + /*skip_encoding_empty_trailers=*/ + false); } void ConnectionImpl::StreamImpl::encodeDataHelper(Buffer::Instance& data, bool end_stream, @@ -502,7 +513,7 @@ void ConnectionImpl::StreamImpl::encodeDataHelper(Buffer::Instance& data, bool e local_end_stream_ = end_stream; parent_.stats_.pending_send_bytes_.add(data.length()); - pending_send_data_.move(data); + pending_send_data_->move(data); if (data_deferred_) { int rc = nghttp2_session_resume_data(parent_.session_, stream_id_); ASSERT(rc == 0); @@ -514,7 +525,7 @@ void ConnectionImpl::StreamImpl::encodeDataHelper(Buffer::Instance& data, bool e // Intended to check through coverage that this error case is tested return; } - if (local_end_stream_ && pending_send_data_.length() > 0) { + if (local_end_stream_ && pending_send_data_->length() > 0) { createPendingFlushTimer(); } } @@ -576,6 +587,12 @@ void ConnectionImpl::StreamImpl::onMetadataDecoded(MetadataMapPtr&& metadata_map } } +void ConnectionImpl::StreamImpl::setAccount(Buffer::BufferMemoryAccountSharedPtr account) { + buffer_memory_account_ = account; + pending_recv_data_->bindAccount(buffer_memory_account_); + pending_send_data_->bindAccount(buffer_memory_account_); +} + ConnectionImpl::ConnectionImpl(Network::Connection& connection, CodecStats& stats, Random::RandomGenerator& random_generator, const envoy::config::core::v3::Http2ProtocolOptions& http2_options, @@ -707,7 +724,7 @@ int ConnectionImpl::onData(int32_t stream_id, const uint8_t* data, size_t len) { StreamImpl* stream = getStream(stream_id); // If this results in buffering too much data, the watermark buffer will call // pendingRecvBufferHighWatermark, resulting in ++read_disable_count_ - stream->pending_recv_data_.add(data, len); + stream->pending_recv_data_->add(data, len); // Update the window to the peer unless some consumer of this stream's data has hit a flow control // limit and disabled reads on this stream if (!stream->buffersOverrun()) { @@ -862,10 +879,10 @@ Status ConnectionImpl::onFrameReceived(const nghttp2_frame* frame) { // It's possible that we are waiting to send a deferred reset, so only raise data if local // is not complete. if (!stream->deferred_reset_) { - stream->decoder().decodeData(stream->pending_recv_data_, stream->remote_end_stream_); + stream->decoder().decodeData(*stream->pending_recv_data_, stream->remote_end_stream_); } - stream->pending_recv_data_.drain(stream->pending_recv_data_.length()); + stream->pending_recv_data_->drain(stream->pending_recv_data_->length()); break; } case NGHTTP2_RST_STREAM: { diff --git a/source/common/http/http2/codec_impl.h b/source/common/http/http2/codec_impl.h index 6f4a3a6f0270e..9a767e1c8337b 100644 --- a/source/common/http/http2/codec_impl.h +++ b/source/common/http/http2/codec_impl.h @@ -8,6 +8,7 @@ #include #include +#include "envoy/buffer/buffer.h" #include "envoy/common/random_generator.h" #include "envoy/common/scope_tracker.h" #include "envoy/config/core/v3/protocol.pb.h" @@ -17,6 +18,7 @@ #include "common/buffer/buffer_impl.h" #include "common/buffer/watermark_buffer.h" +#include "common/common/assert.h" #include "common/common/linked_object.h" #include "common/common/logger.h" #include "common/common/statusor.h" @@ -229,7 +231,7 @@ class ConnectionImpl : public virtual Connection, void removeCallbacks(StreamCallbacks& callbacks) override { removeCallbacksHelper(callbacks); } void resetStream(StreamResetReason reason) override; void readDisable(bool disable) override; - uint32_t bufferLimit() override { return pending_recv_data_.highWatermark(); } + uint32_t bufferLimit() override { return pending_recv_data_->highWatermark(); } const Network::Address::InstanceConstSharedPtr& connectionLocalAddress() override { return parent_.connection_.addressProvider().localAddress(); } @@ -237,6 +239,7 @@ class ConnectionImpl : public virtual Connection, void setFlushTimeout(std::chrono::milliseconds timeout) override { stream_idle_timeout_ = timeout; } + void setAccount(Buffer::BufferMemoryAccountSharedPtr account) override; // ScopeTrackedObject void dumpState(std::ostream& os, int indent_level) const override; @@ -258,8 +261,8 @@ class ConnectionImpl : public virtual Connection, } void setWriteBufferWatermarks(uint32_t high_watermark) { - pending_recv_data_.setWatermarks(high_watermark); - pending_send_data_.setWatermarks(high_watermark); + pending_recv_data_->setWatermarks(high_watermark); + pending_send_data_->setWatermarks(high_watermark); } // If the receive buffer encounters watermark callbacks, enable/disable reads on this stream. @@ -293,19 +296,14 @@ class ConnectionImpl : public virtual Connection, uint32_t unconsumed_bytes_{0}; uint32_t read_disable_count_{0}; + Buffer::BufferMemoryAccountSharedPtr buffer_memory_account_; // Note that in current implementation the watermark callbacks of the pending_recv_data_ are // never called. The watermark value is set to the size of the stream window. As a result this // watermark can never overflow because the peer can never send more bytes than the stream // window without triggering protocol error and this buffer is drained after each DATA frame was // dispatched through the filter chain. See source/docs/flow_control.md for more information. - Buffer::WatermarkBuffer pending_recv_data_{ - [this]() -> void { this->pendingRecvBufferLowWatermark(); }, - [this]() -> void { this->pendingRecvBufferHighWatermark(); }, - []() -> void { /* TODO(adisuissa): Handle overflow watermark */ }}; - Buffer::WatermarkBuffer pending_send_data_{ - [this]() -> void { this->pendingSendBufferLowWatermark(); }, - [this]() -> void { this->pendingSendBufferHighWatermark(); }, - []() -> void { /* TODO(adisuissa): Handle overflow watermark */ }}; + Buffer::InstancePtr pending_recv_data_; + Buffer::InstancePtr pending_send_data_; HeaderMapPtr pending_trailers_to_encode_; std::unique_ptr metadata_decoder_; std::unique_ptr metadata_encoder_; diff --git a/source/common/http/http2/conn_pool.cc b/source/common/http/http2/conn_pool.cc index fe8dca2c4247f..bc137ba39a8f9 100644 --- a/source/common/http/http2/conn_pool.cc +++ b/source/common/http/http2/conn_pool.cc @@ -33,9 +33,9 @@ allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_ host, priority, dispatcher, options, transport_socket_options, random_generator, state, [](HttpConnPoolImplBase* pool) { return std::make_unique(*pool); }, [](Upstream::Host::CreateConnectionData& data, HttpConnPoolImplBase* pool) { - CodecClientPtr codec{new CodecClientProd( - CodecClient::Type::HTTP2, std::move(data.connection_), data.host_description_, - pool->dispatcher(), pool->randomGenerator())}; + CodecClientPtr codec{new CodecClientProd(CodecType::HTTP2, std::move(data.connection_), + data.host_description_, pool->dispatcher(), + pool->randomGenerator())}; return codec; }, std::vector{Protocol::Http2}); diff --git a/source/common/http/http3/codec_stats.h b/source/common/http/http3/codec_stats.h index 327b345f1d468..c9dcf6f395a29 100644 --- a/source/common/http/http3/codec_stats.h +++ b/source/common/http/http3/codec_stats.h @@ -11,18 +11,13 @@ namespace Http3 { /** * All stats for the HTTP/3 codec. @see stats_macros.h - * TODO(danzh) populate all of them in codec. */ #define ALL_HTTP3_CODEC_STATS(COUNTER, GAUGE) \ COUNTER(dropped_headers_with_underscores) \ - COUNTER(header_overflow) \ COUNTER(requests_rejected_with_underscores_in_headers) \ - COUNTER(rx_messaging_error) \ COUNTER(rx_reset) \ - COUNTER(trailers) \ COUNTER(tx_reset) \ - COUNTER(metadata_not_supported_error) \ - GAUGE(streams_active, Accumulate) + COUNTER(metadata_not_supported_error) /** * Wrapper struct for the HTTP/3 codec stats. @see stats_macros.h diff --git a/source/common/http/http3/conn_pool.cc b/source/common/http/http3/conn_pool.cc index 3f1fbcbe5c1a1..ec920aa646d24 100644 --- a/source/common/http/http3/conn_pool.cc +++ b/source/common/http/http3/conn_pool.cc @@ -12,50 +12,46 @@ #include "common/network/utility.h" #include "common/runtime/runtime_features.h" -#ifdef ENVOY_ENABLE_QUIC -#include "common/quic/client_connection_factory_impl.h" -#include "common/quic/envoy_quic_utils.h" -#include "common/quic/quic_transport_socket_factory.h" -#else -#error "http3 conn pool should not be built with QUIC disabled" -#endif - namespace Envoy { namespace Http { namespace Http3 { -// Http3 subclass of FixedHttpConnPoolImpl which exists to store quic data. -class Http3ConnPoolImpl : public FixedHttpConnPoolImpl { -public: - Http3ConnPoolImpl(Upstream::HostConstSharedPtr host, Upstream::ResourcePriority priority, - Event::Dispatcher& dispatcher, - const Network::ConnectionSocket::OptionsSharedPtr& options, - const Network::TransportSocketOptionsSharedPtr& transport_socket_options, - Random::RandomGenerator& random_generator, - Upstream::ClusterConnectivityState& state, CreateClientFn client_fn, - CreateCodecFn codec_fn, std::vector protocol, - TimeSource& time_source) - : FixedHttpConnPoolImpl(host, priority, dispatcher, options, transport_socket_options, - random_generator, state, client_fn, codec_fn, protocol) { - auto source_address = host_->cluster().sourceAddress(); - if (!source_address.get()) { - auto host_address = host->address(); - source_address = Network::Utility::getLocalAddress(host_address->ip()->version()); - } - Network::TransportSocketFactory& transport_socket_factory = host->transportSocketFactory(); - quic_info_ = std::make_unique( - dispatcher, transport_socket_factory, time_source, source_address); - Quic::configQuicInitialFlowControlWindow( - host_->cluster().http3Options().quic_protocol_options(), quic_info_->quic_config_); - } +void Http3ConnPoolImpl::setQuicConfigFromClusterConfig(const Upstream::ClusterInfo& cluster, + quic::QuicConfig& quic_config) { + quic::QuicTime::Delta crypto_timeout = + quic::QuicTime::Delta::FromMilliseconds(cluster.connectTimeout().count()); + quic_config.set_max_time_before_crypto_handshake(crypto_timeout); + int32_t max_streams = + cluster.http3Options().quic_protocol_options().max_concurrent_streams().value(); + quic_config.SetMaxBidirectionalStreamsToSend(max_streams); + quic_config.SetMaxUnidirectionalStreamsToSend(max_streams); + Quic::configQuicInitialFlowControlWindow(cluster.http3Options().quic_protocol_options(), + quic_config); +} - // Make sure all connections are torn down before quic_info_ is deleted. - ~Http3ConnPoolImpl() override { destructAllConnections(); } +Http3ConnPoolImpl::Http3ConnPoolImpl( + Upstream::HostConstSharedPtr host, Upstream::ResourcePriority priority, + Event::Dispatcher& dispatcher, const Network::ConnectionSocket::OptionsSharedPtr& options, + const Network::TransportSocketOptionsSharedPtr& transport_socket_options, + Random::RandomGenerator& random_generator, Upstream::ClusterConnectivityState& state, + CreateClientFn client_fn, CreateCodecFn codec_fn, std::vector protocol, + TimeSource& time_source) + : FixedHttpConnPoolImpl(host, priority, dispatcher, options, transport_socket_options, + random_generator, state, client_fn, codec_fn, protocol) { + auto source_address = host_->cluster().sourceAddress(); + if (!source_address.get()) { + auto host_address = host->address(); + source_address = Network::Utility::getLocalAddress(host_address->ip()->version()); + } + Network::TransportSocketFactory& transport_socket_factory = host->transportSocketFactory(); + quic_info_ = std::make_unique( + dispatcher, transport_socket_factory, time_source, source_address, + host->cluster().perConnectionBufferLimitBytes()); + setQuicConfigFromClusterConfig(host_->cluster(), quic_info_->quic_config_); +} - // Store quic helpers which can be shared between connections and must live - // beyond the lifetime of individual connections. - std::unique_ptr quic_info_; -}; +// Make sure all connections are torn down before quic_info_ is deleted. +Http3ConnPoolImpl::~Http3ConnPoolImpl() { destructAllConnections(); } ConnectionPool::InstancePtr allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_generator, @@ -81,13 +77,13 @@ allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_ source_address = Network::Utility::getLocalAddress(host_address->ip()->version()); } data.connection_ = Quic::createQuicNetworkConnection( - *h3_pool->quic_info_, pool->dispatcher(), host_address, source_address); + h3_pool->quicInfo(), pool->dispatcher(), host_address, source_address); return std::make_unique(*pool, data); }, [](Upstream::Host::CreateConnectionData& data, HttpConnPoolImplBase* pool) { - CodecClientPtr codec{new CodecClientProd( - CodecClient::Type::HTTP3, std::move(data.connection_), data.host_description_, - pool->dispatcher(), pool->randomGenerator())}; + CodecClientPtr codec{new CodecClientProd(CodecType::HTTP3, std::move(data.connection_), + data.host_description_, pool->dispatcher(), + pool->randomGenerator())}; return codec; }, std::vector{Protocol::Http3}, time_source); diff --git a/source/common/http/http3/conn_pool.h b/source/common/http/http3/conn_pool.h index 6f3196253048a..674063e3c9411 100644 --- a/source/common/http/http3/conn_pool.h +++ b/source/common/http/http3/conn_pool.h @@ -7,6 +7,14 @@ #include "common/http/codec_client.h" #include "common/http/conn_pool_base.h" +#ifdef ENVOY_ENABLE_QUIC +#include "common/quic/client_connection_factory_impl.h" +#include "common/quic/envoy_quic_utils.h" +#include "common/quic/quic_transport_socket_factory.h" +#else +#error "http3 conn pool should not be built with QUIC disabled" +#endif + namespace Envoy { namespace Http { namespace Http3 { @@ -26,6 +34,33 @@ class ActiveClient : public MultiplexedActiveClientBase { data) {} }; +// Http3 subclass of FixedHttpConnPoolImpl which exists to store quic data. +class Http3ConnPoolImpl : public FixedHttpConnPoolImpl { +public: + Http3ConnPoolImpl(Upstream::HostConstSharedPtr host, Upstream::ResourcePriority priority, + Event::Dispatcher& dispatcher, + const Network::ConnectionSocket::OptionsSharedPtr& options, + const Network::TransportSocketOptionsSharedPtr& transport_socket_options, + Random::RandomGenerator& random_generator, + Upstream::ClusterConnectivityState& state, CreateClientFn client_fn, + CreateCodecFn codec_fn, std::vector protocol, + TimeSource& time_source); + + ~Http3ConnPoolImpl() override; + + // Set relevant fields in quic_config based on the cluster configuration + // supplied in cluster. + static void setQuicConfigFromClusterConfig(const Upstream::ClusterInfo& cluster, + quic::QuicConfig& quic_config); + + Quic::PersistentQuicInfoImpl& quicInfo() { return *quic_info_; } + +private: + // Store quic helpers which can be shared between connections and must live + // beyond the lifetime of individual connections. + std::unique_ptr quic_info_; +}; + ConnectionPool::InstancePtr allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_generator, Upstream::HostConstSharedPtr host, Upstream::ResourcePriority priority, diff --git a/source/common/http/mixed_conn_pool.cc b/source/common/http/mixed_conn_pool.cc index 26322fb992e9d..07fcfea475aee 100644 --- a/source/common/http/mixed_conn_pool.cc +++ b/source/common/http/mixed_conn_pool.cc @@ -16,8 +16,7 @@ Envoy::ConnectionPool::ActiveClientPtr HttpConnPoolImplMixed::instantiateActiveC CodecClientPtr HttpConnPoolImplMixed::createCodecClient(Upstream::Host::CreateConnectionData& data) { - auto protocol = - protocol_ == Protocol::Http11 ? CodecClient::Type::HTTP1 : CodecClient::Type::HTTP2; + auto protocol = protocol_ == Protocol::Http11 ? CodecType::HTTP1 : CodecType::HTTP2; CodecClientPtr codec{new CodecClientProd(protocol, std::move(data.connection_), data.host_description_, dispatcher_, random_generator_)}; return codec; diff --git a/source/common/network/udp_listener_impl.cc b/source/common/network/udp_listener_impl.cc index 5b5b98b340874..940ae25f9073f 100644 --- a/source/common/network/udp_listener_impl.cc +++ b/source/common/network/udp_listener_impl.cc @@ -75,7 +75,12 @@ void UdpListenerImpl::handleReadCallback() { const Api::IoErrorPtr result = Utility::readPacketsFromSocket( socket_->ioHandle(), *socket_->addressProvider().localAddress(), *this, time_source_, config_.prefer_gro_, packets_dropped_); - // TODO(mattklein123): Handle no error when we limit the number of packets read. + if (result == nullptr) { + // No error. The number of reads was limited by read rate. There are more packets to read. + // Register to read more in the next event loop. + socket_->ioHandle().activateFileEvents(Event::FileReadyType::Read); + return; + } if (result->getErrorCode() != Api::IoError::IoErrorCode::Again) { // TODO(mattklein123): When rate limited logging is implemented log this at error level // on a periodic basis. diff --git a/source/common/network/udp_listener_impl.h b/source/common/network/udp_listener_impl.h index aac7aa1c957de..afd78713eb896 100644 --- a/source/common/network/udp_listener_impl.h +++ b/source/common/network/udp_listener_impl.h @@ -46,6 +46,9 @@ class UdpListenerImpl : public BaseListenerImpl, MonotonicTime receive_time) override; uint64_t maxDatagramSize() const override { return config_.max_rx_datagram_size_; } void onDatagramsDropped(uint32_t dropped) override { cb_.onDatagramsDropped(dropped); } + size_t numPacketsExpectedPerEventLoop() const override { + return cb_.numPacketsExpectedPerEventLoop(); + } protected: void handleWriteCallback(); diff --git a/source/common/network/utility.cc b/source/common/network/utility.cc index d8c59a78d8d26..c2c58af71c086 100644 --- a/source/common/network/utility.cc +++ b/source/common/network/utility.cc @@ -23,6 +23,7 @@ #include "common/network/io_socket_error_impl.h" #include "common/protobuf/protobuf.h" #include "common/protobuf/utility.h" +#include "common/runtime/runtime_features.h" #include "absl/container/fixed_array.h" #include "absl/strings/match.h" @@ -576,10 +577,10 @@ void passPayloadToProcessor(uint64_t bytes_read, Buffer::InstancePtr buffer, Api::IoCallUint64Result Utility::readFromSocket(IoHandle& handle, const Address::Instance& local_address, UdpPacketProcessor& udp_packet_processor, - MonotonicTime receive_time, bool prefer_gro, + MonotonicTime receive_time, bool use_gro, uint32_t* packets_dropped) { - if (prefer_gro && handle.supportsUdpGro()) { + if (use_gro) { Buffer::InstancePtr buffer = std::make_unique(); IoHandle::RecvMsgOutput output(1, packets_dropped); @@ -696,11 +697,24 @@ Api::IoErrorPtr Utility::readPacketsFromSocket(IoHandle& handle, UdpPacketProcessor& udp_packet_processor, TimeSource& time_source, bool prefer_gro, uint32_t& packets_dropped) { + // Read at least one time, and attempt to read numPacketsExpectedPerEventLoop() packets unless + // this goes over MAX_NUM_PACKETS_PER_EVENT_LOOP. + size_t num_packets_to_read = std::min( + MAX_NUM_PACKETS_PER_EVENT_LOOP, udp_packet_processor.numPacketsExpectedPerEventLoop()); + const bool use_gro = prefer_gro && handle.supportsUdpGro(); + size_t num_reads = + use_gro ? (num_packets_to_read / NUM_DATAGRAMS_PER_GRO_RECEIVE) + : (handle.supportsMmsg() ? (num_packets_to_read / NUM_DATAGRAMS_PER_MMSG_RECEIVE) + : num_packets_to_read); + // Make sure to read at least once. + num_reads = std::max(1, num_reads); + bool honor_read_limit = + Runtime::runtimeFeatureEnabled("envoy.reloadable_features.udp_per_event_loop_read_limit"); do { const uint32_t old_packets_dropped = packets_dropped; const MonotonicTime receive_time = time_source.monotonicTime(); Api::IoCallUint64Result result = Utility::readFromSocket( - handle, local_address, udp_packet_processor, receive_time, prefer_gro, &packets_dropped); + handle, local_address, udp_packet_processor, receive_time, use_gro, &packets_dropped); if (!result.ok()) { // No more to read or encountered a system error. @@ -723,6 +737,12 @@ Api::IoErrorPtr Utility::readPacketsFromSocket(IoHandle& handle, delta); udp_packet_processor.onDatagramsDropped(delta); } + if (honor_read_limit) { + --num_reads; + } + if (num_reads == 0) { + return std::move(result.err_); + } } while (true); } diff --git a/source/common/network/utility.h b/source/common/network/utility.h index 2e58c1d9606bc..c892a8efb4bba 100644 --- a/source/common/network/utility.h +++ b/source/common/network/utility.h @@ -61,11 +61,17 @@ class UdpPacketProcessor { * the size of datagrams received, they will be dropped. */ virtual uint64_t maxDatagramSize() const PURE; + + /** + * An estimated number of packets to read in each read event. + */ + virtual size_t numPacketsExpectedPerEventLoop() const PURE; }; static const uint64_t DEFAULT_UDP_MAX_DATAGRAM_SIZE = 1500; static const uint64_t NUM_DATAGRAMS_PER_GRO_RECEIVE = 16; static const uint64_t NUM_DATAGRAMS_PER_MMSG_RECEIVE = 16; +static const uint64_t MAX_NUM_PACKETS_PER_EVENT_LOOP = 6000; /** * Wrapper which resolves UDP socket proto config with defaults. @@ -362,18 +368,20 @@ class Utility { static Api::IoCallUint64Result readFromSocket(IoHandle& handle, const Address::Instance& local_address, UdpPacketProcessor& udp_packet_processor, - MonotonicTime receive_time, bool prefer_gro, + MonotonicTime receive_time, bool use_gro, uint32_t* packets_dropped); /** - * Read available packets from a given UDP socket and pass the packet to a given - * UdpPacketProcessor. + * Read some packets from a given UDP socket and pass the packet to a given + * UdpPacketProcessor. Read no more than MAX_NUM_PACKETS_PER_EVENT_LOOP packets. * @param handle is the UDP socket to read from. * @param local_address is the socket's local address used to populate port. * @param udp_packet_processor is the callback to receive the packets. * @param time_source is the time source used to generate the time stamp of the received packets. * @param prefer_gro supplies whether to use GRO if the OS supports it. * @param packets_dropped is the output parameter for number of packets dropped in kernel. + * Return the io error encountered or nullptr if no io error but read stopped + * because of MAX_NUM_PACKETS_PER_EVENT_LOOP. * * TODO(mattklein123): Allow the number of packets read to be limited for fairness. Currently * this function will always return an error, even if EAGAIN. In the future diff --git a/source/common/protobuf/utility.cc b/source/common/protobuf/utility.cc index 1dfed5a161719..806aca2aec969 100644 --- a/source/common/protobuf/utility.cc +++ b/source/common/protobuf/utility.cc @@ -108,7 +108,9 @@ ProtobufWkt::Value parseYamlNode(const YAML::Node& node) { case YAML::NodeType::Map: { auto& struct_fields = *value.mutable_struct_value()->mutable_fields(); for (const auto& it : node) { - struct_fields[it.first.as()] = parseYamlNode(it.second); + if (it.first.Tag() != "!ignore") { + struct_fields[it.first.as()] = parseYamlNode(it.second); + } } break; } @@ -1056,4 +1058,43 @@ std::string TypeUtil::descriptorFullNameToTypeUrl(absl::string_view type) { return "type.googleapis.com/" + std::string(type); } +void StructUtil::update(ProtobufWkt::Struct& obj, const ProtobufWkt::Struct& with) { + auto& obj_fields = *obj.mutable_fields(); + + for (const auto& [key, val] : with.fields()) { + auto& obj_key = obj_fields[key]; + + // If the types are different, the last one wins. + const auto val_kind = val.kind_case(); + if (val_kind != obj_key.kind_case()) { + obj_key = val; + continue; + } + + // Otherwise, the strategy depends on the value kind. + switch (val.kind_case()) { + // For scalars, the last one wins. + case ProtobufWkt::Value::kNullValue: + case ProtobufWkt::Value::kNumberValue: + case ProtobufWkt::Value::kStringValue: + case ProtobufWkt::Value::kBoolValue: + obj_key = val; + break; + // If we got a structure, recursively update. + case ProtobufWkt::Value::kStructValue: + update(*obj_key.mutable_struct_value(), val.struct_value()); + break; + // For lists, append the new values. + case ProtobufWkt::Value::kListValue: { + auto& obj_key_vec = *obj_key.mutable_list_value()->mutable_values(); + const auto& vals = val.list_value().values(); + obj_key_vec.MergeFrom(vals); + break; + } + case ProtobufWkt::Value::KIND_NOT_SET: + break; + } + } +} + } // namespace Envoy diff --git a/source/common/protobuf/utility.h b/source/common/protobuf/utility.h index cacafc69238aa..2feee5bcadb2f 100644 --- a/source/common/protobuf/utility.h +++ b/source/common/protobuf/utility.h @@ -665,6 +665,25 @@ class TimestampUtil { ProtobufWkt::Timestamp& timestamp); }; +class StructUtil { +public: + /** + * Recursively updates in-place a protobuf structure with keys from another + * object. + * + * The merging strategy is the following. If a key from \p other does not + * exists, it's just copied into \p obj. If the key exists but has a + * different type, it is replaced by the new value. Otherwise: + * - for scalar values (null, string, number, boolean) are replaced with the new value + * - for lists: new values are added to the current list + * - for structures: recursively apply this scheme + * + * @param obj the object to update in-place + * @param with the object to update \p obj with + */ + static void update(ProtobufWkt::Struct& obj, const ProtobufWkt::Struct& with); +}; + } // namespace Envoy namespace std { diff --git a/source/common/quic/active_quic_listener.cc b/source/common/quic/active_quic_listener.cc index ff1f8d2c64624..75d32d5792cfe 100644 --- a/source/common/quic/active_quic_listener.cc +++ b/source/common/quic/active_quic_listener.cc @@ -17,6 +17,7 @@ #include "common/quic/envoy_quic_proof_source.h" #include "common/quic/envoy_quic_utils.h" #include "common/quic/envoy_quic_utils.h" +#include "common/quic/quic_network_connection.h" #include "common/runtime/runtime_features.h" namespace Envoy { @@ -26,17 +27,20 @@ ActiveQuicListener::ActiveQuicListener( uint32_t worker_index, uint32_t concurrency, Event::Dispatcher& dispatcher, Network::UdpConnectionHandler& parent, Network::ListenerConfig& listener_config, const quic::QuicConfig& quic_config, Network::Socket::OptionsSharedPtr options, - bool kernel_worker_routing, const envoy::config::core::v3::RuntimeFeatureFlag& enabled) + bool kernel_worker_routing, const envoy::config::core::v3::RuntimeFeatureFlag& enabled, + uint32_t packets_received_to_connection_count_ratio) : ActiveQuicListener(worker_index, concurrency, dispatcher, parent, listener_config.listenSocketFactory().getListenSocket(), listener_config, - quic_config, std::move(options), kernel_worker_routing, enabled) {} + quic_config, std::move(options), kernel_worker_routing, enabled, + packets_received_to_connection_count_ratio) {} ActiveQuicListener::ActiveQuicListener( uint32_t worker_index, uint32_t concurrency, Event::Dispatcher& dispatcher, Network::UdpConnectionHandler& parent, Network::SocketSharedPtr listen_socket, Network::ListenerConfig& listener_config, const quic::QuicConfig& quic_config, Network::Socket::OptionsSharedPtr options, bool kernel_worker_routing, - const envoy::config::core::v3::RuntimeFeatureFlag& enabled) + const envoy::config::core::v3::RuntimeFeatureFlag& enabled, + uint32_t packets_to_read_to_connection_count_ratio) : Server::ActiveUdpListenerBase( worker_index, concurrency, parent, *listen_socket, dispatcher.createUdpListener( @@ -44,7 +48,8 @@ ActiveQuicListener::ActiveQuicListener( listener_config.udpListenerConfig()->config().downstream_socket_config()), &listener_config), dispatcher_(dispatcher), version_manager_(quic::CurrentSupportedVersions()), - kernel_worker_routing_(kernel_worker_routing) { + kernel_worker_routing_(kernel_worker_routing), + packets_to_read_to_connection_count_ratio_(packets_to_read_to_connection_count_ratio) { // This flag fix a QUICHE issue which may crash Envoy during connection close. SetQuicReloadableFlag(quic_single_ack_in_packet2, true); // Do not include 32-byte per-entry overhead while counting header size. @@ -215,9 +220,18 @@ uint32_t ActiveQuicListener::destination(const Network::UdpRecvData& data) const return connection_id_snippet % concurrency_; } +size_t ActiveQuicListener::numPacketsExpectedPerEventLoop() const { + // Expect each session to read packets_to_read_to_connection_count_ratio_ number of packets in + // this read event. + return quic_dispatcher_->NumSessions() * packets_to_read_to_connection_count_ratio_; +} + ActiveQuicListenerFactory::ActiveQuicListenerFactory( const envoy::config::listener::v3::QuicProtocolOptions& config, uint32_t concurrency) - : concurrency_(concurrency), enabled_(config.enabled()) { + : concurrency_(concurrency), enabled_(config.enabled()), + packets_to_read_to_connection_count_ratio_( + PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, packets_to_read_to_connection_count_ratio, + DEFAULT_PACKETS_TO_READ_PER_CONNECTION)) { uint64_t idle_network_timeout_ms = config.has_idle_timeout() ? DurationUtil::durationToMilliseconds(config.idle_timeout()) : 300000; @@ -300,9 +314,9 @@ Network::ConnectionHandler::ActiveUdpListenerPtr ActiveQuicListenerFactory::crea } #endif - return std::make_unique(worker_index, concurrency_, disptacher, parent, - config, quic_config_, std::move(options), - kernel_worker_routing, enabled_); + return std::make_unique( + worker_index, concurrency_, disptacher, parent, config, quic_config_, std::move(options), + kernel_worker_routing, enabled_, packets_to_read_to_connection_count_ratio_); } // namespace Quic } // namespace Quic diff --git a/source/common/quic/active_quic_listener.h b/source/common/quic/active_quic_listener.h index fb3f1b78cf4c6..7c2264d2da9e1 100644 --- a/source/common/quic/active_quic_listener.h +++ b/source/common/quic/active_quic_listener.h @@ -28,13 +28,15 @@ class ActiveQuicListener : public Envoy::Server::ActiveUdpListenerBase, Network::UdpConnectionHandler& parent, Network::ListenerConfig& listener_config, const quic::QuicConfig& quic_config, Network::Socket::OptionsSharedPtr options, bool kernel_worker_routing, - const envoy::config::core::v3::RuntimeFeatureFlag& enabled); + const envoy::config::core::v3::RuntimeFeatureFlag& enabled, + uint32_t packets_to_read_to_connection_count_ratio); ActiveQuicListener(uint32_t worker_index, uint32_t concurrency, Event::Dispatcher& dispatcher, Network::UdpConnectionHandler& parent, Network::SocketSharedPtr listen_socket, Network::ListenerConfig& listener_config, const quic::QuicConfig& quic_config, Network::Socket::OptionsSharedPtr options, bool kernel_worker_routing, - const envoy::config::core::v3::RuntimeFeatureFlag& enabled); + const envoy::config::core::v3::RuntimeFeatureFlag& enabled, + uint32_t packets_to_read_to_connection_count_ratio); ~ActiveQuicListener() override; @@ -52,6 +54,7 @@ class ActiveQuicListener : public Envoy::Server::ActiveUdpListenerBase, Network::UdpPacketWriter& udpPacketWriter() override { return *udp_packet_writer_; } void onDataWorker(Network::UdpRecvData&& data) override; uint32_t destination(const Network::UdpRecvData& data) const override; + size_t numPacketsExpectedPerEventLoop() const override; // ActiveListenerImplBase void pauseListening() override; @@ -73,6 +76,8 @@ class ActiveQuicListener : public Envoy::Server::ActiveUdpListenerBase, // The number of runs of the event loop in which at least one CHLO was buffered. // TODO(ggreenway): Consider making this a published stat, or some variation of this information. uint64_t event_loops_with_buffered_chlo_for_test_{0}; + + uint32_t packets_to_read_to_connection_count_ratio_; }; using ActiveQuicListenerPtr = std::unique_ptr; @@ -97,6 +102,7 @@ class ActiveQuicListenerFactory : public Network::ActiveUdpListenerFactory, const uint32_t concurrency_; absl::once_flag install_bpf_once_; envoy::config::core::v3::RuntimeFeatureFlag enabled_; + const uint32_t packets_to_read_to_connection_count_ratio_; }; } // namespace Quic diff --git a/source/common/quic/client_connection_factory_impl.cc b/source/common/quic/client_connection_factory_impl.cc index 829ea0ddc67eb..fcc871f382670 100644 --- a/source/common/quic/client_connection_factory_impl.cc +++ b/source/common/quic/client_connection_factory_impl.cc @@ -42,11 +42,13 @@ std::shared_ptr PersistentQuicInfoImpl::cryptoConf PersistentQuicInfoImpl::PersistentQuicInfoImpl( Event::Dispatcher& dispatcher, Network::TransportSocketFactory& transport_socket_factory, - TimeSource& time_source, Network::Address::InstanceConstSharedPtr server_addr) + TimeSource& time_source, Network::Address::InstanceConstSharedPtr server_addr, + uint32_t buffer_limit) : conn_helper_(dispatcher), alarm_factory_(dispatcher, *conn_helper_.GetClock()), server_id_{getConfig(transport_socket_factory).serverNameIndication(), static_cast(server_addr->ip()->port()), false}, - transport_socket_factory_(transport_socket_factory), time_source_(time_source) { + transport_socket_factory_(transport_socket_factory), time_source_(time_source), + buffer_limit_(buffer_limit) { quiche::FlagRegistry::getInstance(); } @@ -69,12 +71,10 @@ createQuicNetworkConnection(Http::PersistentQuicInfo& info, Event::Dispatcher& d ASSERT(!info_impl->supported_versions_.empty()); // QUICHE client session always use the 1st version to start handshake. - // TODO(alyssawilk) pass in ClusterInfo::perConnectionBufferLimitBytes() for - // send_buffer_limit instead of using 0. auto ret = std::make_unique( info_impl->quic_config_, info_impl->supported_versions_, std::move(connection), info_impl->server_id_, std::move(config), &info_impl->push_promise_index_, dispatcher, - /*send_buffer_limit=*/0); + info_impl->buffer_limit_); return ret; } diff --git a/source/common/quic/client_connection_factory_impl.h b/source/common/quic/client_connection_factory_impl.h index ad8334589af36..991249bebee47 100644 --- a/source/common/quic/client_connection_factory_impl.h +++ b/source/common/quic/client_connection_factory_impl.h @@ -20,7 +20,8 @@ struct PersistentQuicInfoImpl : public Http::PersistentQuicInfo { PersistentQuicInfoImpl(Event::Dispatcher& dispatcher, Network::TransportSocketFactory& transport_socket_factory, TimeSource& time_source, - Network::Address::InstanceConstSharedPtr server_addr); + Network::Address::InstanceConstSharedPtr server_addr, + uint32_t buffer_limit); // Returns the most recent crypto config from transport_socket_factory_; std::shared_ptr cryptoConfig(); @@ -42,6 +43,8 @@ struct PersistentQuicInfoImpl : public Http::PersistentQuicInfo { const quic::ParsedQuicVersionVector supported_versions_{quic::CurrentSupportedVersions()}; // TODO(alyssawilk) actually set this up properly. quic::QuicConfig quic_config_; + // The cluster buffer limits. + const uint32_t buffer_limit_; // This arguably should not be shared across connections but as Envoy doesn't // support push promise it's really moot point. quic::QuicClientPushPromiseIndex push_promise_index_; diff --git a/source/common/quic/envoy_quic_client_connection.cc b/source/common/quic/envoy_quic_client_connection.cc index c9046d45b73a6..ac30a8d8b9cf1 100644 --- a/source/common/quic/envoy_quic_client_connection.cc +++ b/source/common/quic/envoy_quic_client_connection.cc @@ -119,7 +119,10 @@ void EnvoyQuicClientConnection::onFileEvent(uint32_t events) { Api::IoErrorPtr err = Network::Utility::readPacketsFromSocket( connectionSocket()->ioHandle(), *connectionSocket()->addressProvider().localAddress(), *this, dispatcher_.timeSource(), true, packets_dropped_); - // TODO(danzh): Handle no error when we limit the number of packets read. + if (err == nullptr) { + connectionSocket()->ioHandle().activateFileEvents(Event::FileReadyType::Read); + return; + } if (err->getErrorCode() != Api::IoError::IoErrorCode::Again) { ENVOY_CONN_LOG(error, "recvmsg result {}: {}", *this, static_cast(err->getErrorCode()), err->getErrorDetails()); diff --git a/source/common/quic/envoy_quic_client_connection.h b/source/common/quic/envoy_quic_client_connection.h index dc3587a3b05c4..21000adce734c 100644 --- a/source/common/quic/envoy_quic_client_connection.h +++ b/source/common/quic/envoy_quic_client_connection.h @@ -53,6 +53,9 @@ class EnvoyQuicClientConnection : public quic::QuicConnection, void onDatagramsDropped(uint32_t) override { // TODO(mattklein123): Emit a stat for this. } + size_t numPacketsExpectedPerEventLoop() const override { + return DEFAULT_PACKETS_TO_READ_PER_CONNECTION; + } // Register file event and apply socket options. void setUpConnectionSocket(); diff --git a/source/common/quic/envoy_quic_client_stream.cc b/source/common/quic/envoy_quic_client_stream.cc index 6cb99fbc02d6d..1098a39b79fd5 100644 --- a/source/common/quic/envoy_quic_client_stream.cc +++ b/source/common/quic/envoy_quic_client_stream.cc @@ -256,11 +256,15 @@ void EnvoyQuicClientStream::maybeDecodeTrailers() { } void EnvoyQuicClientStream::OnStreamReset(const quic::QuicRstStreamFrame& frame) { + ENVOY_STREAM_LOG(debug, "received reset code={}", *this, frame.error_code); + stats_.rx_reset_.inc(); quic::QuicSpdyClientStream::OnStreamReset(frame); runResetCallbacks(quicRstErrorToEnvoyRemoteResetReason(frame.error_code)); } void EnvoyQuicClientStream::Reset(quic::QuicRstStreamErrorCode error) { + ENVOY_STREAM_LOG(debug, "sending reset code={}", *this, error); + stats_.tx_reset_.inc(); // Upper layers expect calling resetStream() to immediately raise reset callbacks. runResetCallbacks(quicRstErrorToEnvoyLocalResetReason(error)); quic::QuicSpdyClientStream::Reset(error); diff --git a/source/common/quic/envoy_quic_client_stream.h b/source/common/quic/envoy_quic_client_stream.h index a722accc6bfc2..b7e3f21d0d05f 100644 --- a/source/common/quic/envoy_quic_client_stream.h +++ b/source/common/quic/envoy_quic_client_stream.h @@ -1,5 +1,7 @@ #pragma once +#include "envoy/buffer/buffer.h" + #if defined(__GNUC__) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" @@ -46,6 +48,10 @@ class EnvoyQuicClientStream : public quic::QuicSpdyClientStream, // Http::Stream void resetStream(Http::StreamResetReason reason) override; void setFlushTimeout(std::chrono::milliseconds) override {} + + void setAccount(Buffer::BufferMemoryAccountSharedPtr) override { + // TODO(kbaichoo): implement account tracking for QUIC. + } // quic::QuicSpdyStream void OnBodyAvailable() override; void OnStreamReset(const quic::QuicRstStreamFrame& frame) override; diff --git a/source/common/quic/envoy_quic_proof_verifier.h b/source/common/quic/envoy_quic_proof_verifier.h index f12d570848f01..cdbfc70109b6a 100644 --- a/source/common/quic/envoy_quic_proof_verifier.h +++ b/source/common/quic/envoy_quic_proof_verifier.h @@ -12,7 +12,9 @@ namespace Quic { class EnvoyQuicProofVerifier : public EnvoyQuicProofVerifierBase { public: EnvoyQuicProofVerifier(Envoy::Ssl::ClientContextSharedPtr&& context) - : context_(std::move(context)) {} + : context_(std::move(context)) { + ASSERT(context_.get()); + } // EnvoyQuicProofVerifierBase quic::QuicAsyncStatus diff --git a/source/common/quic/envoy_quic_server_stream.cc b/source/common/quic/envoy_quic_server_stream.cc index 0c90cf66f6c97..74b439e246253 100644 --- a/source/common/quic/envoy_quic_server_stream.cc +++ b/source/common/quic/envoy_quic_server_stream.cc @@ -266,11 +266,15 @@ bool EnvoyQuicServerStream::OnStopSending(quic::QuicRstStreamErrorCode error) { } void EnvoyQuicServerStream::OnStreamReset(const quic::QuicRstStreamFrame& frame) { + ENVOY_STREAM_LOG(debug, "received reset code={}", *this, frame.error_code); + stats_.rx_reset_.inc(); quic::QuicSpdyServerStreamBase::OnStreamReset(frame); runResetCallbacks(quicRstErrorToEnvoyRemoteResetReason(frame.error_code)); } void EnvoyQuicServerStream::Reset(quic::QuicRstStreamErrorCode error) { + ENVOY_STREAM_LOG(debug, "sending reset code={}", *this, error); + stats_.tx_reset_.inc(); // Upper layers expect calling resetStream() to immediately raise reset callbacks. runResetCallbacks(quicRstErrorToEnvoyLocalResetReason(error)); quic::QuicSpdyServerStreamBase::Reset(error); diff --git a/source/common/quic/envoy_quic_server_stream.h b/source/common/quic/envoy_quic_server_stream.h index f86857cc800df..ac44a083b1f4b 100644 --- a/source/common/quic/envoy_quic_server_stream.h +++ b/source/common/quic/envoy_quic_server_stream.h @@ -54,6 +54,10 @@ class EnvoyQuicServerStream : public quic::QuicSpdyServerStreamBase, void setFlushTimeout(std::chrono::milliseconds) override { // TODO(mattklein123): Actually implement this for HTTP/3 similar to HTTP/2. } + + void setAccount(Buffer::BufferMemoryAccountSharedPtr) override { + // TODO(kbaichoo): implement account tracking for QUIC. + } // quic::QuicSpdyStream void OnBodyAvailable() override; bool OnStopSending(quic::QuicRstStreamErrorCode error) override; diff --git a/source/common/quic/quic_filter_manager_connection_impl.h b/source/common/quic/quic_filter_manager_connection_impl.h index 7922810e362c9..f0f666434c7fa 100644 --- a/source/common/quic/quic_filter_manager_connection_impl.h +++ b/source/common/quic/quic_filter_manager_connection_impl.h @@ -32,6 +32,8 @@ class TestPauseFilterForQuic; namespace Quic { +class QuicNetworkConnectionTest; + // Act as a Network::Connection to HCM and a FilterManager to FilterFactoryCb. class QuicFilterManagerConnectionImpl : public Network::ConnectionImplBase, public SendBufferMonitor { @@ -171,6 +173,7 @@ class QuicFilterManagerConnectionImpl : public Network::ConnectionImplBase, private: friend class Envoy::TestPauseFilterForQuic; + friend class Envoy::Quic::QuicNetworkConnectionTest; // Called when aggregated buffered bytes across all the streams exceeds high watermark. void onSendBufferHighWatermark(); diff --git a/source/common/quic/quic_network_connection.h b/source/common/quic/quic_network_connection.h index 66c51a7d52150..78ea81402e5c5 100644 --- a/source/common/quic/quic_network_connection.h +++ b/source/common/quic/quic_network_connection.h @@ -9,6 +9,9 @@ namespace Envoy { namespace Quic { +// Read ~32k bytes per connection by default, which is about the same as TCP. +static const uint32_t DEFAULT_PACKETS_TO_READ_PER_CONNECTION = 32u; + // A base class of both the client and server connections which keeps stats and // connection socket. class QuicNetworkConnection : protected Logger::Loggable { diff --git a/source/common/router/config_impl.cc b/source/common/router/config_impl.cc index 3195a260af3cb..bd823acfba7bb 100644 --- a/source/common/router/config_impl.cc +++ b/source/common/router/config_impl.cc @@ -626,24 +626,29 @@ void RouteEntryImplBase::finalizeResponseHeaders(Http::ResponseHeaderMap& header } Http::HeaderTransforms -RouteEntryImplBase::responseHeaderTransforms(const StreamInfo::StreamInfo& stream_info) const { +RouteEntryImplBase::responseHeaderTransforms(const StreamInfo::StreamInfo& stream_info, + bool do_formatting) const { Http::HeaderTransforms transforms; if (!vhost_.globalRouteConfig().mostSpecificHeaderMutationsWins()) { // Append user-specified request headers from most to least specific: route-level headers, // virtual host level headers and finally global connection manager level headers. - mergeTransforms(transforms, response_headers_parser_->getHeaderTransforms(stream_info)); - mergeTransforms(transforms, vhost_.responseHeaderParser().getHeaderTransforms(stream_info)); - mergeTransforms( - transforms, - vhost_.globalRouteConfig().responseHeaderParser().getHeaderTransforms(stream_info)); + mergeTransforms(transforms, + response_headers_parser_->getHeaderTransforms(stream_info, do_formatting)); + mergeTransforms(transforms, + vhost_.responseHeaderParser().getHeaderTransforms(stream_info, do_formatting)); + mergeTransforms(transforms, + vhost_.globalRouteConfig().responseHeaderParser().getHeaderTransforms( + stream_info, do_formatting)); } else { // Most specific mutations (route-level) take precedence by being applied // last: if a header is specified at all levels, the last one applied wins. - mergeTransforms( - transforms, - vhost_.globalRouteConfig().responseHeaderParser().getHeaderTransforms(stream_info)); - mergeTransforms(transforms, vhost_.responseHeaderParser().getHeaderTransforms(stream_info)); - mergeTransforms(transforms, response_headers_parser_->getHeaderTransforms(stream_info)); + mergeTransforms(transforms, + vhost_.globalRouteConfig().responseHeaderParser().getHeaderTransforms( + stream_info, do_formatting)); + mergeTransforms(transforms, + vhost_.responseHeaderParser().getHeaderTransforms(stream_info, do_formatting)); + mergeTransforms(transforms, + response_headers_parser_->getHeaderTransforms(stream_info, do_formatting)); } return transforms; } @@ -1047,9 +1052,10 @@ RouteEntryImplBase::WeightedClusterEntry::WeightedClusterEntry( } Http::HeaderTransforms RouteEntryImplBase::WeightedClusterEntry::responseHeaderTransforms( - const StreamInfo::StreamInfo& stream_info) const { - auto transforms = response_headers_parser_->getHeaderTransforms(stream_info); - mergeTransforms(transforms, DynamicRouteEntry::responseHeaderTransforms(stream_info)); + const StreamInfo::StreamInfo& stream_info, bool do_formatting) const { + auto transforms = response_headers_parser_->getHeaderTransforms(stream_info, do_formatting); + mergeTransforms(transforms, + DynamicRouteEntry::responseHeaderTransforms(stream_info, do_formatting)); return transforms; } diff --git a/source/common/router/config_impl.h b/source/common/router/config_impl.h index 7f9abd11c02fc..5a317c25ccfce 100644 --- a/source/common/router/config_impl.h +++ b/source/common/router/config_impl.h @@ -106,7 +106,8 @@ class SslRedirector : public DirectResponseEntry { // Router::DirectResponseEntry void finalizeResponseHeaders(Http::ResponseHeaderMap&, const StreamInfo::StreamInfo&) const override {} - Http::HeaderTransforms responseHeaderTransforms(const StreamInfo::StreamInfo&) const override { + Http::HeaderTransforms responseHeaderTransforms(const StreamInfo::StreamInfo&, + bool) const override { return {}; } std::string newPath(const Http::RequestHeaderMap& headers) const override; @@ -512,8 +513,8 @@ class RouteEntryImplBase : public RouteEntry, bool insert_envoy_original_path) const override; void finalizeResponseHeaders(Http::ResponseHeaderMap& headers, const StreamInfo::StreamInfo& stream_info) const override; - Http::HeaderTransforms - responseHeaderTransforms(const StreamInfo::StreamInfo& stream_info) const override; + Http::HeaderTransforms responseHeaderTransforms(const StreamInfo::StreamInfo& stream_info, + bool do_formatting = true) const override; const Http::HashPolicy* hashPolicy() const override { return hash_policy_.get(); } const HedgePolicy& hedgePolicy() const override { return hedge_policy_; } @@ -647,9 +648,9 @@ class RouteEntryImplBase : public RouteEntry, const StreamInfo::StreamInfo& stream_info) const override { return parent_->finalizeResponseHeaders(headers, stream_info); } - Http::HeaderTransforms - responseHeaderTransforms(const StreamInfo::StreamInfo& stream_info) const override { - return parent_->responseHeaderTransforms(stream_info); + Http::HeaderTransforms responseHeaderTransforms(const StreamInfo::StreamInfo& stream_info, + bool do_formatting = true) const override { + return parent_->responseHeaderTransforms(stream_info, do_formatting); } const CorsPolicy* corsPolicy() const override { return parent_->corsPolicy(); } @@ -777,8 +778,8 @@ class RouteEntryImplBase : public RouteEntry, response_headers_parser_->evaluateHeaders(headers, stream_info); DynamicRouteEntry::finalizeResponseHeaders(headers, stream_info); } - Http::HeaderTransforms - responseHeaderTransforms(const StreamInfo::StreamInfo& stream_info) const override; + Http::HeaderTransforms responseHeaderTransforms(const StreamInfo::StreamInfo& stream_info, + bool do_formatting = true) const override; const RouteSpecificFilterConfig* perFilterConfig(const std::string& name) const override; diff --git a/source/common/router/delegating_route_impl.cc b/source/common/router/delegating_route_impl.cc index af4c61304c698..bdc8ce1dde02f 100644 --- a/source/common/router/delegating_route_impl.cc +++ b/source/common/router/delegating_route_impl.cc @@ -25,8 +25,9 @@ void DelegatingRouteEntry::finalizeResponseHeaders( } Http::HeaderTransforms -DelegatingRouteEntry::responseHeaderTransforms(const StreamInfo::StreamInfo& stream_info) const { - return base_route_->routeEntry()->responseHeaderTransforms(stream_info); +DelegatingRouteEntry::responseHeaderTransforms(const StreamInfo::StreamInfo& stream_info, + bool do_formatting) const { + return base_route_->routeEntry()->responseHeaderTransforms(stream_info, do_formatting); } const std::string& DelegatingRouteEntry::clusterName() const { diff --git a/source/common/router/delegating_route_impl.h b/source/common/router/delegating_route_impl.h index ebe791c37c755..53721b394c53d 100644 --- a/source/common/router/delegating_route_impl.h +++ b/source/common/router/delegating_route_impl.h @@ -46,8 +46,8 @@ class DelegatingRouteEntry : public Router::RouteEntry { // Router::ResponseEntry void finalizeResponseHeaders(Http::ResponseHeaderMap& headers, const StreamInfo::StreamInfo& stream_info) const override; - Http::HeaderTransforms - responseHeaderTransforms(const StreamInfo::StreamInfo& stream_info) const override; + Http::HeaderTransforms responseHeaderTransforms(const StreamInfo::StreamInfo& stream_info, + bool do_formatting = true) const override; // Router::RouteEntry const std::string& clusterName() const override; diff --git a/source/common/router/header_parser.cc b/source/common/router/header_parser.cc index e05460bfb0c17..5a46d92d7c152 100644 --- a/source/common/router/header_parser.cc +++ b/source/common/router/header_parser.cc @@ -296,17 +296,25 @@ void HeaderParser::evaluateHeaders(Http::HeaderMap& headers, } } -Http::HeaderTransforms -HeaderParser::getHeaderTransforms(const StreamInfo::StreamInfo& stream_info) const { +Http::HeaderTransforms HeaderParser::getHeaderTransforms(const StreamInfo::StreamInfo& stream_info, + bool do_formatting) const { Http::HeaderTransforms transforms; for (const auto& [key, entry] : headers_to_add_) { - const std::string value = entry.formatter_->format(stream_info); - if (!value.empty()) { + if (do_formatting) { + const std::string value = entry.formatter_->format(stream_info); + if (!value.empty()) { + if (entry.formatter_->append()) { + transforms.headers_to_append.push_back({key, value}); + } else { + transforms.headers_to_overwrite.push_back({key, value}); + } + } + } else { if (entry.formatter_->append()) { - transforms.headers_to_append.push_back({key, value}); + transforms.headers_to_append.push_back({key, entry.original_value_}); } else { - transforms.headers_to_overwrite.push_back({key, value}); + transforms.headers_to_overwrite.push_back({key, entry.original_value_}); } } } diff --git a/source/common/router/header_parser.h b/source/common/router/header_parser.h index 8e4d5548fce03..03b1dc65b00c3 100644 --- a/source/common/router/header_parser.h +++ b/source/common/router/header_parser.h @@ -55,8 +55,11 @@ class HeaderParser { * Same as evaluateHeaders, but returns the modifications that would have been made rather than * modifying an existing HeaderMap. * @param stream_info contains additional information about the request. + * @param do_formatting whether or not to evaluate configured transformations; if false, returns + * original values instead. */ - Http::HeaderTransforms getHeaderTransforms(const StreamInfo::StreamInfo& stream_info) const; + Http::HeaderTransforms getHeaderTransforms(const StreamInfo::StreamInfo& stream_info, + bool do_formatting = true) const; protected: HeaderParser() = default; diff --git a/source/common/router/upstream_request.cc b/source/common/router/upstream_request.cc index 3213f156aa812..355f577a38bc7 100644 --- a/source/common/router/upstream_request.cc +++ b/source/common/router/upstream_request.cc @@ -371,6 +371,9 @@ void UpstreamRequest::onPoolReady( ENVOY_STREAM_LOG(debug, "pool ready", *parent_.callbacks()); upstream_ = std::move(upstream); + // Have the upstream use the account of the downstream. + upstream_->setAccount(parent_.callbacks()->account()); + if (parent_.requestVcluster()) { // The cluster increases its upstream_rq_total_ counter right before firing this onPoolReady // callback. Hence, the upstream request increases the virtual cluster's upstream_rq_total_ stat diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 0f8995fdb3f12..82f9b46eea368 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -86,6 +86,7 @@ constexpr const char* runtime_features[] = { "envoy.reloadable_features.strip_port_from_connect", "envoy.reloadable_features.treat_host_like_authority", "envoy.reloadable_features.treat_upstream_connect_timeout_as_connect_failure", + "envoy.reloadable_features.udp_per_event_loop_read_limit", "envoy.reloadable_features.upstream_host_weight_change_causes_rebuild", "envoy.reloadable_features.use_observable_cluster_name", "envoy.reloadable_features.vhds_heartbeats", @@ -110,6 +111,8 @@ constexpr const char* disabled_runtime_features[] = { "envoy.reloadable_features.remove_legacy_json", // Sentinel and test flag. "envoy.reloadable_features.test_feature_false", + // TODO(kbaichoo): Remove when this is no longer test only. + "envoy.test_only.per_stream_buffer_accounting", // Allows the use of ExtensionWithMatcher to wrap a HTTP filter with a match tree. "envoy.reloadable_features.experimental_matching_api", }; diff --git a/source/common/singleton/threadsafe_singleton.h b/source/common/singleton/threadsafe_singleton.h index 5b55dc0af5170..c60a4bb06be31 100644 --- a/source/common/singleton/threadsafe_singleton.h +++ b/source/common/singleton/threadsafe_singleton.h @@ -67,10 +67,10 @@ template class InjectableSingleton { static void clear() { loader_ = nullptr; } protected: - static T* loader_; + static std::atomic loader_; }; -template T* InjectableSingleton::loader_ = nullptr; +template std::atomic InjectableSingleton::loader_ = nullptr; template class ScopedInjectableLoader { public: diff --git a/source/common/tcp_proxy/upstream.cc b/source/common/tcp_proxy/upstream.cc index fc0335775afdb..22b939a30bc47 100644 --- a/source/common/tcp_proxy/upstream.cc +++ b/source/common/tcp_proxy/upstream.cc @@ -193,7 +193,7 @@ void TcpConnPool::onPoolReady(Tcp::ConnectionPool::ConnectionDataPtr&& conn_data HttpConnPool::HttpConnPool(Upstream::ThreadLocalCluster& thread_local_cluster, Upstream::LoadBalancerContext* context, const TunnelingConfig& config, Tcp::ConnectionPool::UpstreamCallbacks& upstream_callbacks, - Http::CodecClient::Type type) + Http::CodecType type) : config_(config), type_(type), upstream_callbacks_(upstream_callbacks) { conn_pool_ = thread_local_cluster.httpConnPool(Upstream::ResourcePriority::Default, absl::nullopt, context); @@ -209,7 +209,7 @@ HttpConnPool::~HttpConnPool() { void HttpConnPool::newStream(GenericConnectionPoolCallbacks& callbacks) { callbacks_ = &callbacks; - if (type_ == Http::CodecClient::Type::HTTP1) { + if (type_ == Http::CodecType::HTTP1) { upstream_ = std::make_unique(upstream_callbacks_, config_); } else { upstream_ = std::make_unique(upstream_callbacks_, config_); diff --git a/source/common/tcp_proxy/upstream.h b/source/common/tcp_proxy/upstream.h index bdf31648b034d..3047441a9a4fb 100644 --- a/source/common/tcp_proxy/upstream.h +++ b/source/common/tcp_proxy/upstream.h @@ -48,12 +48,11 @@ class HttpConnPool : public GenericConnPool, public Http::ConnectionPool::Callba HttpConnPool(Upstream::ThreadLocalCluster& thread_local_cluster, Upstream::LoadBalancerContext* context, const TunnelingConfig& config, - Tcp::ConnectionPool::UpstreamCallbacks& upstream_callbacks, - Http::CodecClient::Type type); + Tcp::ConnectionPool::UpstreamCallbacks& upstream_callbacks, Http::CodecType type); ~HttpConnPool() override; // HTTP/3 upstreams are not supported at the moment. - bool valid() const { return conn_pool_ != nullptr && type_ <= Http::CodecClient::Type::HTTP2; } + bool valid() const { return conn_pool_ != nullptr && type_ <= Http::CodecType::HTTP2; } // GenericConnPool void newStream(GenericConnectionPoolCallbacks& callbacks) override; @@ -97,7 +96,7 @@ class HttpConnPool : public GenericConnPool, public Http::ConnectionPool::Callba const Network::Address::InstanceConstSharedPtr& local_address, Ssl::ConnectionInfoConstSharedPtr ssl_info); const TunnelingConfig config_; - Http::CodecClient::Type type_; + Http::CodecType type_; Http::ConnectionPool::Instance* conn_pool_{}; Http::ConnectionPool::Cancellable* upstream_handle_{}; GenericConnectionPoolCallbacks* callbacks_{}; diff --git a/source/common/upstream/health_checker_impl.cc b/source/common/upstream/health_checker_impl.cc index 7b64ed5fda851..608c02e2b7a22 100644 --- a/source/common/upstream/health_checker_impl.cc +++ b/source/common/upstream/health_checker_impl.cc @@ -197,13 +197,13 @@ bool HttpHealthCheckerImpl::HttpStatusChecker::inRange(uint64_t http_status) con return false; } -Http::Protocol codecClientTypeToProtocol(Http::CodecClient::Type codec_client_type) { +Http::Protocol codecClientTypeToProtocol(Http::CodecType codec_client_type) { switch (codec_client_type) { - case Http::CodecClient::Type::HTTP1: + case Http::CodecType::HTTP1: return Http::Protocol::Http11; - case Http::CodecClient::Type::HTTP2: + case Http::CodecType::HTTP2: return Http::Protocol::Http2; - case Http::CodecClient::Type::HTTP3: + case Http::CodecType::HTTP3: return Http::Protocol::Http3; default: NOT_REACHED_GCOVR_EXCL_LINE; @@ -422,15 +422,15 @@ void HttpHealthCheckerImpl::HttpActiveHealthCheckSession::onTimeout() { } } -Http::CodecClient::Type +Http::CodecType HttpHealthCheckerImpl::codecClientType(const envoy::type::v3::CodecClientType& type) { switch (type) { case envoy::type::v3::HTTP3: - return Http::CodecClient::Type::HTTP3; + return Http::CodecType::HTTP3; case envoy::type::v3::HTTP2: - return Http::CodecClient::Type::HTTP2; + return Http::CodecType::HTTP2; case envoy::type::v3::HTTP1: - return Http::CodecClient::Type::HTTP1; + return Http::CodecType::HTTP1; default: NOT_REACHED_GCOVR_EXCL_LINE; } @@ -892,8 +892,8 @@ void GrpcHealthCheckerImpl::GrpcActiveHealthCheckSession::logHealthCheckStatus( Http::CodecClientPtr ProdGrpcHealthCheckerImpl::createCodecClient(Upstream::Host::CreateConnectionData& data) { return std::make_unique( - Http::CodecClient::Type::HTTP2, std::move(data.connection_), data.host_description_, - dispatcher_, random_generator_); + Http::CodecType::HTTP2, std::move(data.connection_), data.host_description_, dispatcher_, + random_generator_); } std::ostream& operator<<(std::ostream& out, HealthState state) { diff --git a/source/common/upstream/health_checker_impl.h b/source/common/upstream/health_checker_impl.h index a156c77f9847f..fdc337d142f6d 100644 --- a/source/common/upstream/health_checker_impl.h +++ b/source/common/upstream/health_checker_impl.h @@ -159,7 +159,7 @@ class HttpHealthCheckerImpl : public HealthCheckerImplBase { return envoy::data::core::v3::HTTP; } - Http::CodecClient::Type codecClientType(const envoy::type::v3::CodecClientType& type); + Http::CodecType codecClientType(const envoy::type::v3::CodecClientType& type); const std::string path_; const std::string host_value_; @@ -168,7 +168,7 @@ class HttpHealthCheckerImpl : public HealthCheckerImplBase { const HttpStatusChecker http_status_checker_; protected: - const Http::CodecClient::Type codec_client_type_; + const Http::CodecType codec_client_type_; Random::RandomGenerator& random_generator_; }; diff --git a/source/docs/response_header_transforms.md b/source/docs/response_header_transforms.md index 33390129ebc75..0d0ca01116b11 100644 --- a/source/docs/response_header_transforms.md +++ b/source/docs/response_header_transforms.md @@ -26,3 +26,6 @@ Http::FilterHeadersStatus MyFilter::decodeHeaders( "local_reply"); } ``` + +If you want to retrieve the original values rather than the formatted ones, pass +`/*do_formatting=*/false` as the second argument to `responseHeaderTransforms`. diff --git a/source/extensions/BUILD b/source/extensions/BUILD index 779d1695d3b7c..5d4f6c8a9b745 100644 --- a/source/extensions/BUILD +++ b/source/extensions/BUILD @@ -1 +1,6 @@ licenses(["notice"]) # Apache 2 + +exports_files([ + "extensions_metadata.yaml", + "extensions_build_config.bzl", +]) diff --git a/source/extensions/access_loggers/file/BUILD b/source/extensions/access_loggers/file/BUILD index 4fc1a97c8bfb2..d053296f6a9b8 100644 --- a/source/extensions/access_loggers/file/BUILD +++ b/source/extensions/access_loggers/file/BUILD @@ -15,12 +15,10 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.access_loggers", # TODO(#9953) determine if this is core or should be cleaned up. extra_visibility = [ "//test:__subpackages__", ], - security_posture = "robust_to_untrusted_downstream", deps = [ "//include/envoy/registry", "//source/common/config:config_provider_lib", diff --git a/source/extensions/access_loggers/grpc/BUILD b/source/extensions/access_loggers/grpc/BUILD index ebffc533ba20a..043c5dc898a1d 100644 --- a/source/extensions/access_loggers/grpc/BUILD +++ b/source/extensions/access_loggers/grpc/BUILD @@ -97,13 +97,11 @@ envoy_cc_extension( name = "http_config", srcs = ["http_config.cc"], hdrs = ["http_config.h"], - category = "envoy.access_loggers", # TODO(#9953) clean up. extra_visibility = [ "//test/common/access_log:__subpackages__", "//test/integration:__subpackages__", ], - security_posture = "robust_to_untrusted_downstream", deps = [ ":config_utils", "//include/envoy/server:access_log_config_interface", @@ -121,13 +119,11 @@ envoy_cc_extension( name = "tcp_config", srcs = ["tcp_config.cc"], hdrs = ["tcp_config.h"], - category = "envoy.access_loggers", # TODO(#9953) clean up. extra_visibility = [ "//test/common/access_log:__subpackages__", "//test/integration:__subpackages__", ], - security_posture = "robust_to_untrusted_downstream", deps = [ ":config_utils", "//include/envoy/server:access_log_config_interface", diff --git a/source/extensions/access_loggers/open_telemetry/BUILD b/source/extensions/access_loggers/open_telemetry/BUILD index 0c4c07a036964..cb85c6957fbf0 100644 --- a/source/extensions/access_loggers/open_telemetry/BUILD +++ b/source/extensions/access_loggers/open_telemetry/BUILD @@ -61,13 +61,11 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.access_loggers", # TODO(#9953) clean up. extra_visibility = [ "//test/common/access_log:__subpackages__", "//test/integration:__subpackages__", ], - security_posture = "robust_to_untrusted_downstream", deps = [ "//include/envoy/server:access_log_config_interface", "//source/common/common:assert_lib", diff --git a/source/extensions/access_loggers/stream/BUILD b/source/extensions/access_loggers/stream/BUILD index a35d7ba9cece1..f78092b3aedb3 100644 --- a/source/extensions/access_loggers/stream/BUILD +++ b/source/extensions/access_loggers/stream/BUILD @@ -12,11 +12,9 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.access_loggers", extra_visibility = [ "//test:__subpackages__", ], - security_posture = "robust_to_untrusted_downstream", deps = [ "//include/envoy/registry", "//source/common/config:config_provider_lib", diff --git a/source/extensions/access_loggers/wasm/BUILD b/source/extensions/access_loggers/wasm/BUILD index 0ed93bef9607d..ebe064a25c6f4 100644 --- a/source/extensions/access_loggers/wasm/BUILD +++ b/source/extensions/access_loggers/wasm/BUILD @@ -26,9 +26,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.access_loggers", - security_posture = "unknown", - status = "alpha", deps = [ ":wasm_access_log_lib", "//include/envoy/registry", diff --git a/source/extensions/bootstrap/wasm/BUILD b/source/extensions/bootstrap/wasm/BUILD index fe58c86f94c02..be38ee803f4aa 100644 --- a/source/extensions/bootstrap/wasm/BUILD +++ b/source/extensions/bootstrap/wasm/BUILD @@ -16,9 +16,6 @@ envoy_cc_extension( hdrs = [ "config.h", ], - category = "envoy.bootstrap", - security_posture = "unknown", - status = "alpha", deps = [ "//include/envoy/registry", "//include/envoy/server:bootstrap_extension_config_interface", @@ -29,7 +26,6 @@ envoy_cc_extension( "//source/common/config:datasource_lib", "//source/common/protobuf:utility_lib", "//source/extensions/common/wasm:wasm_lib", - "//source/extensions/grpc_credentials:well_known_names", "@envoy_api//envoy/extensions/wasm/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/clusters/aggregate/BUILD b/source/extensions/clusters/aggregate/BUILD index 473f140b30da7..38f702f15543c 100644 --- a/source/extensions/clusters/aggregate/BUILD +++ b/source/extensions/clusters/aggregate/BUILD @@ -15,8 +15,6 @@ envoy_cc_extension( "cluster.h", "lb_context.h", ], - category = "envoy.clusters", - security_posture = "requires_trusted_downstream_and_upstream", deps = [ "//source/common/upstream:cluster_factory_lib", "//source/common/upstream:upstream_includes", diff --git a/source/extensions/clusters/dynamic_forward_proxy/BUILD b/source/extensions/clusters/dynamic_forward_proxy/BUILD index 3a6fdf9f10804..36d74421839a1 100644 --- a/source/extensions/clusters/dynamic_forward_proxy/BUILD +++ b/source/extensions/clusters/dynamic_forward_proxy/BUILD @@ -12,8 +12,6 @@ envoy_cc_extension( name = "cluster", srcs = ["cluster.cc"], hdrs = ["cluster.h"], - category = "envoy.clusters", - security_posture = "robust_to_untrusted_downstream", deps = [ "//source/common/network:transport_socket_options_lib", "//source/common/upstream:cluster_factory_lib", diff --git a/source/extensions/clusters/redis/BUILD b/source/extensions/clusters/redis/BUILD index 54577e1483e39..829f517516d24 100644 --- a/source/extensions/clusters/redis/BUILD +++ b/source/extensions/clusters/redis/BUILD @@ -42,8 +42,6 @@ envoy_cc_extension( "redis_cluster.cc", "redis_cluster.h", ], - category = "envoy.clusters", - security_posture = "requires_trusted_downstream_and_upstream", deps = [ "redis_cluster_lb", "//include/envoy/api:api_interface", diff --git a/source/extensions/common/crypto/BUILD b/source/extensions/common/crypto/BUILD index d33b7986b519d..4cd26ae3701ae 100644 --- a/source/extensions/common/crypto/BUILD +++ b/source/extensions/common/crypto/BUILD @@ -18,7 +18,6 @@ envoy_cc_extension( "crypto_impl.h", "utility_impl.h", ], - category = "DELIBERATELY_OMITTED", external_deps = [ "ssl", ], @@ -27,8 +26,6 @@ envoy_cc_extension( "//test/common/config:__subpackages__", "//test/common/crypto:__subpackages__", ], - security_posture = "unknown", - undocumented = True, deps = [ "//include/envoy/buffer:buffer_interface", "//source/common/common:assert_lib", diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc index c2f6cd82a8581..2aa86bbb80ba4 100644 --- a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc @@ -4,6 +4,7 @@ #include "common/config/utility.h" #include "common/http/utility.h" +#include "common/network/resolver_impl.h" #include "common/network/utility.h" // TODO(mattklein123): Move DNS family helpers to a smaller include. @@ -20,8 +21,8 @@ DnsCacheImpl::DnsCacheImpl( const envoy::extensions::common::dynamic_forward_proxy::v3::DnsCacheConfig& config) : main_thread_dispatcher_(main_thread_dispatcher), dns_lookup_family_(Upstream::getDnsLookupFamilyFromEnum(config.dns_lookup_family())), - resolver_(main_thread_dispatcher.createDnsResolver({}, config.use_tcp_for_dns_lookups())), - tls_slot_(tls), scope_(root_scope.createScope(fmt::format("dns_cache.{}.", config.name()))), + resolver_(selectDnsResolver(config, main_thread_dispatcher)), tls_slot_(tls), + scope_(root_scope.createScope(fmt::format("dns_cache.{}.", config.name()))), stats_(generateDnsCacheStats(*scope_)), resource_manager_(*scope_, loader, config.name(), config.dns_cache_circuit_breaker()), refresh_interval_(PROTOBUF_GET_MS_OR_DEFAULT(config, dns_refresh_rate, 60000)), @@ -46,6 +47,22 @@ DnsCacheImpl::~DnsCacheImpl() { } } +Network::DnsResolverSharedPtr DnsCacheImpl::selectDnsResolver( + const envoy::extensions::common::dynamic_forward_proxy::v3::DnsCacheConfig& config, + Event::Dispatcher& main_thread_dispatcher) { + if (config.has_dns_resolver()) { + const auto& resolver_addrs = config.dns_resolver().resolvers(); + std::vector resolvers; + resolvers.reserve(resolver_addrs.size()); + for (const auto& resolver_addr : resolver_addrs) { + resolvers.push_back(Network::Address::resolveProtoAddress(resolver_addr)); + } + return main_thread_dispatcher.createDnsResolver(resolvers, config.use_tcp_for_dns_lookups()); + } + + return main_thread_dispatcher.createDnsResolver({}, config.use_tcp_for_dns_lookups()); +} + DnsCacheStats DnsCacheImpl::generateDnsCacheStats(Stats::Scope& scope) { return {ALL_DNS_CACHE_STATS(POOL_COUNTER(scope), POOL_GAUGE(scope))}; } diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h index c076c79404113..1f5237dc035e5 100644 --- a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h @@ -46,6 +46,9 @@ class DnsCacheImpl : public DnsCache, Logger::LoggableonGrpcCreateInitialMetadata(token_, initial_metadata); + } void onReceiveInitialMetadata(Http::ResponseHeaderMapPtr&& metadata) override { context_->onGrpcReceiveInitialMetadataWrapper(token_, std::move(metadata)); } diff --git a/source/extensions/common/wasm/plugin.cc b/source/extensions/common/wasm/plugin.cc index b01ac0324f53f..fe773a3b1e100 100644 --- a/source/extensions/common/wasm/plugin.cc +++ b/source/extensions/common/wasm/plugin.cc @@ -2,8 +2,6 @@ #include "envoy/common/exception.h" -#include "extensions/common/wasm/well_known_names.h" - #include "include/proxy-wasm/wasm.h" namespace Envoy { @@ -24,8 +22,7 @@ WasmConfig::WasmConfig(const envoy::extensions::wasm::v3::PluginConfig& config) // since it directly accesses Envoy's env vars and we should not modify Envoy's env vars here. // TODO(mathetake): Once proxy_get_map_values(type::EnvironmentVariables, ..) call is supported, // then remove this restriction. - if (config.vm_config().runtime() == WasmRuntimeNames::get().Null && - !envs.key_values().empty()) { + if (config.vm_config().runtime() == "envoy.wasm.runtime.null" && !envs.key_values().empty()) { throw EnvoyException("envoy.extensions.wasm.v3.VmConfig.EnvironmentVariables.key_values must " "not be set for NullVm."); } diff --git a/source/extensions/common/wasm/wasm.h b/source/extensions/common/wasm/wasm.h index a02c1b4c627c2..a44da29e56492 100644 --- a/source/extensions/common/wasm/wasm.h +++ b/source/extensions/common/wasm/wasm.h @@ -24,7 +24,6 @@ #include "extensions/common/wasm/plugin.h" #include "extensions/common/wasm/wasm_extension.h" #include "extensions/common/wasm/wasm_vm.h" -#include "extensions/common/wasm/well_known_names.h" #include "include/proxy-wasm/exports.h" #include "include/proxy-wasm/wasm.h" diff --git a/source/extensions/common/wasm/wasm_vm.cc b/source/extensions/common/wasm/wasm_vm.cc index c8d3fb618e45e..ca6538de5cf8b 100644 --- a/source/extensions/common/wasm/wasm_vm.cc +++ b/source/extensions/common/wasm/wasm_vm.cc @@ -7,7 +7,6 @@ #include "extensions/common/wasm/ext/envoy_null_vm_wasm_api.h" #include "extensions/common/wasm/wasm_extension.h" #include "extensions/common/wasm/wasm_runtime_factory.h" -#include "extensions/common/wasm/well_known_names.h" #include "include/proxy-wasm/null_plugin.h" diff --git a/source/extensions/common/wasm/well_known_names.h b/source/extensions/common/wasm/well_known_names.h deleted file mode 100644 index 3904868bba880..0000000000000 --- a/source/extensions/common/wasm/well_known_names.h +++ /dev/null @@ -1,37 +0,0 @@ -#pragma once - -#include - -#include "common/singleton/const_singleton.h" - -namespace Envoy { -namespace Extensions { -namespace Common { -namespace Wasm { - -/** - * Well-known wasm runtime names. - * NOTE: New wasm runtimes should use the well known name: envoy.wasm.runtime.name. - */ -class WasmRuntimeValues { -public: - // Wasmtime (https://github.com/bytecodealliance/wasmtime). - const std::string Wasmtime = "envoy.wasm.runtime.wasmtime"; - // WAVM (https://github.com/WAVM/WAVM) Wasm VM. - const std::string Wavm = "envoy.wasm.runtime.wavm"; - // Null sandbox: modules must be compiled into envoy and registered name is given in the - // DataSource.inline_string. - const std::string Null = "envoy.wasm.runtime.null"; - // V8-based (https://v8.dev) WebAssembly runtime. - const std::string V8 = "envoy.wasm.runtime.v8"; - - // Filter state name - const std::string FilterState = "envoy.wasm"; -}; - -using WasmRuntimeNames = ConstSingleton; - -} // namespace Wasm -} // namespace Common -} // namespace Extensions -} // namespace Envoy diff --git a/source/extensions/compression/brotli/compressor/BUILD b/source/extensions/compression/brotli/compressor/BUILD index cee2e36945f5e..d190709060059 100644 --- a/source/extensions/compression/brotli/compressor/BUILD +++ b/source/extensions/compression/brotli/compressor/BUILD @@ -25,8 +25,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.compression.compressor", - security_posture = "robust_to_untrusted_downstream", deps = [ ":compressor_lib", "//source/common/http:headers_lib", diff --git a/source/extensions/compression/brotli/decompressor/BUILD b/source/extensions/compression/brotli/decompressor/BUILD index 3667300a8392f..22ae257a84ab9 100644 --- a/source/extensions/compression/brotli/decompressor/BUILD +++ b/source/extensions/compression/brotli/decompressor/BUILD @@ -27,8 +27,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.compression.decompressor", - security_posture = "robust_to_untrusted_downstream", deps = [ ":decompressor_lib", "//source/common/http:headers_lib", diff --git a/source/extensions/compression/gzip/compressor/BUILD b/source/extensions/compression/gzip/compressor/BUILD index 39a7e7c6e9d73..1274b4d8e6ea7 100644 --- a/source/extensions/compression/gzip/compressor/BUILD +++ b/source/extensions/compression/gzip/compressor/BUILD @@ -26,8 +26,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.compression.compressor", - security_posture = "robust_to_untrusted_downstream", deps = [ ":compressor_lib", "//source/common/http:headers_lib", diff --git a/source/extensions/compression/gzip/decompressor/BUILD b/source/extensions/compression/gzip/decompressor/BUILD index 0a1d8766031b9..541aa8bf8df96 100644 --- a/source/extensions/compression/gzip/decompressor/BUILD +++ b/source/extensions/compression/gzip/decompressor/BUILD @@ -29,8 +29,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.compression.decompressor", - security_posture = "robust_to_untrusted_downstream", deps = [ ":zlib_decompressor_impl_lib", "//source/common/http:headers_lib", diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index ee8ab2a61b716..d64a97928582f 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -67,6 +67,7 @@ EXTENSIONS = { "envoy.filters.http.admission_control": "//source/extensions/filters/http/admission_control:config", "envoy.filters.http.aws_lambda": "//source/extensions/filters/http/aws_lambda:config", "envoy.filters.http.aws_request_signing": "//source/extensions/filters/http/aws_request_signing:config", + "envoy.filters.http.bandwidth_limit": "//source/extensions/filters/http/bandwidth_limit:config", "envoy.filters.http.buffer": "//source/extensions/filters/http/buffer:config", "envoy.filters.http.cache": "//source/extensions/filters/http/cache:config", "envoy.filters.http.cdn_loop": "//source/extensions/filters/http/cdn_loop:config", @@ -100,6 +101,7 @@ EXTENSIONS = { "envoy.filters.http.ratelimit": "//source/extensions/filters/http/ratelimit:config", "envoy.filters.http.rbac": "//source/extensions/filters/http/rbac:config", "envoy.filters.http.router": "//source/extensions/filters/http/router:config", + "envoy.filters.http.set_metadata": "//source/extensions/filters/http/set_metadata:config", "envoy.filters.http.squash": "//source/extensions/filters/http/squash:config", "envoy.filters.http.tap": "//source/extensions/filters/http/tap:config", "envoy.filters.http.wasm": "//source/extensions/filters/http/wasm:config", diff --git a/source/extensions/extensions_metadata.yaml b/source/extensions/extensions_metadata.yaml new file mode 100644 index 0000000000000..56570a9404282 --- /dev/null +++ b/source/extensions/extensions_metadata.yaml @@ -0,0 +1,681 @@ +envoy.access_loggers.file: + categories: + - envoy.access_loggers + security_posture: robust_to_untrusted_downstream + status: stable +envoy.access_loggers.http_grpc: + categories: + - envoy.access_loggers + security_posture: robust_to_untrusted_downstream + status: stable +envoy.access_loggers.open_telemetry: + categories: + - envoy.access_loggers + security_posture: robust_to_untrusted_downstream + status: stable +envoy.access_loggers.stream: + categories: + - envoy.access_loggers + security_posture: robust_to_untrusted_downstream + status: stable +envoy.access_loggers.tcp_grpc: + categories: + - envoy.access_loggers + security_posture: robust_to_untrusted_downstream + status: stable +envoy.access_loggers.wasm: + categories: + - envoy.access_loggers + security_posture: unknown + status: alpha +envoy.bootstrap.wasm: + categories: + - envoy.bootstrap + security_posture: unknown + status: alpha +envoy.cache.simple_http_cache: + categories: + - envoy.filters.http.cache + security_posture: robust_to_untrusted_downstream_and_upstream + status: wip +envoy.clusters.aggregate: + categories: + - envoy.clusters + security_posture: requires_trusted_downstream_and_upstream + status: stable +envoy.clusters.dynamic_forward_proxy: + categories: + - envoy.clusters + security_posture: robust_to_untrusted_downstream + status: stable +envoy.clusters.redis: + categories: + - envoy.clusters + security_posture: requires_trusted_downstream_and_upstream + status: stable +envoy.compression.brotli.compressor: + categories: + - envoy.compression.compressor + security_posture: robust_to_untrusted_downstream + status: stable +envoy.compression.brotli.decompressor: + categories: + - envoy.compression.decompressor + security_posture: robust_to_untrusted_downstream + status: stable +envoy.compression.gzip.compressor: + categories: + - envoy.compression.compressor + security_posture: robust_to_untrusted_downstream + status: stable +envoy.compression.gzip.decompressor: + categories: + - envoy.compression.decompressor + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.adaptive_concurrency: + categories: + - envoy.filters.http + security_posture: unknown + status: alpha +envoy.filters.http.admission_control: + categories: + - envoy.filters.http + security_posture: unknown + status: alpha +envoy.filters.http.aws_lambda: + categories: + - envoy.filters.http + security_posture: requires_trusted_downstream_and_upstream + status: alpha +envoy.filters.http.aws_request_signing: + categories: + - envoy.filters.http + security_posture: requires_trusted_downstream_and_upstream + status: alpha +envoy.filters.http.bandwidth_limit: + categories: + - envoy.filters.http + security_posture: unknown + status: stable +envoy.filters.http.buffer: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.cache: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream_and_upstream + status: wip +envoy.filters.http.cdn_loop: + categories: + - envoy.filters.http + security_posture: unknown + status: alpha +envoy.filters.http.composite: + categories: + - envoy.filters.http + security_posture: unknown + status: stable +envoy.filters.http.compressor: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.cors: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.csrf: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.decompressor: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream_and_upstream + status: stable +envoy.filters.http.dynamic_forward_proxy: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.dynamo: + categories: + - envoy.filters.http + security_posture: requires_trusted_downstream_and_upstream + status: stable +envoy.filters.http.ext_authz: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.ext_proc: + categories: + - envoy.filters.http + security_posture: unknown + status: alpha +envoy.filters.http.fault: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.grpc_http1_bridge: + categories: + - envoy.filters.http + security_posture: unknown + status: stable +envoy.filters.http.grpc_http1_reverse_bridge: + categories: + - envoy.filters.http + security_posture: unknown + status: alpha +envoy.filters.http.grpc_json_transcoder: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.grpc_stats: + categories: + - envoy.filters.http + security_posture: unknown + status: alpha +envoy.filters.http.grpc_web: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.gzip: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.header_to_metadata: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.health_check: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.ip_tagging: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.jwt_authn: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: alpha +envoy.filters.http.kill_request: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.local_ratelimit: + categories: + - envoy.filters.http + security_posture: unknown + status: stable +envoy.filters.http.lua: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.oauth2: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: alpha +envoy.filters.http.on_demand: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.original_src: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: alpha +envoy.filters.http.ratelimit: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.rbac: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.router: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.set_metadata: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream_and_upstream + status: stable +envoy.filters.http.squash: + categories: + - envoy.filters.http + security_posture: requires_trusted_downstream_and_upstream + status: stable +envoy.filters.http.tap: + categories: + - envoy.filters.http + security_posture: requires_trusted_downstream_and_upstream + status: alpha +envoy.filters.http.wasm: + categories: + - envoy.filters.http + security_posture: unknown + status: alpha +envoy.filters.listener.http_inspector: + categories: + - envoy.filters.listener + security_posture: requires_trusted_downstream_and_upstream + status: stable +envoy.filters.listener.original_dst: + categories: + - envoy.filters.listener + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.listener.original_src: + categories: + - envoy.filters.listener + security_posture: robust_to_untrusted_downstream + status: alpha +envoy.filters.listener.proxy_protocol: + categories: + - envoy.filters.listener + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.listener.tls_inspector: + categories: + - envoy.filters.listener + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.network.client_ssl_auth: + categories: + - envoy.filters.network + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.network.direct_response: + categories: + - envoy.filters.network + security_posture: unknown + status: stable +envoy.filters.network.dubbo_proxy: + categories: + - envoy.filters.network + security_posture: requires_trusted_downstream_and_upstream + status: alpha +envoy.filters.network.echo: + categories: + - envoy.filters.network + security_posture: unknown + status: stable +envoy.filters.network.ext_authz: + categories: + - envoy.filters.network + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.network.http_connection_manager: + categories: + - envoy.filters.network + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.network.kafka_broker: + categories: + - envoy.filters.network + security_posture: requires_trusted_downstream_and_upstream + status: wip +envoy.filters.network.local_ratelimit: + categories: + - envoy.filters.network + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.network.mongo_proxy: + categories: + - envoy.filters.network + security_posture: requires_trusted_downstream_and_upstream + status: stable +envoy.filters.network.mysql_proxy: + categories: + - envoy.filters.network + security_posture: requires_trusted_downstream_and_upstream + status: alpha +envoy.filters.network.postgres_proxy: + categories: + - envoy.filters.network + security_posture: requires_trusted_downstream_and_upstream + status: stable +envoy.filters.network.ratelimit: + categories: + - envoy.filters.network + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.network.rbac: + categories: + - envoy.filters.network + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.network.redis_proxy: + categories: + - envoy.filters.network + security_posture: requires_trusted_downstream_and_upstream + status: stable +envoy.filters.network.rocketmq_proxy: + categories: + - envoy.filters.network + security_posture: requires_trusted_downstream_and_upstream + status: alpha +envoy.filters.network.sni_cluster: + categories: + - envoy.filters.network + security_posture: unknown + status: stable +envoy.filters.network.sni_dynamic_forward_proxy: + categories: + - envoy.filters.network + security_posture: unknown + status: alpha +envoy.filters.network.tcp_proxy: + categories: + - envoy.filters.network + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.network.thrift_proxy: + categories: + - envoy.filters.network + security_posture: requires_trusted_downstream_and_upstream + status: stable +envoy.filters.network.wasm: + categories: + - envoy.filters.network + security_posture: unknown + status: alpha +envoy.filters.network.zookeeper_proxy: + categories: + - envoy.filters.network + security_posture: requires_trusted_downstream_and_upstream + status: alpha +envoy.filters.thrift.ratelimit: + categories: + - envoy.thrift_proxy.filters + security_posture: requires_trusted_downstream_and_upstream + status: alpha +envoy.filters.thrift.router: + categories: + - envoy.thrift_proxy.filters + security_posture: requires_trusted_downstream_and_upstream + status: stable +envoy.filters.udp_listener.dns_filter: + categories: + - envoy.filters.udp_listener + security_posture: robust_to_untrusted_downstream + status: alpha +envoy.filters.udp_listener.udp_proxy: + categories: + - envoy.filters.udp_listener + security_posture: robust_to_untrusted_downstream + status: stable +envoy.grpc_credentials.aws_iam: + categories: + - envoy.grpc_credentials + security_posture: data_plane_agnostic + status: alpha +envoy.grpc_credentials.file_based_metadata: + categories: + - envoy.grpc_credentials + security_posture: data_plane_agnostic + status: alpha +envoy.health_checkers.redis: + categories: + - envoy.health_checkers + security_posture: requires_trusted_downstream_and_upstream + status: stable +envoy.http.original_ip_detection.custom_header: + categories: + - envoy.http.original_ip_detection + security_posture: robust_to_untrusted_downstream + status: stable +envoy.http.original_ip_detection.xff: + categories: + - envoy.http.original_ip_detection + security_posture: robust_to_untrusted_downstream + status: stable +envoy.http.stateful_header_formatters.preserve_case: + categories: + - envoy.http.stateful_header_formatters + security_posture: robust_to_untrusted_downstream_and_upstream + status: stable +envoy.internal_redirect_predicates.allow_listed_routes: + categories: + - envoy.internal_redirect_predicates + security_posture: robust_to_untrusted_downstream_and_upstream + status: stable +envoy.internal_redirect_predicates.previous_routes: + categories: + - envoy.internal_redirect_predicates + security_posture: robust_to_untrusted_downstream_and_upstream + status: stable +envoy.internal_redirect_predicates.safe_cross_scheme: + categories: + - envoy.internal_redirect_predicates + security_posture: robust_to_untrusted_downstream_and_upstream + status: stable +envoy.io_socket.user_space: + categories: + - envoy.io_socket + security_posture: unknown + status: wip + undocumented: true +envoy.matching.common_inputs.environment_variable: + categories: + - envoy.matching.common_inputs + security_posture: robust_to_untrusted_downstream + status: stable +envoy.matching.input_matchers.consistent_hashing: + categories: + - envoy.matching.input_matchers + security_posture: robust_to_untrusted_downstream + status: stable +envoy.rate_limit_descriptors.expr: + categories: + - envoy.rate_limit_descriptors + security_posture: unknown + status: stable +envoy.request_id.uuid: + categories: + - envoy.request_id + security_posture: robust_to_untrusted_downstream_and_upstream + status: stable +envoy.resource_monitors.fixed_heap: + categories: + - envoy.resource_monitors + security_posture: data_plane_agnostic + status: alpha +envoy.resource_monitors.injected_resource: + categories: + - envoy.resource_monitors + security_posture: data_plane_agnostic + status: alpha +envoy.retry_host_predicates.omit_canary_hosts: + categories: + - envoy.retry_host_predicates + security_posture: robust_to_untrusted_downstream + status: stable +envoy.retry_host_predicates.omit_host_metadata: + categories: + - envoy.retry_host_predicates + security_posture: robust_to_untrusted_downstream + status: stable +envoy.retry_host_predicates.previous_hosts: + categories: + - envoy.retry_host_predicates + security_posture: robust_to_untrusted_downstream + status: stable +envoy.retry_priorities.previous_priorities: + categories: + - envoy.retry_priorities + security_posture: robust_to_untrusted_downstream + status: stable +envoy.stat_sinks.dog_statsd: + categories: + - envoy.stats_sinks + security_posture: data_plane_agnostic + status: stable +envoy.stat_sinks.hystrix: + categories: + - envoy.stats_sinks + security_posture: data_plane_agnostic + status: stable +envoy.stat_sinks.metrics_service: + categories: + - envoy.stats_sinks + security_posture: data_plane_agnostic + status: stable +envoy.stat_sinks.statsd: + categories: + - envoy.stats_sinks + security_posture: data_plane_agnostic + status: stable +envoy.stat_sinks.wasm: + categories: + - envoy.stats_sinks + security_posture: data_plane_agnostic + status: alpha +envoy.tls.cert_validator.spiffe: + categories: + - envoy.tls.cert_validator + security_posture: unknown + status: wip +envoy.tracers.datadog: + categories: + - envoy.tracers + security_posture: robust_to_untrusted_downstream + status: stable +envoy.tracers.dynamic_ot: + categories: + - envoy.tracers + security_posture: robust_to_untrusted_downstream + status: stable +envoy.tracers.lightstep: + categories: + - envoy.tracers + security_posture: robust_to_untrusted_downstream + status: stable +envoy.tracers.opencensus: + categories: + - envoy.tracers + security_posture: robust_to_untrusted_downstream + status: stable +envoy.tracers.skywalking: + categories: + - envoy.tracers + security_posture: robust_to_untrusted_downstream + status: wip +envoy.tracers.xray: + categories: + - envoy.tracers + security_posture: robust_to_untrusted_downstream + status: stable +envoy.tracers.zipkin: + categories: + - envoy.tracers + security_posture: robust_to_untrusted_downstream + status: stable +envoy.transport_sockets.alts: + categories: + - envoy.transport_sockets.downstream + - envoy.transport_sockets.upstream + security_posture: robust_to_untrusted_downstream_and_upstream + status: stable +envoy.transport_sockets.raw_buffer: + categories: + - envoy.transport_sockets.downstream + - envoy.transport_sockets.upstream + security_posture: requires_trusted_downstream_and_upstream + status: stable +envoy.transport_sockets.starttls: + categories: + - envoy.transport_sockets.downstream + - envoy.transport_sockets.upstream + security_posture: robust_to_untrusted_downstream_and_upstream + status: stable +envoy.transport_sockets.tap: + categories: + - envoy.transport_sockets.downstream + - envoy.transport_sockets.upstream + security_posture: requires_trusted_downstream_and_upstream + status: alpha +envoy.transport_sockets.tls: + categories: + - envoy.transport_sockets.downstream + - envoy.transport_sockets.upstream + security_posture: robust_to_untrusted_downstream_and_upstream + status: stable +envoy.transport_sockets.upstream_proxy_protocol: + categories: + - envoy.transport_sockets.upstream + security_posture: robust_to_untrusted_downstream_and_upstream + status: stable +envoy.upstreams.http.generic: + categories: + - envoy.upstreams + security_posture: robust_to_untrusted_downstream + status: stable +envoy.upstreams.http.http: + categories: + - envoy.upstreams + security_posture: robust_to_untrusted_downstream + status: stable +envoy.upstreams.http.http_protocol_options: + categories: + - envoy.upstreams + security_posture: robust_to_untrusted_downstream + status: stable +envoy.upstreams.http.tcp: + categories: + - envoy.upstreams + security_posture: robust_to_untrusted_downstream + status: stable +envoy.upstreams.tcp.generic: + categories: + - envoy.upstreams + security_posture: robust_to_untrusted_downstream + status: stable +envoy.wasm.runtime.null: + categories: + - envoy.wasm.runtime + security_posture: unknown + status: alpha +envoy.wasm.runtime.v8: + categories: + - envoy.wasm.runtime + security_posture: unknown + status: alpha +envoy.wasm.runtime.wasmtime: + categories: + - envoy.wasm.runtime + security_posture: unknown + status: alpha +envoy.wasm.runtime.wavm: + categories: + - envoy.wasm.runtime + security_posture: unknown + status: alpha +envoy.watchdog.profile_action: + categories: + - envoy.guarddog_actions + security_posture: data_plane_agnostic + status: alpha diff --git a/source/extensions/filters/http/adaptive_concurrency/BUILD b/source/extensions/filters/http/adaptive_concurrency/BUILD index 7662d09bc1fd8..8baef84564d7e 100644 --- a/source/extensions/filters/http/adaptive_concurrency/BUILD +++ b/source/extensions/filters/http/adaptive_concurrency/BUILD @@ -30,9 +30,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", - security_posture = "unknown", - status = "alpha", deps = [ "//include/envoy/registry", "//source/extensions/filters/http:well_known_names", diff --git a/source/extensions/filters/http/admission_control/BUILD b/source/extensions/filters/http/admission_control/BUILD index 9bfd7c4505361..f7b60baf0ef51 100644 --- a/source/extensions/filters/http/admission_control/BUILD +++ b/source/extensions/filters/http/admission_control/BUILD @@ -1,6 +1,7 @@ load( "//bazel:envoy_build_system.bzl", "envoy_cc_extension", + "envoy_cc_library", "envoy_extension_package", ) @@ -11,7 +12,7 @@ licenses(["notice"]) # Apache 2 envoy_extension_package() -envoy_cc_extension( +envoy_cc_library( name = "admission_control_filter_lib", srcs = [ "admission_control.cc", @@ -21,8 +22,6 @@ envoy_cc_extension( "admission_control.h", "thread_local_controller.h", ], - category = "envoy.filters.http", - security_posture = "unknown", deps = [ "//include/envoy/http:filter_interface", "//include/envoy/runtime:runtime_interface", @@ -41,9 +40,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", - security_posture = "unknown", - status = "alpha", deps = [ "//include/envoy/registry", "//source/common/common:enum_to_int", diff --git a/source/extensions/filters/http/aws_lambda/BUILD b/source/extensions/filters/http/aws_lambda/BUILD index 1001ba3d87cbf..43544b5eccee7 100644 --- a/source/extensions/filters/http/aws_lambda/BUILD +++ b/source/extensions/filters/http/aws_lambda/BUILD @@ -37,9 +37,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", - security_posture = "requires_trusted_downstream_and_upstream", - status = "alpha", deps = [ ":aws_lambda_filter_lib", "//include/envoy/registry", diff --git a/source/extensions/filters/http/aws_lambda/aws_lambda_filter.cc b/source/extensions/filters/http/aws_lambda/aws_lambda_filter.cc index 6a908ff117e49..ace0940304571 100644 --- a/source/extensions/filters/http/aws_lambda/aws_lambda_filter.cc +++ b/source/extensions/filters/http/aws_lambda/aws_lambda_filter.cc @@ -333,6 +333,10 @@ void Filter::dejsonizeResponse(Http::ResponseHeaderMap& headers, const Buffer::I return; } + // Use JSON as the default content-type. If the response headers have a different content-type + // set, that will be used instead. + headers.setReferenceContentType(Http::Headers::get().ContentTypeValues.Json); + for (auto&& kv : json_resp.headers()) { // ignore H2 pseudo-headers (if any) if (kv.first[0] == ':') { @@ -348,7 +352,6 @@ void Filter::dejsonizeResponse(Http::ResponseHeaderMap& headers, const Buffer::I if (json_resp.status_code() != 0) { headers.setStatus(json_resp.status_code()); } - headers.setReferenceContentType(Http::Headers::get().ContentTypeValues.Json); if (!json_resp.body().empty()) { if (json_resp.is_base64_encoded()) { body.add(Base64::decode(json_resp.body())); diff --git a/source/extensions/filters/http/aws_request_signing/BUILD b/source/extensions/filters/http/aws_request_signing/BUILD index f0222a4b954bc..b1bcf820a85e5 100644 --- a/source/extensions/filters/http/aws_request_signing/BUILD +++ b/source/extensions/filters/http/aws_request_signing/BUILD @@ -29,9 +29,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", - security_posture = "requires_trusted_downstream_and_upstream", - status = "alpha", deps = [ ":aws_request_signing_filter_lib", "//include/envoy/registry", diff --git a/source/extensions/filters/http/bandwidth_limit/BUILD b/source/extensions/filters/http/bandwidth_limit/BUILD new file mode 100644 index 0000000000000..723e3aa158f38 --- /dev/null +++ b/source/extensions/filters/http/bandwidth_limit/BUILD @@ -0,0 +1,46 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_cc_library", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +# Local Bandwidthlimit HTTP L7 filter +# Public docs: docs/root/configuration/http_filters/bandwidth_limit_filter.rst + +envoy_extension_package() + +envoy_cc_library( + name = "bandwidth_limit_lib", + srcs = ["bandwidth_limit.cc"], + hdrs = ["bandwidth_limit.h"], + deps = [ + "//include/envoy/http:codes_interface", + "//include/envoy/server:filter_config_interface", + "//include/envoy/stats:stats_macros", + "//source/common/common:shared_token_bucket_impl_lib", + "//source/common/common:utility_lib", + "//source/common/http:header_utility_lib", + "//source/common/http:headers_lib", + "//source/common/router:header_parser_lib", + "//source/common/runtime:runtime_lib", + "//source/common/stats:timespan_lib", + "//source/extensions/filters/http/common:stream_rate_limiter_lib", + "@envoy_api//envoy/extensions/filters/http/bandwidth_limit/v3alpha:pkg_cc_proto", + ], +) + +envoy_cc_extension( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + deps = [ + ":bandwidth_limit_lib", + "//include/envoy/http:filter_interface", + "//source/common/protobuf:utility_lib", + "//source/extensions/filters/http/common:factory_base_lib", + "@envoy_api//envoy/extensions/filters/http/bandwidth_limit/v3alpha:pkg_cc_proto", + ], +) diff --git a/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.cc b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.cc new file mode 100644 index 0000000000000..23735fe9b1664 --- /dev/null +++ b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.cc @@ -0,0 +1,202 @@ +#include "extensions/filters/http/bandwidth_limit/bandwidth_limit.h" + +#include +#include + +#include "envoy/http/codes.h" + +#include "common/http/utility.h" +#include "common/stats/timespan_impl.h" + +using envoy::extensions::filters::http::bandwidth_limit::v3alpha::BandwidthLimit; +using Envoy::Extensions::HttpFilters::Common::StreamRateLimiter; + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace BandwidthLimitFilter { + +FilterConfig::FilterConfig(const BandwidthLimit& config, Stats::Scope& scope, + Runtime::Loader& runtime, TimeSource& time_source, bool per_route) + : runtime_(runtime), time_source_(time_source), enable_mode_(config.enable_mode()), + limit_kbps_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, limit_kbps, 0)), + fill_interval_(std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT( + config, fill_interval, StreamRateLimiter::DefaultFillInterval.count()))), + enabled_(config.runtime_enabled(), runtime), + stats_(generateStats(config.stat_prefix(), scope)) { + if (per_route && !config.has_limit_kbps()) { + throw EnvoyException("bandwidthlimitfilter: limit must be set for per route filter config"); + } + + // The token bucket is configured with a max token count of the number of + // bytes per second, and refills at the same rate, so that we have a per + // second limit which refills gradually in 1/fill_interval increments. + token_bucket_ = std::make_shared( + StreamRateLimiter::kiloBytesToBytes(limit_kbps_), time_source, + StreamRateLimiter::kiloBytesToBytes(limit_kbps_)); +} + +BandwidthLimitStats FilterConfig::generateStats(const std::string& prefix, Stats::Scope& scope) { + const std::string final_prefix = prefix + ".http_bandwidth_limit"; + return {ALL_BANDWIDTH_LIMIT_STATS(POOL_COUNTER_PREFIX(scope, final_prefix), + POOL_GAUGE_PREFIX(scope, final_prefix), + POOL_HISTOGRAM_PREFIX(scope, final_prefix))}; +} + +// BandwidthLimiter members + +Http::FilterHeadersStatus BandwidthLimiter::decodeHeaders(Http::RequestHeaderMap&, bool) { + const auto& config = getConfig(); + + if (config.enabled() && (config.enableMode() & BandwidthLimit::REQUEST)) { + config.stats().request_enabled_.inc(); + request_limiter_ = std::make_unique( + config.limit(), decoder_callbacks_->decoderBufferLimit(), + [this] { decoder_callbacks_->onDecoderFilterAboveWriteBufferHighWatermark(); }, + [this] { decoder_callbacks_->onDecoderFilterBelowWriteBufferLowWatermark(); }, + [this](Buffer::Instance& data, bool end_stream) { + if (end_stream) { + updateStatsOnDecodeFinish(); + } + decoder_callbacks_->injectDecodedDataToFilterChain(data, end_stream); + }, + [this] { + updateStatsOnDecodeFinish(); + decoder_callbacks_->continueDecoding(); + }, + [config](uint64_t len) { config.stats().request_allowed_size_.set(len); }, + const_cast(&config)->timeSource(), decoder_callbacks_->dispatcher(), + decoder_callbacks_->scope(), config.tokenBucket(), config.fillInterval()); + } + + return Http::FilterHeadersStatus::Continue; +} + +Http::FilterDataStatus BandwidthLimiter::decodeData(Buffer::Instance& data, bool end_stream) { + if (request_limiter_ != nullptr) { + const auto& config = getConfig(); + + if (!request_latency_) { + request_latency_ = std::make_unique( + config.stats().request_transfer_duration_, + const_cast(&config)->timeSource()); + config.stats().request_pending_.inc(); + } + config.stats().request_incoming_size_.set(data.length()); + + request_limiter_->writeData(data, end_stream); + return Http::FilterDataStatus::StopIterationNoBuffer; + } + ENVOY_LOG(debug, "BandwidthLimiter : request_limiter not set."); + return Http::FilterDataStatus::Continue; +} + +Http::FilterTrailersStatus BandwidthLimiter::decodeTrailers(Http::RequestTrailerMap&) { + if (request_limiter_ != nullptr) { + if (request_limiter_->onTrailers()) { + return Http::FilterTrailersStatus::StopIteration; + } else { + updateStatsOnDecodeFinish(); + return Http::FilterTrailersStatus::Continue; + } + } + return Http::FilterTrailersStatus::Continue; +} + +Http::FilterHeadersStatus BandwidthLimiter::encodeHeaders(Http::ResponseHeaderMap&, bool) { + auto& config = getConfig(); + + if (config.enabled() && (config.enableMode() & BandwidthLimit::RESPONSE)) { + config.stats().response_enabled_.inc(); + + response_limiter_ = std::make_unique( + config.limit(), encoder_callbacks_->encoderBufferLimit(), + [this] { encoder_callbacks_->onEncoderFilterAboveWriteBufferHighWatermark(); }, + [this] { encoder_callbacks_->onEncoderFilterBelowWriteBufferLowWatermark(); }, + [this](Buffer::Instance& data, bool end_stream) { + if (end_stream) { + updateStatsOnEncodeFinish(); + } + encoder_callbacks_->injectEncodedDataToFilterChain(data, end_stream); + }, + [this] { + updateStatsOnEncodeFinish(); + encoder_callbacks_->continueEncoding(); + }, + [config](uint64_t len) { config.stats().response_allowed_size_.set(len); }, + const_cast(&config)->timeSource(), encoder_callbacks_->dispatcher(), + encoder_callbacks_->scope(), config.tokenBucket(), config.fillInterval()); + } + + return Http::FilterHeadersStatus::Continue; +} + +Http::FilterDataStatus BandwidthLimiter::encodeData(Buffer::Instance& data, bool end_stream) { + if (response_limiter_ != nullptr) { + const auto& config = getConfig(); + + if (!response_latency_) { + response_latency_ = std::make_unique( + config.stats().response_transfer_duration_, + const_cast(&config)->timeSource()); + config.stats().response_pending_.inc(); + } + config.stats().response_incoming_size_.set(data.length()); + + response_limiter_->writeData(data, end_stream); + return Http::FilterDataStatus::StopIterationNoBuffer; + } + ENVOY_LOG(debug, "BandwidthLimiter : response_limiter not set"); + return Http::FilterDataStatus::Continue; +} + +Http::FilterTrailersStatus BandwidthLimiter::encodeTrailers(Http::ResponseTrailerMap&) { + if (response_limiter_ != nullptr) { + if (response_limiter_->onTrailers()) { + return Http::FilterTrailersStatus::StopIteration; + } else { + updateStatsOnEncodeFinish(); + return Http::FilterTrailersStatus::Continue; + } + } + return Http::FilterTrailersStatus::Continue; +} + +void BandwidthLimiter::updateStatsOnDecodeFinish() { + if (request_latency_) { + request_latency_->complete(); + request_latency_.reset(); + getConfig().stats().request_pending_.dec(); + } +} + +void BandwidthLimiter::updateStatsOnEncodeFinish() { + if (response_latency_) { + response_latency_->complete(); + response_latency_.reset(); + getConfig().stats().response_pending_.dec(); + } +} + +const FilterConfig& BandwidthLimiter::getConfig() const { + const auto* config = Http::Utility::resolveMostSpecificPerFilterConfig( + "envoy.filters.http.bandwidth_limit", decoder_callbacks_->route()); + if (config) { + return *config; + } + return *config_; +} + +void BandwidthLimiter::onDestroy() { + if (request_limiter_ != nullptr) { + request_limiter_->destroy(); + } + if (response_limiter_ != nullptr) { + response_limiter_->destroy(); + } +} + +} // namespace BandwidthLimitFilter +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h new file mode 100644 index 0000000000000..d3b44e75380af --- /dev/null +++ b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h @@ -0,0 +1,151 @@ +#pragma once + +#include +#include +#include +#include + +#include "envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.pb.h" +#include "envoy/http/filter.h" +#include "envoy/runtime/runtime.h" +#include "envoy/stats/scope.h" +#include "envoy/stats/stats_macros.h" +#include "envoy/stats/timespan.h" + +#include "common/common/assert.h" +#include "common/common/shared_token_bucket_impl.h" +#include "common/http/header_map_impl.h" +#include "common/router/header_parser.h" +#include "common/runtime/runtime_protos.h" + +#include "extensions/filters/http/common/stream_rate_limiter.h" + +#include "absl/synchronization/mutex.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace BandwidthLimitFilter { + +/** + * All bandwidth limit stats. @see stats_macros.h + */ +#define ALL_BANDWIDTH_LIMIT_STATS(COUNTER, GAUGE, HISTOGRAM) \ + COUNTER(request_enabled) \ + COUNTER(response_enabled) \ + GAUGE(request_pending, Accumulate) \ + GAUGE(response_pending, Accumulate) \ + GAUGE(request_incoming_size, Accumulate) \ + GAUGE(response_incoming_size, Accumulate) \ + GAUGE(request_allowed_size, Accumulate) \ + GAUGE(response_allowed_size, Accumulate) \ + HISTOGRAM(request_transfer_duration, Milliseconds) \ + HISTOGRAM(response_transfer_duration, Milliseconds) + +/** + * Struct definition for all bandwidth limit stats. @see stats_macros.h + */ +struct BandwidthLimitStats { + ALL_BANDWIDTH_LIMIT_STATS(GENERATE_COUNTER_STRUCT, GENERATE_GAUGE_STRUCT, + GENERATE_HISTOGRAM_STRUCT) +}; + +/** + * Configuration for the HTTP bandwidth limit filter. + */ +class FilterConfig : public ::Envoy::Router::RouteSpecificFilterConfig { +public: + using EnableMode = + envoy::extensions::filters::http::bandwidth_limit::v3alpha::BandwidthLimit_EnableMode; + + FilterConfig( + const envoy::extensions::filters::http::bandwidth_limit::v3alpha::BandwidthLimit& config, + Stats::Scope& scope, Runtime::Loader& runtime, TimeSource& time_source, + bool per_route = false); + ~FilterConfig() override = default; + Runtime::Loader& runtime() { return runtime_; } + BandwidthLimitStats& stats() const { return stats_; } + TimeSource& timeSource() { return time_source_; } + // Must call enabled() before calling limit(). + uint64_t limit() const { return limit_kbps_; } + bool enabled() const { return enabled_.enabled(); } + EnableMode enableMode() const { return enable_mode_; }; + const std::shared_ptr tokenBucket() const { return token_bucket_; } + std::chrono::milliseconds fillInterval() const { return fill_interval_; } + +private: + friend class FilterTest; + + static BandwidthLimitStats generateStats(const std::string& prefix, Stats::Scope& scope); + + Runtime::Loader& runtime_; + TimeSource& time_source_; + const EnableMode enable_mode_; + const uint64_t limit_kbps_; + const std::chrono::milliseconds fill_interval_; + const Runtime::FeatureFlag enabled_; + mutable BandwidthLimitStats stats_; + // Filter chain's shared token bucket + std::shared_ptr token_bucket_; +}; + +using FilterConfigSharedPtr = std::shared_ptr; + +/** + * HTTP bandwidth limit filter. Depending on the route configuration, this + * filter calls consults with local token bucket before allowing further filter + * iteration. + */ +class BandwidthLimiter : public Http::StreamFilter, Logger::Loggable { +public: + BandwidthLimiter(FilterConfigSharedPtr config) : config_(config) {} + + // Http::StreamDecoderFilter + Http::FilterHeadersStatus decodeHeaders(Http::RequestHeaderMap&, bool) override; + Http::FilterDataStatus decodeData(Buffer::Instance& data, bool end_stream) override; + Http::FilterTrailersStatus decodeTrailers(Http::RequestTrailerMap& trailers) override; + + void setDecoderFilterCallbacks(Http::StreamDecoderFilterCallbacks& callbacks) override { + decoder_callbacks_ = &callbacks; + } + + // Http::StreamEncoderFilter + Http::FilterHeadersStatus encode100ContinueHeaders(Http::ResponseHeaderMap&) override { + return Http::FilterHeadersStatus::Continue; + } + + Http::FilterHeadersStatus encodeHeaders(Http::ResponseHeaderMap&, bool) override; + Http::FilterDataStatus encodeData(Buffer::Instance& data, bool end_stream) override; + Http::FilterTrailersStatus encodeTrailers(Http::ResponseTrailerMap&) override; + + Http::FilterMetadataStatus encodeMetadata(Http::MetadataMap&) override { + return Http::FilterMetadataStatus::Continue; + } + + void setEncoderFilterCallbacks(Http::StreamEncoderFilterCallbacks& callbacks) override { + encoder_callbacks_ = &callbacks; + } + + // Http::StreamFilterBase + void onDestroy() override; + +private: + friend class FilterTest; + const FilterConfig& getConfig() const; + + void updateStatsOnDecodeFinish(); + void updateStatsOnEncodeFinish(); + + Http::StreamDecoderFilterCallbacks* decoder_callbacks_{}; + Http::StreamEncoderFilterCallbacks* encoder_callbacks_{}; + FilterConfigSharedPtr config_; + std::unique_ptr request_limiter_; + std::unique_ptr response_limiter_; + Stats::TimespanPtr request_latency_; + Stats::TimespanPtr response_latency_; +}; + +} // namespace BandwidthLimitFilter +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/bandwidth_limit/config.cc b/source/extensions/filters/http/bandwidth_limit/config.cc new file mode 100644 index 0000000000000..2ed2118fea451 --- /dev/null +++ b/source/extensions/filters/http/bandwidth_limit/config.cc @@ -0,0 +1,43 @@ +#include "extensions/filters/http/bandwidth_limit/config.h" + +#include + +#include "envoy/registry/registry.h" + +#include "common/protobuf/utility.h" + +#include "extensions/filters/http/bandwidth_limit/bandwidth_limit.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace BandwidthLimitFilter { + +Http::FilterFactoryCb BandwidthLimitFilterConfig::createFilterFactoryFromProtoTyped( + const envoy::extensions::filters::http::bandwidth_limit::v3alpha::BandwidthLimit& proto_config, + const std::string&, Server::Configuration::FactoryContext& context) { + FilterConfigSharedPtr filter_config = std::make_shared( + proto_config, context.scope(), context.runtime(), context.timeSource()); + return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { + callbacks.addStreamFilter(std::make_shared(filter_config)); + }; +} + +Router::RouteSpecificFilterConfigConstSharedPtr +BandwidthLimitFilterConfig::createRouteSpecificFilterConfigTyped( + const envoy::extensions::filters::http::bandwidth_limit::v3alpha::BandwidthLimit& proto_config, + Server::Configuration::ServerFactoryContext& context, ProtobufMessage::ValidationVisitor&) { + return std::make_shared(proto_config, context.scope(), context.runtime(), + context.timeSource(), true); +} + +/** + * Static registration for the bandwidth limit filter. @see RegisterFactory. + */ +REGISTER_FACTORY(BandwidthLimitFilterConfig, + Server::Configuration::NamedHttpFilterConfigFactory){"envoy.bandwidth_limit"}; + +} // namespace BandwidthLimitFilter +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/bandwidth_limit/config.h b/source/extensions/filters/http/bandwidth_limit/config.h new file mode 100644 index 0000000000000..e9945bbf745d8 --- /dev/null +++ b/source/extensions/filters/http/bandwidth_limit/config.h @@ -0,0 +1,37 @@ +#pragma once + +#include "envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.pb.h" +#include "envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.pb.validate.h" + +#include "extensions/filters/http/common/factory_base.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace BandwidthLimitFilter { + +/** + * Config registration for the bandwidth limit filter. @see NamedHttpFilterConfigFactory. + */ +class BandwidthLimitFilterConfig + : public Common::FactoryBase< + envoy::extensions::filters::http::bandwidth_limit::v3alpha::BandwidthLimit> { +public: + BandwidthLimitFilterConfig() : FactoryBase("envoy.filters.http.bandwidth_limit") {} + +private: + Http::FilterFactoryCb createFilterFactoryFromProtoTyped( + const envoy::extensions::filters::http::bandwidth_limit::v3alpha::BandwidthLimit& + proto_config, + const std::string& stats_prefix, Server::Configuration::FactoryContext& context) override; + + Router::RouteSpecificFilterConfigConstSharedPtr createRouteSpecificFilterConfigTyped( + const envoy::extensions::filters::http::bandwidth_limit::v3alpha::BandwidthLimit& + proto_config, + Server::Configuration::ServerFactoryContext&, ProtobufMessage::ValidationVisitor&) override; +}; + +} // namespace BandwidthLimitFilter +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/buffer/BUILD b/source/extensions/filters/http/buffer/BUILD index c38b84635d661..c691bc382862b 100644 --- a/source/extensions/filters/http/buffer/BUILD +++ b/source/extensions/filters/http/buffer/BUILD @@ -37,8 +37,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", - security_posture = "robust_to_untrusted_downstream", # Legacy test use. TODO(#9953) clean up. visibility = ["//visibility:public"], deps = [ diff --git a/source/extensions/filters/http/cache/BUILD b/source/extensions/filters/http/cache/BUILD index 0023acfeefbe8..bd205889fdfa1 100644 --- a/source/extensions/filters/http/cache/BUILD +++ b/source/extensions/filters/http/cache/BUILD @@ -101,9 +101,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", - security_posture = "robust_to_untrusted_downstream_and_upstream", - status = "wip", deps = [ ":cache_filter_lib", "//source/extensions/filters/http:well_known_names", diff --git a/source/extensions/filters/http/cache/simple_http_cache/BUILD b/source/extensions/filters/http/cache/simple_http_cache/BUILD index a9a500e1f9b8e..481380f23591f 100644 --- a/source/extensions/filters/http/cache/simple_http_cache/BUILD +++ b/source/extensions/filters/http/cache/simple_http_cache/BUILD @@ -14,9 +14,6 @@ envoy_cc_extension( name = "config", srcs = ["simple_http_cache.cc"], hdrs = ["simple_http_cache.h"], - category = "envoy.filters.http.cache", - security_posture = "robust_to_untrusted_downstream_and_upstream", - status = "wip", deps = [ "//include/envoy/registry", "//include/envoy/runtime:runtime_interface", diff --git a/source/extensions/filters/http/cdn_loop/BUILD b/source/extensions/filters/http/cdn_loop/BUILD index 291f20b3a7256..b42834465a14f 100644 --- a/source/extensions/filters/http/cdn_loop/BUILD +++ b/source/extensions/filters/http/cdn_loop/BUILD @@ -45,9 +45,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", - security_posture = "unknown", - status = "alpha", deps = [ ":filter_lib", ":parser_lib", diff --git a/source/extensions/filters/http/composite/BUILD b/source/extensions/filters/http/composite/BUILD index 0d1493808765c..63cb62a7c7b00 100644 --- a/source/extensions/filters/http/composite/BUILD +++ b/source/extensions/filters/http/composite/BUILD @@ -45,8 +45,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", - security_posture = "unknown", deps = [ "//include/envoy/registry", "//include/envoy/server:filter_config_interface", diff --git a/source/extensions/filters/http/compressor/BUILD b/source/extensions/filters/http/compressor/BUILD index cec12558d4a93..ad18a973c8649 100644 --- a/source/extensions/filters/http/compressor/BUILD +++ b/source/extensions/filters/http/compressor/BUILD @@ -27,8 +27,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", - security_posture = "robust_to_untrusted_downstream", deps = [ ":compressor_filter_lib", "//include/envoy/compression/compressor:compressor_config_interface", diff --git a/source/extensions/filters/http/cors/BUILD b/source/extensions/filters/http/cors/BUILD index 719af988af59a..5eb4f63a57246 100644 --- a/source/extensions/filters/http/cors/BUILD +++ b/source/extensions/filters/http/cors/BUILD @@ -31,12 +31,10 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", # TODO(#9953) clean up. extra_visibility = [ "//test/integration:__subpackages__", ], - security_posture = "robust_to_untrusted_downstream", deps = [ "//include/envoy/registry", "//include/envoy/server:filter_config_interface", diff --git a/source/extensions/filters/http/csrf/BUILD b/source/extensions/filters/http/csrf/BUILD index 9b5af4e5a8788..e8e88ea6fa493 100644 --- a/source/extensions/filters/http/csrf/BUILD +++ b/source/extensions/filters/http/csrf/BUILD @@ -33,8 +33,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", - security_posture = "robust_to_untrusted_downstream", deps = [ "//include/envoy/registry", "//source/extensions/filters/http:well_known_names", diff --git a/source/extensions/filters/http/decompressor/BUILD b/source/extensions/filters/http/decompressor/BUILD index fb69254e476b5..78f76c5573f8f 100644 --- a/source/extensions/filters/http/decompressor/BUILD +++ b/source/extensions/filters/http/decompressor/BUILD @@ -33,8 +33,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", - security_posture = "robust_to_untrusted_downstream_and_upstream", deps = [ ":decompressor_filter_lib", "//include/envoy/compression/decompressor:decompressor_config_interface", diff --git a/source/extensions/filters/http/dynamic_forward_proxy/BUILD b/source/extensions/filters/http/dynamic_forward_proxy/BUILD index 5b0768fe9d2d8..33b202755ec47 100644 --- a/source/extensions/filters/http/dynamic_forward_proxy/BUILD +++ b/source/extensions/filters/http/dynamic_forward_proxy/BUILD @@ -29,8 +29,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", - security_posture = "robust_to_untrusted_downstream", deps = [ "//include/envoy/registry", "//include/envoy/server:filter_config_interface", diff --git a/source/extensions/filters/http/dynamo/BUILD b/source/extensions/filters/http/dynamo/BUILD index 4854329af55c1..0abf478922ff3 100644 --- a/source/extensions/filters/http/dynamo/BUILD +++ b/source/extensions/filters/http/dynamo/BUILD @@ -42,8 +42,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", - security_posture = "requires_trusted_downstream_and_upstream", deps = [ ":dynamo_filter_lib", "//include/envoy/registry", diff --git a/source/extensions/filters/http/ext_authz/BUILD b/source/extensions/filters/http/ext_authz/BUILD index 766e09774d1e2..6314f305e97fa 100644 --- a/source/extensions/filters/http/ext_authz/BUILD +++ b/source/extensions/filters/http/ext_authz/BUILD @@ -40,8 +40,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", - security_posture = "robust_to_untrusted_downstream", deps = [ ":ext_authz", "//include/envoy/registry", diff --git a/source/extensions/filters/http/ext_proc/BUILD b/source/extensions/filters/http/ext_proc/BUILD index 1a0dbe8a05a56..6f4e7a9c4928d 100644 --- a/source/extensions/filters/http/ext_proc/BUILD +++ b/source/extensions/filters/http/ext_proc/BUILD @@ -38,9 +38,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", - security_posture = "unknown", - status = "alpha", deps = [ ":client_lib", ":ext_proc", diff --git a/source/extensions/filters/http/fault/BUILD b/source/extensions/filters/http/fault/BUILD index db2a5a61ed97a..3cfe5b8ee205e 100644 --- a/source/extensions/filters/http/fault/BUILD +++ b/source/extensions/filters/http/fault/BUILD @@ -46,8 +46,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", - security_posture = "robust_to_untrusted_downstream", deps = [ "//include/envoy/registry", "//source/extensions/filters/http:well_known_names", diff --git a/source/extensions/filters/http/grpc_http1_bridge/BUILD b/source/extensions/filters/http/grpc_http1_bridge/BUILD index 4a1154094c647..4685a8d07c9d1 100644 --- a/source/extensions/filters/http/grpc_http1_bridge/BUILD +++ b/source/extensions/filters/http/grpc_http1_bridge/BUILD @@ -33,14 +33,12 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", # Legacy test use. TODO(#9953) clean up. extra_visibility = [ "//source/exe:__pkg__", "//test/integration:__subpackages__", "//test/server:__subpackages__", ], - security_posture = "unknown", deps = [ "//include/envoy/registry", "//include/envoy/server:filter_config_interface", diff --git a/source/extensions/filters/http/grpc_http1_reverse_bridge/BUILD b/source/extensions/filters/http/grpc_http1_reverse_bridge/BUILD index be9226b61f545..c4f65adb09a95 100644 --- a/source/extensions/filters/http/grpc_http1_reverse_bridge/BUILD +++ b/source/extensions/filters/http/grpc_http1_reverse_bridge/BUILD @@ -31,9 +31,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", - security_posture = "unknown", - status = "alpha", deps = [ ":filter_lib", "//include/envoy/http:filter_interface", diff --git a/source/extensions/filters/http/grpc_json_transcoder/BUILD b/source/extensions/filters/http/grpc_json_transcoder/BUILD index 822264c252cf4..c1ae930c7fc88 100644 --- a/source/extensions/filters/http/grpc_json_transcoder/BUILD +++ b/source/extensions/filters/http/grpc_json_transcoder/BUILD @@ -62,8 +62,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", - security_posture = "robust_to_untrusted_downstream", deps = [ "//include/envoy/registry", "//source/extensions/filters/http:well_known_names", diff --git a/source/extensions/filters/http/grpc_stats/BUILD b/source/extensions/filters/http/grpc_stats/BUILD index 10c7558f549f8..078b140e912eb 100644 --- a/source/extensions/filters/http/grpc_stats/BUILD +++ b/source/extensions/filters/http/grpc_stats/BUILD @@ -14,9 +14,6 @@ envoy_cc_extension( name = "config", srcs = ["grpc_stats_filter.cc"], hdrs = ["grpc_stats_filter.h"], - category = "envoy.filters.http", - security_posture = "unknown", - status = "alpha", deps = [ "//include/envoy/registry", "//include/envoy/server:filter_config_interface", diff --git a/source/extensions/filters/http/grpc_web/BUILD b/source/extensions/filters/http/grpc_web/BUILD index 4a7089ca962eb..f0f341b49708a 100644 --- a/source/extensions/filters/http/grpc_web/BUILD +++ b/source/extensions/filters/http/grpc_web/BUILD @@ -32,8 +32,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", - security_posture = "robust_to_untrusted_downstream", deps = [ "//include/envoy/registry", "//include/envoy/server:filter_config_interface", diff --git a/source/extensions/filters/http/gzip/BUILD b/source/extensions/filters/http/gzip/BUILD index d2d9fc86479b4..6503189c9e9df 100644 --- a/source/extensions/filters/http/gzip/BUILD +++ b/source/extensions/filters/http/gzip/BUILD @@ -30,8 +30,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", - security_posture = "robust_to_untrusted_downstream", deps = [ "//source/extensions/filters/http:well_known_names", "//source/extensions/filters/http/common:factory_base_lib", diff --git a/source/extensions/filters/http/header_to_metadata/BUILD b/source/extensions/filters/http/header_to_metadata/BUILD index aa13db4517e15..f0e7a6a1c3d01 100644 --- a/source/extensions/filters/http/header_to_metadata/BUILD +++ b/source/extensions/filters/http/header_to_metadata/BUILD @@ -30,8 +30,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", - security_posture = "robust_to_untrusted_downstream", deps = [ "//include/envoy/registry", "//source/common/protobuf:utility_lib", diff --git a/source/extensions/filters/http/health_check/BUILD b/source/extensions/filters/http/health_check/BUILD index c54f3bf2ad17e..52c1554da2e06 100644 --- a/source/extensions/filters/http/health_check/BUILD +++ b/source/extensions/filters/http/health_check/BUILD @@ -37,14 +37,12 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", # Legacy test use. TODO(#9953) clean up. extra_visibility = [ "//test/common/filter/http:__subpackages__", "//test/integration:__subpackages__", "//test/server:__subpackages__", ], - security_posture = "robust_to_untrusted_downstream", deps = [ "//include/envoy/registry", "//source/common/http:header_utility_lib", diff --git a/source/extensions/filters/http/ip_tagging/BUILD b/source/extensions/filters/http/ip_tagging/BUILD index 2c75ece83a991..443d168101f46 100644 --- a/source/extensions/filters/http/ip_tagging/BUILD +++ b/source/extensions/filters/http/ip_tagging/BUILD @@ -33,12 +33,10 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", # TODO(#9953) clean up. extra_visibility = [ "//test/integration:__subpackages__", ], - security_posture = "robust_to_untrusted_downstream", deps = [ "//include/envoy/registry", "//source/common/protobuf:utility_lib", diff --git a/source/extensions/filters/http/jwt_authn/BUILD b/source/extensions/filters/http/jwt_authn/BUILD index 0d5895dfbd5a6..d90c7cd27631e 100644 --- a/source/extensions/filters/http/jwt_authn/BUILD +++ b/source/extensions/filters/http/jwt_authn/BUILD @@ -20,6 +20,33 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "stats_lib", + hdrs = ["stats.h"], + deps = [ + "//include/envoy/stats:stats_macros", + ], +) + +envoy_cc_library( + name = "jwks_async_fetcher_lib", + srcs = ["jwks_async_fetcher.cc"], + hdrs = ["jwks_async_fetcher.h"], + external_deps = [ + "jwt_verify_lib", + ], + deps = [ + ":stats_lib", + "//include/envoy/server:factory_context_interface", + "//source/common/common:minimal_logger_lib", + "//source/common/init:target_lib", + "//source/common/protobuf:utility_lib", + "//source/common/tracing:http_tracer_lib", + "//source/extensions/filters/http/common:jwks_fetcher_lib", + "@envoy_api//envoy/extensions/filters/http/jwt_authn/v3:pkg_cc_proto", + ], +) + envoy_cc_library( name = "jwks_cache_lib", srcs = ["jwks_cache.cc"], @@ -28,9 +55,8 @@ envoy_cc_library( "jwt_verify_lib", ], deps = [ - "//source/common/common:minimal_logger_lib", + "jwks_async_fetcher_lib", "//source/common/config:datasource_lib", - "//source/common/protobuf:utility_lib", "@envoy_api//envoy/extensions/filters/http/jwt_authn/v3:pkg_cc_proto", ], ) @@ -58,7 +84,7 @@ envoy_cc_library( "jwt_verify_lib", ], deps = [ - ":filter_config_interface", + ":filter_config_lib", ":matchers_lib", "//include/envoy/http:filter_interface", "//source/common/http:headers_lib", @@ -70,9 +96,6 @@ envoy_cc_extension( name = "config", srcs = ["filter_factory.cc"], hdrs = ["filter_factory.h"], - category = "envoy.filters.http", - security_posture = "robust_to_untrusted_downstream", - status = "alpha", deps = [ ":filter_lib", "//include/envoy/registry", @@ -109,7 +132,7 @@ envoy_cc_library( ) envoy_cc_library( - name = "filter_config_interface", + name = "filter_config_lib", srcs = ["filter_config.cc"], hdrs = ["filter_config.h"], deps = [ diff --git a/source/extensions/filters/http/jwt_authn/authenticator.cc b/source/extensions/filters/http/jwt_authn/authenticator.cc index ff30bf5760ea1..fd960636ac41d 100644 --- a/source/extensions/filters/http/jwt_authn/authenticator.cc +++ b/source/extensions/filters/http/jwt_authn/authenticator.cc @@ -3,7 +3,6 @@ #include "envoy/http/async_client.h" #include "common/common/assert.h" -#include "common/common/base64.h" #include "common/common/enum_to_int.h" #include "common/common/logger.h" #include "common/http/message_impl.h" @@ -222,6 +221,7 @@ void AuthenticatorImpl::startVerify() { } void AuthenticatorImpl::onJwksSuccess(google::jwt_verify::JwksPtr&& jwks) { + jwks_cache_.stats().jwks_fetch_success_.inc(); const Status status = jwks_data_->setRemoteJwks(std::move(jwks))->getStatus(); if (status != Status::Ok) { doneWithStatus(status); @@ -230,7 +230,10 @@ void AuthenticatorImpl::onJwksSuccess(google::jwt_verify::JwksPtr&& jwks) { } } -void AuthenticatorImpl::onJwksError(Failure) { doneWithStatus(Status::JwksFetchFail); } +void AuthenticatorImpl::onJwksError(Failure) { + jwks_cache_.stats().jwks_fetch_failed_.inc(); + doneWithStatus(Status::JwksFetchFail); +} void AuthenticatorImpl::onDestroy() { if (fetcher_) { @@ -249,12 +252,9 @@ void AuthenticatorImpl::verifyKey() { // Forward the payload const auto& provider = jwks_data_->getJwtProvider(); - if (!provider.forward_payload_header().empty()) { - std::string payload_with_padding = jwt_->payload_str_base64url_; - Base64::completePadding(payload_with_padding); headers_->addCopy(Http::LowerCaseString(provider.forward_payload_header()), - payload_with_padding); + jwt_->payload_str_base64url_); } if (!provider.forward()) { diff --git a/source/extensions/filters/http/jwt_authn/authenticator.h b/source/extensions/filters/http/jwt_authn/authenticator.h index 928a8045b843e..04ee62cede1e1 100644 --- a/source/extensions/filters/http/jwt_authn/authenticator.h +++ b/source/extensions/filters/http/jwt_authn/authenticator.h @@ -2,7 +2,6 @@ #include "envoy/server/filter_config.h" -#include "extensions/filters/http/common/jwks_fetcher.h" #include "extensions/filters/http/jwt_authn/extractor.h" #include "extensions/filters/http/jwt_authn/jwks_cache.h" @@ -21,11 +20,6 @@ using AuthenticatorCallback = std::function; -/** - * CreateJwksFetcherCb is a callback interface for creating a JwksFetcher instance. - */ -using CreateJwksFetcherCb = std::function; - /** * Authenticator object to handle all JWT authentication flow. */ diff --git a/source/extensions/filters/http/jwt_authn/filter_config.cc b/source/extensions/filters/http/jwt_authn/filter_config.cc index a35ddf038824d..475d993daaeca 100644 --- a/source/extensions/filters/http/jwt_authn/filter_config.cc +++ b/source/extensions/filters/http/jwt_authn/filter_config.cc @@ -19,8 +19,7 @@ FilterConfigImpl::FilterConfigImpl( ENVOY_LOG(debug, "Loaded JwtAuthConfig: {}", proto_config_.DebugString()); - jwks_cache_ = - JwksCache::create(proto_config_, time_source_, context.api(), context.threadLocal()); + jwks_cache_ = JwksCache::create(proto_config_, context, Common::JwksFetcher::create, stats_); std::vector names; for (const auto& it : proto_config_.requirement_map()) { diff --git a/source/extensions/filters/http/jwt_authn/filter_config.h b/source/extensions/filters/http/jwt_authn/filter_config.h index d659dbf95135a..78f0e23d98783 100644 --- a/source/extensions/filters/http/jwt_authn/filter_config.h +++ b/source/extensions/filters/http/jwt_authn/filter_config.h @@ -9,6 +9,7 @@ #include "envoy/thread_local/thread_local.h" #include "extensions/filters/http/jwt_authn/matcher.h" +#include "extensions/filters/http/jwt_authn/stats.h" #include "extensions/filters/http/jwt_authn/verifier.h" #include "absl/container/flat_hash_map.h" @@ -18,21 +19,6 @@ namespace Extensions { namespace HttpFilters { namespace JwtAuthn { -/** - * All stats for the Jwt Authn filter. @see stats_macros.h - */ -#define ALL_JWT_AUTHN_FILTER_STATS(COUNTER) \ - COUNTER(allowed) \ - COUNTER(cors_preflight_bypassed) \ - COUNTER(denied) - -/** - * Wrapper struct for jwt_authn filter stats. @see stats_macros.h - */ -struct JwtAuthnFilterStats { - ALL_JWT_AUTHN_FILTER_STATS(GENERATE_COUNTER_STRUCT) -}; - /** * The per-route filter config */ diff --git a/source/extensions/filters/http/jwt_authn/jwks_async_fetcher.cc b/source/extensions/filters/http/jwt_authn/jwks_async_fetcher.cc new file mode 100644 index 0000000000000..0ec5bbdce98f8 --- /dev/null +++ b/source/extensions/filters/http/jwt_authn/jwks_async_fetcher.cc @@ -0,0 +1,98 @@ +#include "extensions/filters/http/jwt_authn/jwks_async_fetcher.h" + +#include "common/protobuf/utility.h" +#include "common/tracing/http_tracer_impl.h" + +using envoy::extensions::filters::http::jwt_authn::v3::RemoteJwks; + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace JwtAuthn { +namespace { + +// Default cache expiration time in 5 minutes. +constexpr int PubkeyCacheExpirationSec = 600; + +} // namespace + +JwksAsyncFetcher::JwksAsyncFetcher(const RemoteJwks& remote_jwks, + Server::Configuration::FactoryContext& context, + CreateJwksFetcherCb create_fetcher_fn, + JwtAuthnFilterStats& stats, JwksDoneFetched done_fn) + : remote_jwks_(remote_jwks), context_(context), create_fetcher_fn_(create_fetcher_fn), + stats_(stats), done_fn_(done_fn), cache_duration_(getCacheDuration(remote_jwks)), + debug_name_(absl::StrCat("Jwks async fetching url=", remote_jwks_.http_uri().uri())) { + // if async_fetch is not enabled, do nothing. + if (!remote_jwks_.has_async_fetch()) { + return; + } + + cache_duration_timer_ = context_.dispatcher().createTimer([this]() -> void { fetch(); }); + + // For fast_listener, just trigger a fetch, not register with init_manager. + if (remote_jwks_.async_fetch().fast_listener()) { + fetch(); + return; + } + + // Register to init_manager, force the listener to wait for the fetching. + init_target_ = std::make_unique(debug_name_, [this]() -> void { fetch(); }); + context_.initManager().add(*init_target_); +} + +std::chrono::seconds JwksAsyncFetcher::getCacheDuration(const RemoteJwks& remote_jwks) { + if (remote_jwks.has_cache_duration()) { + return std::chrono::seconds(DurationUtil::durationToSeconds(remote_jwks.cache_duration())); + } + return std::chrono::seconds(PubkeyCacheExpirationSec); +} + +void JwksAsyncFetcher::fetch() { + if (fetcher_) { + fetcher_->cancel(); + } + + ENVOY_LOG(debug, "{}: started", debug_name_); + fetcher_ = create_fetcher_fn_(context_.clusterManager()); + fetcher_->fetch(remote_jwks_.http_uri(), Tracing::NullSpan::instance(), *this); +} + +void JwksAsyncFetcher::handleFetchDone() { + if (init_target_) { + init_target_->ready(); + init_target_.reset(); + } + + cache_duration_timer_->enableTimer(cache_duration_); +} + +void JwksAsyncFetcher::onJwksSuccess(google::jwt_verify::JwksPtr&& jwks) { + stats_.jwks_fetch_success_.inc(); + + done_fn_(std::move(jwks)); + handleFetchDone(); + + // Note: not to free fetcher_ within onJwksSuccess or onJwksError function. + // They are passed to fetcher_->fetch() and are called by fetcher_ after fetch is done. + // After calling these callback functions, fetch_ calls its reset() function. + // If fetcher_ is freed by the callback, calling reset() will crash. + + // Not need to free fetcher_. At the next fetch(), it will be freed with a cancel() call. + // The cancel() is needed to cancel the old call before the new one is created. + // But it is a no-op if the call is completed. +} + +void JwksAsyncFetcher::onJwksError(Failure) { + stats_.jwks_fetch_failed_.inc(); + + ENVOY_LOG(warn, "{}: failed", debug_name_); + handleFetchDone(); + + // Note: not to free fetcher_ in this function. Please see comment at onJwksSuccess. +} + +} // namespace JwtAuthn +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/jwt_authn/jwks_async_fetcher.h b/source/extensions/filters/http/jwt_authn/jwks_async_fetcher.h new file mode 100644 index 0000000000000..c5909909b44ed --- /dev/null +++ b/source/extensions/filters/http/jwt_authn/jwks_async_fetcher.h @@ -0,0 +1,85 @@ +#pragma once + +#include + +#include "envoy/extensions/filters/http/jwt_authn/v3/config.pb.h" +#include "envoy/server/factory_context.h" + +#include "common/common/logger.h" +#include "common/init/target_impl.h" + +#include "extensions/filters/http/common/jwks_fetcher.h" +#include "extensions/filters/http/jwt_authn/stats.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace JwtAuthn { + +/** + * CreateJwksFetcherCb is a callback interface for creating a JwksFetcher instance. + */ +using CreateJwksFetcherCb = std::function; +/** + * JwksDoneFetched is a callback interface to set a Jwks when fetch is done. + */ +using JwksDoneFetched = std::function; + +// This class handles fetching Jwks asynchronously. +// It will be no-op if async_fetch is not enabled. +// At its constructor, it will start to fetch Jwks, register with init_manager if not fast_listener. +// and handle fetching response. When cache is expired, it will fetch again. +// When a Jwks is fetched, done_fn is called to set the Jwks. +class JwksAsyncFetcher : public Logger::Loggable, + public Common::JwksFetcher::JwksReceiver { +public: + JwksAsyncFetcher(const envoy::extensions::filters::http::jwt_authn::v3::RemoteJwks& remote_jwks, + Server::Configuration::FactoryContext& context, CreateJwksFetcherCb fetcher_fn, + JwtAuthnFilterStats& stats, JwksDoneFetched done_fn); + + // Get the remote Jwks cache duration. + static std::chrono::seconds + getCacheDuration(const envoy::extensions::filters::http::jwt_authn::v3::RemoteJwks& remote_jwks); + +private: + // Fetch the Jwks + void fetch(); + // Handle fetch done. + void handleFetchDone(); + + // Override the functions from Common::JwksFetcher::JwksReceiver + void onJwksSuccess(google::jwt_verify::JwksPtr&& jwks) override; + void onJwksError(Failure reason) override; + + // the remote Jwks config + const envoy::extensions::filters::http::jwt_authn::v3::RemoteJwks& remote_jwks_; + // the factory context + Server::Configuration::FactoryContext& context_; + // the jwks fetcher creator function + const CreateJwksFetcherCb create_fetcher_fn_; + // stats + JwtAuthnFilterStats& stats_; + // the Jwks done function. + const JwksDoneFetched done_fn_; + + // The Jwks fetcher object + Common::JwksFetcherPtr fetcher_; + + // The cache duration. + const std::chrono::seconds cache_duration_; + // The timer to trigger fetch due to cache duration. + Envoy::Event::TimerPtr cache_duration_timer_; + + // The init target. + std::unique_ptr init_target_; + + // Used in logs. + const std::string debug_name_; +}; + +using JwksAsyncFetcherPtr = std::unique_ptr; + +} // namespace JwtAuthn +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/jwt_authn/jwks_cache.cc b/source/extensions/filters/http/jwt_authn/jwks_cache.cc index 7204199fe3e59..48199f0f6d4ff 100644 --- a/source/extensions/filters/http/jwt_authn/jwks_cache.cc +++ b/source/extensions/filters/http/jwt_authn/jwks_cache.cc @@ -7,7 +7,6 @@ #include "common/common/logger.h" #include "common/config/datasource.h" -#include "common/protobuf/utility.h" #include "absl/container/node_hash_map.h" #include "jwt_verify_lib/check_audience.h" @@ -23,16 +22,12 @@ namespace HttpFilters { namespace JwtAuthn { namespace { -// Default cache expiration time in 5 minutes. -constexpr int PubkeyCacheExpirationSec = 600; - -using JwksSharedPtr = std::shared_ptr<::google::jwt_verify::Jwks>; - class JwksDataImpl : public JwksCache::JwksData, public Logger::Loggable { public: - JwksDataImpl(const JwtProvider& jwt_provider, TimeSource& time_source, Api::Api& api, - ThreadLocal::SlotAllocator& tls) - : jwt_provider_(jwt_provider), time_source_(time_source), tls_(tls) { + JwksDataImpl(const JwtProvider& jwt_provider, Server::Configuration::FactoryContext& context, + CreateJwksFetcherCb fetcher_cb, JwtAuthnFilterStats& stats) + : jwt_provider_(jwt_provider), time_source_(context.timeSource()), + tls_(context.threadLocal()) { std::vector audiences; for (const auto& aud : jwt_provider_.audiences()) { @@ -42,7 +37,8 @@ class JwksDataImpl : public JwksCache::JwksData, public Logger::Loggable(); }); - const auto inline_jwks = Config::DataSource::read(jwt_provider_.local_jwks(), true, api); + const auto inline_jwks = + Config::DataSource::read(jwt_provider_.local_jwks(), true, context.api()); if (!inline_jwks.empty()) { auto jwks = ::google::jwt_verify::Jwks::createFrom(inline_jwks, ::google::jwt_verify::Jwks::JWKS); @@ -50,7 +46,14 @@ class JwksDataImpl : public JwksCache::JwksData, public Logger::Loggable( + jwt_provider_.remote_jwks(), context, fetcher_cb, stats, + [this](google::jwt_verify::JwksPtr&& jwks) { setJwksToAllThreads(std::move(jwks)); }); } } } @@ -65,44 +68,32 @@ class JwksDataImpl : public JwksCache::JwksData, public Logger::Loggable= tls_->expire_; } - const ::google::jwt_verify::Jwks* setRemoteJwks(::google::jwt_verify::JwksPtr&& jwks) override { + const ::google::jwt_verify::Jwks* setRemoteJwks(JwksConstPtr&& jwks) override { // convert unique_ptr to shared_ptr - JwksSharedPtr shared_jwks(jwks.release()); + JwksConstSharedPtr shared_jwks = std::move(jwks); tls_->jwks_ = shared_jwks; - tls_->expire_ = getRemoteJwksExpirationTime(); + tls_->expire_ = time_source_.monotonicTime() + + JwksAsyncFetcher::getCacheDuration(jwt_provider_.remote_jwks()); return shared_jwks.get(); } private: struct ThreadLocalCache : public ThreadLocal::ThreadLocalObject { // The jwks object. - JwksSharedPtr jwks_; + JwksConstSharedPtr jwks_; // The pubkey expiration time. MonotonicTime expire_; }; // Set jwks shared_ptr to all threads. - void setJwksToAllThreads(::google::jwt_verify::JwksPtr&& jwks, - std::chrono::steady_clock::time_point expire) { - JwksSharedPtr shared_jwks(jwks.release()); - tls_.runOnAllThreads([shared_jwks, expire](OptRef obj) { + void setJwksToAllThreads(JwksConstPtr&& jwks) { + JwksConstSharedPtr shared_jwks = std::move(jwks); + tls_.runOnAllThreads([shared_jwks](OptRef obj) { obj->jwks_ = shared_jwks; - obj->expire_ = expire; + obj->expire_ = std::chrono::steady_clock::time_point::max(); }); } - // Get the expiration time for a remote Jwks - std::chrono::steady_clock::time_point getRemoteJwksExpirationTime() const { - auto expire = time_source_.monotonicTime(); - if (jwt_provider_.has_remote_jwks() && jwt_provider_.remote_jwks().has_cache_duration()) { - expire += std::chrono::milliseconds( - DurationUtil::durationToMilliseconds(jwt_provider_.remote_jwks().cache_duration())); - } else { - expire += std::chrono::seconds(PubkeyCacheExpirationSec); - } - return expire; - } - // The jwt provider config. const JwtProvider& jwt_provider_; // Check audience object @@ -111,6 +102,8 @@ class JwksDataImpl : public JwksCache::JwksData, public Logger::Loggable tls_; + // async fetcher + JwksAsyncFetcherPtr async_fetcher_; }; using JwksDataImplPtr = std::unique_ptr; @@ -118,11 +111,12 @@ using JwksDataImplPtr = std::unique_ptr; class JwksCacheImpl : public JwksCache { public: // Load the config from envoy config. - JwksCacheImpl(const JwtAuthentication& config, TimeSource& time_source, Api::Api& api, - ThreadLocal::SlotAllocator& tls) { + JwksCacheImpl(const JwtAuthentication& config, Server::Configuration::FactoryContext& context, + CreateJwksFetcherCb fetcher_fn, JwtAuthnFilterStats& stats) + : stats_(stats) { for (const auto& it : config.providers()) { const auto& provider = it.second; - auto jwks_data = std::make_unique(provider, time_source, api, tls); + auto jwks_data = std::make_unique(provider, context, fetcher_fn, stats); if (issuer_ptr_map_.find(provider.issuer()) == issuer_ptr_map_.end()) { issuer_ptr_map_.emplace(provider.issuer(), jwks_data.get()); } @@ -148,6 +142,8 @@ class JwksCacheImpl : public JwksCache { NOT_REACHED_GCOVR_EXCL_LINE; } + JwtAuthnFilterStats& stats() override { return stats_; } + private: JwksData* findIssuerMap(const std::string& issuer) { const auto& it = issuer_ptr_map_.find(issuer); @@ -157,6 +153,8 @@ class JwksCacheImpl : public JwksCache { return it->second; } + // stats + JwtAuthnFilterStats& stats_; // The Jwks data map indexed by provider. absl::node_hash_map jwks_data_map_; // The Jwks data pointer map indexed by issuer. @@ -167,8 +165,9 @@ class JwksCacheImpl : public JwksCache { JwksCachePtr JwksCache::create(const envoy::extensions::filters::http::jwt_authn::v3::JwtAuthentication& config, - TimeSource& time_source, Api::Api& api, ThreadLocal::SlotAllocator& tls) { - return std::make_unique(config, time_source, api, tls); + Server::Configuration::FactoryContext& context, CreateJwksFetcherCb fetcher_fn, + JwtAuthnFilterStats& stats) { + return std::make_unique(config, context, fetcher_fn, stats); } } // namespace JwtAuthn diff --git a/source/extensions/filters/http/jwt_authn/jwks_cache.h b/source/extensions/filters/http/jwt_authn/jwks_cache.h index e3b56348a7407..186401c44c965 100644 --- a/source/extensions/filters/http/jwt_authn/jwks_cache.h +++ b/source/extensions/filters/http/jwt_authn/jwks_cache.h @@ -6,6 +6,10 @@ #include "envoy/extensions/filters/http/jwt_authn/v3/config.pb.h" #include "envoy/thread_local/thread_local.h" +#include "extensions/filters/http/common/jwks_fetcher.h" +#include "extensions/filters/http/jwt_authn/jwks_async_fetcher.h" +#include "extensions/filters/http/jwt_authn/stats.h" + #include "jwt_verify_lib/jwks.h" namespace Envoy { @@ -16,6 +20,9 @@ namespace JwtAuthn { class JwksCache; using JwksCachePtr = std::unique_ptr; +using JwksConstPtr = std::unique_ptr; +using JwksConstSharedPtr = std::shared_ptr; + /** * Interface to access all configured Jwt rules and their cached Jwks objects. * It only caches Jwks specified in the config. @@ -57,8 +64,7 @@ class JwksCache { virtual bool isExpired() const PURE; // Set a remote Jwks. - virtual const ::google::jwt_verify::Jwks* - setRemoteJwks(::google::jwt_verify::JwksPtr&& jwks) PURE; + virtual const ::google::jwt_verify::Jwks* setRemoteJwks(JwksConstPtr&& jwks) PURE; }; // Lookup issuer cache map. The cache only stores Jwks specified in the config. @@ -67,10 +73,13 @@ class JwksCache { // Lookup provider cache map. virtual JwksData* findByProvider(const std::string& provider) PURE; + virtual JwtAuthnFilterStats& stats() PURE; + // Factory function to create an instance. static JwksCachePtr create(const envoy::extensions::filters::http::jwt_authn::v3::JwtAuthentication& config, - TimeSource& time_source, Api::Api& api, ThreadLocal::SlotAllocator& tls); + Server::Configuration::FactoryContext& context, CreateJwksFetcherCb fetcher_fn, + JwtAuthnFilterStats& stats); }; } // namespace JwtAuthn diff --git a/source/extensions/filters/http/jwt_authn/stats.h b/source/extensions/filters/http/jwt_authn/stats.h new file mode 100644 index 0000000000000..f1eccce135606 --- /dev/null +++ b/source/extensions/filters/http/jwt_authn/stats.h @@ -0,0 +1,30 @@ +#pragma once + +#include "envoy/stats/stats_macros.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace JwtAuthn { + +/** + * All stats for the Jwt Authn filter. @see stats_macros.h + */ +#define ALL_JWT_AUTHN_FILTER_STATS(COUNTER) \ + COUNTER(allowed) \ + COUNTER(cors_preflight_bypassed) \ + COUNTER(denied) \ + COUNTER(jwks_fetch_success) \ + COUNTER(jwks_fetch_failed) + +/** + * Wrapper struct for jwt_authn filter stats. @see stats_macros.h + */ +struct JwtAuthnFilterStats { + ALL_JWT_AUTHN_FILTER_STATS(GENERATE_COUNTER_STRUCT) +}; + +} // namespace JwtAuthn +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/kill_request/BUILD b/source/extensions/filters/http/kill_request/BUILD index 09faef0937421..2a7a21ef355e7 100644 --- a/source/extensions/filters/http/kill_request/BUILD +++ b/source/extensions/filters/http/kill_request/BUILD @@ -30,8 +30,6 @@ envoy_cc_extension( name = "kill_request_config", srcs = ["kill_request_config.cc"], hdrs = ["kill_request_config.h"], - category = "envoy.filters.http", - security_posture = "robust_to_untrusted_downstream", deps = [ "//include/envoy/registry", "//source/extensions/filters/http:well_known_names", diff --git a/source/extensions/filters/http/kill_request/kill_request_filter.cc b/source/extensions/filters/http/kill_request/kill_request_filter.cc index 7cf10a23bc1b2..b278fabe235ca 100644 --- a/source/extensions/filters/http/kill_request/kill_request_filter.cc +++ b/source/extensions/filters/http/kill_request/kill_request_filter.cc @@ -61,8 +61,9 @@ Http::FilterHeadersStatus KillRequestFilter::decodeHeaders(Http::RequestHeaderMa if (per_route_kill_settings) { is_correct_direction = per_route_kill_settings->getDirection() == KillRequest::REQUEST; - envoy::type::v3::FractionalPercent probability = per_route_kill_settings->getProbability(); - kill_request_.set_allocated_probability(&probability); + // Allocate the probability into kill_request in case the lifetime of + // per_route_kill_settings does not match that of kill_request_ + kill_request_.mutable_probability()->CopyFrom(per_route_kill_settings->getProbability()); } } diff --git a/source/extensions/filters/http/local_ratelimit/BUILD b/source/extensions/filters/http/local_ratelimit/BUILD index f60271193bc67..d409a424fd6bf 100644 --- a/source/extensions/filters/http/local_ratelimit/BUILD +++ b/source/extensions/filters/http/local_ratelimit/BUILD @@ -36,8 +36,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", - security_posture = "unknown", deps = [ ":local_ratelimit_lib", "//include/envoy/http:filter_interface", diff --git a/source/extensions/filters/http/local_ratelimit/local_ratelimit.cc b/source/extensions/filters/http/local_ratelimit/local_ratelimit.cc index 30ed854930e51..741ec1b3346c0 100644 --- a/source/extensions/filters/http/local_ratelimit/local_ratelimit.cc +++ b/source/extensions/filters/http/local_ratelimit/local_ratelimit.cc @@ -13,18 +13,24 @@ namespace Extensions { namespace HttpFilters { namespace LocalRateLimitFilter { +const std::string& PerConnectionRateLimiter::key() { + CONSTRUCT_ON_FIRST_USE(std::string, "per_connection_local_rate_limiter"); +} + FilterConfig::FilterConfig( const envoy::extensions::filters::http::local_ratelimit::v3::LocalRateLimit& config, const LocalInfo::LocalInfo& local_info, Event::Dispatcher& dispatcher, Stats::Scope& scope, Runtime::Loader& runtime, const bool per_route) : status_(toErrorCode(config.status().code())), stats_(generateStats(config.stat_prefix(), scope)), + fill_interval_(std::chrono::milliseconds( + PROTOBUF_GET_MS_OR_DEFAULT(config.token_bucket(), fill_interval, 0))), + max_tokens_(config.token_bucket().max_tokens()), + tokens_per_fill_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(config.token_bucket(), tokens_per_fill, 1)), + descriptors_(config.descriptors()), + rate_limit_per_connection_(config.local_rate_limit_per_downstream_connection()), rate_limiter_(Filters::Common::LocalRateLimit::LocalRateLimiterImpl( - std::chrono::milliseconds( - PROTOBUF_GET_MS_OR_DEFAULT(config.token_bucket(), fill_interval, 0)), - config.token_bucket().max_tokens(), - PROTOBUF_GET_WRAPPED_OR_DEFAULT(config.token_bucket(), tokens_per_fill, 1), dispatcher, - config.descriptors())), + fill_interval_, max_tokens_, tokens_per_fill_, dispatcher, descriptors_)), local_info_(local_info), runtime_(runtime), filter_enabled_( config.has_filter_enabled() @@ -84,7 +90,7 @@ Http::FilterHeadersStatus Filter::decodeHeaders(Http::RequestHeaderMap& headers, populateDescriptors(descriptors, headers); } - if (config->requestAllowed(descriptors)) { + if (requestAllowed(descriptors)) { config->stats().ok_.inc(); return Http::FilterHeadersStatus::Continue; } @@ -109,6 +115,34 @@ Http::FilterHeadersStatus Filter::decodeHeaders(Http::RequestHeaderMap& headers, return Http::FilterHeadersStatus::StopIteration; } +bool Filter::requestAllowed(absl::Span request_descriptors) { + const auto* config = getConfig(); + return config->rateLimitPerConnection() + ? getPerConnectionRateLimiter().requestAllowed(request_descriptors) + : config->requestAllowed(request_descriptors); +} + +const Filters::Common::LocalRateLimit::LocalRateLimiterImpl& Filter::getPerConnectionRateLimiter() { + const auto* config = getConfig(); + ASSERT(config->rateLimitPerConnection()); + + if (!decoder_callbacks_->streamInfo().filterState()->hasData( + PerConnectionRateLimiter::key())) { + decoder_callbacks_->streamInfo().filterState()->setData( + PerConnectionRateLimiter::key(), + std::make_unique( + config->fillInterval(), config->maxTokens(), config->tokensPerFill(), + decoder_callbacks_->dispatcher(), config->descriptors()), + StreamInfo::FilterState::StateType::ReadOnly, + StreamInfo::FilterState::LifeSpan::Connection); + } + + return decoder_callbacks_->streamInfo() + .filterState() + ->getDataReadOnly(PerConnectionRateLimiter::key()) + .value(); +} + void Filter::populateDescriptors(std::vector& descriptors, Http::RequestHeaderMap& headers) { Router::RouteConstSharedPtr route = decoder_callbacks_->route(); diff --git a/source/extensions/filters/http/local_ratelimit/local_ratelimit.h b/source/extensions/filters/http/local_ratelimit/local_ratelimit.h index af27e23efe4ef..380fd212a5052 100644 --- a/source/extensions/filters/http/local_ratelimit/local_ratelimit.h +++ b/source/extensions/filters/http/local_ratelimit/local_ratelimit.h @@ -42,6 +42,23 @@ struct LocalRateLimitStats { ALL_LOCAL_RATE_LIMIT_STATS(GENERATE_COUNTER_STRUCT) }; +class PerConnectionRateLimiter : public StreamInfo::FilterState::Object { +public: + PerConnectionRateLimiter( + const std::chrono::milliseconds& fill_interval, uint32_t max_tokens, uint32_t tokens_per_fill, + Envoy::Event::Dispatcher& dispatcher, + const Protobuf::RepeatedPtrField< + envoy::extensions::common::ratelimit::v3::LocalRateLimitDescriptor>& descriptor) + : rate_limiter_(fill_interval, max_tokens, tokens_per_fill, dispatcher, descriptor) {} + static const std::string& key(); + const Filters::Common::LocalRateLimit::LocalRateLimiterImpl& value() const { + return rate_limiter_; + } + +private: + Filters::Common::LocalRateLimit::LocalRateLimiterImpl rate_limiter_; +}; + /** * Global configuration for the HTTP local rate limit filter. */ @@ -62,6 +79,15 @@ class FilterConfig : public Router::RouteSpecificFilterConfig { Http::Code status() const { return status_; } uint64_t stage() const { return stage_; } bool hasDescriptors() const { return has_descriptors_; } + const std::chrono::milliseconds& fillInterval() const { return fill_interval_; } + uint32_t maxTokens() const { return max_tokens_; } + uint32_t tokensPerFill() const { return tokens_per_fill_; } + const Protobuf::RepeatedPtrField< + envoy::extensions::common::ratelimit::v3::LocalRateLimitDescriptor>& + descriptors() const { + return descriptors_; + } + bool rateLimitPerConnection() const { return rate_limit_per_connection_; } private: friend class FilterTest; @@ -78,6 +104,13 @@ class FilterConfig : public Router::RouteSpecificFilterConfig { const Http::Code status_; mutable LocalRateLimitStats stats_; + const std::chrono::milliseconds fill_interval_; + const uint32_t max_tokens_; + const uint32_t tokens_per_fill_; + const Protobuf::RepeatedPtrField< + envoy::extensions::common::ratelimit::v3::LocalRateLimitDescriptor> + descriptors_; + const bool rate_limit_per_connection_; Filters::Common::LocalRateLimit::LocalRateLimiterImpl rate_limiter_; const LocalInfo::LocalInfo& local_info_; Runtime::Loader& runtime_; @@ -108,6 +141,8 @@ class Filter : public Http::PassThroughFilter { void populateDescriptors(std::vector& descriptors, Http::RequestHeaderMap& headers); + const Filters::Common::LocalRateLimit::LocalRateLimiterImpl& getPerConnectionRateLimiter(); + bool requestAllowed(absl::Span request_descriptors); const FilterConfig* getConfig() const; FilterConfigSharedPtr config_; diff --git a/source/extensions/filters/http/lua/BUILD b/source/extensions/filters/http/lua/BUILD index 9d6c381a09892..c4683e8263534 100644 --- a/source/extensions/filters/http/lua/BUILD +++ b/source/extensions/filters/http/lua/BUILD @@ -55,8 +55,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", - security_posture = "robust_to_untrusted_downstream", deps = [ "//include/envoy/registry", "//source/extensions/filters/http:well_known_names", diff --git a/source/extensions/filters/http/oauth2/BUILD b/source/extensions/filters/http/oauth2/BUILD index 7fc8a96a6cf31..d7ea3098c4498 100644 --- a/source/extensions/filters/http/oauth2/BUILD +++ b/source/extensions/filters/http/oauth2/BUILD @@ -63,9 +63,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", - security_posture = "robust_to_untrusted_downstream", - status = "alpha", deps = [ ":oauth_lib", "//include/envoy/registry", diff --git a/source/extensions/filters/http/on_demand/BUILD b/source/extensions/filters/http/on_demand/BUILD index 72c5f6b33d564..5c66187881615 100644 --- a/source/extensions/filters/http/on_demand/BUILD +++ b/source/extensions/filters/http/on_demand/BUILD @@ -30,13 +30,11 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", # TODO(#9953) classify and clean up. extra_visibility = [ "//test/common/access_log:__subpackages__", "//test/integration:__subpackages__", ], - security_posture = "robust_to_untrusted_downstream", deps = [ "//include/envoy/registry", "//source/extensions/filters/http:well_known_names", diff --git a/source/extensions/filters/http/original_src/BUILD b/source/extensions/filters/http/original_src/BUILD index 3181285fc50a8..fe0c2adb6b64c 100644 --- a/source/extensions/filters/http/original_src/BUILD +++ b/source/extensions/filters/http/original_src/BUILD @@ -35,9 +35,6 @@ envoy_cc_extension( name = "config", # The extension build system requires a library named config srcs = ["original_src_config_factory.cc"], hdrs = ["original_src_config_factory.h"], - category = "envoy.filters.http", - security_posture = "robust_to_untrusted_downstream", - status = "alpha", deps = [ ":config_lib", ":original_src_lib", diff --git a/source/extensions/filters/http/ratelimit/BUILD b/source/extensions/filters/http/ratelimit/BUILD index 78ec6694d2a55..bc845c26ed5a7 100644 --- a/source/extensions/filters/http/ratelimit/BUILD +++ b/source/extensions/filters/http/ratelimit/BUILD @@ -45,8 +45,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", - security_posture = "robust_to_untrusted_downstream", deps = [ ":ratelimit_lib", "//include/envoy/registry", diff --git a/source/extensions/filters/http/rbac/BUILD b/source/extensions/filters/http/rbac/BUILD index 9cd4d9cbedd8e..a0ef997d5e736 100644 --- a/source/extensions/filters/http/rbac/BUILD +++ b/source/extensions/filters/http/rbac/BUILD @@ -13,12 +13,10 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", # TODO(#9953) clean up. extra_visibility = [ "//test/integration:__subpackages__", ], - security_posture = "robust_to_untrusted_downstream", deps = [ "//include/envoy/registry", "//source/extensions/filters/http:well_known_names", diff --git a/source/extensions/filters/http/router/BUILD b/source/extensions/filters/http/router/BUILD index 3d78b2f303e05..8e268cff166c3 100644 --- a/source/extensions/filters/http/router/BUILD +++ b/source/extensions/filters/http/router/BUILD @@ -15,8 +15,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", - security_posture = "robust_to_untrusted_downstream", # This is core Envoy config. visibility = ["//visibility:public"], deps = [ diff --git a/source/extensions/filters/http/set_metadata/BUILD b/source/extensions/filters/http/set_metadata/BUILD new file mode 100644 index 0000000000000..d4ea191e892d3 --- /dev/null +++ b/source/extensions/filters/http/set_metadata/BUILD @@ -0,0 +1,38 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_cc_library", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_library( + name = "set_metadata_filter_lib", + srcs = ["set_metadata_filter.cc"], + hdrs = ["set_metadata_filter.h"], + deps = [ + "//include/envoy/server:filter_config_interface", + "//source/common/http:utility_lib", + "//source/common/protobuf:utility_lib", + "//source/extensions/filters/http:well_known_names", + "//source/extensions/filters/http/common:pass_through_filter_lib", + "@envoy_api//envoy/extensions/filters/http/set_metadata/v3:pkg_cc_proto", + ], +) + +envoy_cc_extension( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + deps = [ + "//include/envoy/registry", + "//source/common/protobuf:utility_lib", + "//source/extensions/filters/http:well_known_names", + "//source/extensions/filters/http/common:factory_base_lib", + "//source/extensions/filters/http/set_metadata:set_metadata_filter_lib", + "@envoy_api//envoy/extensions/filters/http/set_metadata/v3:pkg_cc_proto", + ], +) diff --git a/source/extensions/filters/http/set_metadata/config.cc b/source/extensions/filters/http/set_metadata/config.cc new file mode 100644 index 0000000000000..2d2f35cd2a7ca --- /dev/null +++ b/source/extensions/filters/http/set_metadata/config.cc @@ -0,0 +1,34 @@ +#include "extensions/filters/http/set_metadata/config.h" + +#include + +#include "envoy/extensions/filters/http/set_metadata/v3/set_metadata.pb.h" +#include "envoy/extensions/filters/http/set_metadata/v3/set_metadata.pb.validate.h" +#include "envoy/registry/registry.h" + +#include "common/protobuf/utility.h" + +#include "extensions/filters/http/set_metadata/set_metadata_filter.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace SetMetadataFilter { + +Http::FilterFactoryCb SetMetadataConfig::createFilterFactoryFromProtoTyped( + const envoy::extensions::filters::http::set_metadata::v3::Config& proto_config, + const std::string&, Server::Configuration::FactoryContext&) { + ConfigSharedPtr filter_config(std::make_shared(proto_config)); + + return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { + callbacks.addStreamDecoderFilter( + Http::StreamDecoderFilterSharedPtr{new SetMetadataFilter(filter_config)}); + }; +} + +REGISTER_FACTORY(SetMetadataConfig, Server::Configuration::NamedHttpFilterConfigFactory); + +} // namespace SetMetadataFilter +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/set_metadata/config.h b/source/extensions/filters/http/set_metadata/config.h new file mode 100644 index 0000000000000..32350b0f6af56 --- /dev/null +++ b/source/extensions/filters/http/set_metadata/config.h @@ -0,0 +1,31 @@ +#pragma once + +#include "envoy/extensions/filters/http/set_metadata/v3/set_metadata.pb.h" +#include "envoy/extensions/filters/http/set_metadata/v3/set_metadata.pb.validate.h" + +#include "extensions/filters/http/common/factory_base.h" +#include "extensions/filters/http/well_known_names.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace SetMetadataFilter { + +/** + * Config registration for the header-to-metadata filter. @see NamedHttpFilterConfigFactory. + */ +class SetMetadataConfig + : public Common::FactoryBase { +public: + SetMetadataConfig() : FactoryBase(HttpFilterNames::get().SetMetadata) {} + +private: + Http::FilterFactoryCb createFilterFactoryFromProtoTyped( + const envoy::extensions::filters::http::set_metadata::v3::Config& proto_config, + const std::string& stats_prefix, Server::Configuration::FactoryContext& context) override; +}; + +} // namespace SetMetadataFilter +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/set_metadata/set_metadata_filter.cc b/source/extensions/filters/http/set_metadata/set_metadata_filter.cc new file mode 100644 index 0000000000000..98295d3cda3d8 --- /dev/null +++ b/source/extensions/filters/http/set_metadata/set_metadata_filter.cc @@ -0,0 +1,49 @@ +#include "extensions/filters/http/set_metadata/set_metadata_filter.h" + +#include "envoy/extensions/filters/http/set_metadata/v3/set_metadata.pb.h" + +#include "common/config/well_known_names.h" +#include "common/http/utility.h" +#include "common/protobuf/protobuf.h" + +#include "extensions/filters/http/well_known_names.h" + +#include "absl/strings/str_format.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace SetMetadataFilter { + +Config::Config(const envoy::extensions::filters::http::set_metadata::v3::Config& proto_config) { + namespace_ = proto_config.metadata_namespace(); + value_ = proto_config.value(); +} + +SetMetadataFilter::SetMetadataFilter(const ConfigSharedPtr config) : config_(config) {} + +SetMetadataFilter::~SetMetadataFilter() = default; + +Http::FilterHeadersStatus SetMetadataFilter::decodeHeaders(Http::RequestHeaderMap&, bool) { + const absl::string_view metadata_namespace = config_->metadataNamespace(); + auto& metadata = *decoder_callbacks_->streamInfo().dynamicMetadata().mutable_filter_metadata(); + ProtobufWkt::Struct& org_fields = metadata[metadata_namespace]; + const ProtobufWkt::Struct& to_merge = config_->value(); + + StructUtil::update(org_fields, to_merge); + + return Http::FilterHeadersStatus::Continue; +} + +Http::FilterDataStatus SetMetadataFilter::decodeData(Buffer::Instance&, bool) { + return Http::FilterDataStatus::Continue; +} + +void SetMetadataFilter::setDecoderFilterCallbacks(Http::StreamDecoderFilterCallbacks& callbacks) { + decoder_callbacks_ = &callbacks; +} + +} // namespace SetMetadataFilter +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/set_metadata/set_metadata_filter.h b/source/extensions/filters/http/set_metadata/set_metadata_filter.h new file mode 100644 index 0000000000000..a5a0957a9f818 --- /dev/null +++ b/source/extensions/filters/http/set_metadata/set_metadata_filter.h @@ -0,0 +1,57 @@ +#pragma once + +#include +#include +#include + +#include "envoy/extensions/filters/http/set_metadata/v3/set_metadata.pb.h" + +#include "common/common/logger.h" + +#include "extensions/filters/http/common/pass_through_filter.h" + +#include "absl/strings/string_view.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace SetMetadataFilter { + +class Config : public ::Envoy::Router::RouteSpecificFilterConfig, + public Logger::Loggable { +public: + Config(const envoy::extensions::filters::http::set_metadata::v3::Config& config); + + absl::string_view metadataNamespace() const { return namespace_; } + const ProtobufWkt::Struct& value() { return value_; } + +private: + std::string namespace_; + ProtobufWkt::Struct value_; +}; + +using ConfigSharedPtr = std::shared_ptr; + +class SetMetadataFilter : public Http::PassThroughDecoderFilter, + public Logger::Loggable { +public: + SetMetadataFilter(const ConfigSharedPtr config); + ~SetMetadataFilter() override; + + // Http::StreamFilterBase + void onDestroy() override {} + + // StreamDecoderFilter + Http::FilterHeadersStatus decodeHeaders(Http::RequestHeaderMap& headers, bool) override; + Http::FilterDataStatus decodeData(Buffer::Instance&, bool) override; + void setDecoderFilterCallbacks(Http::StreamDecoderFilterCallbacks&) override; + +private: + const ConfigSharedPtr config_; + Http::StreamDecoderFilterCallbacks* decoder_callbacks_; +}; + +} // namespace SetMetadataFilter +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/squash/BUILD b/source/extensions/filters/http/squash/BUILD index e486d07f4a890..ef3c4ca805b8a 100644 --- a/source/extensions/filters/http/squash/BUILD +++ b/source/extensions/filters/http/squash/BUILD @@ -37,8 +37,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", - security_posture = "requires_trusted_downstream_and_upstream", deps = [ "//include/envoy/registry", "//source/common/protobuf:utility_lib", diff --git a/source/extensions/filters/http/tap/BUILD b/source/extensions/filters/http/tap/BUILD index 9379579d8b804..e1e0f6407e30f 100644 --- a/source/extensions/filters/http/tap/BUILD +++ b/source/extensions/filters/http/tap/BUILD @@ -52,9 +52,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", - security_posture = "requires_trusted_downstream_and_upstream", - status = "alpha", deps = [ ":tap_config_impl", ":tap_filter_lib", diff --git a/source/extensions/filters/http/wasm/BUILD b/source/extensions/filters/http/wasm/BUILD index e399e89290aa4..db3a6d09196ad 100644 --- a/source/extensions/filters/http/wasm/BUILD +++ b/source/extensions/filters/http/wasm/BUILD @@ -30,9 +30,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.http", - security_posture = "unknown", - status = "alpha", deps = [ ":wasm_filter_lib", "//include/envoy/registry", diff --git a/source/extensions/filters/http/well_known_names.h b/source/extensions/filters/http/well_known_names.h index ddb9e1e6346d4..609e13f5bb496 100644 --- a/source/extensions/filters/http/well_known_names.h +++ b/source/extensions/filters/http/well_known_names.h @@ -14,6 +14,8 @@ class HttpFilterNameValues { public: // Buffer filter const std::string Buffer = "envoy.filters.http.buffer"; + // Bandwidth limit filter + const std::string BandwidthLimit = "envoy.filters.http.bandwidth_limit"; // Cache filter const std::string Cache = "envoy.filters.http.cache"; // CDN Loop filter @@ -88,6 +90,8 @@ class HttpFilterNameValues { const std::string KillRequest = "envoy.filters.http.kill_request"; // External Processing filter const std::string ExternalProcessing = "envoy.filters.http.ext_proc"; + // Set metadata filter + const std::string SetMetadata = "envoy.filters.http.set_metadata"; }; using HttpFilterNames = ConstSingleton; diff --git a/source/extensions/filters/listener/http_inspector/BUILD b/source/extensions/filters/listener/http_inspector/BUILD index 849277d618d12..8426d64e32bf2 100644 --- a/source/extensions/filters/listener/http_inspector/BUILD +++ b/source/extensions/filters/listener/http_inspector/BUILD @@ -31,8 +31,6 @@ envoy_cc_library( envoy_cc_extension( name = "config", srcs = ["config.cc"], - category = "envoy.filters.listener", - security_posture = "requires_trusted_downstream_and_upstream", deps = [ ":http_inspector_lib", "//include/envoy/registry", diff --git a/source/extensions/filters/listener/original_dst/BUILD b/source/extensions/filters/listener/original_dst/BUILD index 62b0b88f001cf..7ee4f6e013c32 100644 --- a/source/extensions/filters/listener/original_dst/BUILD +++ b/source/extensions/filters/listener/original_dst/BUILD @@ -29,12 +29,10 @@ envoy_cc_library( envoy_cc_extension( name = "config", srcs = ["config.cc"], - category = "envoy.filters.listener", # TODO(#9953) clean up. extra_visibility = [ "//test/integration:__subpackages__", ], - security_posture = "robust_to_untrusted_downstream", deps = [ ":original_dst_lib", "//include/envoy/registry", diff --git a/source/extensions/filters/listener/original_src/BUILD b/source/extensions/filters/listener/original_src/BUILD index 26df22093a3cd..4b952500fa44f 100644 --- a/source/extensions/filters/listener/original_src/BUILD +++ b/source/extensions/filters/listener/original_src/BUILD @@ -38,9 +38,6 @@ envoy_cc_extension( name = "config", # The extension build system requires a library named config srcs = ["original_src_config_factory.cc"], hdrs = ["original_src_config_factory.h"], - category = "envoy.filters.listener", - security_posture = "robust_to_untrusted_downstream", - status = "alpha", deps = [ ":config_lib", ":original_src_lib", diff --git a/source/extensions/filters/listener/proxy_protocol/BUILD b/source/extensions/filters/listener/proxy_protocol/BUILD index 66c21a7b27686..6cff8506baed7 100644 --- a/source/extensions/filters/listener/proxy_protocol/BUILD +++ b/source/extensions/filters/listener/proxy_protocol/BUILD @@ -40,12 +40,10 @@ envoy_cc_library( envoy_cc_extension( name = "config", srcs = ["config.cc"], - category = "envoy.filters.listener", # TODO(#9953) clean up. extra_visibility = [ "//test/integration:__subpackages__", ], - security_posture = "robust_to_untrusted_downstream", deps = [ "//include/envoy/registry", "//include/envoy/server:filter_config_interface", diff --git a/source/extensions/filters/listener/tls_inspector/BUILD b/source/extensions/filters/listener/tls_inspector/BUILD index 3f6837524e2b2..109d783b2c4f0 100644 --- a/source/extensions/filters/listener/tls_inspector/BUILD +++ b/source/extensions/filters/listener/tls_inspector/BUILD @@ -35,12 +35,10 @@ envoy_cc_library( envoy_cc_extension( name = "config", srcs = ["config.cc"], - category = "envoy.filters.listener", # TODO(#9953) clean up. extra_visibility = [ "//test/integration:__subpackages__", ], - security_posture = "robust_to_untrusted_downstream", deps = [ "//include/envoy/registry", "//include/envoy/server:filter_config_interface", diff --git a/source/extensions/filters/network/client_ssl_auth/BUILD b/source/extensions/filters/network/client_ssl_auth/BUILD index 184ef95404aa4..de0b01ec4bab0 100644 --- a/source/extensions/filters/network/client_ssl_auth/BUILD +++ b/source/extensions/filters/network/client_ssl_auth/BUILD @@ -40,8 +40,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.network", - security_posture = "robust_to_untrusted_downstream", deps = [ ":client_ssl_auth", "//include/envoy/registry", diff --git a/source/extensions/filters/network/direct_response/BUILD b/source/extensions/filters/network/direct_response/BUILD index 7954de4042115..5a4b40483b4fc 100644 --- a/source/extensions/filters/network/direct_response/BUILD +++ b/source/extensions/filters/network/direct_response/BUILD @@ -28,8 +28,6 @@ envoy_cc_library( envoy_cc_extension( name = "config", srcs = ["config.cc"], - category = "envoy.filters.network", - security_posture = "unknown", deps = [ ":filter", "//include/envoy/registry", diff --git a/source/extensions/filters/network/dubbo_proxy/BUILD b/source/extensions/filters/network/dubbo_proxy/BUILD index e051679a29b6d..49ac684e27208 100644 --- a/source/extensions/filters/network/dubbo_proxy/BUILD +++ b/source/extensions/filters/network/dubbo_proxy/BUILD @@ -96,9 +96,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.network", - security_posture = "requires_trusted_downstream_and_upstream", - status = "alpha", deps = [ ":conn_manager_lib", "//include/envoy/registry", diff --git a/source/extensions/filters/network/echo/BUILD b/source/extensions/filters/network/echo/BUILD index 68270a5dd5e20..2352cb8089531 100644 --- a/source/extensions/filters/network/echo/BUILD +++ b/source/extensions/filters/network/echo/BUILD @@ -28,12 +28,10 @@ envoy_cc_library( envoy_cc_extension( name = "config", srcs = ["config.cc"], - category = "envoy.filters.network", # TODO(#9953) move echo integration test to extensions. extra_visibility = [ "//test/integration:__subpackages__", ], - security_posture = "unknown", deps = [ ":echo", "//include/envoy/registry", diff --git a/source/extensions/filters/network/ext_authz/BUILD b/source/extensions/filters/network/ext_authz/BUILD index 391fe6e21d72c..7ceb93fa7b805 100644 --- a/source/extensions/filters/network/ext_authz/BUILD +++ b/source/extensions/filters/network/ext_authz/BUILD @@ -37,8 +37,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.network", - security_posture = "robust_to_untrusted_downstream", deps = [ "//include/envoy/registry", "//source/common/config:utility_lib", diff --git a/source/extensions/filters/network/http_connection_manager/BUILD b/source/extensions/filters/network/http_connection_manager/BUILD index c2c6a19c2e949..46867ac644bbc 100644 --- a/source/extensions/filters/network/http_connection_manager/BUILD +++ b/source/extensions/filters/network/http_connection_manager/BUILD @@ -18,8 +18,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.network", - security_posture = "robust_to_untrusted_downstream", # This is core Envoy config. visibility = ["//visibility:public"], deps = [ diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index 67003b945d6c2..9995ac7e7fea2 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -307,7 +307,8 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( headers_with_underscores_action_( config.common_http_protocol_options().headers_with_underscores_action()), local_reply_(LocalReply::Factory::create(config.local_reply_config(), context)), - path_with_escaped_slashes_action_(getPathWithEscapedSlashesAction(config, context)) { + path_with_escaped_slashes_action_(getPathWithEscapedSlashesAction(config, context)), + strip_trailing_host_dot_(config.strip_trailing_host_dot()) { // If idle_timeout_ was not configured in common_http_protocol_options, use value in deprecated // idle_timeout field. // TODO(asraa): Remove when idle_timeout is removed. diff --git a/source/extensions/filters/network/http_connection_manager/config.h b/source/extensions/filters/network/http_connection_manager/config.h index b98629ee8ecc2..1e4e93445f61d 100644 --- a/source/extensions/filters/network/http_connection_manager/config.h +++ b/source/extensions/filters/network/http_connection_manager/config.h @@ -175,6 +175,7 @@ class HttpConnectionManagerConfig : Logger::Loggable, const Http::Http1Settings& http1Settings() const override { return http1_settings_; } bool shouldNormalizePath() const override { return normalize_path_; } bool shouldMergeSlashes() const override { return merge_slashes_; } + bool shouldStripTrailingHostDot() const override { return strip_trailing_host_dot_; } Http::StripPortType stripPortType() const override { return strip_port_type_; } envoy::config::core::v3::HttpProtocolOptions::HeadersWithUnderscoresAction headersWithUnderscoresAction() const override { @@ -283,6 +284,7 @@ class HttpConnectionManagerConfig : Logger::Loggable, static const uint64_t RequestHeaderTimeoutMs = 0; const envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager:: PathWithEscapedSlashesAction path_with_escaped_slashes_action_; + const bool strip_trailing_host_dot_; }; /** diff --git a/source/extensions/filters/network/kafka/BUILD b/source/extensions/filters/network/kafka/BUILD index 01c31e63cc9fe..ccc2401ace47e 100644 --- a/source/extensions/filters/network/kafka/BUILD +++ b/source/extensions/filters/network/kafka/BUILD @@ -18,9 +18,6 @@ envoy_cc_extension( name = "kafka_broker_config_lib", srcs = ["broker/config.cc"], hdrs = ["broker/config.h"], - category = "envoy.filters.network", - security_posture = "requires_trusted_downstream_and_upstream", - status = "wip", deps = [ ":kafka_broker_filter_lib", "//source/extensions/filters/network:well_known_names", diff --git a/source/extensions/filters/network/local_ratelimit/BUILD b/source/extensions/filters/network/local_ratelimit/BUILD index 6e10aaff1de32..c0b757dc8c6e1 100644 --- a/source/extensions/filters/network/local_ratelimit/BUILD +++ b/source/extensions/filters/network/local_ratelimit/BUILD @@ -33,8 +33,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.network", - security_posture = "robust_to_untrusted_downstream", deps = [ "//source/extensions/filters/network:well_known_names", "//source/extensions/filters/network/common:factory_base_lib", diff --git a/source/extensions/filters/network/mongo_proxy/BUILD b/source/extensions/filters/network/mongo_proxy/BUILD index ab1956d777cb0..04a32f0729016 100644 --- a/source/extensions/filters/network/mongo_proxy/BUILD +++ b/source/extensions/filters/network/mongo_proxy/BUILD @@ -107,8 +107,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.network", - security_posture = "requires_trusted_downstream_and_upstream", deps = [ ":proxy_lib", "//include/envoy/registry", diff --git a/source/extensions/filters/network/mysql_proxy/BUILD b/source/extensions/filters/network/mysql_proxy/BUILD index d176bad30b970..43d66bdf11c5b 100644 --- a/source/extensions/filters/network/mysql_proxy/BUILD +++ b/source/extensions/filters/network/mysql_proxy/BUILD @@ -107,9 +107,6 @@ envoy_cc_extension( name = "config", srcs = ["mysql_config.cc"], hdrs = ["mysql_config.h"], - category = "envoy.filters.network", - security_posture = "requires_trusted_downstream_and_upstream", - status = "alpha", deps = [ ":filter_lib", "//source/extensions/filters/network:well_known_names", diff --git a/source/extensions/filters/network/postgres_proxy/BUILD b/source/extensions/filters/network/postgres_proxy/BUILD index 398fa80cc67d5..1367aa5c048a0 100644 --- a/source/extensions/filters/network/postgres_proxy/BUILD +++ b/source/extensions/filters/network/postgres_proxy/BUILD @@ -44,9 +44,7 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.network", repository = "@envoy", - security_posture = "requires_trusted_downstream_and_upstream", deps = [ ":filter", "//source/extensions/filters/network:well_known_names", diff --git a/source/extensions/filters/network/ratelimit/BUILD b/source/extensions/filters/network/ratelimit/BUILD index 2ab3b5ac6787c..35694dc418b8f 100644 --- a/source/extensions/filters/network/ratelimit/BUILD +++ b/source/extensions/filters/network/ratelimit/BUILD @@ -39,8 +39,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.network", - security_posture = "robust_to_untrusted_downstream", deps = [ "//include/envoy/registry", "//source/common/config:utility_lib", diff --git a/source/extensions/filters/network/rbac/BUILD b/source/extensions/filters/network/rbac/BUILD index f5a4f38fdc0ed..be7137d0f8d58 100644 --- a/source/extensions/filters/network/rbac/BUILD +++ b/source/extensions/filters/network/rbac/BUILD @@ -13,8 +13,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.network", - security_posture = "robust_to_untrusted_downstream", deps = [ ":rbac_filter", "//include/envoy/registry", diff --git a/source/extensions/filters/network/redis_proxy/BUILD b/source/extensions/filters/network/redis_proxy/BUILD index 7cf695e2a513b..3d70b7f7c3504 100644 --- a/source/extensions/filters/network/redis_proxy/BUILD +++ b/source/extensions/filters/network/redis_proxy/BUILD @@ -120,12 +120,10 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.network", # TODO(#9953) clean up. extra_visibility = [ "//test/integration:__subpackages__", ], - security_posture = "requires_trusted_downstream_and_upstream", deps = [ "//include/envoy/upstream:upstream_interface", "//source/extensions/common/redis:cluster_refresh_manager_lib", diff --git a/source/extensions/filters/network/rocketmq_proxy/BUILD b/source/extensions/filters/network/rocketmq_proxy/BUILD index 4dd07abc6225a..fe9ba3ab5022f 100644 --- a/source/extensions/filters/network/rocketmq_proxy/BUILD +++ b/source/extensions/filters/network/rocketmq_proxy/BUILD @@ -122,9 +122,6 @@ envoy_cc_extension( hdrs = [ "config.h", ], - category = "envoy.filters.network", - security_posture = "requires_trusted_downstream_and_upstream", - status = "alpha", deps = [ ":conn_manager_lib", "//include/envoy/registry", diff --git a/source/extensions/filters/network/sni_cluster/BUILD b/source/extensions/filters/network/sni_cluster/BUILD index 310bf058c1924..f730bd9c49ca6 100644 --- a/source/extensions/filters/network/sni_cluster/BUILD +++ b/source/extensions/filters/network/sni_cluster/BUILD @@ -26,8 +26,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.network", - security_posture = "unknown", deps = [ ":sni_cluster", "//include/envoy/registry", diff --git a/source/extensions/filters/network/sni_dynamic_forward_proxy/BUILD b/source/extensions/filters/network/sni_dynamic_forward_proxy/BUILD index bed8252554bbb..ae0181f77f7fa 100644 --- a/source/extensions/filters/network/sni_dynamic_forward_proxy/BUILD +++ b/source/extensions/filters/network/sni_dynamic_forward_proxy/BUILD @@ -28,9 +28,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.network", - security_posture = "unknown", - status = "alpha", deps = [ ":proxy_filter_lib", "//source/extensions/common/dynamic_forward_proxy:dns_cache_manager_impl", diff --git a/source/extensions/filters/network/tcp_proxy/BUILD b/source/extensions/filters/network/tcp_proxy/BUILD index e1a22d965da99..ea7360966c411 100644 --- a/source/extensions/filters/network/tcp_proxy/BUILD +++ b/source/extensions/filters/network/tcp_proxy/BUILD @@ -15,8 +15,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.network", - security_posture = "robust_to_untrusted_downstream", # This is core Envoy config. visibility = ["//visibility:public"], deps = [ diff --git a/source/extensions/filters/network/thrift_proxy/BUILD b/source/extensions/filters/network/thrift_proxy/BUILD index 47439e4c53eaf..37defe986a917 100644 --- a/source/extensions/filters/network/thrift_proxy/BUILD +++ b/source/extensions/filters/network/thrift_proxy/BUILD @@ -36,8 +36,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.network", - security_posture = "requires_trusted_downstream_and_upstream", deps = [ ":app_exception_lib", ":auto_protocol_lib", diff --git a/source/extensions/filters/network/thrift_proxy/filters/ratelimit/BUILD b/source/extensions/filters/network/thrift_proxy/filters/ratelimit/BUILD index b27da3987272e..0d8f61fe19f83 100644 --- a/source/extensions/filters/network/thrift_proxy/filters/ratelimit/BUILD +++ b/source/extensions/filters/network/thrift_proxy/filters/ratelimit/BUILD @@ -32,9 +32,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.thrift_proxy.filters", - security_posture = "requires_trusted_downstream_and_upstream", - status = "alpha", deps = [ ":ratelimit_lib", "//include/envoy/registry", diff --git a/source/extensions/filters/network/thrift_proxy/router/BUILD b/source/extensions/filters/network/thrift_proxy/router/BUILD index e63f180ecc9eb..a16abad3aeb90 100644 --- a/source/extensions/filters/network/thrift_proxy/router/BUILD +++ b/source/extensions/filters/network/thrift_proxy/router/BUILD @@ -13,8 +13,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.thrift_proxy.filters", - security_posture = "requires_trusted_downstream_and_upstream", deps = [ ":router_lib", "//include/envoy/registry", diff --git a/source/extensions/filters/network/thrift_proxy/router/router_impl.cc b/source/extensions/filters/network/thrift_proxy/router/router_impl.cc index 498a86bbe61c0..5712ea8dd171e 100644 --- a/source/extensions/filters/network/thrift_proxy/router/router_impl.cc +++ b/source/extensions/filters/network/thrift_proxy/router/router_impl.cc @@ -316,6 +316,10 @@ FilterStatus Router::messageEnd() { upstream_request_->transport_->encodeFrame(transport_buffer, *upstream_request_->metadata_, upstream_request_buffer_); + + request_size_ += transport_buffer.length(); + recordClusterScopeHistogram({upstream_rq_size_}, Stats::Histogram::Unit::Bytes, request_size_); + upstream_request_->conn_data_->connection().write(transport_buffer, false); upstream_request_->onRequestComplete(); return FilterStatus::Continue; @@ -324,6 +328,8 @@ FilterStatus Router::messageEnd() { void Router::onUpstreamData(Buffer::Instance& data, bool end_stream) { ASSERT(!upstream_request_->response_complete_); + response_size_ += data.length(); + if (upstream_request_->upgrade_response_ != nullptr) { ENVOY_STREAM_LOG(trace, "reading upgrade response: {} bytes", *callbacks_, data.length()); // Handle upgrade response. @@ -351,6 +357,9 @@ void Router::onUpstreamData(Buffer::Instance& data, bool end_stream) { ThriftFilters::ResponseStatus status = callbacks_->upstreamData(data); if (status == ThriftFilters::ResponseStatus::Complete) { ENVOY_STREAM_LOG(debug, "response complete", *callbacks_); + recordClusterScopeHistogram({upstream_resp_size_}, Stats::Histogram::Unit::Bytes, + response_size_); + switch (callbacks_->responseMetadata()->messageType()) { case MessageType::Reply: incClusterScopeCounter({upstream_resp_reply_}); @@ -373,6 +382,7 @@ void Router::onUpstreamData(Buffer::Instance& data, bool end_stream) { cleanup(); return; } else if (status == ThriftFilters::ResponseStatus::Reset) { + // Note: invalid responses are not accounted in the response size histogram. ENVOY_STREAM_LOG(debug, "upstream reset", *callbacks_); upstream_request_->resetStream(); return; @@ -503,6 +513,7 @@ void Router::UpstreamRequest::onPoolReady(Tcp::ConnectionPool::ConnectionDataPtr upgrade_response_ = protocol_->attemptUpgrade(*transport_, *conn_state_, parent_.upstream_request_buffer_); if (upgrade_response_ != nullptr) { + parent_.request_size_ += parent_.upstream_request_buffer_.length(); conn_data_->connection().write(parent_.upstream_request_buffer_, false); return; } diff --git a/source/extensions/filters/network/thrift_proxy/router/router_impl.h b/source/extensions/filters/network/thrift_proxy/router/router_impl.h index e125c69fce0b8..c12a0a34df8d5 100644 --- a/source/extensions/filters/network/thrift_proxy/router/router_impl.h +++ b/source/extensions/filters/network/thrift_proxy/router/router_impl.h @@ -190,6 +190,8 @@ class Router : public Tcp::ConnectionPool::UpstreamCallbacks, upstream_resp_exception_(stat_name_set_->add("thrift.upstream_resp_exception")), upstream_resp_invalid_type_(stat_name_set_->add("thrift.upstream_resp_invalid_type")), upstream_rq_time_(stat_name_set_->add("thrift.upstream_rq_time")), + upstream_rq_size_(stat_name_set_->add("thrift.upstream_rq_size")), + upstream_resp_size_(stat_name_set_->add("thrift.upstream_resp_size")), passthrough_supported_(false) {} ~Router() override = default; @@ -299,6 +301,8 @@ class Router : public Tcp::ConnectionPool::UpstreamCallbacks, const Stats::StatName upstream_resp_exception_; const Stats::StatName upstream_resp_invalid_type_; const Stats::StatName upstream_rq_time_; + const Stats::StatName upstream_rq_size_; + const Stats::StatName upstream_resp_size_; ThriftFilters::DecoderFilterCallbacks* callbacks_{}; RouteConstSharedPtr route_{}; @@ -309,6 +313,8 @@ class Router : public Tcp::ConnectionPool::UpstreamCallbacks, Buffer::OwnedImpl upstream_request_buffer_; bool passthrough_supported_ : 1; + uint64_t request_size_{}; + uint64_t response_size_{}; }; } // namespace Router diff --git a/source/extensions/filters/network/wasm/BUILD b/source/extensions/filters/network/wasm/BUILD index 2023fd1f48d8d..e8a47db2acc8a 100644 --- a/source/extensions/filters/network/wasm/BUILD +++ b/source/extensions/filters/network/wasm/BUILD @@ -28,9 +28,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.network", - security_posture = "unknown", - status = "alpha", deps = [ ":wasm_filter_lib", "//include/envoy/registry", diff --git a/source/extensions/filters/network/zookeeper_proxy/BUILD b/source/extensions/filters/network/zookeeper_proxy/BUILD index 10d14b23ae88a..9c72e9961dfe9 100644 --- a/source/extensions/filters/network/zookeeper_proxy/BUILD +++ b/source/extensions/filters/network/zookeeper_proxy/BUILD @@ -43,9 +43,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.network", - security_posture = "requires_trusted_downstream_and_upstream", - status = "alpha", deps = [ ":proxy_lib", "//source/extensions/filters/network:well_known_names", diff --git a/source/extensions/filters/udp/dns_filter/BUILD b/source/extensions/filters/udp/dns_filter/BUILD index 210d68496d0da..ab44521fdd408 100644 --- a/source/extensions/filters/udp/dns_filter/BUILD +++ b/source/extensions/filters/udp/dns_filter/BUILD @@ -52,9 +52,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.udp_listener", - security_posture = "robust_to_untrusted_downstream", - status = "alpha", deps = [ ":dns_filter_lib", "//include/envoy/registry", diff --git a/source/extensions/filters/udp/udp_proxy/BUILD b/source/extensions/filters/udp/udp_proxy/BUILD index b939347604fab..bd8fa7e6b3550 100644 --- a/source/extensions/filters/udp/udp_proxy/BUILD +++ b/source/extensions/filters/udp/udp_proxy/BUILD @@ -45,8 +45,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.filters.udp_listener", - security_posture = "robust_to_untrusted_downstream", deps = [ ":udp_proxy_filter_lib", "//include/envoy/registry", diff --git a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc index 3f11d37f0f12e..9d21d784ba575 100644 --- a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc +++ b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc @@ -222,7 +222,10 @@ void UdpProxyFilter::ActiveSession::onReadReady() { const Api::IoErrorPtr result = Network::Utility::readPacketsFromSocket( socket_->ioHandle(), *addresses_.local_, *this, cluster_.filter_.config_->timeSource(), cluster_.filter_.config_->upstreamSocketConfig().prefer_gro_, packets_dropped); - // TODO(mattklein123): Handle no error when we limit the number of packets read. + if (result == nullptr) { + socket_->ioHandle().activateFileEvents(Event::FileReadyType::Read); + return; + } if (result->getErrorCode() != Api::IoError::IoErrorCode::Again) { cluster_.cluster_stats_.sess_rx_errors_.inc(); } diff --git a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h index 4a437152acf90..28f037882cbc2 100644 --- a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h +++ b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h @@ -178,6 +178,10 @@ class UdpProxyFilter : public Network::UdpListenerReadFilter, void onDatagramsDropped(uint32_t dropped) override { cluster_.cluster_stats_.sess_rx_datagrams_dropped_.add(dropped); } + size_t numPacketsExpectedPerEventLoop() const final { + // TODO(mattklein123) change this to a reasonable number if needed. + return Network::MAX_NUM_PACKETS_PER_EVENT_LOOP; + } ClusterInfo& cluster_; const bool use_original_src_ip_; diff --git a/source/extensions/grpc_credentials/BUILD b/source/extensions/grpc_credentials/BUILD deleted file mode 100644 index 40a5e79b39d3b..0000000000000 --- a/source/extensions/grpc_credentials/BUILD +++ /dev/null @@ -1,19 +0,0 @@ -load( - "//bazel:envoy_build_system.bzl", - "envoy_cc_library", - "envoy_extension_package", -) - -licenses(["notice"]) # Apache 2 - -envoy_extension_package() - -envoy_cc_library( - name = "well_known_names", - hdrs = ["well_known_names.h"], - # well known names files are public as long as they exist. - visibility = ["//visibility:public"], - deps = [ - "//source/common/singleton:const_singleton", - ], -) diff --git a/source/extensions/grpc_credentials/aws_iam/BUILD b/source/extensions/grpc_credentials/aws_iam/BUILD index 41e311cc52c55..fa66cc2353992 100644 --- a/source/extensions/grpc_credentials/aws_iam/BUILD +++ b/source/extensions/grpc_credentials/aws_iam/BUILD @@ -14,10 +14,7 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.grpc_credentials", external_deps = ["grpc"], - security_posture = "data_plane_agnostic", - status = "alpha", deps = [ "//include/envoy/grpc:google_grpc_creds_interface", "//include/envoy/registry", @@ -30,7 +27,6 @@ envoy_cc_extension( "//source/extensions/common/aws:region_provider_impl_lib", "//source/extensions/common/aws:signer_impl_lib", "//source/extensions/common/aws:utility_lib", - "//source/extensions/grpc_credentials:well_known_names", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/config/grpc_credential/v3:pkg_cc_proto", ], diff --git a/source/extensions/grpc_credentials/aws_iam/config.cc b/source/extensions/grpc_credentials/aws_iam/config.cc index 345d975fedbda..454e6265204e3 100644 --- a/source/extensions/grpc_credentials/aws_iam/config.cc +++ b/source/extensions/grpc_credentials/aws_iam/config.cc @@ -34,7 +34,7 @@ std::shared_ptr AwsIamGrpcCredentialsFactory::getChann switch (credential.credential_specifier_case()) { case envoy::config::core::v3::GrpcService::GoogleGrpc::CallCredentials:: CredentialSpecifierCase::kFromPlugin: { - if (credential.from_plugin().name() == GrpcCredentialsNames::get().AwsIam) { + if (credential.from_plugin().name() == "envoy.grpc_credentials.aws_iam") { AwsIamGrpcCredentialsFactory credentials_factory; // We don't deal with validation failures here at runtime today, see // https://github.com/envoyproxy/envoy/issues/8010. diff --git a/source/extensions/grpc_credentials/aws_iam/config.h b/source/extensions/grpc_credentials/aws_iam/config.h index 9ac68a0451fc6..819a45598ac36 100644 --- a/source/extensions/grpc_credentials/aws_iam/config.h +++ b/source/extensions/grpc_credentials/aws_iam/config.h @@ -8,7 +8,6 @@ #include "common/http/message_impl.h" #include "extensions/common/aws/signer.h" -#include "extensions/grpc_credentials/well_known_names.h" namespace Envoy { namespace Extensions { @@ -28,7 +27,7 @@ class AwsIamGrpcCredentialsFactory : public Grpc::GoogleGrpcCredentialsFactory { return std::make_unique(); } - std::string name() const override { return GrpcCredentialsNames::get().AwsIam; } + std::string name() const override { return "envoy.grpc_credentials.aws_iam"; } private: static std::string getRegion(const envoy::config::grpc_credential::v3::AwsIamConfig& config); diff --git a/source/extensions/grpc_credentials/example/BUILD b/source/extensions/grpc_credentials/example/BUILD index 8c43f6c275323..582844968ee51 100644 --- a/source/extensions/grpc_credentials/example/BUILD +++ b/source/extensions/grpc_credentials/example/BUILD @@ -26,7 +26,6 @@ envoy_cc_library( "//include/envoy/registry", "//source/common/common:assert_lib", "//source/common/grpc:google_grpc_creds_lib", - "//source/extensions/grpc_credentials:well_known_names", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/grpc_credentials/example/config.h b/source/extensions/grpc_credentials/example/config.h index e6bf0db6ef4b7..33cd85e6f425d 100644 --- a/source/extensions/grpc_credentials/example/config.h +++ b/source/extensions/grpc_credentials/example/config.h @@ -3,8 +3,6 @@ #include "envoy/config/core/v3/grpc_service.pb.h" #include "envoy/grpc/google_grpc_creds.h" -#include "extensions/grpc_credentials/well_known_names.h" - namespace Envoy { namespace Extensions { namespace GrpcCredentials { @@ -32,7 +30,7 @@ class AccessTokenExampleGrpcCredentialsFactory : public Grpc::GoogleGrpcCredenti getChannelCredentials(const envoy::config::core::v3::GrpcService& grpc_service_config, Api::Api& api) override; - std::string name() const override { return GrpcCredentialsNames::get().AccessTokenExample; } + std::string name() const override { return "envoy.grpc_credentials.access_token_example"; } }; /* diff --git a/source/extensions/grpc_credentials/file_based_metadata/BUILD b/source/extensions/grpc_credentials/file_based_metadata/BUILD index 45f065419f87e..5ffe31ce30b08 100644 --- a/source/extensions/grpc_credentials/file_based_metadata/BUILD +++ b/source/extensions/grpc_credentials/file_based_metadata/BUILD @@ -14,17 +14,13 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.grpc_credentials", external_deps = ["grpc"], - security_posture = "data_plane_agnostic", - status = "alpha", deps = [ "//include/envoy/grpc:google_grpc_creds_interface", "//include/envoy/registry", "//source/common/config:utility_lib", "//source/common/grpc:common_lib", "//source/common/grpc:google_grpc_creds_lib", - "//source/extensions/grpc_credentials:well_known_names", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/config/grpc_credential/v3:pkg_cc_proto", ], diff --git a/source/extensions/grpc_credentials/file_based_metadata/config.cc b/source/extensions/grpc_credentials/file_based_metadata/config.cc index 124961d1d57ad..c05f7cc6d611b 100644 --- a/source/extensions/grpc_credentials/file_based_metadata/config.cc +++ b/source/extensions/grpc_credentials/file_based_metadata/config.cc @@ -28,7 +28,7 @@ FileBasedMetadataGrpcCredentialsFactory::getChannelCredentials( switch (credential.credential_specifier_case()) { case envoy::config::core::v3::GrpcService::GoogleGrpc::CallCredentials:: CredentialSpecifierCase::kFromPlugin: { - if (credential.from_plugin().name() == GrpcCredentialsNames::get().FileBasedMetadata) { + if (credential.from_plugin().name() == "envoy.grpc_credentials.file_based_metadata") { FileBasedMetadataGrpcCredentialsFactory file_based_metadata_credentials_factory; // We don't deal with validation failures here at runtime today, see // https://github.com/envoyproxy/envoy/issues/8010. diff --git a/source/extensions/grpc_credentials/file_based_metadata/config.h b/source/extensions/grpc_credentials/file_based_metadata/config.h index c9d7b3dfeeb52..7b844e356500b 100644 --- a/source/extensions/grpc_credentials/file_based_metadata/config.h +++ b/source/extensions/grpc_credentials/file_based_metadata/config.h @@ -6,8 +6,6 @@ #include "common/protobuf/protobuf.h" -#include "extensions/grpc_credentials/well_known_names.h" - namespace Envoy { namespace Extensions { namespace GrpcCredentials { @@ -32,7 +30,7 @@ class FileBasedMetadataGrpcCredentialsFactory : public Grpc::GoogleGrpcCredentia return std::make_unique(); } - std::string name() const override { return GrpcCredentialsNames::get().FileBasedMetadata; } + std::string name() const override { return "envoy.grpc_credentials.file_based_metadata"; } }; class FileBasedMetadataAuthenticator : public grpc::MetadataCredentialsPlugin { diff --git a/source/extensions/grpc_credentials/well_known_names.h b/source/extensions/grpc_credentials/well_known_names.h deleted file mode 100644 index e654414e9b47e..0000000000000 --- a/source/extensions/grpc_credentials/well_known_names.h +++ /dev/null @@ -1,29 +0,0 @@ -#pragma once - -#include - -#include "common/singleton/const_singleton.h" - -namespace Envoy { -namespace Extensions { -namespace GrpcCredentials { - -/** - * Well-known gRPC Credentials names. - * NOTE: New gRPC Credentials should use the well known name: envoy.grpc_credentials.name. - */ -class GrpcCredentialsNameValues { -public: - // Access Token Example. - const std::string AccessTokenExample = "envoy.grpc_credentials.access_token_example"; - // File Based Metadata credentials - const std::string FileBasedMetadata = "envoy.grpc_credentials.file_based_metadata"; - // AWS IAM - const std::string AwsIam = "envoy.grpc_credentials.aws_iam"; -}; - -using GrpcCredentialsNames = ConstSingleton; - -} // namespace GrpcCredentials -} // namespace Extensions -} // namespace Envoy diff --git a/source/extensions/health_checkers/BUILD b/source/extensions/health_checkers/BUILD deleted file mode 100644 index 40a5e79b39d3b..0000000000000 --- a/source/extensions/health_checkers/BUILD +++ /dev/null @@ -1,19 +0,0 @@ -load( - "//bazel:envoy_build_system.bzl", - "envoy_cc_library", - "envoy_extension_package", -) - -licenses(["notice"]) # Apache 2 - -envoy_extension_package() - -envoy_cc_library( - name = "well_known_names", - hdrs = ["well_known_names.h"], - # well known names files are public as long as they exist. - visibility = ["//visibility:public"], - deps = [ - "//source/common/singleton:const_singleton", - ], -) diff --git a/source/extensions/health_checkers/redis/BUILD b/source/extensions/health_checkers/redis/BUILD index 8689cbecb1076..58d1ad4812a3f 100644 --- a/source/extensions/health_checkers/redis/BUILD +++ b/source/extensions/health_checkers/redis/BUILD @@ -31,15 +31,12 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.health_checkers", - security_posture = "requires_trusted_downstream_and_upstream", deps = [ ":redis", ":utility", "//include/envoy/registry", "//include/envoy/server:health_checker_config_interface", "//source/common/common:assert_lib", - "//source/extensions/health_checkers:well_known_names", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/network/redis_proxy/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/health_checkers/redis/v3:pkg_cc_proto", diff --git a/source/extensions/health_checkers/redis/config.h b/source/extensions/health_checkers/redis/config.h index e51cb8b256cbb..919bd158be0b6 100644 --- a/source/extensions/health_checkers/redis/config.h +++ b/source/extensions/health_checkers/redis/config.h @@ -8,7 +8,6 @@ #include "envoy/server/health_checker_config.h" #include "extensions/health_checkers/redis/redis.h" -#include "extensions/health_checkers/well_known_names.h" namespace Envoy { namespace Extensions { @@ -24,7 +23,7 @@ class RedisHealthCheckerFactory : public Server::Configuration::CustomHealthChec createCustomHealthChecker(const envoy::config::core::v3::HealthCheck& config, Server::Configuration::HealthCheckerFactoryContext& context) override; - std::string name() const override { return HealthCheckerNames::get().RedisHealthChecker; } + std::string name() const override { return "envoy.health_checkers.redis"; } ProtobufTypes::MessagePtr createEmptyConfigProto() override { return ProtobufTypes::MessagePtr{new envoy::extensions::health_checkers::redis::v3::Redis()}; } diff --git a/source/extensions/health_checkers/well_known_names.h b/source/extensions/health_checkers/well_known_names.h deleted file mode 100644 index e17e4a2349a40..0000000000000 --- a/source/extensions/health_checkers/well_known_names.h +++ /dev/null @@ -1,25 +0,0 @@ -#pragma once - -#include - -#include "common/singleton/const_singleton.h" - -namespace Envoy { -namespace Extensions { -namespace HealthCheckers { - -/** - * Well-known health checker names. - * NOTE: New health checkers should use the well known name: envoy.health_checkers.name. - */ -class HealthCheckerNameValues { -public: - // Redis health checker. - const std::string RedisHealthChecker = "envoy.health_checkers.redis"; -}; - -using HealthCheckerNames = ConstSingleton; - -} // namespace HealthCheckers -} // namespace Extensions -} // namespace Envoy diff --git a/source/extensions/http/header_formatters/preserve_case/BUILD b/source/extensions/http/header_formatters/preserve_case/BUILD index 6fde9d6725f30..fffdd69fbf5e5 100644 --- a/source/extensions/http/header_formatters/preserve_case/BUILD +++ b/source/extensions/http/header_formatters/preserve_case/BUILD @@ -12,8 +12,6 @@ envoy_cc_extension( name = "preserve_case_formatter", srcs = ["preserve_case_formatter.cc"], hdrs = ["preserve_case_formatter.h"], - category = "envoy.http.stateful_header_formatters", - security_posture = "robust_to_untrusted_downstream_and_upstream", deps = [ "//include/envoy/registry", "@envoy_api//envoy/extensions/http/header_formatters/preserve_case/v3:pkg_cc_proto", diff --git a/source/extensions/http/original_ip_detection/custom_header/BUILD b/source/extensions/http/original_ip_detection/custom_header/BUILD index 13f1a20009999..53c86cdc4afbb 100644 --- a/source/extensions/http/original_ip_detection/custom_header/BUILD +++ b/source/extensions/http/original_ip_detection/custom_header/BUILD @@ -26,8 +26,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.http.original_ip_detection", - security_posture = "robust_to_untrusted_downstream", # This extension is used from core tests. visibility = ["//visibility:public"], deps = [ diff --git a/source/extensions/http/original_ip_detection/xff/BUILD b/source/extensions/http/original_ip_detection/xff/BUILD index a247f485a1f56..546ba74421664 100644 --- a/source/extensions/http/original_ip_detection/xff/BUILD +++ b/source/extensions/http/original_ip_detection/xff/BUILD @@ -26,8 +26,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.http.original_ip_detection", - security_posture = "robust_to_untrusted_downstream", # This extension is core code. visibility = ["//visibility:public"], deps = [ diff --git a/source/extensions/internal_redirect/allow_listed_routes/BUILD b/source/extensions/internal_redirect/allow_listed_routes/BUILD index f3186dde09df6..3e5edbe960126 100644 --- a/source/extensions/internal_redirect/allow_listed_routes/BUILD +++ b/source/extensions/internal_redirect/allow_listed_routes/BUILD @@ -24,12 +24,10 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.internal_redirect_predicates", # TODO(#9953) clean up by moving the redirect test to extensions. extra_visibility = [ "//test/integration:__subpackages__", ], - security_posture = "robust_to_untrusted_downstream_and_upstream", deps = [ ":allow_listed_routes_lib", "//include/envoy/registry", diff --git a/source/extensions/internal_redirect/previous_routes/BUILD b/source/extensions/internal_redirect/previous_routes/BUILD index ada41e1ed237e..d208998603f69 100644 --- a/source/extensions/internal_redirect/previous_routes/BUILD +++ b/source/extensions/internal_redirect/previous_routes/BUILD @@ -24,12 +24,10 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.internal_redirect_predicates", # TODO(#9953) clean up by moving the redirect test to extensions. extra_visibility = [ "//test/integration:__subpackages__", ], - security_posture = "robust_to_untrusted_downstream_and_upstream", deps = [ ":previous_routes_lib", "//include/envoy/registry", diff --git a/source/extensions/internal_redirect/safe_cross_scheme/BUILD b/source/extensions/internal_redirect/safe_cross_scheme/BUILD index 5936010fed94a..13fb41de556f6 100644 --- a/source/extensions/internal_redirect/safe_cross_scheme/BUILD +++ b/source/extensions/internal_redirect/safe_cross_scheme/BUILD @@ -23,12 +23,10 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.internal_redirect_predicates", # TODO(#9953) clean up by moving the redirect test to extensions. extra_visibility = [ "//test/integration:__subpackages__", ], - security_posture = "robust_to_untrusted_downstream_and_upstream", deps = [ ":safe_cross_scheme_lib", "//include/envoy/registry", diff --git a/source/extensions/io_socket/user_space/BUILD b/source/extensions/io_socket/user_space/BUILD index 18a01f6e1eaec..c430f95989417 100644 --- a/source/extensions/io_socket/user_space/BUILD +++ b/source/extensions/io_socket/user_space/BUILD @@ -12,10 +12,6 @@ envoy_extension_package() envoy_cc_extension( name = "config", srcs = ["config.h"], - category = "envoy.io_socket", - security_posture = "unknown", - status = "wip", - undocumented = True, deps = [ ], ) diff --git a/source/extensions/matching/common_inputs/environment_variable/BUILD b/source/extensions/matching/common_inputs/environment_variable/BUILD index cf54b92130a80..2dacd62538db4 100644 --- a/source/extensions/matching/common_inputs/environment_variable/BUILD +++ b/source/extensions/matching/common_inputs/environment_variable/BUILD @@ -22,8 +22,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.matching.common_inputs", - security_posture = "robust_to_untrusted_downstream", deps = [ ":input_lib", "//include/envoy/matcher:matcher_interface", diff --git a/source/extensions/matching/input_matchers/consistent_hashing/BUILD b/source/extensions/matching/input_matchers/consistent_hashing/BUILD index 753f6ae6756a2..0e72af550db3f 100644 --- a/source/extensions/matching/input_matchers/consistent_hashing/BUILD +++ b/source/extensions/matching/input_matchers/consistent_hashing/BUILD @@ -23,8 +23,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.matching.input_matchers", - security_posture = "robust_to_untrusted_downstream", deps = [ ":consistent_hashing_lib", "//include/envoy/matcher:matcher_interface", diff --git a/source/extensions/rate_limit_descriptors/expr/BUILD b/source/extensions/rate_limit_descriptors/expr/BUILD index 088dd84be9c7a..a1f4dc89d881c 100644 --- a/source/extensions/rate_limit_descriptors/expr/BUILD +++ b/source/extensions/rate_limit_descriptors/expr/BUILD @@ -12,14 +12,12 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.rate_limit_descriptors", copts = select({ "//bazel:windows_x86_64": [], # TODO: fix the windows ANTLR build "//conditions:default": [ "-DUSE_CEL_PARSER", ], }), - security_posture = "unknown", deps = [ "//include/envoy/ratelimit:ratelimit_interface", "//include/envoy/registry", diff --git a/source/extensions/request_id/uuid/BUILD b/source/extensions/request_id/uuid/BUILD index 2c09ede8b5594..feb49f7a39014 100644 --- a/source/extensions/request_id/uuid/BUILD +++ b/source/extensions/request_id/uuid/BUILD @@ -16,8 +16,6 @@ envoy_cc_extension( hdrs = [ "config.h", ], - category = "envoy.request_id", - security_posture = "robust_to_untrusted_downstream_and_upstream", visibility = ["//visibility:public"], deps = [ "//include/envoy/http:request_id_extension_interface", diff --git a/source/extensions/resource_monitors/fixed_heap/BUILD b/source/extensions/resource_monitors/fixed_heap/BUILD index 1e856a6b06d7f..f1ce7fa600253 100644 --- a/source/extensions/resource_monitors/fixed_heap/BUILD +++ b/source/extensions/resource_monitors/fixed_heap/BUILD @@ -25,9 +25,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.resource_monitors", - security_posture = "data_plane_agnostic", - status = "alpha", deps = [ ":fixed_heap_monitor", "//include/envoy/registry", diff --git a/source/extensions/resource_monitors/injected_resource/BUILD b/source/extensions/resource_monitors/injected_resource/BUILD index a84b00fbd76b4..50453d2a1bdcf 100644 --- a/source/extensions/resource_monitors/injected_resource/BUILD +++ b/source/extensions/resource_monitors/injected_resource/BUILD @@ -26,14 +26,11 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.resource_monitors", # TODO(#9953) clean up. extra_visibility = [ "//test/integration:__subpackages__", "//test/common/quic/integration:__subpackages__", ], - security_posture = "data_plane_agnostic", - status = "alpha", deps = [ ":injected_resource_monitor", "//include/envoy/registry", diff --git a/source/extensions/retry/host/omit_canary_hosts/BUILD b/source/extensions/retry/host/omit_canary_hosts/BUILD index 734c5df847361..8e3f446d0848b 100644 --- a/source/extensions/retry/host/omit_canary_hosts/BUILD +++ b/source/extensions/retry/host/omit_canary_hosts/BUILD @@ -21,8 +21,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.retry_host_predicates", - security_posture = "robust_to_untrusted_downstream", deps = [ ":omit_canary_hosts_predicate_lib", "//include/envoy/registry", diff --git a/source/extensions/retry/host/omit_host_metadata/BUILD b/source/extensions/retry/host/omit_host_metadata/BUILD index 51813ad4a4b86..92a916dfa28f1 100644 --- a/source/extensions/retry/host/omit_host_metadata/BUILD +++ b/source/extensions/retry/host/omit_host_metadata/BUILD @@ -23,8 +23,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.retry_host_predicates", - security_posture = "robust_to_untrusted_downstream", deps = [ ":omit_host_metadata_predicate_lib", "//include/envoy/registry", diff --git a/source/extensions/retry/host/previous_hosts/BUILD b/source/extensions/retry/host/previous_hosts/BUILD index 81842e7a67889..ae6e4dd7859b3 100644 --- a/source/extensions/retry/host/previous_hosts/BUILD +++ b/source/extensions/retry/host/previous_hosts/BUILD @@ -21,8 +21,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.retry_host_predicates", - security_posture = "robust_to_untrusted_downstream", deps = [ ":previous_hosts_predicate_lib", "//include/envoy/registry", diff --git a/source/extensions/retry/priority/BUILD b/source/extensions/retry/priority/BUILD deleted file mode 100644 index 22d835b407061..0000000000000 --- a/source/extensions/retry/priority/BUILD +++ /dev/null @@ -1,17 +0,0 @@ -load( - "//bazel:envoy_build_system.bzl", - "envoy_cc_library", - "envoy_extension_package", -) - -licenses(["notice"]) # Apache 2 - -envoy_extension_package() - -envoy_cc_library( - name = "well_known_names", - hdrs = ["well_known_names.h"], - deps = [ - "//source/common/singleton:const_singleton", - ], -) diff --git a/source/extensions/retry/priority/previous_priorities/BUILD b/source/extensions/retry/priority/previous_priorities/BUILD index ae736299dc7f1..9867c5903fe62 100644 --- a/source/extensions/retry/priority/previous_priorities/BUILD +++ b/source/extensions/retry/priority/previous_priorities/BUILD @@ -23,14 +23,11 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.retry_priorities", - security_posture = "robust_to_untrusted_downstream", deps = [ ":previous_priorities_lib", "//include/envoy/registry", "//include/envoy/upstream:retry_interface", "//source/common/protobuf", - "//source/extensions/retry/priority:well_known_names", "@envoy_api//envoy/extensions/retry/priority/previous_priorities/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/retry/priority/previous_priorities/config.h b/source/extensions/retry/priority/previous_priorities/config.h index 3976a94f32974..e6c64b94ebf79 100644 --- a/source/extensions/retry/priority/previous_priorities/config.h +++ b/source/extensions/retry/priority/previous_priorities/config.h @@ -6,7 +6,6 @@ #include "common/protobuf/protobuf.h" #include "extensions/retry/priority/previous_priorities/previous_priorities.h" -#include "extensions/retry/priority/well_known_names.h" namespace Envoy { namespace Extensions { @@ -20,9 +19,7 @@ class PreviousPrioritiesRetryPriorityFactory : public Upstream::RetryPriorityFac ProtobufMessage::ValidationVisitor& validation_visitor, uint32_t max_retries) override; - std::string name() const override { - return RetryPriorityValues::get().PreviousPrioritiesRetryPriority; - } + std::string name() const override { return "envoy.retry_priorities.previous_priorities"; } ProtobufTypes::MessagePtr createEmptyConfigProto() override { return ProtobufTypes::MessagePtr(new envoy::extensions::retry::priority::previous_priorities:: diff --git a/source/extensions/retry/priority/well_known_names.h b/source/extensions/retry/priority/well_known_names.h deleted file mode 100644 index 5e20524e8df66..0000000000000 --- a/source/extensions/retry/priority/well_known_names.h +++ /dev/null @@ -1,26 +0,0 @@ -#pragma once - -#include - -#include "common/singleton/const_singleton.h" - -namespace Envoy { -namespace Extensions { -namespace Retry { -namespace Priority { - -/** - * Well-known retry priority load names. - */ -class RetryPriorityNameValues { -public: - // Previous priority retry priority. Excludes previously attempted priorities during retries. - const std::string PreviousPrioritiesRetryPriority = "envoy.retry_priorities.previous_priorities"; -}; - -using RetryPriorityValues = ConstSingleton; - -} // namespace Priority -} // namespace Retry -} // namespace Extensions -} // namespace Envoy diff --git a/source/extensions/stat_sinks/dog_statsd/BUILD b/source/extensions/stat_sinks/dog_statsd/BUILD index a9a269862dd39..7105afc19f24d 100644 --- a/source/extensions/stat_sinks/dog_statsd/BUILD +++ b/source/extensions/stat_sinks/dog_statsd/BUILD @@ -15,8 +15,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.stats_sinks", - security_posture = "data_plane_agnostic", deps = [ "//include/envoy/registry", "//source/common/network:address_lib", diff --git a/source/extensions/stat_sinks/hystrix/BUILD b/source/extensions/stat_sinks/hystrix/BUILD index 1566d97c6de1f..58fa5ed5ea4b4 100644 --- a/source/extensions/stat_sinks/hystrix/BUILD +++ b/source/extensions/stat_sinks/hystrix/BUILD @@ -15,8 +15,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.stats_sinks", - security_posture = "data_plane_agnostic", deps = [ ":hystrix_lib", "//include/envoy/registry", diff --git a/source/extensions/stat_sinks/metrics_service/BUILD b/source/extensions/stat_sinks/metrics_service/BUILD index 28afad7f25ac9..9b95a7760ae4e 100644 --- a/source/extensions/stat_sinks/metrics_service/BUILD +++ b/source/extensions/stat_sinks/metrics_service/BUILD @@ -43,8 +43,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.stats_sinks", - security_posture = "data_plane_agnostic", deps = [ "//include/envoy/registry", "//source/common/common:assert_lib", diff --git a/source/extensions/stat_sinks/statsd/BUILD b/source/extensions/stat_sinks/statsd/BUILD index 8d4c70c3131ab..cdfaddf114db5 100644 --- a/source/extensions/stat_sinks/statsd/BUILD +++ b/source/extensions/stat_sinks/statsd/BUILD @@ -14,8 +14,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.stats_sinks", - security_posture = "data_plane_agnostic", # Legacy test use. TODO(#9953) clean up. deps = [ "//include/envoy/registry", diff --git a/source/extensions/stat_sinks/wasm/BUILD b/source/extensions/stat_sinks/wasm/BUILD index 6c6b6523bb801..e8ee99ab1f29b 100644 --- a/source/extensions/stat_sinks/wasm/BUILD +++ b/source/extensions/stat_sinks/wasm/BUILD @@ -15,9 +15,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.stats_sinks", - security_posture = "data_plane_agnostic", - status = "alpha", deps = [ ":wasm_stat_sink_lib", "//include/envoy/registry", diff --git a/source/extensions/tracers/datadog/BUILD b/source/extensions/tracers/datadog/BUILD index 164a1d73c1f94..0c314374f9683 100644 --- a/source/extensions/tracers/datadog/BUILD +++ b/source/extensions/tracers/datadog/BUILD @@ -34,8 +34,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.tracers", - security_posture = "robust_to_untrusted_downstream", deps = [ ":datadog_tracer_lib", "//source/extensions/tracers/common:factory_base_lib", diff --git a/source/extensions/tracers/dynamic_ot/BUILD b/source/extensions/tracers/dynamic_ot/BUILD index c7ce76f3267f2..8a3bb1937ed38 100644 --- a/source/extensions/tracers/dynamic_ot/BUILD +++ b/source/extensions/tracers/dynamic_ot/BUILD @@ -29,8 +29,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.tracers", - security_posture = "robust_to_untrusted_downstream", deps = [ ":dynamic_opentracing_driver_lib", "//source/extensions/tracers/common:factory_base_lib", diff --git a/source/extensions/tracers/lightstep/BUILD b/source/extensions/tracers/lightstep/BUILD index 72f4ff80f1461..3c9cf3c57e0dc 100644 --- a/source/extensions/tracers/lightstep/BUILD +++ b/source/extensions/tracers/lightstep/BUILD @@ -35,8 +35,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.tracers", - security_posture = "robust_to_untrusted_downstream", deps = [ ":lightstep_tracer_lib", "//source/common/config:datasource_lib", diff --git a/source/extensions/tracers/opencensus/BUILD b/source/extensions/tracers/opencensus/BUILD index a1c414cca9d7e..f661bed6ddf62 100644 --- a/source/extensions/tracers/opencensus/BUILD +++ b/source/extensions/tracers/opencensus/BUILD @@ -16,8 +16,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.tracers", - security_posture = "robust_to_untrusted_downstream", deps = [ ":opencensus_tracer_impl", "//source/extensions/tracers/common:factory_base_lib", diff --git a/source/extensions/tracers/skywalking/BUILD b/source/extensions/tracers/skywalking/BUILD index 41da2c3f61f37..1265cd752cfaf 100644 --- a/source/extensions/tracers/skywalking/BUILD +++ b/source/extensions/tracers/skywalking/BUILD @@ -67,9 +67,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.tracers", - security_posture = "robust_to_untrusted_downstream", - status = "wip", deps = [ ":skywalking_tracer_lib", "//source/common/config:datasource_lib", diff --git a/source/extensions/tracers/xray/BUILD b/source/extensions/tracers/xray/BUILD index 797e8a84e407d..8048e35897e55 100644 --- a/source/extensions/tracers/xray/BUILD +++ b/source/extensions/tracers/xray/BUILD @@ -58,8 +58,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.tracers", - security_posture = "robust_to_untrusted_downstream", deps = [ ":xray_lib", "//source/common/config:datasource_lib", diff --git a/source/extensions/tracers/zipkin/BUILD b/source/extensions/tracers/zipkin/BUILD index 34e00329e121a..0c39a41bf535a 100644 --- a/source/extensions/tracers/zipkin/BUILD +++ b/source/extensions/tracers/zipkin/BUILD @@ -67,12 +67,10 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.tracers", # Legacy test use. TODO(#9953) clean up. extra_visibility = [ "//test/server:__subpackages__", ], - security_posture = "robust_to_untrusted_downstream", deps = [ ":zipkin_lib", "//source/extensions/tracers/common:factory_base_lib", diff --git a/source/extensions/transport_sockets/alts/BUILD b/source/extensions/transport_sockets/alts/BUILD index 587e9a2ecf29d..3e393a3781cc8 100644 --- a/source/extensions/transport_sockets/alts/BUILD +++ b/source/extensions/transport_sockets/alts/BUILD @@ -34,14 +34,9 @@ envoy_cc_extension( hdrs = [ "config.h", ], - category = ( - "envoy.transport_sockets.downstream", - "envoy.transport_sockets.upstream", - ), external_deps = [ "abseil_node_hash_set", ], - security_posture = "robust_to_untrusted_downstream_and_upstream", deps = [ ":tsi_handshaker", ":tsi_socket", diff --git a/source/extensions/transport_sockets/proxy_protocol/BUILD b/source/extensions/transport_sockets/proxy_protocol/BUILD index e268b828524c3..403fb996e900c 100644 --- a/source/extensions/transport_sockets/proxy_protocol/BUILD +++ b/source/extensions/transport_sockets/proxy_protocol/BUILD @@ -13,10 +13,6 @@ envoy_cc_extension( name = "upstream_config", srcs = ["config.cc"], hdrs = ["config.h"], - category = ( - "envoy.transport_sockets.upstream", - ), - security_posture = "robust_to_untrusted_downstream_and_upstream", # header generated in Envoy, so can't be faked deps = [ ":upstream_proxy_protocol", "//include/envoy/network:transport_socket_interface", diff --git a/source/extensions/transport_sockets/raw_buffer/BUILD b/source/extensions/transport_sockets/raw_buffer/BUILD index 94a2bee0a980a..9c9ad99107d71 100644 --- a/source/extensions/transport_sockets/raw_buffer/BUILD +++ b/source/extensions/transport_sockets/raw_buffer/BUILD @@ -14,11 +14,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = ( - "envoy.transport_sockets.downstream", - "envoy.transport_sockets.upstream", - ), - security_posture = "requires_trusted_downstream_and_upstream", # This is core Envoy config. visibility = ["//visibility:public"], deps = [ diff --git a/source/extensions/transport_sockets/starttls/BUILD b/source/extensions/transport_sockets/starttls/BUILD index f3414c9837e37..31b016d9d97cc 100644 --- a/source/extensions/transport_sockets/starttls/BUILD +++ b/source/extensions/transport_sockets/starttls/BUILD @@ -15,11 +15,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = ( - "envoy.transport_sockets.downstream", - "envoy.transport_sockets.upstream", - ), - security_posture = "robust_to_untrusted_downstream_and_upstream", visibility = ["//visibility:public"], deps = [ ":starttls_socket_lib", diff --git a/source/extensions/transport_sockets/tap/BUILD b/source/extensions/transport_sockets/tap/BUILD index e97cb4f1255c3..6875dfe77b82e 100644 --- a/source/extensions/transport_sockets/tap/BUILD +++ b/source/extensions/transport_sockets/tap/BUILD @@ -51,17 +51,11 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = ( - "envoy.transport_sockets.downstream", - "envoy.transport_sockets.upstream", - ), # TODO(#9953) clean up. extra_visibility = [ "//test/common/access_log:__subpackages__", "//test/integration:__subpackages__", ], - security_posture = "requires_trusted_downstream_and_upstream", - status = "alpha", deps = [ ":tap_config_impl", ":tap_lib", diff --git a/source/extensions/transport_sockets/tls/BUILD b/source/extensions/transport_sockets/tls/BUILD index 29dfa2be2a9dc..766ba6ff34914 100644 --- a/source/extensions/transport_sockets/tls/BUILD +++ b/source/extensions/transport_sockets/tls/BUILD @@ -15,11 +15,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = ( - "envoy.transport_sockets.downstream", - "envoy.transport_sockets.upstream", - ), - security_posture = "robust_to_untrusted_downstream_and_upstream", # TLS is core functionality. visibility = ["//visibility:public"], deps = [ diff --git a/source/extensions/transport_sockets/tls/cert_validator/spiffe/BUILD b/source/extensions/transport_sockets/tls/cert_validator/spiffe/BUILD index d6f74254f3a07..812d2f17b7d1c 100644 --- a/source/extensions/transport_sockets/tls/cert_validator/spiffe/BUILD +++ b/source/extensions/transport_sockets/tls/cert_validator/spiffe/BUILD @@ -16,14 +16,11 @@ envoy_cc_extension( hdrs = [ "spiffe_validator.h", ], - category = "envoy.tls.cert_validator", external_deps = [ "ssl", "abseil_base", "abseil_hash", ], - security_posture = "unknown", - status = "wip", visibility = ["//visibility:public"], deps = [ "//include/envoy/ssl:context_config_interface", diff --git a/source/extensions/transport_sockets/tls/context_config_impl.cc b/source/extensions/transport_sockets/tls/context_config_impl.cc index 40f113c95fefc..d20f55d817e84 100644 --- a/source/extensions/transport_sockets/tls/context_config_impl.cc +++ b/source/extensions/transport_sockets/tls/context_config_impl.cc @@ -26,8 +26,8 @@ namespace { std::vector getTlsCertificateConfigProviders( const envoy::extensions::transport_sockets::tls::v3::CommonTlsContext& config, Server::Configuration::TransportSocketFactoryContext& factory_context) { + std::vector providers; if (!config.tls_certificates().empty()) { - std::vector providers; for (const auto& tls_certificate : config.tls_certificates()) { if (!tls_certificate.has_private_key_provider() && !tls_certificate.has_certificate_chain() && !tls_certificate.has_private_key()) { @@ -39,20 +39,22 @@ std::vector getTlsCertificateConf return providers; } if (!config.tls_certificate_sds_secret_configs().empty()) { - const auto& sds_secret_config = config.tls_certificate_sds_secret_configs(0); - if (sds_secret_config.has_sds_config()) { - // Fetch dynamic secret. - return {factory_context.secretManager().findOrCreateTlsCertificateProvider( - sds_secret_config.sds_config(), sds_secret_config.name(), factory_context)}; - } else { - // Load static secret. - auto secret_provider = factory_context.secretManager().findStaticTlsCertificateProvider( - sds_secret_config.name()); - if (!secret_provider) { - throw EnvoyException(fmt::format("Unknown static secret: {}", sds_secret_config.name())); + for (const auto& sds_secret_config : config.tls_certificate_sds_secret_configs()) { + if (sds_secret_config.has_sds_config()) { + // Fetch dynamic secret. + providers.push_back(factory_context.secretManager().findOrCreateTlsCertificateProvider( + sds_secret_config.sds_config(), sds_secret_config.name(), factory_context)); + } else { + // Load static secret. + auto secret_provider = factory_context.secretManager().findStaticTlsCertificateProvider( + sds_secret_config.name()); + if (!secret_provider) { + throw EnvoyException(fmt::format("Unknown static secret: {}", sds_secret_config.name())); + } + providers.push_back(secret_provider); } - return {secret_provider}; } + return providers; } return {}; } @@ -247,25 +249,27 @@ Ssl::CertificateValidationContextConfigPtr ContextConfigImpl::getCombinedValidat } void ContextConfigImpl::setSecretUpdateCallback(std::function callback) { - if (!tls_certificate_providers_.empty()) { - // Once tls_certificate_config_ receives new secret, this callback updates - // ContextConfigImpl::tls_certificate_config_ with new secret. - tc_update_callback_handle_ = - tls_certificate_providers_[0]->addUpdateCallback([this, callback]() { - // This breaks multiple certificate support, but today SDS is only single cert. - // TODO(htuch): Fix this when SDS goes multi-cert. + // When any of tls_certificate_providers_ receives a new secret, this callback updates + // ContextConfigImpl::tls_certificate_configs_ with new secret. + for (const auto& tls_certificate_provider : tls_certificate_providers_) { + tc_update_callback_handles_.push_back( + tls_certificate_provider->addUpdateCallback([this, callback]() { tls_certificate_configs_.clear(); - tls_certificate_configs_.emplace_back(*tls_certificate_providers_[0]->secret(), nullptr, - api_); + for (const auto& tls_certificate_provider : tls_certificate_providers_) { + auto* secret = tls_certificate_provider->secret(); + if (secret != nullptr) { + tls_certificate_configs_.emplace_back(*secret, nullptr, api_); + } + } callback(); - }); + })); } if (certificate_validation_context_provider_) { if (default_cvc_) { // Once certificate_validation_context_provider_ receives new secret, this callback updates // ContextConfigImpl::validation_context_config_ with a combined certificate validation - // context. The combined certificate validation context is created by merging new secret into - // default_cvc_. + // context. The combined certificate validation context is created by merging new secret + // into default_cvc_. cvc_update_callback_handle_ = certificate_validation_context_provider_->addUpdateCallback([this, callback]() { validation_context_config_ = getCombinedValidationContextConfig( diff --git a/source/extensions/transport_sockets/tls/context_config_impl.h b/source/extensions/transport_sockets/tls/context_config_impl.h index 85ef556d874b7..549575aa0937e 100644 --- a/source/extensions/transport_sockets/tls/context_config_impl.h +++ b/source/extensions/transport_sockets/tls/context_config_impl.h @@ -87,7 +87,7 @@ class ContextConfigImpl : public virtual Ssl::ContextConfig { default_cvc_; std::vector tls_certificate_providers_; // Handle for TLS certificate dynamic secret callback. - Envoy::Common::CallbackHandlePtr tc_update_callback_handle_; + std::vector tc_update_callback_handles_; Secret::CertificateValidationContextConfigProviderSharedPtr certificate_validation_context_provider_; // Handle for certificate validation context dynamic secret callback. diff --git a/source/extensions/transport_sockets/tls/utility.cc b/source/extensions/transport_sockets/tls/utility.cc index 10688dd6052b2..1dfa5df48b696 100644 --- a/source/extensions/transport_sockets/tls/utility.cc +++ b/source/extensions/transport_sockets/tls/utility.cc @@ -14,6 +14,31 @@ namespace Extensions { namespace TransportSockets { namespace Tls { +#if BORINGSSL_API_VERSION < 10 +static constexpr absl::string_view SSL_ERROR_NONE_MESSAGE = "NONE"; +static constexpr absl::string_view SSL_ERROR_SSL_MESSAGE = "SSL"; +static constexpr absl::string_view SSL_ERROR_WANT_READ_MESSAGE = "WANT_READ"; +static constexpr absl::string_view SSL_ERROR_WANT_WRITE_MESSAGE = "WANT_WRITE"; +static constexpr absl::string_view SSL_ERROR_WANT_X509_LOOPUP_MESSAGE = "WANT_X509_LOOKUP"; +static constexpr absl::string_view SSL_ERROR_SYSCALL_MESSAGE = "SYSCALL"; +static constexpr absl::string_view SSL_ERROR_ZERO_RETURN_MESSAGE = "ZERO_RETURN"; +static constexpr absl::string_view SSL_ERROR_WANT_CONNECT_MESSAGE = "WANT_CONNECT"; +static constexpr absl::string_view SSL_ERROR_WANT_ACCEPT_MESSAGE = "WANT_ACCEPT"; +static constexpr absl::string_view SSL_ERROR_WANT_CHANNEL_ID_LOOKUP_MESSAGE = + "WANT_CHANNEL_ID_LOOKUP"; +static constexpr absl::string_view SSL_ERROR_PENDING_SESSION_MESSAGE = "PENDING_SESSION"; +static constexpr absl::string_view SSL_ERROR_PENDING_CERTIFICATE_MESSAGE = "PENDING_CERTIFICATE"; +static constexpr absl::string_view SSL_ERROR_WANT_PRIVATE_KEY_OPERATION_MESSAGE = + "WANT_PRIVATE_KEY_OPERATION"; +static constexpr absl::string_view SSL_ERROR_PENDING_TICKET_MESSAGE = "PENDING_TICKET"; +static constexpr absl::string_view SSL_ERROR_EARLY_DATA_REJECTED_MESSAGE = "EARLY_DATA_REJECTED"; +static constexpr absl::string_view SSL_ERROR_WANT_CERTIFICATE_VERIFY_MESSAGE = + "WANT_CERTIFICATE_VERIFY"; +static constexpr absl::string_view SSL_ERROR_HANDOFF_MESSAGE = "HANDOFF"; +static constexpr absl::string_view SSL_ERROR_HANDBACK_MESSAGE = "HANDBACK"; +#endif +static constexpr absl::string_view SSL_ERROR_UNKNOWN_ERROR_MESSAGE = "UNKNOWN_ERROR"; + Envoy::Ssl::CertificateDetailsPtr Utility::certificateDetails(X509* cert, const std::string& path, TimeSource& time_source) { Envoy::Ssl::CertificateDetailsPtr certificate_details = @@ -255,6 +280,9 @@ absl::optional Utility::getLastCryptoError() { } absl::string_view Utility::getErrorDescription(int err) { +#if BORINGSSL_API_VERSION < 10 + // TODO(davidben): Remove this and the corresponding SSL_ERROR_*_MESSAGE constants when the FIPS + // build is updated to a later version. switch (err) { case SSL_ERROR_NONE: return SSL_ERROR_NONE_MESSAGE; @@ -292,10 +320,15 @@ absl::string_view Utility::getErrorDescription(int err) { return SSL_ERROR_HANDOFF_MESSAGE; case SSL_ERROR_HANDBACK: return SSL_ERROR_HANDBACK_MESSAGE; - default: - ENVOY_BUG(false, "Unknown BoringSSL error had occurred"); - return SSL_ERROR_UNKNOWN_ERROR_MESSAGE; } +#else + const char* description = SSL_error_description(err); + if (description) { + return description; + } +#endif + ENVOY_BUG(false, "Unknown BoringSSL error had occurred"); + return SSL_ERROR_UNKNOWN_ERROR_MESSAGE; } } // namespace Tls diff --git a/source/extensions/transport_sockets/tls/utility.h b/source/extensions/transport_sockets/tls/utility.h index 2b4c2ab2dff28..3bb48b2aa1481 100644 --- a/source/extensions/transport_sockets/tls/utility.h +++ b/source/extensions/transport_sockets/tls/utility.h @@ -17,29 +17,6 @@ namespace TransportSockets { namespace Tls { namespace Utility { -static constexpr absl::string_view SSL_ERROR_NONE_MESSAGE = "NONE"; -static constexpr absl::string_view SSL_ERROR_SSL_MESSAGE = "SSL"; -static constexpr absl::string_view SSL_ERROR_WANT_READ_MESSAGE = "WANT_READ"; -static constexpr absl::string_view SSL_ERROR_WANT_WRITE_MESSAGE = "WANT_WRITE"; -static constexpr absl::string_view SSL_ERROR_WANT_X509_LOOPUP_MESSAGE = "WANT_X509_LOOKUP"; -static constexpr absl::string_view SSL_ERROR_SYSCALL_MESSAGE = "SYSCALL"; -static constexpr absl::string_view SSL_ERROR_ZERO_RETURN_MESSAGE = "ZERO_RETURN"; -static constexpr absl::string_view SSL_ERROR_WANT_CONNECT_MESSAGE = "WANT_CONNECT"; -static constexpr absl::string_view SSL_ERROR_WANT_ACCEPT_MESSAGE = "WANT_ACCEPT"; -static constexpr absl::string_view SSL_ERROR_WANT_CHANNEL_ID_LOOKUP_MESSAGE = - "WANT_CHANNEL_ID_LOOKUP"; -static constexpr absl::string_view SSL_ERROR_PENDING_SESSION_MESSAGE = "PENDING_SESSION"; -static constexpr absl::string_view SSL_ERROR_PENDING_CERTIFICATE_MESSAGE = "PENDING_CERTIFICATE"; -static constexpr absl::string_view SSL_ERROR_WANT_PRIVATE_KEY_OPERATION_MESSAGE = - "WANT_PRIVATE_KEY_OPERATION"; -static constexpr absl::string_view SSL_ERROR_PENDING_TICKET_MESSAGE = "PENDING_TICKET"; -static constexpr absl::string_view SSL_ERROR_EARLY_DATA_REJECTED_MESSAGE = "EARLY_DATA_REJECTED"; -static constexpr absl::string_view SSL_ERROR_WANT_CERTIFICATE_VERIFY_MESSAGE = - "WANT_CERTIFICATE_VERIFY"; -static constexpr absl::string_view SSL_ERROR_HANDOFF_MESSAGE = "HANDOFF"; -static constexpr absl::string_view SSL_ERROR_HANDBACK_MESSAGE = "HANDBACK"; -static constexpr absl::string_view SSL_ERROR_UNKNOWN_ERROR_MESSAGE = "UNKNOWN_ERROR"; - Envoy::Ssl::CertificateDetailsPtr certificateDetails(X509* cert, const std::string& path, TimeSource& time_source); diff --git a/source/extensions/upstreams/http/BUILD b/source/extensions/upstreams/http/BUILD index 198a0b12b4fc1..247274b2fa85b 100644 --- a/source/extensions/upstreams/http/BUILD +++ b/source/extensions/upstreams/http/BUILD @@ -12,8 +12,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.upstreams", - security_posture = "robust_to_untrusted_downstream", # This is core Envoy config. visibility = ["//visibility:public"], deps = [ diff --git a/source/extensions/upstreams/http/generic/BUILD b/source/extensions/upstreams/http/generic/BUILD index 1e2c0d2119e7a..759f4626f205b 100644 --- a/source/extensions/upstreams/http/generic/BUILD +++ b/source/extensions/upstreams/http/generic/BUILD @@ -16,8 +16,6 @@ envoy_cc_extension( hdrs = [ "config.h", ], - category = "envoy.upstreams", - security_posture = "robust_to_untrusted_downstream", visibility = ["//visibility:public"], deps = [ "//source/extensions/upstreams/http/http:upstream_request_lib", diff --git a/source/extensions/upstreams/http/http/BUILD b/source/extensions/upstreams/http/http/BUILD index 132d065cabb3b..4a4bd1be575f0 100644 --- a/source/extensions/upstreams/http/http/BUILD +++ b/source/extensions/upstreams/http/http/BUILD @@ -17,8 +17,6 @@ envoy_cc_extension( hdrs = [ "config.h", ], - category = "envoy.upstreams", - security_posture = "robust_to_untrusted_downstream", visibility = ["//visibility:public"], deps = [ ":upstream_request_lib", diff --git a/source/extensions/upstreams/http/http/upstream_request.h b/source/extensions/upstreams/http/http/upstream_request.h index 94f5a0a0fda6f..c36dbb4f1a97a 100644 --- a/source/extensions/upstreams/http/http/upstream_request.h +++ b/source/extensions/upstreams/http/http/upstream_request.h @@ -82,6 +82,10 @@ class HttpUpstream : public Router::GenericUpstream, public Envoy::Http::StreamC request_encoder_->getStream().resetStream(Envoy::Http::StreamResetReason::LocalReset); } + void setAccount(Buffer::BufferMemoryAccountSharedPtr account) override { + request_encoder_->getStream().setAccount(std::move(account)); + } + // Http::StreamCallbacks void onResetStream(Envoy::Http::StreamResetReason reason, absl::string_view transport_failure_reason) override { diff --git a/source/extensions/upstreams/http/tcp/BUILD b/source/extensions/upstreams/http/tcp/BUILD index 46169ea4b14cc..95b2d94dbae47 100644 --- a/source/extensions/upstreams/http/tcp/BUILD +++ b/source/extensions/upstreams/http/tcp/BUILD @@ -17,8 +17,6 @@ envoy_cc_extension( hdrs = [ "config.h", ], - category = "envoy.upstreams", - security_posture = "robust_to_untrusted_downstream", visibility = ["//visibility:public"], deps = [ ":upstream_request_lib", diff --git a/source/extensions/upstreams/http/tcp/upstream_request.h b/source/extensions/upstreams/http/tcp/upstream_request.h index 3a002f4b7a128..bffc7ee447d1d 100644 --- a/source/extensions/upstreams/http/tcp/upstream_request.h +++ b/source/extensions/upstreams/http/tcp/upstream_request.h @@ -74,6 +74,7 @@ class TcpUpstream : public Router::GenericUpstream, void encodeTrailers(const Envoy::Http::RequestTrailerMap&) override; void readDisable(bool disable) override; void resetStream() override; + void setAccount(Buffer::BufferMemoryAccountSharedPtr) override {} // Tcp::ConnectionPool::UpstreamCallbacks void onUpstreamData(Buffer::Instance& data, bool end_stream) override; diff --git a/source/extensions/upstreams/tcp/generic/BUILD b/source/extensions/upstreams/tcp/generic/BUILD index 2320d1ea51ef1..673d44aeae318 100644 --- a/source/extensions/upstreams/tcp/generic/BUILD +++ b/source/extensions/upstreams/tcp/generic/BUILD @@ -16,8 +16,6 @@ envoy_cc_extension( hdrs = [ "config.h", ], - category = "envoy.upstreams", - security_posture = "robust_to_untrusted_downstream", visibility = ["//visibility:public"], deps = [ "//source/common/http:codec_client_lib", diff --git a/source/extensions/upstreams/tcp/generic/config.cc b/source/extensions/upstreams/tcp/generic/config.cc index 1a71c3c5bc04b..7d708c4a666a1 100644 --- a/source/extensions/upstreams/tcp/generic/config.cc +++ b/source/extensions/upstreams/tcp/generic/config.cc @@ -18,8 +18,8 @@ TcpProxy::GenericConnPoolPtr GenericConnPoolFactory::createGenericConnPool( if (config.has_value()) { auto pool_type = ((thread_local_cluster.info()->features() & Upstream::ClusterInfo::Features::HTTP2) != 0) - ? Http::CodecClient::Type::HTTP2 - : Http::CodecClient::Type::HTTP1; + ? Http::CodecType::HTTP2 + : Http::CodecType::HTTP1; auto ret = std::make_unique( thread_local_cluster, context, config.value(), upstream_callbacks, pool_type); return (ret->valid() ? std::move(ret) : nullptr); diff --git a/source/extensions/wasm_runtime/null/BUILD b/source/extensions/wasm_runtime/null/BUILD index e66dce75d6f3d..1dbb4846e20b4 100644 --- a/source/extensions/wasm_runtime/null/BUILD +++ b/source/extensions/wasm_runtime/null/BUILD @@ -11,9 +11,6 @@ envoy_extension_package() envoy_cc_extension( name = "config", srcs = ["config.cc"], - category = "envoy.wasm.runtime", - security_posture = "unknown", - status = "alpha", deps = [ "//include/envoy/registry", "//source/extensions/common/wasm:wasm_runtime_factory_interface", diff --git a/source/extensions/wasm_runtime/v8/BUILD b/source/extensions/wasm_runtime/v8/BUILD index 8024375f64463..45dd6833558f5 100644 --- a/source/extensions/wasm_runtime/v8/BUILD +++ b/source/extensions/wasm_runtime/v8/BUILD @@ -12,9 +12,6 @@ envoy_extension_package() envoy_cc_extension( name = "config", srcs = ["config.cc"], - category = "envoy.wasm.runtime", - security_posture = "unknown", - status = "alpha", deps = [ "//include/envoy/registry", "//source/extensions/common/wasm:wasm_runtime_factory_interface", diff --git a/source/extensions/wasm_runtime/wasmtime/BUILD b/source/extensions/wasm_runtime/wasmtime/BUILD index 47923bd0caa34..83ee6552fe39a 100644 --- a/source/extensions/wasm_runtime/wasmtime/BUILD +++ b/source/extensions/wasm_runtime/wasmtime/BUILD @@ -12,9 +12,6 @@ envoy_extension_package() envoy_cc_extension( name = "config", srcs = ["config.cc"], - category = "envoy.wasm.runtime", - security_posture = "unknown", - status = "alpha", deps = [ "//include/envoy/registry", "//source/extensions/common/wasm:wasm_runtime_factory_interface", diff --git a/source/extensions/wasm_runtime/wavm/BUILD b/source/extensions/wasm_runtime/wavm/BUILD index f2b8c69ae785d..cca25e7aaace4 100644 --- a/source/extensions/wasm_runtime/wavm/BUILD +++ b/source/extensions/wasm_runtime/wavm/BUILD @@ -12,9 +12,6 @@ envoy_extension_package() envoy_cc_extension( name = "config", srcs = ["config.cc"], - category = "envoy.wasm.runtime", - security_posture = "unknown", - status = "alpha", deps = [ "//include/envoy/registry", "//source/extensions/common/wasm:wasm_runtime_factory_interface", diff --git a/source/extensions/watchdog/profile_action/BUILD b/source/extensions/watchdog/profile_action/BUILD index 8da916b007ad9..6c0ab2f392d6a 100644 --- a/source/extensions/watchdog/profile_action/BUILD +++ b/source/extensions/watchdog/profile_action/BUILD @@ -33,9 +33,6 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], - category = "envoy.guarddog_actions", - security_posture = "data_plane_agnostic", - status = "alpha", deps = [ ":profile_action_lib", "//include/envoy/registry", diff --git a/source/server/active_udp_listener.h b/source/server/active_udp_listener.h index e122b65d2e42e..08472bc853c57 100644 --- a/source/server/active_udp_listener.h +++ b/source/server/active_udp_listener.h @@ -8,6 +8,8 @@ #include "envoy/network/listen_socket.h" #include "envoy/network/listener.h" +#include "common/network/utility.h" + #include "server/active_listener_base.h" namespace Envoy { @@ -82,6 +84,10 @@ class ActiveRawUdpListener : public ActiveUdpListenerBase, void onWriteReady(const Network::Socket& socket) override; void onReceiveError(Api::IoError::IoErrorCode error_code) override; Network::UdpPacketWriter& udpPacketWriter() override { return *udp_packet_writer_; } + size_t numPacketsExpectedPerEventLoop() const final { + // TODO(mattklein123) change this to a reasonable number if needed. + return Network::MAX_NUM_PACKETS_PER_EVENT_LOOP; + } // Network::UdpWorker void onDataWorker(Network::UdpRecvData&& data) override; diff --git a/source/server/admin/admin.h b/source/server/admin/admin.h index 84d5236e212bc..5faa06b141438 100644 --- a/source/server/admin/admin.h +++ b/source/server/admin/admin.h @@ -177,6 +177,7 @@ class AdminImpl : public Admin, const Http::Http1Settings& http1Settings() const override { return http1_settings_; } bool shouldNormalizePath() const override { return true; } bool shouldMergeSlashes() const override { return true; } + bool shouldStripTrailingHostDot() const override { return false; } Http::StripPortType stripPortType() const override { return Http::StripPortType::None; } envoy::config::core::v3::HttpProtocolOptions::HeadersWithUnderscoresAction headersWithUnderscoresAction() const override { diff --git a/source/server/server.h b/source/server/server.h index 0c6e027d58ecd..066fb07a844ca 100644 --- a/source/server/server.h +++ b/source/server/server.h @@ -73,8 +73,8 @@ struct ServerCompilationSettingsStats { COUNTER(static_unknown_fields) \ COUNTER(dropped_stat_flushes) \ GAUGE(concurrency, NeverImport) \ - GAUGE(days_until_first_cert_expiring, Accumulate) \ - GAUGE(seconds_until_first_ocsp_response_expiring, Accumulate) \ + GAUGE(days_until_first_cert_expiring, NeverImport) \ + GAUGE(seconds_until_first_ocsp_response_expiring, NeverImport) \ GAUGE(hot_restart_epoch, NeverImport) \ /* hot_restart_generation is an Accumulate gauge; we omit it here for testing dynamics. */ \ GAUGE(live, NeverImport) \ diff --git a/test/common/buffer/buffer_fuzz.cc b/test/common/buffer/buffer_fuzz.cc index 493f31b8b31db..1e8faab75182f 100644 --- a/test/common/buffer/buffer_fuzz.cc +++ b/test/common/buffer/buffer_fuzz.cc @@ -193,6 +193,7 @@ class StringBuffer : public Buffer::Instance { // WatermarkBuffer implementations. ASSERT(false); } + uint32_t highWatermark() const override { return 0; } bool highWatermarkTriggered() const override { return false; } diff --git a/test/common/common/base64_test.cc b/test/common/common/base64_test.cc index ff9382191e896..04ad1dfb24b9e 100644 --- a/test/common/common/base64_test.cc +++ b/test/common/common/base64_test.cc @@ -132,47 +132,6 @@ TEST(Base64Test, BinaryBufferEncode) { EXPECT_EQ("AAECAwgKCQCqvN4=", Base64::encode(buffer, 30)); } -TEST(Base64Test, CompletePadding) { - struct CompletePaddingBase64UrlTestCases { - std::string base64, base64_with_padding; - }; - - // For base64 encoding, there are only three length needed to test - // - 3n bytes => 4n bytes, no padding needed - // - 3n + 1 bytes => 4n + 2 bytes, 2 padding needed - // - 3n + 2 bytes => 4n + 3 bytes, 1 padding needed - CompletePaddingBase64UrlTestCases testCases[3] = { - // Payload text(3n bytes): - {"eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG8iLCJpYXQiOjE1MTYyMzkwMjJ" - "9", - // No padding added. - "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG8iLCJpYXQiOjE1MTYyMzkwMjJ" - "9"}, - // Payload text(3n + 1 bytes): - {"eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2" - "MjM5MDIyfQ", - // 2 padding added. - "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2" - "MjM5MDIyfQ=="}, - // Payload text(3n + 2 bytes): - {"eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lZSIsImlhdCI6MTUx" - "NjIzOTAyMn0", - // 1 padding added. - "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lZSIsImlhdCI6MTUx" - "NjIzOTAyMn0="}}; - for (auto& tc : testCases) { - // Ensure these two base64 binaries are equivalent after decoding. - EXPECT_EQ(Base64::decodeWithoutPadding(tc.base64), - Base64::decodeWithoutPadding(tc.base64_with_padding)); - // Ensure the `base64_with_padding` is correctly padded. - EXPECT_NE(Base64::decode(tc.base64_with_padding), ""); - - std::string base64_padded = tc.base64; - Base64::completePadding(base64_padded); - EXPECT_EQ(base64_padded, tc.base64_with_padding); - } -} - TEST(Base64UrlTest, EncodeString) { EXPECT_EQ("", Base64Url::encode("", 0)); EXPECT_EQ("AAA", Base64Url::encode("\0\0", 2)); diff --git a/test/common/common/scope_tracker_test.cc b/test/common/common/scope_tracker_test.cc index 1c5451660f049..ee53c88a537ce 100644 --- a/test/common/common/scope_tracker_test.cc +++ b/test/common/common/scope_tracker_test.cc @@ -18,7 +18,7 @@ using testing::_; TEST(ScopeTrackerScopeStateTest, ShouldManageTrackedObjectOnDispatcherStack) { Api::ApiPtr api(Api::createApiForTest()); Event::DispatcherPtr dispatcher(api->allocateDispatcher("test_thread")); - MockScopedTrackedObject tracked_object; + MockScopeTrackedObject tracked_object; { ScopeTrackerScopeState scope(&tracked_object, *dispatcher); // Check that the tracked_object is on the tracked object stack diff --git a/test/common/event/dispatcher_impl_test.cc b/test/common/event/dispatcher_impl_test.cc index fab86c343f1bf..9525ed6ad61c1 100644 --- a/test/common/event/dispatcher_impl_test.cc +++ b/test/common/event/dispatcher_impl_test.cc @@ -593,7 +593,7 @@ TEST_F(DispatcherImplTest, Timer) { TEST_F(DispatcherImplTest, TimerWithScope) { TimerPtr timer; - MockScopedTrackedObject scope; + MockScopeTrackedObject scope; dispatcher_->post([this, &timer, &scope]() { { // Expect a call to dumpState. The timer will call onFatalError during diff --git a/test/common/event/file_event_impl_test.cc b/test/common/event/file_event_impl_test.cc index 64abdc349ea9b..1dc8eae1984dd 100644 --- a/test/common/event/file_event_impl_test.cc +++ b/test/common/event/file_event_impl_test.cc @@ -35,6 +35,17 @@ class FileEventImplTest : public testing::Test { ASSERT_EQ(sizeof(data), static_cast(result.rc_)); } + void clearReadable() { + // Read the data from the socket so it is no longer readable. + char buffer[10]; + struct iovec vec { + buffer, sizeof(buffer) + }; + const Api::SysCallSizeResult result = os_sys_calls_.readv(fds_[0], &vec, 1); + EXPECT_LT(0, static_cast(result.rc_)); + EXPECT_GT(sizeof(buffer), static_cast(result.rc_)); + } + void TearDown() override { os_sys_calls_.close(fds_[0]); os_sys_calls_.close(fds_[1]); @@ -253,18 +264,17 @@ TEST_F(FileEventImplTest, EdgeTrigger) { #endif TEST_F(FileEventImplTest, LevelTrigger) { + testing::InSequence s; ReadyWatcher read_event; - EXPECT_CALL(read_event, ready()).Times(2); ReadyWatcher write_event; - EXPECT_CALL(write_event, ready()).Times(2); - int count = 2; + int count = 0; Event::FileEventPtr file_event = dispatcher_->createFileEvent( fds_[0], [&](uint32_t events) -> void { - if (count-- == 0) { + ASSERT(count > 0); + if (--count == 0) { dispatcher_->exit(); - return; } if (events & FileReadyType::Read) { read_event.ready(); @@ -276,14 +286,43 @@ TEST_F(FileEventImplTest, LevelTrigger) { }, FileTriggerType::Level, FileReadyType::Read | FileReadyType::Write); + // Expect events to be delivered twice since count=2 and level events are delivered on each + // iteration until the fd state changes. + EXPECT_CALL(read_event, ready()); + EXPECT_CALL(write_event, ready()); + EXPECT_CALL(read_event, ready()); + EXPECT_CALL(write_event, ready()); + count = 2; + dispatcher_->run(Event::Dispatcher::RunType::Block); + + // Change the event mask to just Write and verify that only that event is delivered. + EXPECT_CALL(read_event, ready()).Times(0); + EXPECT_CALL(write_event, ready()); + file_event->setEnabled(FileReadyType::Write); + count = 1; + dispatcher_->run(Event::Dispatcher::RunType::Block); + + // Activate read, and verify it is delivered despite not being part of the enabled event mask. + EXPECT_CALL(read_event, ready()); + EXPECT_CALL(write_event, ready()); + file_event->activate(FileReadyType::Read); + count = 1; + dispatcher_->run(Event::Dispatcher::RunType::Block); + + // Activate read and then call setEnabled. Verify that the read event is not delivered; setEnabled + // clears events from explicit calls to activate. + EXPECT_CALL(read_event, ready()).Times(0); + EXPECT_CALL(write_event, ready()); + file_event->activate(FileReadyType::Read); + file_event->setEnabled(FileReadyType::Write); + count = 1; dispatcher_->run(Event::Dispatcher::RunType::Block); } TEST_F(FileEventImplTest, SetEnabled) { + testing::InSequence s; ReadyWatcher read_event; - EXPECT_CALL(read_event, ready()).Times(2); ReadyWatcher write_event; - EXPECT_CALL(write_event, ready()).Times(2); const FileTriggerType trigger = Event::PlatformDefaultTriggerType; @@ -300,17 +339,125 @@ TEST_F(FileEventImplTest, SetEnabled) { }, trigger, FileReadyType::Read | FileReadyType::Write); + EXPECT_CALL(read_event, ready()); file_event->setEnabled(FileReadyType::Read); dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + EXPECT_CALL(write_event, ready()); file_event->setEnabled(FileReadyType::Write); dispatcher_->run(Event::Dispatcher::RunType::NonBlock); file_event->setEnabled(0); dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + EXPECT_CALL(read_event, ready()); + EXPECT_CALL(write_event, ready()); + file_event->setEnabled(FileReadyType::Read | FileReadyType::Write); + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + + // Run a manual event to ensure that previous expectations are satisfied before moving on. + ReadyWatcher manual_event; + EXPECT_CALL(manual_event, ready()); + manual_event.ready(); + + clearReadable(); + + file_event->setEnabled(FileReadyType::Read); + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + + EXPECT_CALL(write_event, ready()); + file_event->setEnabled(FileReadyType::Write); + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + + EXPECT_CALL(write_event, ready()); + file_event->setEnabled(FileReadyType::Read | FileReadyType::Write); + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + + // Repeat the previous registration, verify that write event is delivered again. + EXPECT_CALL(write_event, ready()); file_event->setEnabled(FileReadyType::Read | FileReadyType::Write); dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + + // Synthetic read events are delivered even if the active registration doesn't contain them. + EXPECT_CALL(read_event, ready()); + file_event->activate(FileReadyType::Read); + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + + // Run a manual event to ensure that previous expectations are satisfied before moving on. + EXPECT_CALL(manual_event, ready()); + manual_event.ready(); + + // Do a read activation followed setEnabled to verify that the activation is cleared. + EXPECT_CALL(write_event, ready()); + file_event->activate(FileReadyType::Read); + file_event->setEnabled(FileReadyType::Read | FileReadyType::Write); + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + + // Repeat the previous steps but with the same input to setEnabled to verify that the activation + // is cleared even in cases where the setEnable mask hasn't changed. + EXPECT_CALL(write_event, ready()); + file_event->activate(FileReadyType::Read); + file_event->setEnabled(FileReadyType::Read | FileReadyType::Write); + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); +} + +TEST_F(FileEventImplTest, RegisterIfEmulatedEdge) { + // Test only applies if using EmulatedEdge trigger mode + if constexpr (PlatformDefaultTriggerType != FileTriggerType::EmulatedEdge) { + return; + } + + testing::InSequence s; + ReadyWatcher read_event; + ReadyWatcher write_event; + + const FileTriggerType trigger = Event::PlatformDefaultTriggerType; + + Event::FileEventPtr file_event = dispatcher_->createFileEvent( + fds_[0], + [&](uint32_t events) -> void { + if (events & FileReadyType::Read) { + read_event.ready(); + } + + if (events & FileReadyType::Write) { + write_event.ready(); + } + }, + trigger, FileReadyType::Read | FileReadyType::Write); + + EXPECT_CALL(read_event, ready()).Times(0); + EXPECT_CALL(write_event, ready()).Times(0); + file_event->unregisterEventIfEmulatedEdge(FileReadyType::Read | FileReadyType::Write); + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + + EXPECT_CALL(read_event, ready()); + file_event->registerEventIfEmulatedEdge(FileReadyType::Read); + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + + EXPECT_CALL(write_event, ready()); + file_event->registerEventIfEmulatedEdge(FileReadyType::Write); + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + + EXPECT_CALL(read_event, ready()); + file_event->registerEventIfEmulatedEdge(FileReadyType::Read | FileReadyType::Write); + file_event->unregisterEventIfEmulatedEdge(FileReadyType::Write); + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + + EXPECT_CALL(read_event, ready()).Times(0); + EXPECT_CALL(write_event, ready()).Times(0); + file_event->unregisterEventIfEmulatedEdge(FileReadyType::Read); + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + + EXPECT_CALL(read_event, ready()); + EXPECT_CALL(write_event, ready()); + file_event->registerEventIfEmulatedEdge(FileReadyType::Read | FileReadyType::Write); + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + + // Events are delivered once due to auto unregistration after they are delivered. + EXPECT_CALL(read_event, ready()).Times(0); + EXPECT_CALL(write_event, ready()).Times(0); + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); } } // namespace diff --git a/test/common/event/scaled_range_timer_manager_impl_test.cc b/test/common/event/scaled_range_timer_manager_impl_test.cc index 90ad37070a86f..ca89c39fbd971 100644 --- a/test/common/event/scaled_range_timer_manager_impl_test.cc +++ b/test/common/event/scaled_range_timer_manager_impl_test.cc @@ -299,7 +299,7 @@ class ScaledRangeTimerManagerTestWithScope : public ScaledRangeTimerManagerTest, public testing::WithParamInterface { public: ScopeTrackedObject* getScope() { return GetParam() ? &scope_ : nullptr; } - MockScopedTrackedObject scope_; + MockScopeTrackedObject scope_; }; TEST_P(ScaledRangeTimerManagerTestWithScope, ReRegisterOnCallback) { diff --git a/test/common/grpc/BUILD b/test/common/grpc/BUILD index ed8fb7f6170de..78fe8522438e2 100644 --- a/test/common/grpc/BUILD +++ b/test/common/grpc/BUILD @@ -172,7 +172,6 @@ envoy_cc_test( deps = [ ":grpc_client_integration_test_harness_lib", "//source/common/grpc:async_client_lib", - "//source/extensions/grpc_credentials:well_known_names", "//source/extensions/grpc_credentials/example:config", ] + envoy_select_google_grpc(["//source/common/grpc:google_async_client_lib"]), ) diff --git a/test/common/grpc/grpc_client_integration_test.cc b/test/common/grpc/grpc_client_integration_test.cc index 0f3ff252a5891..578f489b848b0 100644 --- a/test/common/grpc/grpc_client_integration_test.cc +++ b/test/common/grpc/grpc_client_integration_test.cc @@ -3,8 +3,6 @@ #include "common/grpc/google_async_client_impl.h" -#include "extensions/grpc_credentials/well_known_names.h" - #endif #include "test/common/grpc/grpc_client_integration_test_harness.h" @@ -452,8 +450,7 @@ INSTANTIATE_TEST_SUITE_P(SslIpVersionsClientType, GrpcAccessTokenClientIntegrati TEST_P(GrpcAccessTokenClientIntegrationTest, AccessTokenAuthRequest) { SKIP_IF_GRPC_CLIENT(ClientType::EnvoyGrpc); access_token_value_ = "accesstokenvalue"; - credentials_factory_name_ = - Extensions::GrpcCredentials::GrpcCredentialsNames::get().AccessTokenExample; + credentials_factory_name_ = "envoy.grpc_credentials.access_token_example"; initialize(); auto request = createRequest(empty_metadata_); request->sendReply(); @@ -464,8 +461,7 @@ TEST_P(GrpcAccessTokenClientIntegrationTest, AccessTokenAuthRequest) { TEST_P(GrpcAccessTokenClientIntegrationTest, AccessTokenAuthStream) { SKIP_IF_GRPC_CLIENT(ClientType::EnvoyGrpc); access_token_value_ = "accesstokenvalue"; - credentials_factory_name_ = - Extensions::GrpcCredentials::GrpcCredentialsNames::get().AccessTokenExample; + credentials_factory_name_ = "envoy.grpc_credentials.access_token_example"; initialize(); auto stream = createStream(empty_metadata_); stream->sendServerInitialMetadata(empty_metadata_); @@ -480,8 +476,7 @@ TEST_P(GrpcAccessTokenClientIntegrationTest, MultipleAccessTokens) { SKIP_IF_GRPC_CLIENT(ClientType::EnvoyGrpc); access_token_value_ = "accesstokenvalue"; access_token_value_2_ = "accesstokenvalue2"; - credentials_factory_name_ = - Extensions::GrpcCredentials::GrpcCredentialsNames::get().AccessTokenExample; + credentials_factory_name_ = "envoy.grpc_credentials.access_token_example"; initialize(); auto request = createRequest(empty_metadata_); request->sendReply(); @@ -493,8 +488,7 @@ TEST_P(GrpcAccessTokenClientIntegrationTest, ExtraCredentialParams) { SKIP_IF_GRPC_CLIENT(ClientType::EnvoyGrpc); access_token_value_ = "accesstokenvalue"; refresh_token_value_ = "refreshtokenvalue"; - credentials_factory_name_ = - Extensions::GrpcCredentials::GrpcCredentialsNames::get().AccessTokenExample; + credentials_factory_name_ = "envoy.grpc_credentials.access_token_example"; initialize(); auto request = createRequest(empty_metadata_); request->sendReply(); @@ -504,8 +498,7 @@ TEST_P(GrpcAccessTokenClientIntegrationTest, ExtraCredentialParams) { // Validate that no access token still works TEST_P(GrpcAccessTokenClientIntegrationTest, NoAccessTokens) { SKIP_IF_GRPC_CLIENT(ClientType::EnvoyGrpc); - credentials_factory_name_ = - Extensions::GrpcCredentials::GrpcCredentialsNames::get().AccessTokenExample; + credentials_factory_name_ = "envoy.grpc_credentials.access_token_example"; initialize(); auto request = createRequest(empty_metadata_); request->sendReply(); diff --git a/test/common/grpc/grpc_client_integration_test_harness.h b/test/common/grpc/grpc_client_integration_test_harness.h index 7a611435607c1..3145e6b89b84a 100644 --- a/test/common/grpc/grpc_client_integration_test_harness.h +++ b/test/common/grpc/grpc_client_integration_test_harness.h @@ -246,7 +246,7 @@ class GrpcClientIntegrationTest : public GrpcClientIntegrationParamTest { virtual void initialize() { if (fake_upstream_ == nullptr) { FakeUpstreamConfig config(test_time_.timeSystem()); - config.upstream_protocol_ = FakeHttpConnection::Type::HTTP2; + config.upstream_protocol_ = Http::CodecType::HTTP2; fake_upstream_ = std::make_unique(0, ipVersion(), config); } switch (clientType()) { @@ -524,7 +524,7 @@ class GrpcSslClientIntegrationTest : public GrpcClientIntegrationTest { async_client_transport_socket_ = mock_host_description_->socket_factory_->createTransportSocket(nullptr); FakeUpstreamConfig config(test_time_.timeSystem()); - config.upstream_protocol_ = FakeHttpConnection::Type::HTTP2; + config.upstream_protocol_ = Http::CodecType::HTTP2; fake_upstream_ = std::make_unique(createUpstreamSslContext(), 0, ipVersion(), config); diff --git a/test/common/http/BUILD b/test/common/http/BUILD index ee91fd614c683..615595df3b935 100644 --- a/test/common/http/BUILD +++ b/test/common/http/BUILD @@ -214,7 +214,8 @@ envoy_cc_test( ], shard_count = 3, deps = [ - ":ip_detection_extensions_lib", + ":custom_header_extension_lib", + ":xff_extension_lib", "//source/common/http:conn_manager_lib", "//source/common/http:context_lib", "//source/extensions/access_loggers/common:file_access_log_lib", @@ -239,7 +240,8 @@ envoy_cc_test( name = "conn_manager_utility_test", srcs = ["conn_manager_utility_test.cc"], deps = [ - ":ip_detection_extensions_lib", + ":custom_header_extension_lib", + ":xff_extension_lib", "//source/common/common:random_generator_lib", "//source/common/event:dispatcher_lib", "//source/common/http:conn_manager_lib", @@ -418,8 +420,6 @@ envoy_cc_test( ], ) -# TODO(alyssawilk): The conn pool grid should work correctly without H3. Circle back and figure -# out the right way of handling the lack of H3 in the build. envoy_cc_test( name = "conn_pool_grid_test", srcs = envoy_select_enable_http3(["conn_pool_grid_test.cc"]), @@ -497,11 +497,19 @@ envoy_cc_fuzz_test( ) envoy_cc_test_library( - name = "ip_detection_extensions_lib", - srcs = ["ip_detection_extensions.cc"], - hdrs = ["ip_detection_extensions.h"], + name = "custom_header_extension_lib", + srcs = ["custom_header_extension.cc"], + hdrs = ["custom_header_extension.h"], deps = [ "//source/extensions/http/original_ip_detection/custom_header:custom_header_lib", + ], +) + +envoy_cc_test_library( + name = "xff_extension_lib", + srcs = ["xff_extension.cc"], + hdrs = ["xff_extension.h"], + deps = [ "//source/extensions/http/original_ip_detection/xff:xff_lib", ], ) diff --git a/test/common/http/alternate_protocols_cache_impl_test.cc b/test/common/http/alternate_protocols_cache_impl_test.cc index dc0f95848d2b3..88347e98bac3e 100644 --- a/test/common/http/alternate_protocols_cache_impl_test.cc +++ b/test/common/http/alternate_protocols_cache_impl_test.cc @@ -23,29 +23,31 @@ class AlternateProtocolsCacheImplTest : public testing::Test, public Event::Test const std::string alpn1_ = "alpn1"; const std::string alpn2_ = "alpn2"; + const MonotonicTime expiration1_ = simTime().monotonicTime() + Seconds(5); + const MonotonicTime expiration2_ = simTime().monotonicTime() + Seconds(10); + const AlternateProtocolsCacheImpl::Origin origin1_ = {https_, hostname1_, port1_}; const AlternateProtocolsCacheImpl::Origin origin2_ = {https_, hostname2_, port2_}; - const AlternateProtocolsCacheImpl::AlternateProtocol protocol1_ = {alpn1_, hostname1_, port1_}; - const AlternateProtocolsCacheImpl::AlternateProtocol protocol2_ = {alpn2_, hostname2_, port2_}; + const AlternateProtocolsCacheImpl::AlternateProtocol protocol1_ = {alpn1_, hostname1_, port1_, + expiration1_}; + const AlternateProtocolsCacheImpl::AlternateProtocol protocol2_ = {alpn2_, hostname2_, port2_, + expiration2_}; const std::vector protocols1_ = {protocol1_}; const std::vector protocols2_ = {protocol2_}; - - const MonotonicTime expiration1_ = simTime().monotonicTime() + Seconds(5); - const MonotonicTime expiration2_ = simTime().monotonicTime() + Seconds(10); }; TEST_F(AlternateProtocolsCacheImplTest, Init) { EXPECT_EQ(0, protocols_.size()); } TEST_F(AlternateProtocolsCacheImplTest, SetAlternatives) { EXPECT_EQ(0, protocols_.size()); - protocols_.setAlternatives(origin1_, protocols1_, expiration1_); + protocols_.setAlternatives(origin1_, protocols1_); EXPECT_EQ(1, protocols_.size()); } TEST_F(AlternateProtocolsCacheImplTest, FindAlternatives) { - protocols_.setAlternatives(origin1_, protocols1_, expiration1_); + protocols_.setAlternatives(origin1_, protocols1_); OptRef> protocols = protocols_.findAlternatives(origin1_); ASSERT_TRUE(protocols.has_value()); @@ -53,8 +55,8 @@ TEST_F(AlternateProtocolsCacheImplTest, FindAlternatives) { } TEST_F(AlternateProtocolsCacheImplTest, FindAlternativesAfterReplacement) { - protocols_.setAlternatives(origin1_, protocols1_, expiration1_); - protocols_.setAlternatives(origin1_, protocols2_, expiration2_); + protocols_.setAlternatives(origin1_, protocols1_); + protocols_.setAlternatives(origin1_, protocols2_); OptRef> protocols = protocols_.findAlternatives(origin1_); ASSERT_TRUE(protocols.has_value()); @@ -63,8 +65,8 @@ TEST_F(AlternateProtocolsCacheImplTest, FindAlternativesAfterReplacement) { } TEST_F(AlternateProtocolsCacheImplTest, FindAlternativesForMultipleOrigins) { - protocols_.setAlternatives(origin1_, protocols1_, expiration1_); - protocols_.setAlternatives(origin2_, protocols2_, expiration2_); + protocols_.setAlternatives(origin1_, protocols1_); + protocols_.setAlternatives(origin2_, protocols2_); OptRef> protocols = protocols_.findAlternatives(origin1_); ASSERT_TRUE(protocols.has_value()); @@ -75,7 +77,7 @@ TEST_F(AlternateProtocolsCacheImplTest, FindAlternativesForMultipleOrigins) { } TEST_F(AlternateProtocolsCacheImplTest, FindAlternativesAfterExpiration) { - protocols_.setAlternatives(origin1_, protocols1_, expiration1_); + protocols_.setAlternatives(origin1_, protocols1_); simTime().setMonotonicTime(expiration1_ + Seconds(1)); OptRef> protocols = protocols_.findAlternatives(origin1_); @@ -83,6 +85,37 @@ TEST_F(AlternateProtocolsCacheImplTest, FindAlternativesAfterExpiration) { EXPECT_EQ(0, protocols_.size()); } +TEST_F(AlternateProtocolsCacheImplTest, FindAlternativesAfterPartialExpiration) { + protocols_.setAlternatives(origin1_, {protocol1_, protocol2_}); + simTime().setMonotonicTime(expiration1_ + Seconds(1)); + OptRef> protocols = + protocols_.findAlternatives(origin1_); + ASSERT_TRUE(protocols.has_value()); + EXPECT_EQ(protocols2_.size(), protocols->size()); + EXPECT_EQ(protocols2_, protocols.ref()); +} + +TEST_F(AlternateProtocolsCacheImplTest, FindAlternativesAfterTruncation) { + AlternateProtocolsCacheImpl::AlternateProtocol protocol = protocol1_; + + std::vector expected_protocols; + for (size_t i = 0; i < 10; ++i) { + protocol.port_++; + expected_protocols.push_back(protocol); + } + std::vector full_protocols = expected_protocols; + protocol.port_++; + full_protocols.push_back(protocol); + full_protocols.push_back(protocol); + + protocols_.setAlternatives(origin1_, full_protocols); + OptRef> protocols = + protocols_.findAlternatives(origin1_); + ASSERT_TRUE(protocols.has_value()); + EXPECT_EQ(10, protocols->size()); + EXPECT_EQ(expected_protocols, protocols.ref()); +} + } // namespace } // namespace Http } // namespace Envoy diff --git a/test/common/http/codec_client_test.cc b/test/common/http/codec_client_test.cc index 06621f778ba98..55e0b3c4d221a 100644 --- a/test/common/http/codec_client_test.cc +++ b/test/common/http/codec_client_test.cc @@ -56,8 +56,8 @@ class CodecClientTest : public Event::TestUsingSimulatedTime, public testing::Te Network::ClientConnectionPtr connection{connection_}; EXPECT_CALL(dispatcher_, createTimer_(_)); - client_ = std::make_unique(CodecClient::Type::HTTP1, std::move(connection), - codec_, nullptr, host_, dispatcher_); + client_ = std::make_unique(CodecType::HTTP1, std::move(connection), codec_, + nullptr, host_, dispatcher_); ON_CALL(*connection_, streamInfo()).WillByDefault(ReturnRef(stream_info_)); } @@ -87,8 +87,8 @@ TEST_F(CodecClientTest, NotCallDetectEarlyCloseWhenReadDiabledUsingHttp3) { auto codec = new Http::MockClientConnection(); EXPECT_CALL(dispatcher_, createTimer_(_)); - client_ = std::make_unique(CodecClient::Type::HTTP3, std::move(connection), - codec, nullptr, host_, dispatcher_); + client_ = std::make_unique(CodecType::HTTP3, std::move(connection), codec, + nullptr, host_, dispatcher_); } TEST_F(CodecClientTest, BasicHeaderOnlyResponse) { @@ -308,9 +308,8 @@ class CodecNetworkTest : public Event::TestUsingSimulatedTime, client_connection_->addConnectionCallbacks(client_callbacks_); codec_ = new Http::MockClientConnection(); - client_ = - std::make_unique(CodecClient::Type::HTTP1, std::move(client_connection), - codec_, nullptr, host_, *dispatcher_); + client_ = std::make_unique(CodecType::HTTP1, std::move(client_connection), + codec_, nullptr, host_, *dispatcher_); int expected_callbacks = 2; EXPECT_CALL(listener_callbacks_, onAccept_(_)) diff --git a/test/common/http/codec_impl_corpus/clusterfuzz-testcase-minimized-codec_impl_fuzz_test-5766628005642240 b/test/common/http/codec_impl_corpus/clusterfuzz-testcase-minimized-codec_impl_fuzz_test-5766628005642240 new file mode 100644 index 0000000000000..584b156e01ec7 --- /dev/null +++ b/test/common/http/codec_impl_corpus/clusterfuzz-testcase-minimized-codec_impl_fuzz_test-5766628005642240 @@ -0,0 +1,192 @@ +actions { + server_drain { + } +} +actions { + new_stream { + request_headers { + headers { + key: ":method" + value: "GET" + } + headers { + key: ":path" + value: "/" + } + headers { + key: ":scheme" + value: "E" + } + headers { + key: ":authority" + value: "V" + } + } + } +} +actions { + server_drain { + } +} +actions { + client_drain { + } +} +actions { + swap_buffer { + } +} +actions { + swap_buffer { + } +} +actions { + client_drain { + } +} +actions { + server_drain { + } +} +actions { + server_drain { + } +} +actions { + stream_action { + response { + headers { + } + } + } +} +actions { + server_drain { + } +} +actions { + stream_action { + response { + data: 1751212400 + } + } +} +actions { + swap_buffer { + } +} +actions { + stream_action { + response { + data: 8963 + } + } +} +actions { + quiesce_drain { + } +} +actions { + stream_action { + response { + data: 9 + } + } +} +actions { + stream_action { + response { + data: 9 + } + } +} +actions { + client_drain { + } +} +actions { + stream_action { + response { + data: 3 + } + } +} +actions { + quiesce_drain { + } +} +actions { + quiesce_drain { + } +} +actions { + stream_action { + response { + data: 3 + } + } +} +actions { + stream_action { + request { + data: 0 + } + } +} +actions { + quiesce_drain { + } +} +actions { + stream_action { + response { + data: 9 + end_stream: true + } + } +} +actions { + quiesce_drain { + } +} +actions { + stream_action { + response { + data: 3 + } + } +} +actions { + client_drain { + } +} +actions { + stream_action { + request { + data: 7 + } + } +} +actions { + stream_action { + stream_id: 2097152 + response { + data: 9 + } + } +} +actions { + stream_action { + response { + data: 0 + } + } +} +actions { + mutate { + } +} +actions { + swap_buffer { + } +} diff --git a/test/common/http/codec_impl_fuzz_test.cc b/test/common/http/codec_impl_fuzz_test.cc index d0fbdca11a4f8..8471fed61f472 100644 --- a/test/common/http/codec_impl_fuzz_test.cc +++ b/test/common/http/codec_impl_fuzz_test.cc @@ -361,7 +361,13 @@ class HttpStream : public LinkedObject { } } // Perform the stream action. - directionalAction(request_, stream_action.request()); + // The request_.request_encoder_ is initialized from the response_.response_decoder_. + // Fuzz test codec_impl_fuzz_test-5766628005642240 created a situation where the response + // stream was in closed state leading to the state.request_encoder_ in directionalAction() + // kData case no longer being a valid address. + if (response_.stream_state_ != HttpStream::StreamState::Closed) { + directionalAction(request_, stream_action.request()); + } break; } case test::common::http::StreamAction::kResponse: { diff --git a/test/common/http/common.h b/test/common/http/common.h index 3a28e9b369b87..06385cbc6c12a 100644 --- a/test/common/http/common.h +++ b/test/common/http/common.h @@ -17,7 +17,7 @@ namespace Envoy { class CodecClientForTest : public Http::CodecClient { public: using DestroyCb = std::function; - CodecClientForTest(CodecClient::Type type, Network::ClientConnectionPtr&& connection, + CodecClientForTest(Http::CodecType type, Network::ClientConnectionPtr&& connection, Http::ClientConnection* codec, DestroyCb destroy_cb, Upstream::HostDescriptionConstSharedPtr host, Event::Dispatcher& dispatcher) : CodecClient(type, std::move(connection), host, dispatcher), destroy_cb_(destroy_cb) { diff --git a/test/common/http/conn_manager_impl_fuzz_test.cc b/test/common/http/conn_manager_impl_fuzz_test.cc index a62a208315ae4..dd9c7e1525984 100644 --- a/test/common/http/conn_manager_impl_fuzz_test.cc +++ b/test/common/http/conn_manager_impl_fuzz_test.cc @@ -198,6 +198,7 @@ class FuzzConfig : public ConnectionManagerConfig { const Http::Http1Settings& http1Settings() const override { return http1_settings_; } bool shouldNormalizePath() const override { return false; } bool shouldMergeSlashes() const override { return false; } + bool shouldStripTrailingHostDot() const override { return false; } Http::StripPortType stripPortType() const override { return Http::StripPortType::None; } envoy::config::core::v3::HttpProtocolOptions::HeadersWithUnderscoresAction headersWithUnderscoresAction() const override { diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index 80e1364e23aba..20d5abb112cde 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -1193,6 +1193,47 @@ TEST_F(HttpConnectionManagerImplTest, RouteShouldUseNormalizedHost) { filter_callbacks_.connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); } +// Observe that we strip the trailing dot. +TEST_F(HttpConnectionManagerImplTest, StripTrailingHostDot) { + setup(false, ""); + // Enable removal of host's trailing dot. + strip_trailing_host_dot_ = true; + const std::string original_host = "host."; + const std::string updated_host = "host"; + // Set up the codec. + Buffer::OwnedImpl fake_input("1234"); + conn_manager_->createCodec(fake_input); + // Create a new stream. + decoder_ = &conn_manager_->newStream(response_encoder_); + RequestHeaderMapPtr headers{new TestRequestHeaderMapImpl{ + {":authority", original_host}, {":path", "/"}, {":method", "GET"}}}; + RequestHeaderMap* updated_headers = headers.get(); + decoder_->decodeHeaders(std::move(headers), true); + EXPECT_EQ(updated_host, updated_headers->getHostValue()); + // Clean up. + filter_callbacks_.connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); +} + +TEST_F(HttpConnectionManagerImplTest, HostWithoutTrailingDot) { + setup(false, ""); + // Enable removal of host's trailing dot. + strip_trailing_host_dot_ = true; + const std::string original_host = "host"; + const std::string updated_host = "host"; + // Set up the codec. + Buffer::OwnedImpl fake_input("1234"); + conn_manager_->createCodec(fake_input); + // Create a new stream. + decoder_ = &conn_manager_->newStream(response_encoder_); + RequestHeaderMapPtr headers{new TestRequestHeaderMapImpl{ + {":authority", original_host}, {":path", "/"}, {":method", "GET"}}}; + RequestHeaderMap* updated_headers = headers.get(); + decoder_->decodeHeaders(std::move(headers), true); + EXPECT_EQ(updated_host, updated_headers->getHostValue()); + // Clean up. + filter_callbacks_.connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); +} + TEST_F(HttpConnectionManagerImplTest, DateHeaderNotPresent) { setup(false, ""); setUpEncoderAndDecoder(false, false); diff --git a/test/common/http/conn_manager_impl_test_2.cc b/test/common/http/conn_manager_impl_test_2.cc index f660286015139..3c71e80667695 100644 --- a/test/common/http/conn_manager_impl_test_2.cc +++ b/test/common/http/conn_manager_impl_test_2.cc @@ -1,5 +1,5 @@ #include "test/common/http/conn_manager_impl_test_base.h" -#include "test/common/http/ip_detection_extensions.h" +#include "test/common/http/custom_header_extension.h" #include "test/test_common/logging.h" #include "test/test_common/test_runtime.h" diff --git a/test/common/http/conn_manager_impl_test_base.cc b/test/common/http/conn_manager_impl_test_base.cc index cd9fa25b98e67..92e74ac4c97a5 100644 --- a/test/common/http/conn_manager_impl_test_base.cc +++ b/test/common/http/conn_manager_impl_test_base.cc @@ -2,7 +2,7 @@ #include "extensions/request_id/uuid/config.h" -#include "test/common/http/ip_detection_extensions.h" +#include "test/common/http/xff_extension.h" using testing::AtLeast; using testing::InSequence; diff --git a/test/common/http/conn_manager_impl_test_base.h b/test/common/http/conn_manager_impl_test_base.h index c8e174853fb4e..18b04e5176032 100644 --- a/test/common/http/conn_manager_impl_test_base.h +++ b/test/common/http/conn_manager_impl_test_base.h @@ -136,6 +136,7 @@ class HttpConnectionManagerImplTest : public testing::Test, public ConnectionMan const Http::Http1Settings& http1Settings() const override { return http1_settings_; } bool shouldNormalizePath() const override { return normalize_path_; } bool shouldMergeSlashes() const override { return merge_slashes_; } + bool shouldStripTrailingHostDot() const override { return strip_trailing_host_dot_; } Http::StripPortType stripPortType() const override { return strip_port_type_; } const RequestIDExtensionSharedPtr& requestIDExtension() override { return request_id_extension_; } envoy::config::core::v3::HttpProtocolOptions::HeadersWithUnderscoresAction @@ -197,7 +198,7 @@ class HttpConnectionManagerImplTest : public testing::Test, public ConnectionMan std::make_shared>()}; TracingConnectionManagerConfigPtr tracing_config_; SlowDateProviderImpl date_provider_{test_time_.timeSystem()}; - MockStream stream_; + NiceMock stream_; Http::StreamCallbacks* stream_callbacks_{nullptr}; NiceMock cluster_manager_; NiceMock overload_manager_; @@ -230,6 +231,7 @@ class HttpConnectionManagerImplTest : public testing::Test, public ConnectionMan PathWithEscapedSlashesAction path_with_escaped_slashes_action_{ envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager:: KEEP_UNCHANGED}; + bool strip_trailing_host_dot_ = false; }; } // namespace Http diff --git a/test/common/http/conn_manager_utility_test.cc b/test/common/http/conn_manager_utility_test.cc index 7dc2a957ebad9..a53c9fe67a090 100644 --- a/test/common/http/conn_manager_utility_test.cc +++ b/test/common/http/conn_manager_utility_test.cc @@ -14,7 +14,8 @@ #include "extensions/request_id/uuid/config.h" -#include "test/common/http/ip_detection_extensions.h" +#include "test/common/http/custom_header_extension.h" +#include "test/common/http/xff_extension.h" #include "test/mocks/http/mocks.h" #include "test/mocks/local_info/mocks.h" #include "test/mocks/network/mocks.h" @@ -1502,6 +1503,51 @@ TEST_F(ConnectionManagerUtilityTest, RemovePort) { EXPECT_EQ(header_map_none.getHostValue(), "host:9999"); } +// maybeNormalizeHost() removes trailing dot of host from host header. +TEST_F(ConnectionManagerUtilityTest, RemoveTrailingDot) { + ON_CALL(config_, shouldStripTrailingHostDot()).WillByDefault(Return(true)); + TestRequestHeaderMapImpl original_headers; + original_headers.setHost("host."); + + TestRequestHeaderMapImpl header_map(original_headers); + ConnectionManagerUtility::maybeNormalizeHost(header_map, config_, 0); + EXPECT_EQ(header_map.getHostValue(), "host"); + + ON_CALL(config_, stripPortType()).WillByDefault(Return(Http::StripPortType::None)); + ON_CALL(config_, shouldStripTrailingHostDot()).WillByDefault(Return(true)); + TestRequestHeaderMapImpl original_headers_with_port; + original_headers_with_port.setHost("host.:443"); + + TestRequestHeaderMapImpl header_map_with_port(original_headers_with_port); + ConnectionManagerUtility::maybeNormalizeHost(header_map_with_port, config_, 443); + EXPECT_EQ(header_map_with_port.getHostValue(), "host:443"); + + ON_CALL(config_, stripPortType()).WillByDefault(Return(Http::StripPortType::MatchingHost)); + ON_CALL(config_, shouldStripTrailingHostDot()).WillByDefault(Return(true)); + TestRequestHeaderMapImpl original_headers_strip_port; + original_headers_strip_port.setHost("host.:443"); + + TestRequestHeaderMapImpl header_map_strip_port(original_headers_strip_port); + ConnectionManagerUtility::maybeNormalizeHost(header_map_strip_port, config_, 443); + EXPECT_EQ(header_map_strip_port.getHostValue(), "host"); + + ON_CALL(config_, shouldStripTrailingHostDot()).WillByDefault(Return(true)); + TestRequestHeaderMapImpl original_headers_no_dot; + original_headers_no_dot.setHost("host"); + + TestRequestHeaderMapImpl header_map_no_dot(original_headers_no_dot); + ConnectionManagerUtility::maybeNormalizeHost(header_map_no_dot, config_, 0); + EXPECT_EQ(header_map_no_dot.getHostValue(), "host"); + + ON_CALL(config_, shouldStripTrailingHostDot()).WillByDefault(Return(false)); + TestRequestHeaderMapImpl original_headers_none; + original_headers_none.setHost("host."); + + TestRequestHeaderMapImpl header_map_none(original_headers_none); + ConnectionManagerUtility::maybeNormalizeHost(header_map_none, config_, 0); + EXPECT_EQ(header_map_none.getHostValue(), "host."); +} + // maybeNormalizePath() does not touch escaped slashes when configured to KEEP_UNCHANGED. TEST_F(ConnectionManagerUtilityTest, KeepEscapedSlashesWhenConfigured) { ON_CALL(config_, pathWithEscapedSlashesAction()) diff --git a/test/common/http/conn_pool_grid_test.cc b/test/common/http/conn_pool_grid_test.cc index e25551aaaf5dc..d52fd391620af 100644 --- a/test/common/http/conn_pool_grid_test.cc +++ b/test/common/http/conn_pool_grid_test.cc @@ -122,9 +122,8 @@ class ConnectivityGridTestBase : public Event::TestUsingSimulatedTime, public te void addHttp3AlternateProtocol() { AlternateProtocolsCacheImpl::Origin origin("https", "hostname", 9000); const std::vector protocols = { - {"h3-29", "", origin.port_}}; - alternate_protocols_->setAlternatives(origin, protocols, - simTime().monotonicTime() + Seconds(5)); + {"h3-29", "", origin.port_, simTime().monotonicTime() + Seconds(5)}}; + alternate_protocols_->setAlternatives(origin, protocols); } const Network::ConnectionSocket::OptionsSharedPtr socket_options_; @@ -513,8 +512,8 @@ TEST_F(ConnectivityGridWithAlternateProtocolsCacheImplTest, SuccessWithoutHttp3) TEST_F(ConnectivityGridWithAlternateProtocolsCacheImplTest, SuccessWithExpiredHttp3) { AlternateProtocolsCacheImpl::Origin origin("https", "hostname", 9000); const std::vector protocols = { - {"h3-29", "", origin.port_}}; - alternate_protocols_->setAlternatives(origin, protocols, simTime().monotonicTime() + Seconds(5)); + {"h3-29", "", origin.port_, simTime().monotonicTime() + Seconds(5)}}; + alternate_protocols_->setAlternatives(origin, protocols); simTime().setMonotonicTime(simTime().monotonicTime() + Seconds(10)); EXPECT_EQ(grid_.first(), nullptr); @@ -536,8 +535,8 @@ TEST_F(ConnectivityGridWithAlternateProtocolsCacheImplTest, SuccessWithExpiredHt TEST_F(ConnectivityGridWithAlternateProtocolsCacheImplTest, SuccessWithoutHttp3NoMatchingHostname) { AlternateProtocolsCacheImpl::Origin origin("https", "hostname", 9000); const std::vector protocols = { - {"h3-29", "otherhostname", origin.port_}}; - alternate_protocols_->setAlternatives(origin, protocols, simTime().monotonicTime() + Seconds(5)); + {"h3-29", "otherhostname", origin.port_, simTime().monotonicTime() + Seconds(5)}}; + alternate_protocols_->setAlternatives(origin, protocols); EXPECT_EQ(grid_.first(), nullptr); @@ -557,8 +556,8 @@ TEST_F(ConnectivityGridWithAlternateProtocolsCacheImplTest, SuccessWithoutHttp3N TEST_F(ConnectivityGridWithAlternateProtocolsCacheImplTest, SuccessWithoutHttp3NoMatchingPort) { AlternateProtocolsCacheImpl::Origin origin("https", "hostname", 9000); const std::vector protocols = { - {"h3-29", "", origin.port_ + 1}}; - alternate_protocols_->setAlternatives(origin, protocols, simTime().monotonicTime() + Seconds(5)); + {"h3-29", "", origin.port_ + 1, simTime().monotonicTime() + Seconds(5)}}; + alternate_protocols_->setAlternatives(origin, protocols); EXPECT_EQ(grid_.first(), nullptr); @@ -577,8 +576,8 @@ TEST_F(ConnectivityGridWithAlternateProtocolsCacheImplTest, SuccessWithoutHttp3N TEST_F(ConnectivityGridWithAlternateProtocolsCacheImplTest, SuccessWithoutHttp3NoMatchingAlpn) { AlternateProtocolsCacheImpl::Origin origin("https", "hostname", 9000); const std::vector protocols = { - {"http/2", "", origin.port_}}; - alternate_protocols_->setAlternatives(origin, protocols, simTime().monotonicTime() + Seconds(5)); + {"http/2", "", origin.port_, simTime().monotonicTime() + Seconds(5)}}; + alternate_protocols_->setAlternatives(origin, protocols); EXPECT_EQ(grid_.first(), nullptr); diff --git a/test/common/http/ip_detection_extensions.cc b/test/common/http/custom_header_extension.cc similarity index 74% rename from test/common/http/ip_detection_extensions.cc rename to test/common/http/custom_header_extension.cc index 95c2abec84888..33bb73e363064 100644 --- a/test/common/http/ip_detection_extensions.cc +++ b/test/common/http/custom_header_extension.cc @@ -1,14 +1,9 @@ -#include "ip_detection_extensions.h" +#include "test/common/http/custom_header_extension.h" #include "extensions/http/original_ip_detection/custom_header/custom_header.h" -#include "extensions/http/original_ip_detection/xff/xff.h" namespace Envoy { -Http::OriginalIPDetectionSharedPtr getXFFExtension(uint32_t hops) { - return std::make_shared(hops); -} - Http::OriginalIPDetectionSharedPtr getCustomHeaderExtension(const std::string& header_name) { return std::make_shared< Extensions::Http::OriginalIPDetection::CustomHeader::CustomHeaderIPDetection>(header_name); diff --git a/test/common/http/ip_detection_extensions.h b/test/common/http/custom_header_extension.h similarity index 86% rename from test/common/http/ip_detection_extensions.h rename to test/common/http/custom_header_extension.h index 32863f20a1a10..638f291eb0f45 100644 --- a/test/common/http/ip_detection_extensions.h +++ b/test/common/http/custom_header_extension.h @@ -5,7 +5,6 @@ // This helper is used to escape namespace pollution issues. namespace Envoy { -Http::OriginalIPDetectionSharedPtr getXFFExtension(uint32_t hops); Http::OriginalIPDetectionSharedPtr getCustomHeaderExtension(const std::string& header_name); Http::OriginalIPDetectionSharedPtr getCustomHeaderExtension(const std::string& header_name, diff --git a/test/common/http/filter_manager_test.cc b/test/common/http/filter_manager_test.cc index b90a099367efb..477184462d001 100644 --- a/test/common/http/filter_manager_test.cc +++ b/test/common/http/filter_manager_test.cc @@ -28,8 +28,8 @@ class FilterManagerTest : public testing::Test { public: void initialize() { filter_manager_ = std::make_unique( - filter_manager_callbacks_, dispatcher_, connection_, 0, true, 10000, filter_factory_, - local_reply_, protocol_, time_source_, filter_state_, + filter_manager_callbacks_, dispatcher_, connection_, 0, nullptr, true, 10000, + filter_factory_, local_reply_, protocol_, time_source_, filter_state_, StreamInfo::FilterState::LifeSpan::Connection); } diff --git a/test/common/http/header_utility_test.cc b/test/common/http/header_utility_test.cc index a529f7011b010..c926034f3fd05 100644 --- a/test/common/http/header_utility_test.cc +++ b/test/common/http/header_utility_test.cc @@ -114,6 +114,34 @@ TEST_F(HeaderUtilityTest, RemovePortsFromHostConnectLegacy) { } } +// Host's trailing dot from host header get removed. +TEST_F(HeaderUtilityTest, RemoveTrailingDotFromHost) { + const std::vector> host_headers{ + {"localhost", "localhost"}, // w/o dot + {"localhost.", "localhost"}, // name w/ dot + {"", ""}, // empty + {"192.168.1.1", "192.168.1.1"}, // ipv4 + {"abc.com", "abc.com"}, // dns w/o dot + {"abc.com.", "abc.com"}, // dns w/ dot + {"abc.com:443", "abc.com:443"}, // dns port w/o dot + {"abc.com.:443", "abc.com:443"}, // dns port w/ dot + {"[fc00::1]", "[fc00::1]"}, // ipv6 + {":", ":"}, // malformed string #1 + {"]:", "]:"}, // malformed string #2 + {":abc", ":abc"}, // malformed string #3 + {".", ""}, // malformed string #4 + {"..", "."}, // malformed string #5 + {".123", ".123"}, // malformed string #6 + {".:.", ".:"} // malformed string #7 + }; + + for (const auto& host_pair : host_headers) { + auto& host_header = hostHeaderEntry(host_pair.first); + HeaderUtility::stripTrailingHostDot(headers_); + EXPECT_EQ(host_header.value().getStringView(), host_pair.second); + } +} + TEST(GetAllOfHeaderAsStringTest, All) { const LowerCaseString test_header("test"); { @@ -495,7 +523,9 @@ invert_match: true EXPECT_FALSE(HeaderUtility::matchHeaders(unmatching_headers, header_data)); } -TEST(MatchHeadersTest, HeaderPresentMatch) { +// Test the case present_match is true. Expected true when +// header matched, expected false when no header matched. +TEST(MatchHeadersTest, HeaderPresentMatchWithTrueValue) { TestRequestHeaderMapImpl matching_headers{{"match-header", "123"}}; TestRequestHeaderMapImpl unmatching_headers{{"nonmatch-header", "1234"}, {"other-nonmatch-header", "123.456"}}; @@ -512,7 +542,28 @@ present_match: true EXPECT_FALSE(HeaderUtility::matchHeaders(unmatching_headers, header_data)); } -TEST(MatchHeadersTest, HeaderPresentInverseMatch) { +// Test the case present_match is false. Expected false when +// header matched, expected true when no header matched. +TEST(MatchHeadersTest, HeaderPresentMatchWithFalseValue) { + TestRequestHeaderMapImpl matching_headers{{"match-header", "123"}}; + TestRequestHeaderMapImpl unmatching_headers{{"nonmatch-header", "1234"}, + {"other-nonmatch-header", "123.456"}}; + + const std::string yaml = R"EOF( +name: match-header +present_match: false + )EOF"; + + std::vector header_data; + header_data.push_back( + std::make_unique(parseHeaderMatcherFromYaml(yaml))); + EXPECT_FALSE(HeaderUtility::matchHeaders(matching_headers, header_data)); + EXPECT_TRUE(HeaderUtility::matchHeaders(unmatching_headers, header_data)); +} + +// Test the case present_match is true and invert_match is true. Expected true when +// no header matched, expected false when header matched. +TEST(MatchHeadersTest, HeaderPresentInverseMatchWithTrueValue) { TestRequestHeaderMapImpl unmatching_headers{{"match-header", "123"}}; TestRequestHeaderMapImpl matching_headers{{"nonmatch-header", "1234"}, {"other-nonmatch-header", "123.456"}}; @@ -530,6 +581,26 @@ invert_match: true EXPECT_FALSE(HeaderUtility::matchHeaders(unmatching_headers, header_data)); } +// Test the case present_match is true and invert_match is true. Expected false when +// no header matched, expected true when header matched. +TEST(MatchHeadersTest, HeaderPresentInverseMatchWithFalseValue) { + TestRequestHeaderMapImpl unmatching_headers{{"match-header", "123"}}; + TestRequestHeaderMapImpl matching_headers{{"nonmatch-header", "1234"}, + {"other-nonmatch-header", "123.456"}}; + + const std::string yaml = R"EOF( +name: match-header +present_match: false +invert_match: true + )EOF"; + + std::vector header_data; + header_data.push_back( + std::make_unique(parseHeaderMatcherFromYaml(yaml))); + EXPECT_FALSE(HeaderUtility::matchHeaders(matching_headers, header_data)); + EXPECT_TRUE(HeaderUtility::matchHeaders(unmatching_headers, header_data)); +} + TEST(MatchHeadersTest, HeaderPrefixMatch) { TestRequestHeaderMapImpl matching_headers{{"match-header", "value123"}}; TestRequestHeaderMapImpl unmatching_headers{{"match-header", "123value"}}; diff --git a/test/common/http/http1/conn_pool_test.cc b/test/common/http/http1/conn_pool_test.cc index fee006d726d35..2c461685b04e7 100644 --- a/test/common/http/http1/conn_pool_test.cc +++ b/test/common/http/http1/conn_pool_test.cc @@ -106,7 +106,7 @@ class ConnPoolImplForTest : public Event::TestUsingSimulatedTime, public FixedHt test_client.client_dispatcher_ = api_->allocateDispatcher("test_thread"); Network::ClientConnectionPtr connection{test_client.connection_}; test_client.codec_client_ = new CodecClientForTest( - CodecClient::Type::HTTP1, std::move(connection), test_client.codec_, + CodecType::HTTP1, std::move(connection), test_client.codec_, [this](CodecClient* codec_client) -> void { for (auto i = test_clients_.begin(); i != test_clients_.end(); i++) { if (i->codec_client_ == codec_client) { diff --git a/test/common/http/http2/codec_impl_test.cc b/test/common/http/http2/codec_impl_test.cc index 5572f3f5dcb09..0752c0796fff6 100644 --- a/test/common/http/http2/codec_impl_test.cc +++ b/test/common/http/http2/codec_impl_test.cc @@ -1278,7 +1278,7 @@ TEST_P(Http2CodecImplFlowControlTest, TestFlowControlInPendingSendData) { // Now that the flow control window is full, further data causes the send buffer to back up. Buffer::OwnedImpl more_long_data(std::string(initial_stream_window, 'a')); request_encoder_->encodeData(more_long_data, false); - EXPECT_EQ(initial_stream_window, client_->getStream(1)->pending_send_data_.length()); + EXPECT_EQ(initial_stream_window, client_->getStream(1)->pending_send_data_->length()); EXPECT_EQ(initial_stream_window, TestUtility::findGauge(client_stats_store_, "http2.pending_send_bytes")->value()); EXPECT_EQ(initial_stream_window, server_->getStream(1)->unconsumed_bytes_); @@ -1287,7 +1287,7 @@ TEST_P(Http2CodecImplFlowControlTest, TestFlowControlInPendingSendData) { EXPECT_CALL(callbacks, onAboveWriteBufferHighWatermark()); Buffer::OwnedImpl last_byte("!"); request_encoder_->encodeData(last_byte, false); - EXPECT_EQ(initial_stream_window + 1, client_->getStream(1)->pending_send_data_.length()); + EXPECT_EQ(initial_stream_window + 1, client_->getStream(1)->pending_send_data_->length()); EXPECT_EQ(initial_stream_window + 1, TestUtility::findGauge(client_stats_store_, "http2.pending_send_bytes")->value()); @@ -1332,7 +1332,7 @@ TEST_P(Http2CodecImplFlowControlTest, TestFlowControlInPendingSendData) { EXPECT_CALL(callbacks2, onBelowWriteBufferLowWatermark()).Times(0); EXPECT_CALL(callbacks3, onBelowWriteBufferLowWatermark()); server_->getStream(1)->readDisable(false); - EXPECT_EQ(0, client_->getStream(1)->pending_send_data_.length()); + EXPECT_EQ(0, client_->getStream(1)->pending_send_data_->length()); EXPECT_EQ(0, TestUtility::findGauge(client_stats_store_, "http2.pending_send_bytes")->value()); // The extra 1 byte sent won't trigger another window update, so the final window should be the // initial window minus the last 1 byte flush from the client to server. diff --git a/test/common/http/http2/conn_pool_test.cc b/test/common/http/http2/conn_pool_test.cc index d8b34b43b63d0..f29d0d83209eb 100644 --- a/test/common/http/http2/conn_pool_test.cc +++ b/test/common/http/http2/conn_pool_test.cc @@ -131,7 +131,7 @@ class Http2ConnPoolImplTest : public Event::TestUsingSimulatedTime, public testi auto cluster = std::make_shared>(); Network::ClientConnectionPtr connection{test_client.connection_}; test_client.codec_client_ = new CodecClientForTest( - CodecClient::Type::HTTP1, std::move(connection), test_client.codec_, + CodecType::HTTP1, std::move(connection), test_client.codec_, [this](CodecClient*) -> void { onClientDestroy(); }, Upstream::makeTestHost(cluster, "tcp://127.0.0.1:9000", simTime()), *test_client.client_dispatcher_); diff --git a/test/common/http/http3/BUILD b/test/common/http/http3/BUILD new file mode 100644 index 0000000000000..b2ab320ff0aff --- /dev/null +++ b/test/common/http/http3/BUILD @@ -0,0 +1,33 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_test", + "envoy_package", + "envoy_select_enable_http3", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_cc_test( + name = "conn_pool_test", + srcs = envoy_select_enable_http3(["conn_pool_test.cc"]), + tags = ["nofips"], + deps = envoy_select_enable_http3([ + "//source/common/event:dispatcher_lib", + "//source/common/http/http3:conn_pool_lib", + "//source/common/network:utility_lib", + "//source/common/upstream:upstream_includes", + "//source/common/upstream:upstream_lib", + "//test/common/http:common_lib", + "//test/common/upstream:utility_lib", + "//test/mocks/event:event_mocks", + "//test/mocks/http:http_mocks", + "//test/mocks/network:network_mocks", + "//test/mocks/runtime:runtime_mocks", + "//test/mocks/server:transport_socket_factory_context_mocks", + "//test/mocks/upstream:cluster_info_mocks", + "//test/mocks/upstream:transport_socket_match_mocks", + "//test/test_common:test_runtime_lib", + ]), +) diff --git a/test/common/http/http3/conn_pool_test.cc b/test/common/http/http3/conn_pool_test.cc new file mode 100644 index 0000000000000..3c16ae1d6eb78 --- /dev/null +++ b/test/common/http/http3/conn_pool_test.cc @@ -0,0 +1,89 @@ +#include "common/http/http3/conn_pool.h" +#include "common/quic/quic_transport_socket_factory.h" + +#include "test/common/upstream/utility.h" +#include "test/mocks/common.h" +#include "test/mocks/event/mocks.h" +#include "test/mocks/server/transport_socket_factory_context.h" +#include "test/mocks/ssl/mocks.h" +#include "test/mocks/upstream/cluster_info.h" +#include "test/mocks/upstream/host.h" +#include "test/test_common/simulated_time_system.h" + +using testing::NiceMock; +using testing::Return; + +namespace Envoy { +namespace Http { +namespace Http3 { + +TEST(Convert, Basic) { + NiceMock cluster_info; + quic::QuicConfig config; + + EXPECT_CALL(cluster_info, connectTimeout).WillOnce(Return(std::chrono::milliseconds(42))); + auto* protocol_options = cluster_info.http3_options_.mutable_quic_protocol_options(); + protocol_options->mutable_max_concurrent_streams()->set_value(43); + protocol_options->mutable_initial_stream_window_size()->set_value(65555); + + Http3ConnPoolImpl::setQuicConfigFromClusterConfig(cluster_info, config); + + EXPECT_EQ(config.max_time_before_crypto_handshake(), quic::QuicTime::Delta::FromMilliseconds(42)); + EXPECT_EQ(config.GetMaxBidirectionalStreamsToSend(), + protocol_options->max_concurrent_streams().value()); + EXPECT_EQ(config.GetMaxUnidirectionalStreamsToSend(), + protocol_options->max_concurrent_streams().value()); + EXPECT_EQ(config.GetInitialMaxStreamDataBytesIncomingBidirectionalToSend(), + protocol_options->initial_stream_window_size().value()); +} + +class Http3ConnPoolImplTest : public Event::TestUsingSimulatedTime, public testing::Test { +public: + void initialize() { + EXPECT_CALL(mockHost(), address()).WillRepeatedly(Return(test_address_)); + EXPECT_CALL(mockHost(), transportSocketFactory()).WillRepeatedly(testing::ReturnRef(factory_)); + new Event::MockSchedulableCallback(&dispatcher_); + Network::ConnectionSocket::OptionsSharedPtr options; + Network::TransportSocketOptionsSharedPtr transport_options; + pool_ = allocateConnPool(dispatcher_, random_, host_, Upstream::ResourcePriority::Default, + options, transport_options, state_, simTime()); + } + + Upstream::MockHost& mockHost() { return static_cast(*host_); } + + NiceMock dispatcher_; + std::shared_ptr cluster_{new NiceMock()}; + Upstream::HostSharedPtr host_{new NiceMock}; + NiceMock random_; + Upstream::ClusterConnectivityState state_; + Network::Address::InstanceConstSharedPtr test_address_ = + Network::Utility::resolveUrl("tcp://127.0.0.1:3000"); + NiceMock context_; + Quic::QuicClientTransportSocketFactory factory_{ + std::unique_ptr(new NiceMock), + context_}; + ConnectionPool::InstancePtr pool_; +}; + +TEST_F(Http3ConnPoolImplTest, CreationWithBufferLimits) { + EXPECT_CALL(mockHost().cluster_, perConnectionBufferLimitBytes); + initialize(); +} + +TEST_F(Http3ConnPoolImplTest, CreationWithConfig) { + // Set a couple of options from setQuicConfigFromClusterConfig to make sure they are applied. + auto* options = mockHost().cluster_.http3_options_.mutable_quic_protocol_options(); + options->mutable_max_concurrent_streams()->set_value(15); + options->mutable_initial_stream_window_size()->set_value(65555); + initialize(); + + Quic::PersistentQuicInfoImpl& info = static_cast(pool_.get())->quicInfo(); + EXPECT_EQ(info.quic_config_.GetMaxUnidirectionalStreamsToSend(), + options->max_concurrent_streams().value()); + EXPECT_EQ(info.quic_config_.GetInitialMaxStreamDataBytesIncomingBidirectionalToSend(), + options->initial_stream_window_size().value()); +} + +} // namespace Http3 +} // namespace Http +} // namespace Envoy diff --git a/test/common/http/xff_extension.cc b/test/common/http/xff_extension.cc new file mode 100644 index 0000000000000..3cca217790578 --- /dev/null +++ b/test/common/http/xff_extension.cc @@ -0,0 +1,11 @@ +#include "test/common/http/xff_extension.h" + +#include "extensions/http/original_ip_detection/xff/xff.h" + +namespace Envoy { + +Http::OriginalIPDetectionSharedPtr getXFFExtension(uint32_t hops) { + return std::make_shared(hops); +} + +} // namespace Envoy diff --git a/test/common/http/xff_extension.h b/test/common/http/xff_extension.h new file mode 100644 index 0000000000000..a8b203848452a --- /dev/null +++ b/test/common/http/xff_extension.h @@ -0,0 +1,10 @@ +#pragma once + +#include "envoy/http/original_ip_detection.h" + +// This helper is used to escape namespace pollution issues. +namespace Envoy { + +Http::OriginalIPDetectionSharedPtr getXFFExtension(uint32_t hops); + +} // namespace Envoy diff --git a/test/common/network/udp_fuzz.cc b/test/common/network/udp_fuzz.cc index 792f1f93286b6..ca1d2d41c3be5 100644 --- a/test/common/network/udp_fuzz.cc +++ b/test/common/network/udp_fuzz.cc @@ -44,6 +44,7 @@ class FuzzUdpListenerCallbacks : public Network::UdpListenerCallbacks { void onDatagramsDropped(uint32_t dropped) override; uint32_t workerIndex() const override; Network::UdpPacketWriter& udpPacketWriter() override; + size_t numPacketsExpectedPerEventLoop() const override; private: UdpFuzz* my_upf_; @@ -178,6 +179,10 @@ void FuzzUdpListenerCallbacks::onDatagramsDropped(uint32_t dropped) { UNREFERENCED_PARAMETER(dropped); } +size_t FuzzUdpListenerCallbacks::numPacketsExpectedPerEventLoop() const { + return Network::MAX_NUM_PACKETS_PER_EVENT_LOOP; +} + DEFINE_FUZZER(const uint8_t* buf, size_t len) { UdpFuzz udp_instance(buf, len); } } // namespace } // namespace Envoy diff --git a/test/common/network/udp_listener_impl_test.cc b/test/common/network/udp_listener_impl_test.cc index d200af1a0a170..1ac4dcb2936f5 100644 --- a/test/common/network/udp_listener_impl_test.cc +++ b/test/common/network/udp_listener_impl_test.cc @@ -56,6 +56,8 @@ class UdpListenerImplTest : public UdpListenerImplTestBase { // Return the real version by default. ON_CALL(override_syscall_, supportsMmsg()) .WillByDefault(Return(os_calls.latched().supportsMmsg())); + ON_CALL(listener_callbacks_, numPacketsExpectedPerEventLoop()) + .WillByDefault(Return(MAX_NUM_PACKETS_PER_EVENT_LOOP)); // Set listening socket options. server_socket_->addOptions(SocketOptionFactory::buildIpPacketInfoOptions()); @@ -63,6 +65,12 @@ class UdpListenerImplTest : public UdpListenerImplTestBase { if (Api::OsSysCallsSingleton::get().supportsUdpGro()) { server_socket_->addOptions(SocketOptionFactory::buildUdpGroOptions()); } + std::unique_ptr options = + std::make_unique(); + options->push_back(std::make_shared( + envoy::config::core::v3::SocketOption::STATE_BOUND, + ENVOY_MAKE_SOCKET_OPTION_NAME(SOL_SOCKET, SO_RCVBUF), 4 * 1024 * 1024)); + server_socket_->addOptions(std::move(options)); envoy::config::core::v3::UdpSocketConfig config; if (prefer_gro) { config.mutable_prefer_gro()->set_value(prefer_gro); @@ -71,11 +79,22 @@ class UdpListenerImplTest : public UdpListenerImplTestBase { std::make_unique(dispatcherImpl(), server_socket_, listener_callbacks_, dispatcherImpl().timeSource(), config); udp_packet_writer_ = std::make_unique(server_socket_->ioHandle()); + int get_recvbuf_size = 0; + socklen_t int_size = static_cast(sizeof(get_recvbuf_size)); + const Api::SysCallIntResult result2 = + server_socket_->getSocketOption(SOL_SOCKET, SO_RCVBUF, &get_recvbuf_size, &int_size); + EXPECT_EQ(0, result2.rc_); + // Kernel increases the buffer size to allow bookkeeping overhead. + if (get_recvbuf_size < 4 * 1024 * 1024) { + recvbuf_large_enough_ = false; + } + ON_CALL(listener_callbacks_, udpPacketWriter()).WillByDefault(ReturnRef(*udp_packet_writer_)); } NiceMock override_syscall_; TestThreadsafeSingletonInjector os_calls{&override_syscall_}; + bool recvbuf_large_enough_{true}; }; INSTANTIATE_TEST_SUITE_P(IpVersions, UdpListenerImplTest, @@ -145,7 +164,7 @@ TEST_P(UdpListenerImplTest, UseActualDstUdp) { dispatcher_->run(Event::Dispatcher::RunType::Block); } -// Test a large datagram that gets dropped using recvmmsg if supported. +// Test a large datagram that gets dropped using recvmsg or recvmmsg if supported. TEST_P(UdpListenerImplTest, LargeDatagramRecvmmsg) { setup(); @@ -172,33 +191,75 @@ TEST_P(UdpListenerImplTest, LargeDatagramRecvmmsg) { EXPECT_EQ(2, listener_->packetsDropped()); } -// Test a large datagram that gets dropped using recvmsg. -TEST_P(UdpListenerImplTest, LargeDatagramRecvmsg) { +TEST_P(UdpListenerImplTest, LimitNumberOfReadsPerLoop) { setup(); + if (!Runtime::runtimeFeatureEnabled("envoy.reloadable_features.udp_per_event_loop_read_limit")) { + return; + } + const uint64_t num_packets_per_read = + Api::OsSysCallsSingleton::get().supportsMmsg() ? NUM_DATAGRAMS_PER_MMSG_RECEIVE : 1u; + + size_t num_packets_expected_per_loop{32u}; + // These packets should be read in more than 3 loops. + const std::string payload1(10, 'a'); + for (uint64_t i = 0; i < 2 * num_packets_expected_per_loop; ++i) { + client_.write(payload1, *send_to_addr_); + } + const std::string last_piece("bbb"); + client_.write(last_piece, *send_to_addr_); - ON_CALL(override_syscall_, supportsMmsg()).WillByDefault(Return(false)); - - // This will get dropped. - const std::string first(4096, 'a'); - client_.write(first, *send_to_addr_); - const std::string second("second"); - client_.write(second, *send_to_addr_); - // This will get dropped. - const std::string third(4096, 'b'); - client_.write(third, *send_to_addr_); + EXPECT_CALL(listener_callbacks_, onReadReady()).Times(testing::AtLeast(3u)); + EXPECT_CALL(listener_callbacks_, numPacketsExpectedPerEventLoop()) + .WillRepeatedly(Return(num_packets_expected_per_loop)); + EXPECT_CALL(listener_callbacks_, onData(_)) + .WillRepeatedly(Invoke([&](const UdpRecvData& data) -> void { + validateRecvCallbackParams(data, num_packets_per_read); + if (last_piece == data.buffer_->toString()) { + dispatcher_->exit(); + } + })); + dispatcher_->run(Event::Dispatcher::RunType::Block); + num_packets_received_by_listener_ = 0u; + num_packets_expected_per_loop = 0u; + std::string payload2(10, 'c'); + // This packet should be read. + client_.write(payload2, *send_to_addr_); EXPECT_CALL(listener_callbacks_, onReadReady()); - EXPECT_CALL(listener_callbacks_, onDatagramsDropped(_)).Times(AtLeast(1)); + EXPECT_CALL(listener_callbacks_, numPacketsExpectedPerEventLoop()) + .WillRepeatedly(Return(num_packets_expected_per_loop)); EXPECT_CALL(listener_callbacks_, onData(_)).WillOnce(Invoke([&](const UdpRecvData& data) -> void { - validateRecvCallbackParams( - data, Api::OsSysCallsSingleton::get().supportsMmsg() ? NUM_DATAGRAMS_PER_MMSG_RECEIVE : 1u); - EXPECT_EQ(data.buffer_->toString(), second); - + validateRecvCallbackParams(data, num_packets_per_read); + EXPECT_EQ(payload2, data.buffer_->toString()); dispatcher_->exit(); })); + dispatcher_->run(Event::Dispatcher::RunType::Block); + if (!recvbuf_large_enough_) { + // If SO_RCVBUF failed to enlarge receive buffer to 4MB, the rest of test will likely to fail + // because packets may be easily dropped. Skip the rest of the test. + return; + } + num_packets_received_by_listener_ = 0u; + // Though the mocked callback wants to read more, only 6000 reads maximum are allowed. + num_packets_expected_per_loop = MAX_NUM_PACKETS_PER_EVENT_LOOP + 1u; + std::string payload3(10, 'd'); + for (uint64_t i = 0; i < num_packets_expected_per_loop; ++i) { + client_.write(payload3, *send_to_addr_); + } + std::string really_last_piece("eee"); + client_.write(really_last_piece, *send_to_addr_); + EXPECT_CALL(listener_callbacks_, onReadReady()).Times(testing::AtLeast(2u)); + EXPECT_CALL(listener_callbacks_, numPacketsExpectedPerEventLoop()) + .WillRepeatedly(Return(num_packets_expected_per_loop)); + EXPECT_CALL(listener_callbacks_, onData(_)) + .WillRepeatedly(Invoke([&](const UdpRecvData& data) -> void { + validateRecvCallbackParams(data, num_packets_per_read); + if (really_last_piece == data.buffer_->toString()) { + dispatcher_->exit(); + } + })); dispatcher_->run(Event::Dispatcher::RunType::Block); - EXPECT_EQ(2, listener_->packetsDropped()); } #ifdef UDP_GRO @@ -380,13 +441,16 @@ TEST_P(UdpListenerImplTest, UdpListenerRecvMsgError) { EXPECT_CALL(listener_callbacks_, onReceiveError(_)) .WillOnce(Invoke([&](Api::IoError::IoErrorCode err) -> void { ASSERT_EQ(Api::IoError::IoErrorCode::NoSupport, err); - dispatcher_->exit(); })); // Inject mocked OsSysCalls implementation to mock a read failure. Api::MockOsSysCalls os_sys_calls; TestThreadsafeSingletonInjector os_calls(&os_sys_calls); - EXPECT_CALL(os_sys_calls, supportsMmsg()); + EXPECT_CALL(os_sys_calls, supportsMmsg()) + .Times( + (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.udp_per_event_loop_read_limit") + ? 2u + : 1u)); EXPECT_CALL(os_sys_calls, recvmsg(_, _, _)) .WillOnce(Return(Api::SysCallSizeResult{-1, SOCKET_ERROR_NOT_SUP})); diff --git a/test/common/protobuf/utility_test.cc b/test/common/protobuf/utility_test.cc index 8b2a857577c35..45983670a0035 100644 --- a/test/common/protobuf/utility_test.cc +++ b/test/common/protobuf/utility_test.cc @@ -1185,6 +1185,11 @@ TEST_F(ProtobufUtilityTest, ValueUtilLoadFromYamlObject) { "struct_value { fields { key: \"foo\" value { string_value: \"bar\" } } }"); } +TEST_F(ProtobufUtilityTest, ValueUtilLoadFromYamlObjectWithIgnoredEntries) { + EXPECT_EQ(ValueUtil::loadFromYaml("!ignore foo: bar\nbaz: qux").ShortDebugString(), + "struct_value { fields { key: \"baz\" value { string_value: \"qux\" } } }"); +} + TEST(LoadFromYamlExceptionTest, BadConversion) { std::string bad_yaml = R"EOF( admin: @@ -2016,4 +2021,100 @@ TEST(TypeUtilTest, TypeUrlHelperFunction) { TypeUtil::descriptorFullNameToTypeUrl("envoy.config.filter.http.ip_tagging.v2.IPTagging")); } +class StructUtilTest : public ProtobufUtilityTest { +protected: + ProtobufWkt::Struct updateSimpleStruct(const ProtobufWkt::Value& v0, + const ProtobufWkt::Value& v1) { + ProtobufWkt::Struct obj, with; + (*obj.mutable_fields())["key"] = v0; + (*with.mutable_fields())["key"] = v1; + StructUtil::update(obj, with); + EXPECT_EQ(obj.fields().size(), 1); + return obj; + } +}; + +TEST_F(StructUtilTest, StructUtilUpdateScalars) { + { + const auto obj = updateSimpleStruct(ValueUtil::stringValue("v0"), ValueUtil::stringValue("v1")); + EXPECT_EQ(obj.fields().at("key").string_value(), "v1"); + } + + { + const auto obj = updateSimpleStruct(ValueUtil::numberValue(0), ValueUtil::numberValue(1)); + EXPECT_EQ(obj.fields().at("key").number_value(), 1); + } + + { + const auto obj = updateSimpleStruct(ValueUtil::boolValue(false), ValueUtil::boolValue(true)); + EXPECT_EQ(obj.fields().at("key").bool_value(), true); + } + + { + const auto obj = updateSimpleStruct(ValueUtil::nullValue(), ValueUtil::nullValue()); + EXPECT_EQ(obj.fields().at("key").kind_case(), ProtobufWkt::Value::KindCase::kNullValue); + } +} + +TEST_F(StructUtilTest, StructUtilUpdateDifferentKind) { + { + const auto obj = updateSimpleStruct(ValueUtil::stringValue("v0"), ValueUtil::numberValue(1)); + auto& val = obj.fields().at("key"); + EXPECT_EQ(val.kind_case(), ProtobufWkt::Value::KindCase::kNumberValue); + EXPECT_EQ(val.number_value(), 1); + } + + { + const auto obj = + updateSimpleStruct(ValueUtil::structValue(MessageUtil::keyValueStruct("subkey", "v0")), + ValueUtil::stringValue("v1")); + auto& val = obj.fields().at("key"); + EXPECT_EQ(val.kind_case(), ProtobufWkt::Value::KindCase::kStringValue); + EXPECT_EQ(val.string_value(), "v1"); + } +} + +TEST_F(StructUtilTest, StructUtilUpdateList) { + ProtobufWkt::Struct obj, with; + auto& list = *(*obj.mutable_fields())["key"].mutable_list_value(); + list.add_values()->set_string_value("v0"); + + auto& with_list = *(*with.mutable_fields())["key"].mutable_list_value(); + with_list.add_values()->set_number_value(1); + const auto v2 = MessageUtil::keyValueStruct("subkey", "str"); + *with_list.add_values()->mutable_struct_value() = v2; + + StructUtil::update(obj, with); + ASSERT_THAT(obj.fields().size(), 1); + const auto& list_vals = list.values(); + EXPECT_TRUE(ValueUtil::equal(list_vals[0], ValueUtil::stringValue("v0"))); + EXPECT_TRUE(ValueUtil::equal(list_vals[1], ValueUtil::numberValue(1))); + EXPECT_TRUE(ValueUtil::equal(list_vals[2], ValueUtil::structValue(v2))); +} + +TEST_F(StructUtilTest, StructUtilUpdateNewKey) { + ProtobufWkt::Struct obj, with; + (*obj.mutable_fields())["key0"].set_number_value(1); + (*with.mutable_fields())["key1"].set_number_value(1); + StructUtil::update(obj, with); + + const auto& fields = obj.fields(); + EXPECT_TRUE(ValueUtil::equal(fields.at("key0"), ValueUtil::numberValue(1))); + EXPECT_TRUE(ValueUtil::equal(fields.at("key1"), ValueUtil::numberValue(1))); +} + +TEST_F(StructUtilTest, StructUtilUpdateRecursiveStruct) { + ProtobufWkt::Struct obj, with; + *(*obj.mutable_fields())["tags"].mutable_struct_value() = + MessageUtil::keyValueStruct("tag0", "1"); + *(*with.mutable_fields())["tags"].mutable_struct_value() = + MessageUtil::keyValueStruct("tag1", "1"); + StructUtil::update(obj, with); + + ASSERT_EQ(obj.fields().at("tags").kind_case(), ProtobufWkt::Value::KindCase::kStructValue); + const auto& tags = obj.fields().at("tags").struct_value().fields(); + EXPECT_TRUE(ValueUtil::equal(tags.at("tag0"), ValueUtil::stringValue("1"))); + EXPECT_TRUE(ValueUtil::equal(tags.at("tag1"), ValueUtil::stringValue("1"))); +} + } // namespace Envoy diff --git a/test/common/quic/active_quic_listener_test.cc b/test/common/quic/active_quic_listener_test.cc index b9df504e88a44..ffb12a05b56bf 100644 --- a/test/common/quic/active_quic_listener_test.cc +++ b/test/common/quic/active_quic_listener_test.cc @@ -267,6 +267,7 @@ class ActiveQuicListenerTest : public QuicMultiVersionTest { enabled: default_value: true runtime_key: quic.enabled + packets_to_read_to_connection_count_ratio: 50 )EOF", connection_window_size_, stream_window_size_); } @@ -323,13 +324,13 @@ TEST_P(ActiveQuicListenerTest, FailSocketOptionUponCreation) { auto options = std::make_shared>(); options->emplace_back(std::move(option)); quic_listener_.reset(); - EXPECT_THROW_WITH_REGEX( - (void)std::make_unique( - 0, 1, *dispatcher_, connection_handler_, listen_socket_, listener_config_, quic_config_, - options, false, - ActiveQuicListenerFactoryPeer::runtimeEnabled( - static_cast(listener_factory_.get()))), - Network::CreateListenerException, "Failed to apply socket options."); + EXPECT_THROW_WITH_REGEX((void)std::make_unique( + 0, 1, *dispatcher_, connection_handler_, listen_socket_, + listener_config_, quic_config_, options, false, + ActiveQuicListenerFactoryPeer::runtimeEnabled( + static_cast(listener_factory_.get())), + 32u), + Network::CreateListenerException, "Failed to apply socket options."); } TEST_P(ActiveQuicListenerTest, ReceiveCHLO) { @@ -342,6 +343,10 @@ TEST_P(ActiveQuicListenerTest, ReceiveCHLO) { dispatcher_->run(Event::Dispatcher::RunType::NonBlock); EXPECT_FALSE(buffered_packets->HasChlosBuffered()); EXPECT_NE(0u, quic_dispatcher_->NumSessions()); + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.udp_per_event_loop_read_limit")) { + EXPECT_EQ(50 * quic_dispatcher_->NumSessions(), + quic_listener_->numPacketsExpectedPerEventLoop()); + } const quic::QuicSession* session = quic::test::QuicDispatcherPeer::FindSession(quic_dispatcher_, connection_id); ASSERT(session != nullptr); @@ -389,7 +394,15 @@ TEST_P(ActiveQuicListenerTest, ProcessBufferedChlos) { quic::QuicBufferedPacketStore* const buffered_packets = quic::test::QuicDispatcherPeer::GetBufferedPackets(quic_dispatcher_); const uint32_t count = (ActiveQuicListener::kNumSessionsToCreatePerLoop * 2) + 1; - maybeConfigureMocks(count); + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.udp_per_event_loop_read_limit")) { + maybeConfigureMocks(count + 1); + // Create 1 session to increase number of packet to read in the next read event. + sendCHLO(quic::test::TestConnectionId()); + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + EXPECT_NE(0u, quic_dispatcher_->NumSessions()); + } else { + maybeConfigureMocks(count); + } // Generate one more CHLO than can be processed immediately. for (size_t i = 1; i <= count; ++i) { @@ -407,6 +420,11 @@ TEST_P(ActiveQuicListenerTest, ProcessBufferedChlos) { } EXPECT_FALSE(buffered_packets->HasChlosBuffered()); EXPECT_NE(0u, quic_dispatcher_->NumSessions()); + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.udp_per_event_loop_read_limit")) { + EXPECT_EQ(50 * quic_dispatcher_->NumSessions(), + quic_listener_->numPacketsExpectedPerEventLoop()); + } + readFromClientSockets(); } diff --git a/test/common/quic/client_connection_factory_impl_test.cc b/test/common/quic/client_connection_factory_impl_test.cc index 87e6111cc870e..b7d26e2e2f40c 100644 --- a/test/common/quic/client_connection_factory_impl_test.cc +++ b/test/common/quic/client_connection_factory_impl_test.cc @@ -27,6 +27,10 @@ class QuicNetworkConnectionTest : public Event::TestUsingSimulatedTime, public t context_); } + uint32_t highWatermark(EnvoyQuicClientSession* session) { + return session->write_buffer_watermark_simulation_.highWatermark(); + } + NiceMock dispatcher_; std::shared_ptr cluster_{new NiceMock()}; Upstream::HostSharedPtr host_{new NiceMock}; @@ -41,7 +45,7 @@ class QuicNetworkConnectionTest : public Event::TestUsingSimulatedTime, public t TEST_F(QuicNetworkConnectionTest, BufferLimits) { initialize(); - PersistentQuicInfoImpl info{dispatcher_, *factory_, simTime(), test_address_}; + PersistentQuicInfoImpl info{dispatcher_, *factory_, simTime(), test_address_, 45}; std::unique_ptr client_connection = createQuicNetworkConnection(info, dispatcher_, test_address_, test_address_); @@ -50,6 +54,7 @@ TEST_F(QuicNetworkConnectionTest, BufferLimits) { client_connection->connect(); EXPECT_TRUE(client_connection->connecting()); ASSERT(session != nullptr); + EXPECT_EQ(highWatermark(session), 45); EXPECT_EQ(absl::nullopt, session->unixSocketPeerCredentials()); EXPECT_EQ(absl::nullopt, session->lastRoundTripTime()); client_connection->close(Network::ConnectionCloseType::NoFlush); diff --git a/test/common/router/BUILD b/test/common/router/BUILD index 858621a096c4d..eabd2de01e82d 100644 --- a/test/common/router/BUILD +++ b/test/common/router/BUILD @@ -29,6 +29,7 @@ envoy_cc_test_library( "//source/common/http:header_map_lib", "//source/common/http:headers_lib", "//source/common/router:config_lib", + "//source/common/router:string_accessor_lib", "//source/common/stream_info:filter_state_lib", "//test/extensions/filters/http/common:empty_http_filter_config_lib", "//test/fuzz:utility_lib", diff --git a/test/common/router/config_impl_test.cc b/test/common/router/config_impl_test.cc index 5911d90c50dd9..ee1ce6c82d7f6 100644 --- a/test/common/router/config_impl_test.cc +++ b/test/common/router/config_impl_test.cc @@ -18,6 +18,7 @@ #include "common/http/headers.h" #include "common/network/address_impl.h" #include "common/router/config_impl.h" +#include "common/router/string_accessor_impl.h" #include "common/stream_info/filter_state_impl.h" #include "test/common/router/route_fuzz.pb.h" @@ -1790,6 +1791,52 @@ TEST_F(RouteMatcherTest, TestAddRemoveResponseHeadersAppendMostSpecificWins) { Http::LowerCaseString("x-vhost-remove"))); } +TEST_F(RouteMatcherTest, TestResponseHeaderTransformsDoFormatting) { + factory_context_.cluster_manager_.initializeClusters({"default"}, {}); + const std::string yaml = R"EOF( +virtual_hosts: + - name: default + domains: ["*"] + routes: + - match: + prefix: "/" + route: + cluster: "default" +response_headers_to_add: + - header: + key: x-has-variable + value: "%PER_REQUEST_STATE(testing)%" + append: false +)EOF"; + NiceMock stream_info; + + Envoy::StreamInfo::FilterStateSharedPtr filter_state( + std::make_shared( + Envoy::StreamInfo::FilterState::LifeSpan::FilterChain)); + filter_state->setData("testing", std::make_unique("test_value"), + StreamInfo::FilterState::StateType::ReadOnly, + StreamInfo::FilterState::LifeSpan::FilterChain); + ON_CALL(stream_info, filterState()).WillByDefault(ReturnRef(filter_state)); + ON_CALL(Const(stream_info), filterState()).WillByDefault(ReturnRef(*filter_state)); + + TestConfigImpl config(parseRouteConfigurationFromYaml(yaml), factory_context_, true); + + Http::TestRequestHeaderMapImpl req_headers = + genHeaders("www.lyft.com", "/new_endpoint/foo", "GET"); + const RouteEntry* route = config.route(req_headers, 0)->routeEntry(); + Http::TestResponseHeaderMapImpl headers; + route->finalizeResponseHeaders(headers, stream_info); + + auto transforms = route->responseHeaderTransforms(stream_info, /*do_formatting=*/true); + EXPECT_THAT(transforms.headers_to_overwrite, + ElementsAre(Pair(Http::LowerCaseString("x-has-variable"), "test_value"))); + + transforms = route->responseHeaderTransforms(stream_info, /*do_formatting=*/false); + EXPECT_THAT( + transforms.headers_to_overwrite, + ElementsAre(Pair(Http::LowerCaseString("x-has-variable"), "%PER_REQUEST_STATE(testing)%"))); +} + TEST_F(RouteMatcherTest, TestAddGlobalResponseHeaderRemoveFromRoute) { const std::string yaml = R"EOF( virtual_hosts: diff --git a/test/common/router/header_formatter_test.cc b/test/common/router/header_formatter_test.cc index 6ffb945b39486..47385fb9dad99 100644 --- a/test/common/router/header_formatter_test.cc +++ b/test/common/router/header_formatter_test.cc @@ -1438,7 +1438,7 @@ response_headers_to_remove: ["x-foo-header"] EXPECT_EQ("bar", header_map.get_("x-foo-header")); } -TEST(HeaderParserTest, GetHeaderTransforms) { +TEST(HeaderParserTest, GetHeaderTransformsWithFormatting) { const std::string yaml = R"EOF( match: { prefix: "/new_endpoint" } route: @@ -1452,6 +1452,10 @@ match: { prefix: "/new_endpoint" } key: "x-bar-header" value: "bar" append: false + - header: + key: "x-per-request-header" + value: "%PER_REQUEST_STATE(testing)%" + append: false response_headers_to_remove: ["x-baz-header"] )EOF"; @@ -1460,11 +1464,67 @@ response_headers_to_remove: ["x-baz-header"] HeaderParser::configure(route.response_headers_to_add(), route.response_headers_to_remove()); NiceMock stream_info; + Envoy::StreamInfo::FilterStateSharedPtr filter_state( + std::make_shared( + Envoy::StreamInfo::FilterState::LifeSpan::FilterChain)); + filter_state->setData("testing", std::make_unique("test_value"), + StreamInfo::FilterState::StateType::ReadOnly, + StreamInfo::FilterState::LifeSpan::FilterChain); + ON_CALL(stream_info, filterState()).WillByDefault(ReturnRef(filter_state)); + ON_CALL(Const(stream_info), filterState()).WillByDefault(ReturnRef(*filter_state)); + auto transforms = resp_header_parser->getHeaderTransforms(stream_info); EXPECT_THAT(transforms.headers_to_append, ElementsAre(Pair(Http::LowerCaseString("x-foo-header"), "foo"))); EXPECT_THAT(transforms.headers_to_overwrite, - ElementsAre(Pair(Http::LowerCaseString("x-bar-header"), "bar"))); + ElementsAre(Pair(Http::LowerCaseString("x-bar-header"), "bar"), + Pair(Http::LowerCaseString("x-per-request-header"), "test_value"))); + EXPECT_THAT(transforms.headers_to_remove, ElementsAre(Http::LowerCaseString("x-baz-header"))); +} + +TEST(HeaderParserTest, GetHeaderTransformsOriginalValues) { + const std::string yaml = R"EOF( +match: { prefix: "/new_endpoint" } +route: + cluster: www2 +response_headers_to_add: + - header: + key: "x-foo-header" + value: "foo" + append: true + - header: + key: "x-bar-header" + value: "bar" + append: false + - header: + key: "x-per-request-header" + value: "%PER_REQUEST_STATE(testing)%" + append: false +response_headers_to_remove: ["x-baz-header"] +)EOF"; + + const auto route = parseRouteFromV3Yaml(yaml); + HeaderParserPtr response_header_parser = + HeaderParser::configure(route.response_headers_to_add(), route.response_headers_to_remove()); + NiceMock stream_info; + + Envoy::StreamInfo::FilterStateSharedPtr filter_state( + std::make_shared( + Envoy::StreamInfo::FilterState::LifeSpan::FilterChain)); + filter_state->setData("testing", std::make_unique("test_value"), + StreamInfo::FilterState::StateType::ReadOnly, + StreamInfo::FilterState::LifeSpan::FilterChain); + ON_CALL(stream_info, filterState()).WillByDefault(ReturnRef(filter_state)); + ON_CALL(Const(stream_info), filterState()).WillByDefault(ReturnRef(*filter_state)); + + auto transforms = + response_header_parser->getHeaderTransforms(stream_info, /*do_formatting=*/false); + EXPECT_THAT(transforms.headers_to_append, + ElementsAre(Pair(Http::LowerCaseString("x-foo-header"), "foo"))); + EXPECT_THAT(transforms.headers_to_overwrite, + ElementsAre(Pair(Http::LowerCaseString("x-bar-header"), "bar"), + Pair(Http::LowerCaseString("x-per-request-header"), + "%PER_REQUEST_STATE(testing)%"))); EXPECT_THAT(transforms.headers_to_remove, ElementsAre(Http::LowerCaseString("x-baz-header"))); } diff --git a/test/common/stats/stat_merger_corpus/clusterfuzz-testcase-minimized-stat_merger_fuzz_test-5675276891717632 b/test/common/stats/stat_merger_corpus/clusterfuzz-testcase-minimized-stat_merger_fuzz_test-5675276891717632 new file mode 100644 index 0000000000000..d565d75ac8ee2 --- /dev/null +++ b/test/common/stats/stat_merger_corpus/clusterfuzz-testcase-minimized-stat_merger_fuzz_test-5675276891717632 @@ -0,0 +1 @@ +ÿö .. \ No newline at end of file diff --git a/test/common/stats/stat_merger_fuzz_test.cc b/test/common/stats/stat_merger_fuzz_test.cc index f3e84f14f632a..bd3e6b2dbc071 100644 --- a/test/common/stats/stat_merger_fuzz_test.cc +++ b/test/common/stats/stat_merger_fuzz_test.cc @@ -39,7 +39,7 @@ void testDynamicEncoding(absl::string_view data, SymbolTable& symbol_table) { // TODO(#10008): We should remove the "1 +" below, so we can get empty // segments, which trigger some inconsistent handling as described in that // bug. - uint32_t num_bytes = (1 + data[index]) & 0x7; + uint32_t num_bytes = 1 + (data[index] & 0x7); // Carve out the segment and use the 4th bit from the control-byte to // determine whether to treat this segment symbolic or not. diff --git a/test/common/upstream/health_check_fuzz.cc b/test/common/upstream/health_check_fuzz.cc index d2e37392b296e..6a14b35fe3b9c 100644 --- a/test/common/upstream/health_check_fuzz.cc +++ b/test/common/upstream/health_check_fuzz.cc @@ -343,9 +343,9 @@ void GrpcHealthCheckFuzz::initialize(test::common::upstream::HealthCheckTestCase Event::MockDispatcher dispatcher_; auto time_source = std::make_unique>(); test_session.codec_client_ = new CodecClientForTest( - Http::CodecClient::Type::HTTP1, std::move(conn_data.connection_), - test_session.codec_, nullptr, - Upstream::makeTestHost(cluster, "tcp://127.0.0.1:9000", *time_source), dispatcher_); + Http::CodecType::HTTP1, std::move(conn_data.connection_), test_session.codec_, + nullptr, Upstream::makeTestHost(cluster, "tcp://127.0.0.1:9000", *time_source), + dispatcher_); return test_session.codec_client_; })); expectStreamCreate(); diff --git a/test/common/upstream/health_check_fuzz_test_utils.cc b/test/common/upstream/health_check_fuzz_test_utils.cc index e67e5f7a6cf06..0bfae721dca7b 100644 --- a/test/common/upstream/health_check_fuzz_test_utils.cc +++ b/test/common/upstream/health_check_fuzz_test_utils.cc @@ -52,8 +52,8 @@ void HttpHealthCheckerImplTestBase::expectClientCreate( new NiceMock()}; Event::MockDispatcher dispatcher_; test_session.codec_client_ = new CodecClientForTest( - Http::CodecClient::Type::HTTP1, std::move(conn_data.connection_), - test_session.codec_, nullptr, + Http::CodecType::HTTP1, std::move(conn_data.connection_), test_session.codec_, + nullptr, Upstream::makeTestHost(cluster, "tcp://127.0.0.1:9000", dispatcher_.timeSource()), dispatcher_); return test_session.codec_client_; @@ -129,8 +129,8 @@ void GrpcHealthCheckerImplTestBaseUtils::expectClientCreate(size_t index) { Event::MockDispatcher dispatcher_; test_session.codec_client_ = new CodecClientForTest( - Http::CodecClient::Type::HTTP1, std::move(conn_data.connection_), - test_session.codec_, nullptr, + Http::CodecType::HTTP1, std::move(conn_data.connection_), test_session.codec_, + nullptr, Upstream::makeTestHost(cluster, "tcp://127.0.0.1:9000", dispatcher_.timeSource()), dispatcher_); return test_session.codec_client_; diff --git a/test/common/upstream/health_check_fuzz_test_utils.h b/test/common/upstream/health_check_fuzz_test_utils.h index afc9b0a86cf15..833df01ad95ec 100644 --- a/test/common/upstream/health_check_fuzz_test_utils.h +++ b/test/common/upstream/health_check_fuzz_test_utils.h @@ -37,7 +37,7 @@ class TestHttpHealthCheckerImpl : public HttpHealthCheckerImpl { // HttpHealthCheckerImpl MOCK_METHOD(Http::CodecClient*, createCodecClient_, (Upstream::Host::CreateConnectionData&)); - Http::CodecClient::Type codecClientType() { return codec_client_type_; } + Http::CodecType codecClientType() { return codec_client_type_; } }; class HttpHealthCheckerImplTestBase : public HealthCheckerTestBase { diff --git a/test/common/upstream/health_checker_impl_test.cc b/test/common/upstream/health_checker_impl_test.cc index efb4b3b6c9615..5cba4b980a27c 100644 --- a/test/common/upstream/health_checker_impl_test.cc +++ b/test/common/upstream/health_checker_impl_test.cc @@ -124,7 +124,7 @@ class TestHttpHealthCheckerImpl : public HttpHealthCheckerImpl { // HttpHealthCheckerImpl MOCK_METHOD(Http::CodecClient*, createCodecClient_, (Upstream::Host::CreateConnectionData&)); - Http::CodecClient::Type codecClientType() { return codec_client_type_; } + Http::CodecType codecClientType() { return codec_client_type_; } }; class HttpHealthCheckerImplTest : public Event::TestUsingSimulatedTime, @@ -584,9 +584,9 @@ class HttpHealthCheckerImplTest : public Event::TestUsingSimulatedTime, new NiceMock()}; Event::MockDispatcher dispatcher_; test_session.codec_client_ = new CodecClientForTest( - Http::CodecClient::Type::HTTP1, std::move(conn_data.connection_), - test_session.codec_, nullptr, - Upstream::makeTestHost(cluster, "tcp://127.0.0.1:9000", simTime()), dispatcher_); + Http::CodecType::HTTP1, std::move(conn_data.connection_), test_session.codec_, + nullptr, Upstream::makeTestHost(cluster, "tcp://127.0.0.1:9000", simTime()), + dispatcher_); return test_session.codec_client_; })); } @@ -2530,7 +2530,7 @@ TEST_F(HttpHealthCheckerImplTest, SuccessWithMultipleHostsAndAltPort) { TEST_F(HttpHealthCheckerImplTest, Http2ClusterUseHttp2CodecClient) { setupNoServiceValidationHCWithHttp2(); - EXPECT_EQ(Http::CodecClient::Type::HTTP2, health_checker_->codecClientType()); + EXPECT_EQ(Http::CodecType::HTTP2, health_checker_->codecClientType()); } MATCHER_P(MetadataEq, expected, "") { @@ -3006,7 +3006,7 @@ class ProdHttpHealthCheckerTest : public testing::Test, public HealthCheckerTest TEST_F(ProdHttpHealthCheckerTest, ProdHttpHealthCheckerH1HealthChecking) { setupNoServiceValidationHC(); - EXPECT_EQ(Http::CodecClient::Type::HTTP1, + EXPECT_EQ(Http::CodecType::HTTP1, health_checker_->createCodecClientForTest(std::move(connection_))->type()); } @@ -3028,7 +3028,7 @@ TEST_F(HttpHealthCheckerImplTest, DEPRECATED_FEATURE_TEST(Http1CodecClient)) { allocHealthChecker(yaml, false); addCompletionCallback(); - EXPECT_EQ(Http::CodecClient::Type::HTTP1, health_checker_->codecClientType()); + EXPECT_EQ(Http::CodecType::HTTP1, health_checker_->codecClientType()); } TEST_F(HttpHealthCheckerImplTest, DEPRECATED_FEATURE_TEST(Http2CodecClient)) { @@ -3049,7 +3049,7 @@ TEST_F(HttpHealthCheckerImplTest, DEPRECATED_FEATURE_TEST(Http2CodecClient)) { allocHealthChecker(yaml, false); addCompletionCallback(); - EXPECT_EQ(Http::CodecClient::Type::HTTP2, health_checker_->codecClientType()); + EXPECT_EQ(Http::CodecType::HTTP2, health_checker_->codecClientType()); } TEST_F(HttpHealthCheckerImplTest, DEPRECATED_FEATURE_TEST(ServiceNameMatch)) { @@ -3120,7 +3120,7 @@ TEST_F(HttpHealthCheckerImplTest, DEPRECATED_FEATURE_TEST(ServiceNameMismatch)) TEST_F(ProdHttpHealthCheckerTest, ProdHttpHealthCheckerH2HealthChecking) { setupNoServiceValidationHCWithHttp2(); - EXPECT_EQ(Http::CodecClient::Type::HTTP2, + EXPECT_EQ(Http::CodecType::HTTP2, health_checker_->createCodecClientForTest(std::move(connection_))->type()); } @@ -4122,9 +4122,9 @@ class GrpcHealthCheckerImplTestBase : public Event::TestUsingSimulatedTime, Event::MockDispatcher dispatcher_; test_session.codec_client_ = new CodecClientForTest( - Http::CodecClient::Type::HTTP1, std::move(conn_data.connection_), - test_session.codec_, nullptr, - Upstream::makeTestHost(cluster, "tcp://127.0.0.1:9000", simTime()), dispatcher_); + Http::CodecType::HTTP1, std::move(conn_data.connection_), test_session.codec_, + nullptr, Upstream::makeTestHost(cluster, "tcp://127.0.0.1:9000", simTime()), + dispatcher_); return test_session.codec_client_; })); } diff --git a/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc b/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc index eb594fd1b9935..f2daefe122107 100644 --- a/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc +++ b/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc @@ -22,11 +22,11 @@ namespace { class AccessLogIntegrationTest : public Grpc::VersionedGrpcClientIntegrationParamTest, public HttpIntegrationTest { public: - AccessLogIntegrationTest() : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, ipVersion()) {} + AccessLogIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, ipVersion()) {} void createUpstreams() override { HttpIntegrationTest::createUpstreams(); - addFakeUpstream(FakeHttpConnection::Type::HTTP2); + addFakeUpstream(Http::CodecType::HTTP2); } void initialize() override { diff --git a/test/extensions/access_loggers/grpc/tcp_grpc_access_log_integration_test.cc b/test/extensions/access_loggers/grpc/tcp_grpc_access_log_integration_test.cc index 51fc54f02d781..0e700eb4603ea 100644 --- a/test/extensions/access_loggers/grpc/tcp_grpc_access_log_integration_test.cc +++ b/test/extensions/access_loggers/grpc/tcp_grpc_access_log_integration_test.cc @@ -34,7 +34,7 @@ class TcpGrpcAccessLogIntegrationTest : public Grpc::VersionedGrpcClientIntegrat void createUpstreams() override { BaseIntegrationTest::createUpstreams(); - addFakeUpstream(FakeHttpConnection::Type::HTTP2); + addFakeUpstream(Http::CodecType::HTTP2); } void initialize() override { diff --git a/test/extensions/access_loggers/open_telemetry/access_log_integration_test.cc b/test/extensions/access_loggers/open_telemetry/access_log_integration_test.cc index 107572c662d6f..eead9606d55b1 100644 --- a/test/extensions/access_loggers/open_telemetry/access_log_integration_test.cc +++ b/test/extensions/access_loggers/open_telemetry/access_log_integration_test.cc @@ -49,11 +49,11 @@ namespace { class AccessLogIntegrationTest : public Grpc::VersionedGrpcClientIntegrationParamTest, public HttpIntegrationTest { public: - AccessLogIntegrationTest() : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, ipVersion()) {} + AccessLogIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, ipVersion()) {} void createUpstreams() override { HttpIntegrationTest::createUpstreams(); - addFakeUpstream(FakeHttpConnection::Type::HTTP2); + addFakeUpstream(Http::CodecType::HTTP2); } void initialize() override { diff --git a/test/extensions/access_loggers/wasm/test_data/BUILD b/test/extensions/access_loggers/wasm/test_data/BUILD index f49006867f2f7..ca3e8f847472c 100644 --- a/test/extensions/access_loggers/wasm/test_data/BUILD +++ b/test/extensions/access_loggers/wasm/test_data/BUILD @@ -22,7 +22,6 @@ envoy_cc_library( "//source/common/common:c_smart_ptr_lib", "//source/extensions/common/wasm:wasm_hdr", "//source/extensions/common/wasm:wasm_lib", - "//source/extensions/common/wasm:well_known_names", ], ) diff --git a/test/extensions/bootstrap/wasm/test_data/BUILD b/test/extensions/bootstrap/wasm/test_data/BUILD index 7256f0e7f0dec..ba016610e703e 100644 --- a/test/extensions/bootstrap/wasm/test_data/BUILD +++ b/test/extensions/bootstrap/wasm/test_data/BUILD @@ -31,7 +31,6 @@ envoy_cc_library( "//source/common/common:assert_lib", "//source/common/common:c_smart_ptr_lib", "//source/extensions/common/wasm:wasm_lib", - "//source/extensions/common/wasm:well_known_names", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], ) @@ -49,7 +48,6 @@ envoy_cc_library( "//source/common/common:c_smart_ptr_lib", "//source/extensions/common/wasm:wasm_hdr", "//source/extensions/common/wasm:wasm_lib", - "//source/extensions/common/wasm:well_known_names", ], ) @@ -66,7 +64,6 @@ envoy_cc_library( "//source/common/common:c_smart_ptr_lib", "//source/extensions/common/wasm:wasm_hdr", "//source/extensions/common/wasm:wasm_lib", - "//source/extensions/common/wasm:well_known_names", ], ) diff --git a/test/extensions/bootstrap/wasm/wasm_integration_test.cc b/test/extensions/bootstrap/wasm/wasm_integration_test.cc index 197a45c24ca31..28b8b9dbb31c0 100644 --- a/test/extensions/bootstrap/wasm/wasm_integration_test.cc +++ b/test/extensions/bootstrap/wasm/wasm_integration_test.cc @@ -13,11 +13,11 @@ namespace { class WasmIntegrationTest : public HttpIntegrationTest, public testing::TestWithParam { public: WasmIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, Network::Address::IpVersion::v4) {} + : HttpIntegrationTest(Http::CodecType::HTTP1, Network::Address::IpVersion::v4) {} void createUpstreams() override { HttpIntegrationTest::createUpstreams(); - addFakeUpstream(FakeHttpConnection::Type::HTTP1); + addFakeUpstream(Http::CodecType::HTTP1); } void cleanup() { diff --git a/test/extensions/clusters/aggregate/cluster_integration_test.cc b/test/extensions/clusters/aggregate/cluster_integration_test.cc index 16a1297d0a4db..b2181452bca61 100644 --- a/test/extensions/clusters/aggregate/cluster_integration_test.cc +++ b/test/extensions/clusters/aggregate/cluster_integration_test.cc @@ -124,8 +124,7 @@ const std::string& config() { class AggregateIntegrationTest : public testing::TestWithParam, public HttpIntegrationTest { public: - AggregateIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, GetParam(), config()) { + AggregateIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam(), config()) { use_lds_ = false; } @@ -133,14 +132,14 @@ class AggregateIntegrationTest : public testing::TestWithParamlocalAddress()->ip()->port(), Network::Test::getLoopbackAddressString(GetParam())); diff --git a/test/extensions/common/dynamic_forward_proxy/BUILD b/test/extensions/common/dynamic_forward_proxy/BUILD index 6c2e4d91d7379..b280556cba0ce 100644 --- a/test/extensions/common/dynamic_forward_proxy/BUILD +++ b/test/extensions/common/dynamic_forward_proxy/BUILD @@ -23,6 +23,7 @@ envoy_cc_test( "//test/test_common:simulated_time_system_lib", "//test/test_common:test_runtime_lib", "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/common/dynamic_forward_proxy/v3:pkg_cc_proto", ], ) diff --git a/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc b/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc index bf887397a504c..ad5b0c5272146 100644 --- a/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc +++ b/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc @@ -1,7 +1,9 @@ #include "envoy/config/cluster/v3/cluster.pb.h" +#include "envoy/config/core/v3/resolver.pb.h" #include "envoy/extensions/common/dynamic_forward_proxy/v3/dns_cache.pb.h" #include "common/config/utility.h" +#include "common/network/resolver_impl.h" #include "extensions/common/dynamic_forward_proxy/dns_cache_impl.h" #include "extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.h" @@ -92,6 +94,10 @@ MATCHER_P3(DnsHostInfoEquals, address, resolved_host, is_ip_address, "") { MATCHER(DnsHostInfoAddressIsNull, "") { return arg->address() == nullptr; } +MATCHER_P(CustomDnsResolversSizeEquals, expected_resolvers, "") { + return expected_resolvers.size() == arg.size(); +} + // Basic successful resolution and then re-resolution. TEST_F(DnsCacheImplTest, ResolveSuccess) { initialize(); @@ -780,6 +786,42 @@ TEST(DnsCacheManagerImplTest, LoadViaConfig) { "config specified DNS cache 'foo' with different settings"); } +TEST(DnsCacheConfigOptionsTest, EmtpyDnsResolverConfig) { + NiceMock dispatcher; + std::shared_ptr resolver{std::make_shared()}; + NiceMock tls; + NiceMock random; + NiceMock loader; + Stats::IsolatedStoreImpl store; + + envoy::extensions::common::dynamic_forward_proxy::v3::DnsCacheConfig config; + std::vector expectedEmptyDnsResolvers; + EXPECT_CALL(dispatcher, createDnsResolver(expectedEmptyDnsResolvers, _)) + .WillOnce(Return(resolver)); + DnsCacheImpl dns_cache_(dispatcher, tls, random, loader, store, config); +} + +TEST(DnsCacheConfigOptionsTest, NonEmptyDnsResolverConfig) { + NiceMock dispatcher; + std::shared_ptr resolver{std::make_shared()}; + NiceMock tls; + NiceMock random; + NiceMock loader; + Stats::IsolatedStoreImpl store; + envoy::extensions::common::dynamic_forward_proxy::v3::DnsCacheConfig config; + + envoy::config::core::v3::Address* dns_resolvers = config.mutable_dns_resolver()->add_resolvers(); + dns_resolvers->mutable_socket_address()->set_address("1.2.3.4"); + dns_resolvers->mutable_socket_address()->set_port_value(8080); + + std::vector expected_dns_resolver_config; + expected_dns_resolver_config.push_back(Network::Address::resolveProtoAddress(*dns_resolvers)); + EXPECT_CALL(dispatcher, + createDnsResolver(CustomDnsResolversSizeEquals(expected_dns_resolver_config), _)) + .WillOnce(Return(resolver)); + DnsCacheImpl dns_cache_(dispatcher, tls, random, loader, store, config); +} + // Note: this test is done here, rather than a TYPED_TEST_SUITE in // //test/common/config:utility_test, because we did not want to include an extension type in // non-extension test suites. diff --git a/test/extensions/common/wasm/plugin_test.cc b/test/extensions/common/wasm/plugin_test.cc index 47c72822de1a5..2b219066935f0 100644 --- a/test/extensions/common/wasm/plugin_test.cc +++ b/test/extensions/common/wasm/plugin_test.cc @@ -3,7 +3,6 @@ #include "envoy/common/exception.h" #include "extensions/common/wasm/plugin.h" -#include "extensions/common/wasm/well_known_names.h" #include "test/test_common/environment.h" #include "test/test_common/utility.h" @@ -74,7 +73,7 @@ TEST(TestWasmConfig, EnvKeyException) { TEST(TestWasmConfig, NullVMEnv) { envoy::extensions::wasm::v3::PluginConfig plugin_config; - plugin_config.mutable_vm_config()->set_runtime(WasmRuntimeNames::get().Null); + plugin_config.mutable_vm_config()->set_runtime("envoy.wasm.runtime.null"); (*plugin_config.mutable_vm_config() ->mutable_environment_variables() ->mutable_key_values())["key"] = "value"; diff --git a/test/extensions/common/wasm/test_data/BUILD b/test/extensions/common/wasm/test_data/BUILD index 9160fe94f739f..95db303869db9 100644 --- a/test/extensions/common/wasm/test_data/BUILD +++ b/test/extensions/common/wasm/test_data/BUILD @@ -28,7 +28,6 @@ envoy_cc_library( "//source/common/common:c_smart_ptr_lib", "//source/extensions/common/wasm:wasm_hdr", "//source/extensions/common/wasm:wasm_lib", - "//source/extensions/common/wasm:well_known_names", ], ) @@ -45,7 +44,6 @@ envoy_cc_library( "//source/common/common:c_smart_ptr_lib", "//source/extensions/common/wasm:wasm_hdr", "//source/extensions/common/wasm:wasm_lib", - "//source/extensions/common/wasm:well_known_names", "//source/extensions/common/wasm/ext:envoy_null_plugin", ], ) @@ -63,7 +61,6 @@ envoy_cc_library( "//source/common/common:c_smart_ptr_lib", "//source/extensions/common/wasm:wasm_hdr", "//source/extensions/common/wasm:wasm_lib", - "//source/extensions/common/wasm:well_known_names", "//source/extensions/common/wasm/ext:envoy_null_plugin", ], ) diff --git a/test/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter_integration_test.h b/test/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter_integration_test.h index 264469ff43e1e..35a107450298e 100644 --- a/test/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter_integration_test.h +++ b/test/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter_integration_test.h @@ -36,12 +36,11 @@ class AdaptiveConcurrencyIntegrationTest public Event::TestUsingSimulatedTime, public HttpIntegrationTest { public: - AdaptiveConcurrencyIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, GetParam()) {} + AdaptiveConcurrencyIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP2, GetParam()) {} void customInit() { - setDownstreamProtocol(Http::CodecClient::Type::HTTP2); - setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); + setDownstreamProtocol(Http::CodecType::HTTP2); + setUpstreamProtocol(Http::CodecType::HTTP2); config_helper_.addFilter(ADAPTIVE_CONCURRENCY_CONFIG); initialize(); codec_client_ = makeHttpConnection(lookupPort("http")); diff --git a/test/extensions/filters/http/admission_control/admission_control_integration_test.cc b/test/extensions/filters/http/admission_control/admission_control_integration_test.cc index 3535a136f58b6..a7e8d83f97cfb 100644 --- a/test/extensions/filters/http/admission_control/admission_control_integration_test.cc +++ b/test/extensions/filters/http/admission_control/admission_control_integration_test.cc @@ -33,8 +33,7 @@ class AdmissionControlIntegrationTest : public Event::TestUsingSimulatedTime, public testing::TestWithParam, public HttpIntegrationTest { public: - AdmissionControlIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, GetParam()) {} + AdmissionControlIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam()) {} void SetUp() override {} @@ -133,7 +132,7 @@ TEST_P(AdmissionControlIntegrationTest, HttpTest) { TEST_P(AdmissionControlIntegrationTest, GrpcTest) { autonomous_upstream_ = true; - setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); + setUpstreamProtocol(Http::CodecType::HTTP2); initialize(); // Drop the success rate to a very low value. diff --git a/test/extensions/filters/http/aws_lambda/aws_lambda_filter_integration_test.cc b/test/extensions/filters/http/aws_lambda/aws_lambda_filter_integration_test.cc index 5f5816a7de6a8..db3227d9a2df4 100644 --- a/test/extensions/filters/http/aws_lambda/aws_lambda_filter_integration_test.cc +++ b/test/extensions/filters/http/aws_lambda/aws_lambda_filter_integration_test.cc @@ -16,15 +16,14 @@ namespace { class AwsLambdaFilterIntegrationTest : public testing::TestWithParam, public HttpIntegrationTest { public: - AwsLambdaFilterIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, GetParam()) {} + AwsLambdaFilterIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP2, GetParam()) {} void SetUp() override { // Set these environment variables to quickly sign credentials instead of attempting to query // instance metadata and timing-out. TestEnvironment::setEnvVar("AWS_ACCESS_KEY_ID", "aws-user", 1 /*overwrite*/); TestEnvironment::setEnvVar("AWS_SECRET_ACCESS_KEY", "secret", 1 /*overwrite*/); - setUpstreamProtocol(FakeHttpConnection::Type::HTTP1); + setUpstreamProtocol(Http::CodecType::HTTP1); } void TearDown() override { fake_upstream_connection_.reset(); } diff --git a/test/extensions/filters/http/aws_lambda/aws_lambda_filter_test.cc b/test/extensions/filters/http/aws_lambda/aws_lambda_filter_test.cc index d8fb873cf2972..ccc8114a45e3e 100644 --- a/test/extensions/filters/http/aws_lambda/aws_lambda_filter_test.cc +++ b/test/extensions/filters/http/aws_lambda/aws_lambda_filter_test.cc @@ -519,6 +519,7 @@ TEST_F(AwsLambdaFilterTest, EncodeDataJsonModeTransformToHttp) { EXPECT_FALSE(headers.has(":other")); EXPECT_EQ("awesome value", headers.get_("x-awesome-header")); + EXPECT_EQ("application/json", headers.get_("content-type")); std::vector cookies; headers.iterate([&cookies](const Http::HeaderEntry& entry) { @@ -531,6 +532,40 @@ TEST_F(AwsLambdaFilterTest, EncodeDataJsonModeTransformToHttp) { EXPECT_THAT(cookies, ElementsAre("session-id=42; Secure; HttpOnly", "user=joe")); } +/** + * encodeData() data in JSON mode should respect content-type header. + */ +TEST_F(AwsLambdaFilterTest, EncodeDataJsonModeContentTypeHeader) { + setupFilter({arn_, InvocationMode::Synchronous, false /*passthrough*/}); + filter_->resolveSettings(); + Http::TestResponseHeaderMapImpl headers; + headers.setStatus(200); + filter_->encodeHeaders(headers, false /*end_stream*/); + + constexpr auto json_response = R"EOF( + { + "statusCode": 201, + "headers": {"content-type": "text/plain"} + } + )EOF"; + + Buffer::OwnedImpl encoded_buf; + encoded_buf.add(json_response); + auto on_modify_encoding_buffer = [&encoded_buf](std::function cb) { + cb(encoded_buf); + }; + EXPECT_CALL(encoder_callbacks_, encodingBuffer).WillRepeatedly(Return(&encoded_buf)); + EXPECT_CALL(encoder_callbacks_, modifyEncodingBuffer) + .WillRepeatedly(Invoke(on_modify_encoding_buffer)); + + auto result = filter_->encodeData(encoded_buf, true /*end_stream*/); + EXPECT_EQ(Http::FilterDataStatus::Continue, result); + + ASSERT_NE(nullptr, headers.Status()); + EXPECT_EQ("201", headers.getStatusValue()); + EXPECT_EQ("text/plain", headers.get_("content-type")); +} + /** * encodeData() in JSON mode with a non-empty body should translate the body to plain text if it was * base64-encoded. diff --git a/test/extensions/filters/http/bandwidth_limit/BUILD b/test/extensions/filters/http/bandwidth_limit/BUILD new file mode 100644 index 0000000000000..ec46a1a917808 --- /dev/null +++ b/test/extensions/filters/http/bandwidth_limit/BUILD @@ -0,0 +1,38 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_extension_cc_test( + name = "filter_test", + srcs = ["filter_test.cc"], + extension_name = "envoy.filters.http.bandwidth_limit", + deps = [ + "//source/common/common:utility_lib", + "//source/common/http:header_utility_lib", + "//source/common/http:headers_lib", + "//source/common/router:header_parser_lib", + "//source/common/runtime:runtime_lib", + "//source/extensions/filters/http/bandwidth_limit:bandwidth_limit_lib", + "//test/mocks/server:server_mocks", + "@envoy_api//envoy/extensions/filters/http/bandwidth_limit/v3alpha:pkg_cc_proto", + ], +) + +envoy_extension_cc_test( + name = "config_test", + srcs = ["config_test.cc"], + extension_name = "envoy.filters.http.bandwidth_limit", + deps = [ + "//source/extensions/filters/http/bandwidth_limit:config", + "//test/mocks/server:server_mocks", + ], +) diff --git a/test/extensions/filters/http/bandwidth_limit/config_test.cc b/test/extensions/filters/http/bandwidth_limit/config_test.cc new file mode 100644 index 0000000000000..29675d090fa9b --- /dev/null +++ b/test/extensions/filters/http/bandwidth_limit/config_test.cc @@ -0,0 +1,119 @@ +#include "extensions/filters/http/bandwidth_limit/bandwidth_limit.h" +#include "extensions/filters/http/bandwidth_limit/config.h" + +#include "test/mocks/server/mocks.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace BandwidthLimitFilter { + +using EnableMode = + envoy::extensions::filters::http::bandwidth_limit::v3alpha::BandwidthLimit_EnableMode; + +TEST(Factory, GlobalEmptyConfig) { + const std::string yaml = R"( + stat_prefix: test + )"; + + BandwidthLimitFilterConfig factory; + ProtobufTypes::MessagePtr proto_config = factory.createEmptyRouteConfigProto(); + TestUtility::loadFromYaml(yaml, *proto_config); + + NiceMock context; + + EXPECT_CALL(context.dispatcher_, createTimer_(_)).Times(0); + auto callback = factory.createFilterFactoryFromProto(*proto_config, "stats", context); + Http::MockFilterChainFactoryCallbacks filter_callback; + EXPECT_CALL(filter_callback, addStreamFilter(_)); + callback(filter_callback); +} + +TEST(Factory, RouteSpecificFilterConfig) { + const std::string config_yaml = R"( + stat_prefix: test + enable_mode: REQUEST_AND_RESPONSE + limit_kbps: 10 + fill_interval: 0.1s + )"; + + BandwidthLimitFilterConfig factory; + ProtobufTypes::MessagePtr proto_config = factory.createEmptyRouteConfigProto(); + TestUtility::loadFromYaml(config_yaml, *proto_config); + + NiceMock context; + + EXPECT_CALL(context.dispatcher_, createTimer_(_)).Times(0); + const auto route_config = factory.createRouteSpecificFilterConfig( + *proto_config, context, ProtobufMessage::getNullValidationVisitor()); + const auto* config = dynamic_cast(route_config.get()); + EXPECT_EQ(config->limit(), 10); + EXPECT_EQ(config->fillInterval().count(), 100); + EXPECT_EQ(config->enableMode(), EnableMode::BandwidthLimit_EnableMode_REQUEST_AND_RESPONSE); + EXPECT_FALSE(config->tokenBucket() == nullptr); +} + +TEST(Factory, RouteSpecificFilterConfigDisabledByDefault) { + const std::string config_yaml = R"( + stat_prefix: test + limit_kbps: 10 + fill_interval: 0.1s + )"; + + BandwidthLimitFilterConfig factory; + ProtobufTypes::MessagePtr proto_config = factory.createEmptyRouteConfigProto(); + TestUtility::loadFromYaml(config_yaml, *proto_config); + + NiceMock context; + + EXPECT_CALL(context.dispatcher_, createTimer_(_)).Times(0); + const auto route_config = factory.createRouteSpecificFilterConfig( + *proto_config, context, ProtobufMessage::getNullValidationVisitor()); + const auto* config = dynamic_cast(route_config.get()); + EXPECT_EQ(config->enableMode(), EnableMode::BandwidthLimit_EnableMode_DISABLED); + EXPECT_EQ(config->limit(), 10); + EXPECT_EQ(config->fillInterval().count(), 100); +} + +TEST(Factory, RouteSpecificFilterConfigDefaultFillInterval) { + const std::string config_yaml = R"( + stat_prefix: test + enable_mode: REQUEST_AND_RESPONSE + limit_kbps: 10 + )"; + + BandwidthLimitFilterConfig factory; + ProtobufTypes::MessagePtr proto_config = factory.createEmptyRouteConfigProto(); + TestUtility::loadFromYaml(config_yaml, *proto_config); + + NiceMock context; + + EXPECT_CALL(context.dispatcher_, createTimer_(_)).Times(0); + const auto route_config = factory.createRouteSpecificFilterConfig( + *proto_config, context, ProtobufMessage::getNullValidationVisitor()); + const auto* config = dynamic_cast(route_config.get()); + EXPECT_EQ(config->limit(), 10); + EXPECT_EQ(config->fillInterval().count(), 50); +} + +TEST(Factory, PerRouteConfigNoLimits) { + const std::string config_yaml = R"( + stat_prefix: test + )"; + + BandwidthLimitFilterConfig factory; + ProtobufTypes::MessagePtr proto_config = factory.createEmptyRouteConfigProto(); + TestUtility::loadFromYaml(config_yaml, *proto_config); + + NiceMock context; + EXPECT_THROW(factory.createRouteSpecificFilterConfig(*proto_config, context, + ProtobufMessage::getNullValidationVisitor()), + EnvoyException); +} +} // namespace BandwidthLimitFilter +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/http/bandwidth_limit/filter_test.cc b/test/extensions/filters/http/bandwidth_limit/filter_test.cc new file mode 100644 index 0000000000000..497a9472f8d3e --- /dev/null +++ b/test/extensions/filters/http/bandwidth_limit/filter_test.cc @@ -0,0 +1,558 @@ +#include "envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.pb.h" + +#include "extensions/filters/http/bandwidth_limit/bandwidth_limit.h" + +#include "test/mocks/http/mocks.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::_; +using testing::AnyNumber; +using testing::NiceMock; +using testing::Return; + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace BandwidthLimitFilter { + +class FilterTest : public testing::Test { +public: + FilterTest() = default; + + void setup(const std::string& yaml) { + envoy::extensions::filters::http::bandwidth_limit::v3alpha::BandwidthLimit config; + TestUtility::loadFromYaml(yaml, config); + config_ = std::make_shared(config, stats_, runtime_, time_system_, true); + filter_ = std::make_shared(config_); + filter_->setDecoderFilterCallbacks(decoder_filter_callbacks_); + filter_->setEncoderFilterCallbacks(encoder_filter_callbacks_); + + EXPECT_CALL(decoder_filter_callbacks_.dispatcher_, pushTrackedObject(_)).Times(AnyNumber()); + EXPECT_CALL(decoder_filter_callbacks_.dispatcher_, popTrackedObject(_)).Times(AnyNumber()); + EXPECT_CALL(encoder_filter_callbacks_.dispatcher_, pushTrackedObject(_)).Times(AnyNumber()); + EXPECT_CALL(encoder_filter_callbacks_.dispatcher_, popTrackedObject(_)).Times(AnyNumber()); + } + + uint64_t findCounter(const std::string& name) { + const auto counter = TestUtility::findCounter(stats_, name); + return counter != nullptr ? counter->value() : 0; + } + + uint64_t findGauge(const std::string& name) { + const auto gauge = TestUtility::findGauge(stats_, name); + return gauge != nullptr ? gauge->value() : 0; + } + + NiceMock stats_; + NiceMock decoder_filter_callbacks_; + NiceMock encoder_filter_callbacks_; + NiceMock runtime_; + std::shared_ptr config_; + std::shared_ptr filter_; + Http::TestRequestHeaderMapImpl request_headers_; + Http::TestRequestTrailerMapImpl request_trailers_; + Http::TestResponseHeaderMapImpl response_headers_; + Http::TestResponseTrailerMapImpl response_trailers_; + Buffer::OwnedImpl data_; + Event::SimulatedTimeSystem time_system_; +}; + +TEST_F(FilterTest, Disabled) { + const std::string config_yaml = R"( + stat_prefix: test + runtime_enabled: + default_value: false + runtime_key: foo_key + enable_mode: DISABLED + limit_kbps: 10 + fill_interval: 1s + )"; + setup(fmt::format(config_yaml, "1")); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers_, false)); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers_)); + EXPECT_EQ(0U, findCounter("test.http_bandwidth_limit.request_enabled")); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(response_headers_, false)); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->encodeData(data_, false)); + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->encodeTrailers(response_trailers_)); + EXPECT_EQ(0U, findCounter("test.http_bandwidth_limit.response_enabled")); +} + +TEST_F(FilterTest, LimitOnDecode) { + const std::string config_yaml = R"( + stat_prefix: test + runtime_enabled: + default_value: true + runtime_key: foo_key + enable_mode: REQUEST + limit_kbps: 1 + )"; + setup(fmt::format(config_yaml, "1")); + + ON_CALL(decoder_filter_callbacks_, decoderBufferLimit()).WillByDefault(Return(1100)); + Event::MockTimer* token_timer = + new NiceMock(&decoder_filter_callbacks_.dispatcher_); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers_, true)); + + EXPECT_EQ(1U, findCounter("test.http_bandwidth_limit.request_enabled")); + EXPECT_EQ(1UL, config_->limit()); + EXPECT_EQ(50UL, config_->fillInterval().count()); + + // Send a small amount of data which should be within limit. + Buffer::OwnedImpl data1("hello"); + EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(0), _)); + EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->decodeData(data1, false)); + EXPECT_EQ(1, findGauge("test.http_bandwidth_limit.request_pending")); + EXPECT_EQ(5, findGauge("test.http_bandwidth_limit.request_incoming_size")); + EXPECT_CALL(decoder_filter_callbacks_, + injectDecodedDataToFilterChain(BufferStringEqual("hello"), false)); + token_timer->invokeCallback(); + EXPECT_EQ(5, findGauge("test.http_bandwidth_limit.request_allowed_size")); + + // Advance time by 1s which should refill all tokens. + time_system_.advanceTimeWait(std::chrono::seconds(1)); + + // Send 1126 bytes of data which is 1s + 2 refill cycles of data. + EXPECT_CALL(decoder_filter_callbacks_, onDecoderFilterAboveWriteBufferHighWatermark()); + EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(0), _)); + Buffer::OwnedImpl data2(std::string(1126, 'a')); + EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->decodeData(data2, false)); + EXPECT_EQ(1, findGauge("test.http_bandwidth_limit.request_pending")); + EXPECT_EQ(1126, findGauge("test.http_bandwidth_limit.request_incoming_size")); + + EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(50), _)); + EXPECT_CALL(decoder_filter_callbacks_, onDecoderFilterBelowWriteBufferLowWatermark()); + EXPECT_CALL(decoder_filter_callbacks_, + injectDecodedDataToFilterChain(BufferStringEqual(std::string(1024, 'a')), false)); + token_timer->invokeCallback(); + EXPECT_EQ(1024, findGauge("test.http_bandwidth_limit.request_allowed_size")); + EXPECT_EQ(1126, findGauge("test.http_bandwidth_limit.request_incoming_size")); + + // Fire timer, also advance time. + time_system_.advanceTimeWait(std::chrono::milliseconds(50)); + EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(50), _)); + EXPECT_CALL(decoder_filter_callbacks_, + injectDecodedDataToFilterChain(BufferStringEqual(std::string(51, 'a')), false)); + token_timer->invokeCallback(); + EXPECT_EQ(51, findGauge("test.http_bandwidth_limit.request_allowed_size")); + EXPECT_EQ(1126, findGauge("test.http_bandwidth_limit.request_incoming_size")); + + // Get new data with current data buffered, not end_stream. + Buffer::OwnedImpl data3(std::string(51, 'b')); + EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->decodeData(data3, false)); + EXPECT_EQ(1, findGauge("test.http_bandwidth_limit.request_pending")); + EXPECT_EQ(51, findGauge("test.http_bandwidth_limit.request_incoming_size")); + + // Fire timer, also advance time. + time_system_.advanceTimeWait(std::chrono::milliseconds(50)); + EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(50), _)); + EXPECT_CALL(decoder_filter_callbacks_, + injectDecodedDataToFilterChain(BufferStringEqual(std::string(51, 'a')), false)); + token_timer->invokeCallback(); + EXPECT_EQ(51, findGauge("test.http_bandwidth_limit.request_allowed_size")); + + // Fire timer, also advance time. No timer enable because there is nothing + // buffered. + time_system_.advanceTimeWait(std::chrono::milliseconds(50)); + EXPECT_CALL(decoder_filter_callbacks_, + injectDecodedDataToFilterChain(BufferStringEqual(std::string(51, 'b')), false)); + token_timer->invokeCallback(); + EXPECT_EQ(51, findGauge("test.http_bandwidth_limit.request_allowed_size")); + + // Advance time by 1s for a full refill. + time_system_.advanceTimeWait(std::chrono::seconds(1)); + + // Now send 1024 in one shot with end_stream true which should go through and + // end the stream. + EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(0), _)); + Buffer::OwnedImpl data4(std::string(1024, 'c')); + EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->decodeData(data4, true)); + EXPECT_EQ(1024, findGauge("test.http_bandwidth_limit.request_incoming_size")); + EXPECT_CALL(decoder_filter_callbacks_, + injectDecodedDataToFilterChain(BufferStringEqual(std::string(1024, 'c')), true)); + token_timer->invokeCallback(); + EXPECT_EQ(1024, findGauge("test.http_bandwidth_limit.request_allowed_size")); + EXPECT_EQ(0, findGauge("test.http_bandwidth_limit.request_pending")); + + filter_->onDestroy(); +} + +TEST_F(FilterTest, LimitOnEncode) { + const std::string config_yaml = R"( + stat_prefix: test + runtime_enabled: + default_value: true + runtime_key: foo_key + enable_mode: RESPONSE + limit_kbps: 1 + )"; + setup(fmt::format(config_yaml, "1")); + + ON_CALL(encoder_filter_callbacks_, encoderBufferLimit()).WillByDefault(Return(1100)); + Event::MockTimer* token_timer = + new NiceMock(&encoder_filter_callbacks_.dispatcher_); + + EXPECT_EQ(1UL, config_->limit()); + EXPECT_EQ(50UL, config_->fillInterval().count()); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, + filter_->encode100ContinueHeaders(response_headers_)); + Http::MetadataMap metadata_map; + EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_->encodeMetadata(metadata_map)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(response_headers_, false)); + EXPECT_EQ(1U, findCounter("test.http_bandwidth_limit.response_enabled")); + + // Send a small amount of data which should be within limit. + Buffer::OwnedImpl data1("hello"); + EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(0), _)); + EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->encodeData(data1, false)); + EXPECT_EQ(1, findGauge("test.http_bandwidth_limit.response_pending")); + EXPECT_EQ(5, findGauge("test.http_bandwidth_limit.response_incoming_size")); + EXPECT_CALL(encoder_filter_callbacks_, + injectEncodedDataToFilterChain(BufferStringEqual("hello"), false)); + token_timer->invokeCallback(); + EXPECT_EQ(5, findGauge("test.http_bandwidth_limit.response_allowed_size")); + + // Advance time by 1s which should refill all tokens. + time_system_.advanceTimeWait(std::chrono::seconds(1)); + + // Send 1126 bytes of data which is 1s + 2 refill cycles of data. + EXPECT_CALL(encoder_filter_callbacks_, onEncoderFilterAboveWriteBufferHighWatermark()); + EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(0), _)); + Buffer::OwnedImpl data2(std::string(1126, 'a')); + EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->encodeData(data2, false)); + EXPECT_EQ(1126, findGauge("test.http_bandwidth_limit.response_incoming_size")); + + EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(50), _)); + EXPECT_CALL(encoder_filter_callbacks_, onEncoderFilterBelowWriteBufferLowWatermark()); + EXPECT_CALL(encoder_filter_callbacks_, + injectEncodedDataToFilterChain(BufferStringEqual(std::string(1024, 'a')), false)); + token_timer->invokeCallback(); + EXPECT_EQ(1, findGauge("test.http_bandwidth_limit.response_pending")); + EXPECT_EQ(1126, findGauge("test.http_bandwidth_limit.response_incoming_size")); + EXPECT_EQ(1024, findGauge("test.http_bandwidth_limit.response_allowed_size")); + + // Fire timer, also advance time. + time_system_.advanceTimeWait(std::chrono::milliseconds(50)); + EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(50), _)); + EXPECT_CALL(encoder_filter_callbacks_, + injectEncodedDataToFilterChain(BufferStringEqual(std::string(51, 'a')), false)); + token_timer->invokeCallback(); + EXPECT_EQ(51, findGauge("test.http_bandwidth_limit.response_allowed_size")); + + // Get new data with current data buffered, not end_stream. + Buffer::OwnedImpl data3(std::string(51, 'b')); + EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->encodeData(data3, false)); + + // Fire timer, also advance time. + time_system_.advanceTimeWait(std::chrono::milliseconds(50)); + EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(50), _)); + EXPECT_CALL(encoder_filter_callbacks_, + injectEncodedDataToFilterChain(BufferStringEqual(std::string(51, 'a')), false)); + token_timer->invokeCallback(); + EXPECT_EQ(51, findGauge("test.http_bandwidth_limit.response_allowed_size")); + + // Fire timer, also advance time. No time enable because there is nothing + // buffered. + time_system_.advanceTimeWait(std::chrono::milliseconds(50)); + EXPECT_CALL(encoder_filter_callbacks_, + injectEncodedDataToFilterChain(BufferStringEqual(std::string(51, 'b')), false)); + token_timer->invokeCallback(); + EXPECT_EQ(51, findGauge("test.http_bandwidth_limit.response_allowed_size")); + + // Advance time by 1s for a full refill. + time_system_.advanceTimeWait(std::chrono::seconds(1)); + + // Now send 1024 in one shot with end_stream true which should go through and + // end the stream. + EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(0), _)); + Buffer::OwnedImpl data4(std::string(1024, 'c')); + EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->encodeData(data4, true)); + EXPECT_EQ(1024, findGauge("test.http_bandwidth_limit.response_incoming_size")); + EXPECT_CALL(encoder_filter_callbacks_, + injectEncodedDataToFilterChain(BufferStringEqual(std::string(1024, 'c')), true)); + token_timer->invokeCallback(); + EXPECT_EQ(0, findGauge("test.http_bandwidth_limit.response_pending")); + EXPECT_EQ(1024, findGauge("test.http_bandwidth_limit.response_allowed_size")); + + filter_->onDestroy(); +} + +TEST_F(FilterTest, LimitOnDecodeAndEncode) { + const std::string config_yaml = R"( + stat_prefix: test + runtime_enabled: + default_value: true + runtime_key: foo_key + enable_mode: REQUEST_AND_RESPONSE + limit_kbps: 1 + )"; + setup(fmt::format(config_yaml, "1")); + + ON_CALL(decoder_filter_callbacks_, decoderBufferLimit()).WillByDefault(Return(1050)); + ON_CALL(encoder_filter_callbacks_, encoderBufferLimit()).WillByDefault(Return(1100)); + Event::MockTimer* request_timer = + new NiceMock(&decoder_filter_callbacks_.dispatcher_); + Event::MockTimer* response_timer = + new NiceMock(&encoder_filter_callbacks_.dispatcher_); + + EXPECT_EQ(1UL, config_->limit()); + EXPECT_EQ(50UL, config_->fillInterval().count()); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, + filter_->encode100ContinueHeaders(response_headers_)); + Http::MetadataMap metadata_map; + EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_->decodeMetadata(metadata_map)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers_, false)); + EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_->encodeMetadata(metadata_map)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(response_headers_, false)); + + // Send small amount of data from both sides which should be within initial + // bucket limit. + Buffer::OwnedImpl dec_data1("hello"); + EXPECT_CALL(*request_timer, enableTimer(std::chrono::milliseconds(0), _)); + EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->decodeData(dec_data1, false)); + EXPECT_CALL(decoder_filter_callbacks_, + injectDecodedDataToFilterChain(BufferStringEqual("hello"), false)); + request_timer->invokeCallback(); + + Buffer::OwnedImpl enc_data1("world!"); + EXPECT_CALL(*response_timer, enableTimer(std::chrono::milliseconds(0), _)); + EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->encodeData(enc_data1, false)); + EXPECT_CALL(encoder_filter_callbacks_, + injectEncodedDataToFilterChain(BufferStringEqual("world!"), false)); + response_timer->invokeCallback(); + + // Advance time by 1s which should refill all tokens. + time_system_.advanceTimeWait(std::chrono::seconds(1)); + // Send 1075 bytes of data on request path which is 1s + 1 refill cycle of + // data. Send 102 bytes of data on response path which is 2 refill cycles of + // data. + Buffer::OwnedImpl dec_data2(std::string(1075, 'd')); + Buffer::OwnedImpl enc_data2(std::string(102, 'e')); + + EXPECT_CALL(decoder_filter_callbacks_, onDecoderFilterAboveWriteBufferHighWatermark()); + EXPECT_CALL(*request_timer, enableTimer(std::chrono::milliseconds(0), _)); + EXPECT_CALL(*response_timer, enableTimer(std::chrono::milliseconds(0), _)); + EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->decodeData(dec_data2, false)); + EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->encodeData(enc_data2, false)); + + EXPECT_CALL(*request_timer, enableTimer(std::chrono::milliseconds(50), _)); + EXPECT_CALL(decoder_filter_callbacks_, onDecoderFilterBelowWriteBufferLowWatermark()); + EXPECT_CALL(decoder_filter_callbacks_, + injectDecodedDataToFilterChain(BufferStringEqual(std::string(1024, 'd')), false)); + request_timer->invokeCallback(); + + // Encoder will not be able to write any bytes due to insufficient tokens. + EXPECT_CALL(*response_timer, enableTimer(std::chrono::milliseconds(50), _)); + EXPECT_CALL(encoder_filter_callbacks_, + injectEncodedDataToFilterChain(BufferStringEqual(std::string("")), false)); + response_timer->invokeCallback(); + + // Fire timer, also advance time by 1 unit. + time_system_.advanceTimeWait(std::chrono::milliseconds(50)); + EXPECT_CALL(decoder_filter_callbacks_, + injectDecodedDataToFilterChain(BufferStringEqual(std::string(51, 'd')), false)); + request_timer->invokeCallback(); + // Encoder will not be able to write any bytes due to insufficient tokens. + EXPECT_CALL(*response_timer, enableTimer(std::chrono::milliseconds(50), _)); + EXPECT_CALL(encoder_filter_callbacks_, + injectEncodedDataToFilterChain(BufferStringEqual(std::string("")), false)); + response_timer->invokeCallback(); + + // Fire timer, also advance time by 1 unit. + time_system_.advanceTimeWait(std::chrono::milliseconds(50)); + EXPECT_CALL(*response_timer, enableTimer(std::chrono::milliseconds(50), _)); + EXPECT_CALL(encoder_filter_callbacks_, + injectEncodedDataToFilterChain(BufferStringEqual(std::string(51, 'e')), false)); + response_timer->invokeCallback(); + + // Get new data with current data buffered, not end_stream. + Buffer::OwnedImpl data3(std::string(51, 'b')); + EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->encodeData(data3, false)); + + // Fire timer, also advance time. + time_system_.advanceTimeWait(std::chrono::milliseconds(50)); + EXPECT_CALL(*response_timer, enableTimer(std::chrono::milliseconds(50), _)); + EXPECT_CALL(encoder_filter_callbacks_, + injectEncodedDataToFilterChain(BufferStringEqual(std::string(51, 'e')), false)); + response_timer->invokeCallback(); + + // Fire timer, also advance time. No time enable because there is nothing + // buffered. + time_system_.advanceTimeWait(std::chrono::milliseconds(50)); + EXPECT_CALL(encoder_filter_callbacks_, + injectEncodedDataToFilterChain(BufferStringEqual(std::string(51, 'b')), false)); + response_timer->invokeCallback(); + + // Advance time by 1s for a full refill. + time_system_.advanceTimeWait(std::chrono::seconds(1)); + + // Now send 1024 in total with end_stream true which should go through and end + // the streams. + Buffer::OwnedImpl enc_data4(std::string(960, 'e')); + Buffer::OwnedImpl dec_data4(std::string(51, 'd')); + EXPECT_CALL(*response_timer, enableTimer(std::chrono::milliseconds(0), _)); + EXPECT_CALL(*request_timer, enableTimer(std::chrono::milliseconds(0), _)); + EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->decodeData(dec_data4, true)); + EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->encodeData(enc_data4, true)); + EXPECT_CALL(decoder_filter_callbacks_, + injectDecodedDataToFilterChain(BufferStringEqual(std::string(51, 'd')), true)); + EXPECT_CALL(encoder_filter_callbacks_, + injectEncodedDataToFilterChain(BufferStringEqual(std::string(960, 'e')), true)); + response_timer->invokeCallback(); + request_timer->invokeCallback(); + + filter_->onDestroy(); +} + +TEST_F(FilterTest, WithTrailers) { + const std::string config_yaml = R"( + stat_prefix: test + runtime_enabled: + default_value: true + runtime_key: foo_key + enable_mode: REQUEST_AND_RESPONSE + limit_kbps: 1 + )"; + setup(fmt::format(config_yaml, "1")); + + ON_CALL(decoder_filter_callbacks_, decoderBufferLimit()).WillByDefault(Return(1050)); + ON_CALL(encoder_filter_callbacks_, encoderBufferLimit()).WillByDefault(Return(1100)); + Event::MockTimer* request_timer = + new NiceMock(&decoder_filter_callbacks_.dispatcher_); + Event::MockTimer* response_timer = + new NiceMock(&encoder_filter_callbacks_.dispatcher_); + + EXPECT_EQ(1UL, config_->limit()); + EXPECT_EQ(50UL, config_->fillInterval().count()); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, + filter_->encode100ContinueHeaders(response_headers_)); + Http::MetadataMap metadata_map; + EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_->decodeMetadata(metadata_map)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers_, false)); + EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_->encodeMetadata(metadata_map)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(response_headers_, false)); + + Buffer::OwnedImpl dec_data(std::string(102, 'd')); + Buffer::OwnedImpl enc_data(std::string(56, 'e')); + + EXPECT_EQ(0, findGauge("test.http_bandwidth_limit.request_pending")); + EXPECT_EQ(0, findGauge("test.http_bandwidth_limit.response_pending")); + EXPECT_CALL(*request_timer, enableTimer(std::chrono::milliseconds(0), _)); + EXPECT_CALL(*response_timer, enableTimer(std::chrono::milliseconds(0), _)); + EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->decodeData(dec_data, false)); + EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->encodeData(enc_data, false)); + EXPECT_EQ(1, findGauge("test.http_bandwidth_limit.request_pending")); + EXPECT_EQ(1, findGauge("test.http_bandwidth_limit.response_pending")); + + EXPECT_CALL(*request_timer, enableTimer(std::chrono::milliseconds(50), _)); + EXPECT_CALL(decoder_filter_callbacks_, + injectDecodedDataToFilterChain(BufferStringEqual(std::string(51, 'd')), false)); + EXPECT_EQ(Http::FilterTrailersStatus::StopIteration, filter_->decodeTrailers(request_trailers_)); + request_timer->invokeCallback(); + EXPECT_EQ(1, findGauge("test.http_bandwidth_limit.request_pending")); + + // Fire timer, also advance time by 1 unit. + time_system_.advanceTimeWait(std::chrono::milliseconds(50)); + EXPECT_CALL(decoder_filter_callbacks_, + injectDecodedDataToFilterChain(BufferStringEqual(std::string(51, 'd')), false)); + request_timer->invokeCallback(); + EXPECT_EQ(0, findGauge("test.http_bandwidth_limit.request_pending")); + + // Fire timer, also advance time by 1 unit. + time_system_.advanceTimeWait(std::chrono::milliseconds(50)); + EXPECT_CALL(*response_timer, enableTimer(std::chrono::milliseconds(50), _)); + EXPECT_CALL(encoder_filter_callbacks_, + injectEncodedDataToFilterChain(BufferStringEqual(std::string(51, 'e')), false)); + response_timer->invokeCallback(); + EXPECT_EQ(Http::FilterTrailersStatus::StopIteration, filter_->encodeTrailers(response_trailers_)); + EXPECT_EQ(1, findGauge("test.http_bandwidth_limit.response_pending")); + + time_system_.advanceTimeWait(std::chrono::milliseconds(50)); + EXPECT_CALL(encoder_filter_callbacks_, + injectEncodedDataToFilterChain(BufferStringEqual(std::string(5, 'e')), false)); + response_timer->invokeCallback(); + EXPECT_EQ(0, findGauge("test.http_bandwidth_limit.response_pending")); +} + +TEST_F(FilterTest, WithTrailersNoEndStream) { + const std::string config_yaml = R"( + stat_prefix: test + runtime_enabled: + default_value: true + runtime_key: foo_key + enable_mode: REQUEST_AND_RESPONSE + limit_kbps: 1 + )"; + setup(fmt::format(config_yaml, "1")); + + ON_CALL(decoder_filter_callbacks_, decoderBufferLimit()).WillByDefault(Return(1050)); + ON_CALL(encoder_filter_callbacks_, encoderBufferLimit()).WillByDefault(Return(1100)); + Event::MockTimer* request_timer = + new NiceMock(&decoder_filter_callbacks_.dispatcher_); + Event::MockTimer* response_timer = + new NiceMock(&encoder_filter_callbacks_.dispatcher_); + + EXPECT_EQ(1UL, config_->limit()); + EXPECT_EQ(50UL, config_->fillInterval().count()); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, + filter_->encode100ContinueHeaders(response_headers_)); + Http::MetadataMap metadata_map; + EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_->decodeMetadata(metadata_map)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers_, false)); + EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_->encodeMetadata(metadata_map)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(response_headers_, false)); + + Buffer::OwnedImpl dec_data(std::string(102, 'd')); + Buffer::OwnedImpl enc_data(std::string(56, 'e')); + + EXPECT_EQ(0, findGauge("test.http_bandwidth_limit.request_pending")); + EXPECT_EQ(0, findGauge("test.http_bandwidth_limit.response_pending")); + EXPECT_CALL(*request_timer, enableTimer(std::chrono::milliseconds(0), _)); + EXPECT_CALL(*response_timer, enableTimer(std::chrono::milliseconds(0), _)); + EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->decodeData(dec_data, false)); + EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->encodeData(enc_data, false)); + EXPECT_EQ(1, findGauge("test.http_bandwidth_limit.request_pending")); + EXPECT_EQ(1, findGauge("test.http_bandwidth_limit.response_pending")); + + EXPECT_CALL(*request_timer, enableTimer(std::chrono::milliseconds(50), _)); + EXPECT_CALL(decoder_filter_callbacks_, + injectDecodedDataToFilterChain(BufferStringEqual(std::string(51, 'd')), false)); + request_timer->invokeCallback(); + + // Fire timer, also advance time by 1 unit. + time_system_.advanceTimeWait(std::chrono::milliseconds(50)); + EXPECT_CALL(decoder_filter_callbacks_, + injectDecodedDataToFilterChain(BufferStringEqual(std::string(51, 'd')), false)); + request_timer->invokeCallback(); + EXPECT_EQ(1, findGauge("test.http_bandwidth_limit.request_pending")); + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers_)); + EXPECT_EQ(0, findGauge("test.http_bandwidth_limit.request_pending")); + + // Fire timer, also advance time by 1 unit. + time_system_.advanceTimeWait(std::chrono::milliseconds(50)); + EXPECT_CALL(*response_timer, enableTimer(std::chrono::milliseconds(50), _)); + EXPECT_CALL(encoder_filter_callbacks_, + injectEncodedDataToFilterChain(BufferStringEqual(std::string(51, 'e')), false)); + response_timer->invokeCallback(); + + time_system_.advanceTimeWait(std::chrono::milliseconds(50)); + EXPECT_CALL(encoder_filter_callbacks_, + injectEncodedDataToFilterChain(BufferStringEqual(std::string(5, 'e')), false)); + response_timer->invokeCallback(); + EXPECT_EQ(1, findGauge("test.http_bandwidth_limit.response_pending")); + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->encodeTrailers(response_trailers_)); + EXPECT_EQ(0, findGauge("test.http_bandwidth_limit.response_pending")); +} + +} // namespace BandwidthLimitFilter +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/http/cdn_loop/filter_integration_test.cc b/test/extensions/filters/http/cdn_loop/filter_integration_test.cc index a5db631c5c1ac..5e61749d0bf28 100644 --- a/test/extensions/filters/http/cdn_loop/filter_integration_test.cc +++ b/test/extensions/filters/http/cdn_loop/filter_integration_test.cc @@ -178,10 +178,10 @@ TEST_P(CdnLoopFilterIntegrationTest, CdnLoop2Allowed3Seen) { INSTANTIATE_TEST_SUITE_P(Protocols, CdnLoopFilterIntegrationTest, testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams( - {Http::CodecClient::Type::HTTP1, Http::CodecClient::Type::HTTP2}, + {Http::CodecType::HTTP1, Http::CodecType::HTTP2}, // Upstream doesn't matter, so by testing only 1, // the test is twice as fast. - {FakeHttpConnection::Type::HTTP1})), + {Http::CodecType::HTTP1})), HttpProtocolIntegrationTest::protocolTestParamsToString); } // namespace diff --git a/test/extensions/filters/http/common/compressor/compressor_integration_tests.cc b/test/extensions/filters/http/common/compressor/compressor_integration_tests.cc index 1ee14acf3cd4e..fc351d6994e56 100644 --- a/test/extensions/filters/http/common/compressor/compressor_integration_tests.cc +++ b/test/extensions/filters/http/common/compressor/compressor_integration_tests.cc @@ -129,7 +129,7 @@ ConfigHelper::HttpModifierFunction setRouteUsingWebsocket() { } void WebsocketWithCompressorIntegrationTest::initialize() { - if (upstreamProtocol() != FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() != Http::CodecType::HTTP1) { config_helper_.addConfigModifier( [&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { ConfigHelper::HttpProtocolOptions protocol_options; @@ -140,7 +140,7 @@ void WebsocketWithCompressorIntegrationTest::initialize() { *bootstrap.mutable_static_resources()->mutable_clusters(0), protocol_options); }); } - if (downstreamProtocol() != Http::CodecClient::Type::HTTP1) { + if (downstreamProtocol() != Http::CodecType::HTTP1) { config_helper_.addConfigModifier( [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& hcm) -> void { hcm.mutable_http2_protocol_options()->set_allow_connect(true); }); @@ -205,7 +205,7 @@ TEST_P(WebsocketWithCompressorIntegrationTest, NonWebsocketUpgrade) { performUpgrade(upgradeRequestHeaders("foo", 0), upgradeResponseHeaders("foo")); sendBidirectionalData(); codec_client_->sendData(*request_encoder_, "bye!", false); - if (downstreamProtocol() == Http::CodecClient::Type::HTTP1) { + if (downstreamProtocol() == Http::CodecType::HTTP1) { codec_client_->close(); } else { codec_client_->sendReset(*request_encoder_); @@ -268,7 +268,7 @@ TEST_P(CompressorProxyingConnectIntegrationTest, ProxyConnect) { RELEASE_ASSERT(result, result.message()); ASSERT_TRUE(upstream_request_->waitForHeadersComplete()); EXPECT_EQ(upstream_request_->headers().get(Http::Headers::get().Method)[0]->value(), "CONNECT"); - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() == Http::CodecType::HTTP1) { EXPECT_TRUE(upstream_request_->headers().get(Http::Headers::get().Protocol).empty()); } else { EXPECT_EQ(upstream_request_->headers().get(Http::Headers::get().Protocol)[0]->value(), diff --git a/test/extensions/filters/http/common/compressor/compressor_integration_tests.h b/test/extensions/filters/http/common/compressor/compressor_integration_tests.h index 43bd52e50d740..38d82c04dc589 100644 --- a/test/extensions/filters/http/common/compressor/compressor_integration_tests.h +++ b/test/extensions/filters/http/common/compressor/compressor_integration_tests.h @@ -28,7 +28,7 @@ class WebsocketWithCompressorIntegrationTest : public HttpProtocolIntegrationTes ABSL_MUST_USE_RESULT testing::AssertionResult waitForUpstreamDisconnectOrReset() { - if (upstreamProtocol() != FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() != Http::CodecType::HTTP1) { return upstream_request_->waitForReset(); } else { return fake_upstream_connection_->waitForDisconnect(); diff --git a/test/extensions/filters/http/common/stream_rate_limiter_test.cc b/test/extensions/filters/http/common/stream_rate_limiter_test.cc index d2e57de2d4f98..3048add956433 100644 --- a/test/extensions/filters/http/common/stream_rate_limiter_test.cc +++ b/test/extensions/filters/http/common/stream_rate_limiter_test.cc @@ -142,6 +142,9 @@ TEST_F(StreamRateLimiterTest, RateLimitOnSingleStream) { EXPECT_CALL(decoder_callbacks_, injectDecodedDataToFilterChain(BufferStringEqual(std::string(1024, 'c')), true)); token_timer->invokeCallback(); + + limiter_->destroy(); + EXPECT_EQ(limiter_->destroyed(), true); } } // namespace Common diff --git a/test/extensions/filters/http/composite/composite_filter_integration_test.cc b/test/extensions/filters/http/composite/composite_filter_integration_test.cc index de0abba85a078..b639e8cf3a7b5 100644 --- a/test/extensions/filters/http/composite/composite_filter_integration_test.cc +++ b/test/extensions/filters/http/composite/composite_filter_integration_test.cc @@ -11,8 +11,7 @@ namespace Envoy { class CompositeFilterIntegrationTest : public testing::TestWithParam, public HttpIntegrationTest { public: - CompositeFilterIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, GetParam()) {} + CompositeFilterIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam()) {} void initialize() override { config_helper_.addRuntimeOverride("envoy.reloadable_features.experimental_matching_api", diff --git a/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc b/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc index aaaf2908facf9..a95e6accf74f4 100644 --- a/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc +++ b/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc @@ -14,7 +14,7 @@ class CompressorIntegrationTest : public testing::TestWithParam, public HttpIntegrationTest { public: - CorsFilterIntegrationTest() : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, GetParam()) {} + CorsFilterIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam()) {} void initialize() override { config_helper_.addFilter("name: envoy.filters.http.cors"); diff --git a/test/extensions/filters/http/decompressor/decompressor_filter_integration_test.cc b/test/extensions/filters/http/decompressor/decompressor_filter_integration_test.cc index 4f9b4f8a011df..be1cb8384aa95 100644 --- a/test/extensions/filters/http/decompressor/decompressor_filter_integration_test.cc +++ b/test/extensions/filters/http/decompressor/decompressor_filter_integration_test.cc @@ -14,7 +14,7 @@ namespace Envoy { class DecompressorIntegrationTest : public testing::TestWithParam, public HttpIntegrationTest { public: - DecompressorIntegrationTest() : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, GetParam()) { + DecompressorIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP2, GetParam()) { Extensions::Compression::Gzip::Compressor::GzipCompressorLibraryFactory compressor_library_factory; envoy::extensions::compression::gzip::compressor::v3::Gzip factory_config; @@ -29,7 +29,7 @@ class DecompressorIntegrationTest : public testing::TestWithParam>; class ExtAuthzGrpcIntegrationTest : public Grpc::VersionedGrpcClientIntegrationParamTest, public HttpIntegrationTest { public: - ExtAuthzGrpcIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, ipVersion()) {} + ExtAuthzGrpcIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, ipVersion()) {} void createUpstreams() override { HttpIntegrationTest::createUpstreams(); - addFakeUpstream(FakeHttpConnection::Type::HTTP2); + addFakeUpstream(Http::CodecType::HTTP2); } void initializeConfig(bool disable_with_metadata = false) { @@ -317,9 +316,9 @@ class ExtAuthzGrpcIntegrationTest : public Grpc::VersionedGrpcClientIntegrationP cleanupUpstreamAndDownstream(); } - const std::string expectedCheckRequest(Http::CodecClient::Type downstream_protocol) { + const std::string expectedCheckRequest(Http::CodecType downstream_protocol) { const std::string expected_downstream_protocol = - downstream_protocol == Http::CodecClient::Type::HTTP1 ? "HTTP/1.1" : "HTTP/2"; + downstream_protocol == Http::CodecType::HTTP1 ? "HTTP/1.1" : "HTTP/2"; constexpr absl::string_view expected_format = R"EOF( attributes: request: @@ -336,17 +335,15 @@ class ExtAuthzGrpcIntegrationTest : public Grpc::VersionedGrpcClientIntegrationP expected_downstream_protocol); } - void expectCheckRequestWithBody(Http::CodecClient::Type downstream_protocol, - uint64_t request_size) { + void expectCheckRequestWithBody(Http::CodecType downstream_protocol, uint64_t request_size) { expectCheckRequestWithBodyWithHeaders(downstream_protocol, request_size, Headers{}, Headers{}, Headers{}, Http::TestRequestHeaderMapImpl{}, Http::TestRequestHeaderMapImpl{}); } void expectCheckRequestWithBodyWithHeaders( - Http::CodecClient::Type downstream_protocol, uint64_t request_size, - const Headers& headers_to_add, const Headers& headers_to_append, - const Headers& headers_to_remove, + Http::CodecType downstream_protocol, uint64_t request_size, const Headers& headers_to_add, + const Headers& headers_to_append, const Headers& headers_to_remove, const Http::TestRequestHeaderMapImpl& new_headers_from_upstream, const Http::TestRequestHeaderMapImpl& headers_to_append_multiple) { initializeConfig(); @@ -379,7 +376,7 @@ class ExtAuthzGrpcIntegrationTest : public Grpc::VersionedGrpcClientIntegrationP const std::string& expected_status) { initializeConfig(disable_with_metadata); setDenyAtDisableRuntimeConfig(deny_at_disable, disable_with_metadata); - setDownstreamProtocol(Http::CodecClient::Type::HTTP2); + setDownstreamProtocol(Http::CodecType::HTTP2); HttpIntegrationTest::initialize(); initiateClientConnection(4); if (!deny_at_disable) { @@ -406,11 +403,11 @@ class ExtAuthzGrpcIntegrationTest : public Grpc::VersionedGrpcClientIntegrationP class ExtAuthzHttpIntegrationTest : public HttpIntegrationTest, public TestWithParam { public: - ExtAuthzHttpIntegrationTest() : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, GetParam()) {} + ExtAuthzHttpIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam()) {} void createUpstreams() override { HttpIntegrationTest::createUpstreams(); - addFakeUpstream(FakeHttpConnection::Type::HTTP1); + addFakeUpstream(Http::CodecType::HTTP1); } // By default, HTTP Service uses case sensitive string matcher. @@ -580,28 +577,28 @@ INSTANTIATE_TEST_SUITE_P(IpVersionsCientType, ExtAuthzGrpcIntegrationTest, // HTTP/1.1. TEST_P(ExtAuthzGrpcIntegrationTest, HTTP1DownstreamRequestWithBody) { XDS_DEPRECATED_FEATURE_TEST_SKIP; - expectCheckRequestWithBody(Http::CodecClient::Type::HTTP1, 4); + expectCheckRequestWithBody(Http::CodecType::HTTP1, 4); } // Verifies that the request body is included in the CheckRequest when the downstream protocol is // HTTP/1.1 and the size of the request body is larger than max_request_bytes. TEST_P(ExtAuthzGrpcIntegrationTest, HTTP1DownstreamRequestWithLargeBody) { XDS_DEPRECATED_FEATURE_TEST_SKIP; - expectCheckRequestWithBody(Http::CodecClient::Type::HTTP1, 2048); + expectCheckRequestWithBody(Http::CodecType::HTTP1, 2048); } // Verifies that the request body is included in the CheckRequest when the downstream protocol is // HTTP/2. TEST_P(ExtAuthzGrpcIntegrationTest, HTTP2DownstreamRequestWithBody) { XDS_DEPRECATED_FEATURE_TEST_SKIP; - expectCheckRequestWithBody(Http::CodecClient::Type::HTTP2, 4); + expectCheckRequestWithBody(Http::CodecType::HTTP2, 4); } // Verifies that the request body is included in the CheckRequest when the downstream protocol is // HTTP/2 and the size of the request body is larger than max_request_bytes. TEST_P(ExtAuthzGrpcIntegrationTest, HTTP2DownstreamRequestWithLargeBody) { XDS_DEPRECATED_FEATURE_TEST_SKIP; - expectCheckRequestWithBody(Http::CodecClient::Type::HTTP2, 2048); + expectCheckRequestWithBody(Http::CodecType::HTTP2, 2048); } // Verifies that the original request headers will be added and appended when the authorization @@ -610,7 +607,7 @@ TEST_P(ExtAuthzGrpcIntegrationTest, HTTP2DownstreamRequestWithLargeBody) { TEST_P(ExtAuthzGrpcIntegrationTest, SendHeadersToAddAndToAppendToUpstream) { XDS_DEPRECATED_FEATURE_TEST_SKIP; expectCheckRequestWithBodyWithHeaders( - Http::CodecClient::Type::HTTP1, 4, + Http::CodecType::HTTP1, 4, /*headers_to_add=*/Headers{{"header1", "header1"}}, /*headers_to_append=*/Headers{{"header2", "header2"}}, /*headers_to_remove=*/Headers{{"remove-me", "upstream-should-not-see-me"}}, @@ -646,14 +643,14 @@ TEST_P(ExtAuthzGrpcIntegrationTest, DownstreamHeadersOnSuccess) { initializeConfig(); // Use h1, set up the test. - setDownstreamProtocol(Http::CodecClient::Type::HTTP1); + setDownstreamProtocol(Http::CodecType::HTTP1); HttpIntegrationTest::initialize(); // Start a client connection and request. initiateClientConnection(0); // Wait for the ext_authz request as a result of the client request. - waitForExtAuthzRequest(expectedCheckRequest(Http::CodecClient::Type::HTTP1)); + waitForExtAuthzRequest(expectedCheckRequest(Http::CodecType::HTTP1)); // Send back an ext_authz response with response_headers_to_add set. sendExtAuthzResponse(Headers{}, Headers{}, Headers{}, Http::TestRequestHeaderMapImpl{}, @@ -686,12 +683,11 @@ TEST_P(ExtAuthzHttpIntegrationTest, DefaultCaseSensitiveStringMatcher) { class ExtAuthzLocalReplyIntegrationTest : public HttpIntegrationTest, public TestWithParam { public: - ExtAuthzLocalReplyIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, GetParam()) {} + ExtAuthzLocalReplyIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam()) {} void createUpstreams() override { HttpIntegrationTest::createUpstreams(); - addFakeUpstream(FakeHttpConnection::Type::HTTP1); + addFakeUpstream(Http::CodecType::HTTP1); } void cleanup() { @@ -798,7 +794,7 @@ TEST_P(ExtAuthzLocalReplyIntegrationTest, DeniedHeaderTest) { TEST_P(ExtAuthzGrpcIntegrationTest, GoogleAsyncClientCreation) { XDS_DEPRECATED_FEATURE_TEST_SKIP; initializeConfig(); - setDownstreamProtocol(Http::CodecClient::Type::HTTP2); + setDownstreamProtocol(Http::CodecType::HTTP2); HttpIntegrationTest::initialize(); int expected_grpc_client_creation_count = 0; @@ -811,7 +807,7 @@ TEST_P(ExtAuthzGrpcIntegrationTest, GoogleAsyncClientCreation) { } initiateClientConnection(4, Headers{}, Headers{}); - waitForExtAuthzRequest(expectedCheckRequest(Http::CodecClient::Type::HTTP2)); + waitForExtAuthzRequest(expectedCheckRequest(Http::CodecType::HTTP2)); sendExtAuthzResponse(Headers{}, Headers{}, Headers{}, Http::TestRequestHeaderMapImpl{}, Http::TestRequestHeaderMapImpl{}, Headers{}); diff --git a/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc b/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc index 0e3786158db12..0b9150a3ecaa0 100644 --- a/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc +++ b/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc @@ -41,12 +41,12 @@ using namespace std::chrono_literals; class ExtProcIntegrationTest : public HttpIntegrationTest, public Grpc::GrpcClientIntegrationParamTest { protected: - ExtProcIntegrationTest() : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion()) {} + ExtProcIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP2, ipVersion()) {} void createUpstreams() override { // Need to create a separate "upstream" for the gRPC server HttpIntegrationTest::createUpstreams(); - addFakeUpstream(FakeHttpConnection::Type::HTTP2); + addFakeUpstream(Http::CodecType::HTTP2); } void TearDown() override { @@ -82,8 +82,8 @@ class ExtProcIntegrationTest : public HttpIntegrationTest, ext_proc_filter.mutable_typed_config()->PackFrom(proto_config_); config_helper_.addFilter(MessageUtil::getJsonStringFromMessageOrDie(ext_proc_filter)); }); - setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); - setDownstreamProtocol(Http::CodecClient::Type::HTTP2); + setUpstreamProtocol(Http::CodecType::HTTP2); + setDownstreamProtocol(Http::CodecType::HTTP2); } IntegrationStreamDecoderPtr sendDownstreamRequest( diff --git a/test/extensions/filters/http/fault/fault_filter_integration_test.cc b/test/extensions/filters/http/fault/fault_filter_integration_test.cc index f23633aa308a9..370c662438bee 100644 --- a/test/extensions/filters/http/fault/fault_filter_integration_test.cc +++ b/test/extensions/filters/http/fault/fault_filter_integration_test.cc @@ -393,7 +393,7 @@ TEST_P(FaultIntegrationTestAllProtocols, FaultAbortGrpcConfig) { class FaultIntegrationTestHttp2 : public FaultIntegrationTest {}; INSTANTIATE_TEST_SUITE_P(Protocols, FaultIntegrationTestHttp2, testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams( - {Http::CodecClient::Type::HTTP2}, {FakeHttpConnection::Type::HTTP2})), + {Http::CodecType::HTTP2}, {Http::CodecType::HTTP2})), HttpProtocolIntegrationTest::protocolTestParamsToString); // Rate limiting with trailers received after the body has been flushed. diff --git a/test/extensions/filters/http/grpc_http1_reverse_bridge/reverse_bridge_integration_test.cc b/test/extensions/filters/http/grpc_http1_reverse_bridge/reverse_bridge_integration_test.cc index 911e52eb127d1..43cb6339eef46 100644 --- a/test/extensions/filters/http/grpc_http1_reverse_bridge/reverse_bridge_integration_test.cc +++ b/test/extensions/filters/http/grpc_http1_reverse_bridge/reverse_bridge_integration_test.cc @@ -26,11 +26,10 @@ namespace { class ReverseBridgeIntegrationTest : public testing::TestWithParam, public HttpIntegrationTest { public: - ReverseBridgeIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, GetParam()) {} + ReverseBridgeIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP2, GetParam()) {} void initialize() override { - setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); + setUpstreamProtocol(Http::CodecType::HTTP2); const std::string filter = R"EOF( @@ -57,7 +56,7 @@ name: grpc_http1_reverse_bridge void TearDown() override { fake_upstream_connection_.reset(); } protected: - FakeHttpConnection::Type upstream_protocol_; + Http::CodecType upstream_protocol_; }; INSTANTIATE_TEST_SUITE_P(IpVersions, ReverseBridgeIntegrationTest, @@ -68,7 +67,7 @@ INSTANTIATE_TEST_SUITE_P(IpVersions, ReverseBridgeIntegrationTest, // doesn't enable the bridge. // Regression test of https://github.com/envoyproxy/envoy/issues/9922 TEST_P(ReverseBridgeIntegrationTest, DisabledRoute) { - upstream_protocol_ = FakeHttpConnection::Type::HTTP2; + upstream_protocol_ = Http::CodecType::HTTP2; initialize(); codec_client_ = makeHttpConnection(lookupPort("http")); @@ -117,7 +116,7 @@ TEST_P(ReverseBridgeIntegrationTest, DisabledRoute) { } TEST_P(ReverseBridgeIntegrationTest, EnabledRoute) { - upstream_protocol_ = FakeHttpConnection::Type::HTTP1; + upstream_protocol_ = Http::CodecType::HTTP1; initialize(); codec_client_ = makeHttpConnection(lookupPort("http")); @@ -174,7 +173,7 @@ TEST_P(ReverseBridgeIntegrationTest, EnabledRoute) { } TEST_P(ReverseBridgeIntegrationTest, EnabledRouteBadContentType) { - upstream_protocol_ = FakeHttpConnection::Type::HTTP1; + upstream_protocol_ = Http::CodecType::HTTP1; initialize(); codec_client_ = makeHttpConnection(lookupPort("http")); diff --git a/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc b/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc index 15e669eb33671..065d599f1fdd8 100644 --- a/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc +++ b/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc @@ -30,11 +30,10 @@ class GrpcJsonTranscoderIntegrationTest : public testing::TestWithParam, public HttpIntegrationTest { public: - GrpcJsonTranscoderIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, GetParam()) {} + GrpcJsonTranscoderIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam()) {} void SetUp() override { - setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); + setUpstreamProtocol(Http::CodecType::HTTP2); const std::string filter = R"EOF( name: grpc_json_transcoder @@ -1253,7 +1252,7 @@ class OverrideConfigGrpcJsonTranscoderIntegrationTest : public GrpcJsonTranscode * Global initializer for all integration tests. */ void SetUp() override { - setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); + setUpstreamProtocol(Http::CodecType::HTTP2); // creates filter but doesn't apply it to bookstore services const std::string filter = R"EOF( @@ -1301,7 +1300,7 @@ class BufferLimitsDisabledGrpcJsonTranscoderIntegrationTest : public GrpcJsonTranscoderIntegrationTest { public: void SetUp() override { - setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); + setUpstreamProtocol(Http::CodecType::HTTP2); const std::string filter = R"EOF( name: grpc_json_transcoder diff --git a/test/extensions/filters/http/grpc_web/grpc_web_filter_integration_test.cc b/test/extensions/filters/http/grpc_web/grpc_web_filter_integration_test.cc index 32ca9673445af..701f97662a9e5 100644 --- a/test/extensions/filters/http/grpc_web/grpc_web_filter_integration_test.cc +++ b/test/extensions/filters/http/grpc_web/grpc_web_filter_integration_test.cc @@ -18,7 +18,7 @@ constexpr uint64_t MAX_BUFFERED_PLAINTEXT_LENGTH = 16384; using SkipEncodingEmptyTrailers = bool; using ContentType = std::string; using Accept = std::string; -using TestParams = std::tuple; class GrpcWebFilterIntegrationTest : public testing::TestWithParam, @@ -28,18 +28,18 @@ class GrpcWebFilterIntegrationTest : public testing::TestWithParam, : HttpIntegrationTest(std::get<1>(GetParam()), std::get<0>(GetParam())) {} void SetUp() override { - setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); + setUpstreamProtocol(Http::CodecType::HTTP2); config_helper_.addFilter("name: envoy.filters.http.grpc_web"); } void initialize() override { - if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + if (downstream_protocol_ == Http::CodecType::HTTP1) { config_helper_.addConfigModifier(setEnableDownstreamTrailersHttp1()); } else { skipEncodingEmptyTrailers(http2_skip_encoding_empty_trailers_); } - setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); + setUpstreamProtocol(Http::CodecType::HTTP2); HttpIntegrationTest::initialize(); } @@ -139,13 +139,13 @@ class GrpcWebFilterIntegrationTest : public testing::TestWithParam, "{}_{}_{}_{}_{}", TestUtility::ipTestParamsToString(testing::TestParamInfo( std::get<0>(params.param), params.index)), - std::get<1>(params.param) == Http::CodecClient::Type::HTTP2 ? "Http2" : "Http", + std::get<1>(params.param) == Http::CodecType::HTTP2 ? "Http2" : "Http", std::get<2>(params.param) ? "SkipEncodingEmptyTrailers" : "SubmitEncodingEmptyTrailers", std::get<3>(params.param) == text ? "SendText" : "SendBinary", std::get<4>(params.param) == text ? "AcceptText" : "AcceptBinary"); } - const Envoy::Http::CodecClient::Type downstream_protocol_{std::get<1>(GetParam())}; + const Envoy::Http::CodecType downstream_protocol_{std::get<1>(GetParam())}; const bool http2_skip_encoding_empty_trailers_{std::get<2>(GetParam())}; const ContentType content_type_{std::get<3>(GetParam())}; const Accept accept_{std::get<4>(GetParam())}; @@ -153,12 +153,12 @@ class GrpcWebFilterIntegrationTest : public testing::TestWithParam, INSTANTIATE_TEST_SUITE_P( Params, GrpcWebFilterIntegrationTest, - testing::Combine( - testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), - testing::Values(Http::CodecClient::Type::HTTP1, Http::CodecClient::Type::HTTP2), - testing::Values(SkipEncodingEmptyTrailers{true}, SkipEncodingEmptyTrailers{false}), - testing::Values(ContentType{text}, ContentType{binary}), - testing::Values(Accept{text}, Accept{binary})), + testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + testing::Values(Http::CodecType::HTTP1, Http::CodecType::HTTP2), + testing::Values(SkipEncodingEmptyTrailers{true}, + SkipEncodingEmptyTrailers{false}), + testing::Values(ContentType{text}, ContentType{binary}), + testing::Values(Accept{text}, Accept{binary})), GrpcWebFilterIntegrationTest::testParamsToString); TEST_P(GrpcWebFilterIntegrationTest, GrpcWebTrailersNotDuplicated) { @@ -202,12 +202,12 @@ TEST_P(GrpcWebFilterIntegrationTest, GrpcWebTrailersNotDuplicated) { EXPECT_TRUE(absl::StrContains(response_body, "response1:trailer1")); EXPECT_TRUE(absl::StrContains(response_body, "response2:trailer2")); - if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + if (downstream_protocol_ == Http::CodecType::HTTP1) { // When the downstream protocol is HTTP/1.1 we expect the trailers to be in the response-body. EXPECT_EQ(nullptr, response->trailers()); } - if (downstream_protocol_ == Http::CodecClient::Type::HTTP2) { + if (downstream_protocol_ == Http::CodecType::HTTP2) { if (http2_skip_encoding_empty_trailers_) { // When the downstream protocol is HTTP/2 and the feature-flag to skip encoding empty trailers // is turned on, expect that the trailers are included in the response-body. diff --git a/test/extensions/filters/http/gzip/gzip_filter_integration_test.cc b/test/extensions/filters/http/gzip/gzip_filter_integration_test.cc index bd0edc6d1d560..e17b55409ec89 100644 --- a/test/extensions/filters/http/gzip/gzip_filter_integration_test.cc +++ b/test/extensions/filters/http/gzip/gzip_filter_integration_test.cc @@ -14,7 +14,7 @@ class GzipIntegrationTest : public testing::TestWithParamstats().jwks_fetch_success_.value()); + EXPECT_EQ(0U, filter_config_->stats().jwks_fetch_failed_.value()); } // This test verifies the Jwt is forwarded if "forward" flag is set. @@ -131,6 +134,9 @@ TEST_F(AuthenticatorTest, TestForwardJwt) { // Payload not set by default EXPECT_EQ(out_name_, ""); + + EXPECT_EQ(1U, filter_config_->stats().jwks_fetch_success_.value()); + EXPECT_EQ(0U, filter_config_->stats().jwks_fetch_failed_.value()); } // This test verifies the Jwt payload is set. @@ -181,6 +187,9 @@ TEST_F(AuthenticatorTest, TestWrongIssuer) { Http::TestRequestHeaderMapImpl headers{ {"Authorization", "Bearer " + std::string(OtherGoodToken)}}; expectVerifyStatus(Status::JwtUnknownIssuer, headers); + + EXPECT_EQ(0U, filter_config_->stats().jwks_fetch_success_.value()); + EXPECT_EQ(0U, filter_config_->stats().jwks_fetch_failed_.value()); } // Jwt "iss" is "other.com", "issuer" in JwtProvider is not specified, diff --git a/test/extensions/filters/http/jwt_authn/filter_integration_test.cc b/test/extensions/filters/http/jwt_authn/filter_integration_test.cc index 7adac3450ce60..0a5792d1c1f76 100644 --- a/test/extensions/filters/http/jwt_authn/filter_integration_test.cc +++ b/test/extensions/filters/http/jwt_authn/filter_integration_test.cc @@ -78,6 +78,20 @@ std::string getAuthFilterConfig(const std::string& config_str, bool use_local_jw return MessageUtil::getJsonStringFromMessageOrDie(filter); } +std::string getAsyncFetchFilterConfig(const std::string& config_str, bool fast_listener) { + JwtAuthentication proto_config; + TestUtility::loadFromYaml(config_str, proto_config); + + auto& provider0 = (*proto_config.mutable_providers())[std::string(ProviderName)]; + auto* async_fetch = provider0.mutable_remote_jwks()->mutable_async_fetch(); + async_fetch->set_fast_listener(fast_listener); + + HttpFilter filter; + filter.set_name(HttpFilterNames::get().JwtAuthn); + filter.mutable_typed_config()->PackFrom(proto_config); + return MessageUtil::getJsonStringFromMessageOrDie(filter); +} + std::string getFilterConfig(bool use_local_jwks) { return getAuthFilterConfig(ExampleConfig, use_local_jwks); } @@ -329,6 +343,18 @@ class RemoteJwksIntegrationTest : public HttpProtocolIntegrationTest { initialize(); } + void initializeAsyncFetchFilter(bool fast_listener) { + config_helper_.addFilter(getAsyncFetchFilterConfig(ExampleConfig, fast_listener)); + + config_helper_.addConfigModifier([](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + auto* jwks_cluster = bootstrap.mutable_static_resources()->add_clusters(); + jwks_cluster->MergeFrom(bootstrap.static_resources().clusters()[0]); + jwks_cluster->set_name("pubkey_cluster"); + }); + + initialize(); + } + void waitForJwksResponse(const std::string& status, const std::string& jwks_body) { AssertionResult result = fake_upstreams_[1]->waitForHttpConnection(*dispatcher_, fake_jwks_connection_); @@ -448,6 +474,112 @@ TEST_P(RemoteJwksIntegrationTest, FetchFailedMissingCluster) { cleanup(); } +TEST_P(RemoteJwksIntegrationTest, WithGoodTokenAsyncFetch) { + on_server_init_function_ = [this]() { waitForJwksResponse("200", PublicKey); }; + initializeAsyncFetchFilter(false); + + codec_client_ = makeHttpConnection(lookupPort("http")); + + auto response = codec_client_->makeHeaderOnlyRequest(Http::TestRequestHeaderMapImpl{ + {":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "host"}, + {"Authorization", "Bearer " + std::string(GoodToken)}, + }); + + waitForNextUpstreamRequest(); + + const auto payload_entry = + upstream_request_->headers().get(Http::LowerCaseString("sec-istio-auth-userinfo")); + EXPECT_FALSE(payload_entry.empty()); + EXPECT_EQ(payload_entry[0]->value().getStringView(), ExpectedPayloadValue); + // Verify the token is removed. + EXPECT_TRUE(upstream_request_->headers().get(Http::CustomHeaders::get().Authorization).empty()); + + upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, true); + + ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); + + cleanup(); +} + +TEST_P(RemoteJwksIntegrationTest, WithGoodTokenAsyncFetchFast) { + on_server_init_function_ = [this]() { waitForJwksResponse("200", PublicKey); }; + initializeAsyncFetchFilter(true); + + codec_client_ = makeHttpConnection(lookupPort("http")); + + auto response = codec_client_->makeHeaderOnlyRequest(Http::TestRequestHeaderMapImpl{ + {":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "host"}, + {"Authorization", "Bearer " + std::string(GoodToken)}, + }); + + waitForNextUpstreamRequest(); + + const auto payload_entry = + upstream_request_->headers().get(Http::LowerCaseString("sec-istio-auth-userinfo")); + EXPECT_FALSE(payload_entry.empty()); + EXPECT_EQ(payload_entry[0]->value().getStringView(), ExpectedPayloadValue); + // Verify the token is removed. + EXPECT_TRUE(upstream_request_->headers().get(Http::CustomHeaders::get().Authorization).empty()); + + upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, true); + + ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); + + cleanup(); +} + +TEST_P(RemoteJwksIntegrationTest, WithFailedJwksAsyncFetch) { + on_server_init_function_ = [this]() { waitForJwksResponse("500", ""); }; + initializeAsyncFetchFilter(false); + + codec_client_ = makeHttpConnection(lookupPort("http")); + + auto response = codec_client_->makeHeaderOnlyRequest(Http::TestRequestHeaderMapImpl{ + {":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "host"}, + {"Authorization", "Bearer " + std::string(GoodToken)}, + }); + + ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->complete()); + EXPECT_EQ("401", response->headers().getStatusValue()); + + cleanup(); +} + +TEST_P(RemoteJwksIntegrationTest, WithFailedJwksAsyncFetchFast) { + on_server_init_function_ = [this]() { waitForJwksResponse("500", ""); }; + initializeAsyncFetchFilter(true); + + codec_client_ = makeHttpConnection(lookupPort("http")); + + auto response = codec_client_->makeHeaderOnlyRequest(Http::TestRequestHeaderMapImpl{ + {":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "host"}, + {"Authorization", "Bearer " + std::string(GoodToken)}, + }); + + ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->complete()); + EXPECT_EQ("401", response->headers().getStatusValue()); + + cleanup(); +} + class PerRouteIntegrationTest : public HttpProtocolIntegrationTest { public: void setup(const std::string& filter_config, const PerRouteConfig& per_route) { diff --git a/test/extensions/filters/http/jwt_authn/jwks_async_fetcher_test.cc b/test/extensions/filters/http/jwt_authn/jwks_async_fetcher_test.cc new file mode 100644 index 0000000000000..1d21e254d7f94 --- /dev/null +++ b/test/extensions/filters/http/jwt_authn/jwks_async_fetcher_test.cc @@ -0,0 +1,248 @@ +#include "extensions/filters/http/jwt_authn/jwks_async_fetcher.h" + +#include "test/extensions/filters/http/jwt_authn/test_common.h" +#include "test/mocks/server/factory_context.h" + +using envoy::extensions::filters::http::jwt_authn::v3::RemoteJwks; +using Envoy::Extensions::HttpFilters::Common::JwksFetcher; +using Envoy::Extensions::HttpFilters::Common::JwksFetcherPtr; + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace JwtAuthn { +namespace { + +JwtAuthnFilterStats generateMockStats(Stats::Scope& scope) { + return {ALL_JWT_AUTHN_FILTER_STATS(POOL_COUNTER_PREFIX(scope, ""))}; +} + +class MockJwksFetcher : public Common::JwksFetcher { +public: + using SaveJwksReceiverFn = std::function; + MockJwksFetcher(SaveJwksReceiverFn receiver_fn) : receiver_fn_(receiver_fn) {} + + void cancel() override {} + void fetch(const envoy::config::core::v3::HttpUri&, Tracing::Span&, + JwksReceiver& receiver) override { + receiver_fn_(receiver); + } + +private: + SaveJwksReceiverFn receiver_fn_; +}; + +// TestParam is for fast_listener, +class JwksAsyncFetcherTest : public testing::TestWithParam { +public: + JwksAsyncFetcherTest() : stats_(generateMockStats(context_.scope())) {} + + // init manager is used in is_slow_listener mode + bool initManagerUsed() const { + return config_.has_async_fetch() && !config_.async_fetch().fast_listener(); + } + + void setupAsyncFetcher(const std::string& config_str) { + TestUtility::loadFromYaml(config_str, config_); + if (config_.has_async_fetch()) { + // Param is for fast_listener, + if (GetParam()) { + config_.mutable_async_fetch()->set_fast_listener(true); + } + } + + if (initManagerUsed()) { + EXPECT_CALL(context_.init_manager_, add(_)) + .WillOnce(Invoke([this](const Init::Target& target) { + init_target_handle_ = target.createHandle("test"); + })); + } + + // if async_fetch is enabled, timer is created + if (config_.has_async_fetch()) { + timer_ = new NiceMock(&context_.dispatcher_); + expected_duration_ = JwksAsyncFetcher::getCacheDuration(config_); + } + + async_fetcher_ = std::make_unique( + config_, context_, + [this](Upstream::ClusterManager&) { + return std::make_unique( + [this](Common::JwksFetcher::JwksReceiver& receiver) { + fetch_receiver_array_.push_back(&receiver); + }); + }, + stats_, + [this](google::jwt_verify::JwksPtr&& jwks) { out_jwks_array_.push_back(std::move(jwks)); }); + + if (initManagerUsed()) { + init_target_handle_->initialize(init_watcher_); + } + } + + RemoteJwks config_; + JwksAsyncFetcherPtr async_fetcher_; + NiceMock context_; + JwtAuthnFilterStats stats_; + std::vector fetch_receiver_array_; + std::vector out_jwks_array_; + + Init::TargetHandlePtr init_target_handle_; + NiceMock init_watcher_; + Event::MockTimer* timer_{}; + std::chrono::milliseconds expected_duration_; +}; + +INSTANTIATE_TEST_SUITE_P(JwksAsyncFetcherTest, JwksAsyncFetcherTest, + testing::ValuesIn({false, true})); + +TEST_P(JwksAsyncFetcherTest, TestNotAsyncFetch) { + const char config[] = R"( + http_uri: + uri: https://pubkey_server/pubkey_path + cluster: pubkey_cluster +)"; + + setupAsyncFetcher(config); + // fetch is not called + EXPECT_EQ(fetch_receiver_array_.size(), 0); + // Not Jwks output + EXPECT_EQ(out_jwks_array_.size(), 0); + // init_watcher ready is not called. + init_watcher_.expectReady().Times(0); + + EXPECT_EQ(0U, stats_.jwks_fetch_success_.value()); + EXPECT_EQ(0U, stats_.jwks_fetch_failed_.value()); +} + +TEST_P(JwksAsyncFetcherTest, TestGoodFetch) { + const char config[] = R"( + http_uri: + uri: https://pubkey_server/pubkey_path + cluster: pubkey_cluster + async_fetch: {} +)"; + + setupAsyncFetcher(config); + // Jwks response is not received yet + EXPECT_EQ(out_jwks_array_.size(), 0); + + if (initManagerUsed()) { + // Verify ready is not called. + init_watcher_.expectReady().Times(0); + EXPECT_TRUE(::testing::Mock::VerifyAndClearExpectations(&init_watcher_)); + init_watcher_.expectReady(); + } + + // Trigger the Jwks response + EXPECT_EQ(fetch_receiver_array_.size(), 1); + auto jwks = google::jwt_verify::Jwks::createFrom(PublicKey, google::jwt_verify::Jwks::JWKS); + fetch_receiver_array_[0]->onJwksSuccess(std::move(jwks)); + + // Output 1 jwks. + EXPECT_EQ(out_jwks_array_.size(), 1); + + EXPECT_EQ(1U, stats_.jwks_fetch_success_.value()); + EXPECT_EQ(0U, stats_.jwks_fetch_failed_.value()); +} + +TEST_P(JwksAsyncFetcherTest, TestNetworkFailureFetch) { + const char config[] = R"( + http_uri: + uri: https://pubkey_server/pubkey_path + cluster: pubkey_cluster + async_fetch: {} +)"; + + // Just start the Jwks fetch call + setupAsyncFetcher(config); + // Jwks response is not received yet + EXPECT_EQ(out_jwks_array_.size(), 0); + + if (initManagerUsed()) { + // Verify ready is not called. + init_watcher_.expectReady().Times(0); + EXPECT_TRUE(::testing::Mock::VerifyAndClearExpectations(&init_watcher_)); + // Verify ready is called. + init_watcher_.expectReady(); + } + + // Trigger the Jwks response + EXPECT_EQ(fetch_receiver_array_.size(), 1); + fetch_receiver_array_[0]->onJwksError(Common::JwksFetcher::JwksReceiver::Failure::Network); + + // Output 0 jwks. + EXPECT_EQ(out_jwks_array_.size(), 0); + + EXPECT_EQ(0U, stats_.jwks_fetch_success_.value()); + EXPECT_EQ(1U, stats_.jwks_fetch_failed_.value()); +} + +TEST_P(JwksAsyncFetcherTest, TestGoodFetchAndRefresh) { + const char config[] = R"( + http_uri: + uri: https://pubkey_server/pubkey_path + cluster: pubkey_cluster + async_fetch: {} +)"; + + setupAsyncFetcher(config); + // Initial fetch is successful + EXPECT_EQ(fetch_receiver_array_.size(), 1); + auto jwks = google::jwt_verify::Jwks::createFrom(PublicKey, google::jwt_verify::Jwks::JWKS); + fetch_receiver_array_[0]->onJwksSuccess(std::move(jwks)); + + // Output 1 jwks. + EXPECT_EQ(out_jwks_array_.size(), 1); + + // Expect refresh timer is enabled. + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + timer_->invokeCallback(); + + // refetch again after cache duration interval: successful. + EXPECT_EQ(fetch_receiver_array_.size(), 2); + auto jwks1 = google::jwt_verify::Jwks::createFrom(PublicKey, google::jwt_verify::Jwks::JWKS); + fetch_receiver_array_[1]->onJwksSuccess(std::move(jwks1)); + + // Output 2 jwks. + EXPECT_EQ(out_jwks_array_.size(), 2); + EXPECT_EQ(2U, stats_.jwks_fetch_success_.value()); + EXPECT_EQ(0U, stats_.jwks_fetch_failed_.value()); +} + +TEST_P(JwksAsyncFetcherTest, TestNetworkFailureFetchAndRefresh) { + const char config[] = R"( + http_uri: + uri: https://pubkey_server/pubkey_path + cluster: pubkey_cluster + async_fetch: {} +)"; + + // Just start the Jwks fetch call + setupAsyncFetcher(config); + // first fetch: network failure. + EXPECT_EQ(fetch_receiver_array_.size(), 1); + fetch_receiver_array_[0]->onJwksError(Common::JwksFetcher::JwksReceiver::Failure::Network); + + // Output 0 jwks. + EXPECT_EQ(out_jwks_array_.size(), 0); + + // Expect refresh timer is enabled. + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + timer_->invokeCallback(); + + // refetch again after cache duration interval: network failure. + EXPECT_EQ(fetch_receiver_array_.size(), 2); + fetch_receiver_array_[1]->onJwksError(Common::JwksFetcher::JwksReceiver::Failure::Network); + + // Output 0 jwks. + EXPECT_EQ(out_jwks_array_.size(), 0); + EXPECT_EQ(0U, stats_.jwks_fetch_success_.value()); + EXPECT_EQ(2U, stats_.jwks_fetch_failed_.value()); +} + +} // namespace +} // namespace JwtAuthn +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/http/jwt_authn/jwks_cache_test.cc b/test/extensions/filters/http/jwt_authn/jwks_cache_test.cc index a0fcdde765c15..6c22ec49a54bc 100644 --- a/test/extensions/filters/http/jwt_authn/jwks_cache_test.cc +++ b/test/extensions/filters/http/jwt_authn/jwks_cache_test.cc @@ -9,12 +9,12 @@ #include "extensions/filters/http/jwt_authn/jwks_cache.h" #include "test/extensions/filters/http/jwt_authn/test_common.h" -#include "test/mocks/thread_local/mocks.h" -#include "test/test_common/simulated_time_system.h" +#include "test/mocks/server/factory_context.h" #include "test/test_common/utility.h" using envoy::extensions::filters::http::jwt_authn::v3::JwtAuthentication; using ::google::jwt_verify::Status; +using ::testing::MockFunction; namespace Envoy { namespace Extensions { @@ -22,25 +22,32 @@ namespace HttpFilters { namespace JwtAuthn { namespace { +JwtAuthnFilterStats generateMockStats(Stats::Scope& scope) { + return {ALL_JWT_AUTHN_FILTER_STATS(POOL_COUNTER_PREFIX(scope, ""))}; +} + class JwksCacheTest : public testing::Test { protected: - JwksCacheTest() : api_(Api::createApiForTest()) {} + JwksCacheTest() : stats_(generateMockStats(context_.scope())) {} + void SetUp() override { + // fetcher is only called at async_fetch. In this test, it is never called. + EXPECT_CALL(mock_fetcher_, Call(_)).Times(0); setupCache(ExampleConfig); jwks_ = google::jwt_verify::Jwks::createFrom(PublicKey, google::jwt_verify::Jwks::JWKS); } void setupCache(const std::string& config_str) { TestUtility::loadFromYaml(config_str, config_); - cache_ = JwksCache::create(config_, time_system_, *api_, tls_); + cache_ = JwksCache::create(config_, context_, mock_fetcher_.AsStdFunction(), stats_); } - Event::SimulatedTimeSystem time_system_; JwtAuthentication config_; JwksCachePtr cache_; google::jwt_verify::JwksPtr jwks_; - Api::ApiPtr api_; - ::testing::NiceMock tls_; + MockFunction mock_fetcher_; + NiceMock context_; + JwtAuthnFilterStats stats_; }; // Test findByProvider @@ -84,7 +91,7 @@ TEST_F(JwksCacheTest, TestSetRemoteJwks) { auto& provider0 = (*config_.mutable_providers())[std::string(ProviderName)]; // Set cache_duration to 1 second to test expiration provider0.mutable_remote_jwks()->mutable_cache_duration()->set_seconds(1); - cache_ = JwksCache::create(config_, time_system_, *api_, tls_); + cache_ = JwksCache::create(config_, context_, mock_fetcher_.AsStdFunction(), stats_); auto jwks = cache_->findByIssuer("https://example.com"); EXPECT_TRUE(jwks->getJwksObj() == nullptr); @@ -94,7 +101,7 @@ TEST_F(JwksCacheTest, TestSetRemoteJwks) { EXPECT_FALSE(jwks->isExpired()); // cache duration is 1 second, sleep two seconds to expire it - time_system_.advanceTimeWait(std::chrono::seconds(2)); + context_.time_system_.advanceTimeWait(std::chrono::seconds(2)); EXPECT_TRUE(jwks->isExpired()); } @@ -103,7 +110,7 @@ TEST_F(JwksCacheTest, TestSetRemoteJwksWithDefaultCacheDuration) { auto& provider0 = (*config_.mutable_providers())[std::string(ProviderName)]; // Clear cache_duration to use default one. provider0.mutable_remote_jwks()->clear_cache_duration(); - cache_ = JwksCache::create(config_, time_system_, *api_, tls_); + cache_ = JwksCache::create(config_, context_, mock_fetcher_.AsStdFunction(), stats_); auto jwks = cache_->findByIssuer("https://example.com"); EXPECT_TRUE(jwks->getJwksObj() == nullptr); @@ -120,7 +127,7 @@ TEST_F(JwksCacheTest, TestGoodInlineJwks) { auto local_jwks = provider0.mutable_local_jwks(); local_jwks->set_inline_string(PublicKey); - cache_ = JwksCache::create(config_, time_system_, *api_, tls_); + cache_ = JwksCache::create(config_, context_, mock_fetcher_.AsStdFunction(), stats_); auto jwks = cache_->findByIssuer("https://example.com"); EXPECT_FALSE(jwks->getJwksObj() == nullptr); @@ -134,7 +141,7 @@ TEST_F(JwksCacheTest, TestBadInlineJwks) { auto local_jwks = provider0.mutable_local_jwks(); local_jwks->set_inline_string("BAD-JWKS"); - cache_ = JwksCache::create(config_, time_system_, *api_, tls_); + cache_ = JwksCache::create(config_, context_, mock_fetcher_.AsStdFunction(), stats_); auto jwks = cache_->findByIssuer("https://example.com"); EXPECT_TRUE(jwks->getJwksObj() == nullptr); diff --git a/test/extensions/filters/http/jwt_authn/test_common.h b/test/extensions/filters/http/jwt_authn/test_common.h index b8e070c426854..0f2478ee2a73b 100644 --- a/test/extensions/filters/http/jwt_authn/test_common.h +++ b/test/extensions/filters/http/jwt_authn/test_common.h @@ -176,7 +176,7 @@ const char OtherGoodToken[] = // Expected base64 payload value. const char ExpectedPayloadValue[] = "eyJpc3MiOiJodHRwczovL2V4YW1wbGUuY29tIiwic3ViIjoidGVzdEBleGFtcG" "xlLmNvbSIsImV4cCI6MjAwMTAwMTAwMSwiYXVkIjoiZXhhbXBsZV9zZXJ2" - "aWNlIn0="; + "aWNlIn0"; // Base64 decoded Payload JSON const char ExpectedPayloadJSON[] = R"( diff --git a/test/extensions/filters/http/kill_request/crash_integration_test.cc b/test/extensions/filters/http/kill_request/crash_integration_test.cc index 63da020128495..ec68ab95e298e 100644 --- a/test/extensions/filters/http/kill_request/crash_integration_test.cc +++ b/test/extensions/filters/http/kill_request/crash_integration_test.cc @@ -65,7 +65,7 @@ TEST_P(CrashIntegrationTestAllProtocols, UnwindsTrackedObjectStack) { // - ActiveStream // - Http(1|2)::ConnectionImpl // - Network::ConnectionImpl - const std::string death_string = GetParam().downstream_protocol == Http::CodecClient::Type::HTTP2 + const std::string death_string = GetParam().downstream_protocol == Http::CodecType::HTTP2 ? "ActiveStream.*Http2::ConnectionImpl.*ConnectionImpl" : "ActiveStream.*Http1::ConnectionImpl.*ConnectionImpl"; EXPECT_DEATH(sendRequestAndWaitForResponse(request_headers, 0, default_response_headers_, 1024), diff --git a/test/extensions/filters/http/kill_request/kill_request_filter_test.cc b/test/extensions/filters/http/kill_request/kill_request_filter_test.cc index 581bfa0d34864..2b9483bdc0256 100644 --- a/test/extensions/filters/http/kill_request/kill_request_filter_test.cc +++ b/test/extensions/filters/http/kill_request/kill_request_filter_test.cc @@ -209,6 +209,24 @@ TEST_F(KillRequestFilterTest, KillsBasedOnDirection) { "KillRequestFilter is crashing Envoy!!!"); } +TEST_F(KillRequestFilterTest, PerRouteKillSettingFound) { + envoy::extensions::filters::http::kill_request::v3::KillRequest kill_request; + setUpTest(kill_request); + request_headers_.addCopy("x-envoy-kill-request", "true"); + + envoy::extensions::filters::http::kill_request::v3::KillRequest route_level_kill_request; + // Set probability to zero to make isKillRequestEnabled return false + route_level_kill_request.mutable_probability()->set_numerator(0); + route_level_kill_request.set_kill_request_header("x-custom-kill-request"); + + // Return valid kill setting on the REQUEST direction + const KillSettings kill_settings(route_level_kill_request); + ON_CALL(decoder_filter_callbacks_.route_->route_entry_, + perFilterConfig(Extensions::HttpFilters::HttpFilterNames::get().KillRequest)) + .WillByDefault(Return(&kill_settings)); + ASSERT_EQ(filter_->decodeHeaders(request_headers_, false), Http::FilterHeadersStatus::Continue); +} + } // namespace } // namespace KillRequest } // namespace HttpFilters diff --git a/test/extensions/filters/http/local_ratelimit/BUILD b/test/extensions/filters/http/local_ratelimit/BUILD index 0aeca92a857a8..a945749878304 100644 --- a/test/extensions/filters/http/local_ratelimit/BUILD +++ b/test/extensions/filters/http/local_ratelimit/BUILD @@ -33,3 +33,13 @@ envoy_extension_cc_test( "//test/mocks/server:server_mocks", ], ) + +envoy_extension_cc_test( + name = "local_ratelimit_integration_test", + srcs = ["local_ratelimit_integration_test.cc"], + extension_name = "envoy.filters.http.local_ratelimit", + deps = [ + "//source/extensions/filters/http/local_ratelimit:config", + "//test/integration:http_protocol_integration_lib", + ], +) diff --git a/test/extensions/filters/http/local_ratelimit/filter_test.cc b/test/extensions/filters/http/local_ratelimit/filter_test.cc index cd9b6ec84f7b0..1614b0520aaa1 100644 --- a/test/extensions/filters/http/local_ratelimit/filter_test.cc +++ b/test/extensions/filters/http/local_ratelimit/filter_test.cc @@ -39,7 +39,12 @@ stat_prefix: test header: key: x-local-ratelimited value: 'true' +local_rate_limit_per_downstream_connection: {} )"; +// '{}' used in the yaml config above are position dependent placeholders used for substitutions. +// Different test cases toggle functionality based on these positional placeholder variables +// For instance, fmt::format(config_yaml, "1", "false") substitutes '1' and 'false' for 'max_tokens' +// and 'local_rate_limit_per_downstream_connection' configurations, respectively. class FilterTest : public testing::Test { public: @@ -58,12 +63,18 @@ class FilterTest : public testing::Test { testing::Matcher(Percent(100)))) .WillRepeatedly(testing::Return(enforced)); + ON_CALL(decoder_callbacks_, dispatcher()).WillByDefault(ReturnRef(dispatcher_)); + ON_CALL(decoder_callbacks_2_, dispatcher()).WillByDefault(ReturnRef(dispatcher_)); + envoy::extensions::filters::http::local_ratelimit::v3::LocalRateLimit config; TestUtility::loadFromYaml(yaml, config); config_ = std::make_shared(config, local_info_, dispatcher_, stats_, runtime_, per_route); filter_ = std::make_shared(config_); filter_->setDecoderFilterCallbacks(decoder_callbacks_); + + filter_2_ = std::make_shared(config_); + filter_2_->setDecoderFilterCallbacks(decoder_callbacks_2_); } void setup(const std::string& yaml, const bool enabled = true, const bool enforced = true) { setupPerRoute(yaml, enabled, enforced); @@ -78,25 +89,27 @@ class FilterTest : public testing::Test { Stats::IsolatedStoreImpl stats_; testing::NiceMock decoder_callbacks_; + testing::NiceMock decoder_callbacks_2_; NiceMock dispatcher_; NiceMock runtime_; NiceMock local_info_; std::shared_ptr config_; std::shared_ptr filter_; + std::shared_ptr filter_2_; }; TEST_F(FilterTest, Runtime) { - setup(fmt::format(config_yaml, "1"), false, false); + setup(fmt::format(config_yaml, "1", "false"), false, false); EXPECT_EQ(&runtime_, &(config_->runtime())); } TEST_F(FilterTest, ToErrorCode) { - setup(fmt::format(config_yaml, "1"), false, false); + setup(fmt::format(config_yaml, "1", "false"), false, false); EXPECT_EQ(Http::Code::BadRequest, toErrorCode(400)); } TEST_F(FilterTest, Disabled) { - setup(fmt::format(config_yaml, "1"), false, false); + setup(fmt::format(config_yaml, "1", "false"), false, false); auto headers = Http::TestRequestHeaderMapImpl(); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, false)); EXPECT_EQ(0U, findCounter("test.http_local_rate_limit.enabled")); @@ -104,18 +117,31 @@ TEST_F(FilterTest, Disabled) { } TEST_F(FilterTest, RequestOk) { - setup(fmt::format(config_yaml, "1")); + setup(fmt::format(config_yaml, "1", "false")); auto headers = Http::TestRequestHeaderMapImpl(); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, false)); - EXPECT_EQ(1U, findCounter("test.http_local_rate_limit.enabled")); - EXPECT_EQ(0U, findCounter("test.http_local_rate_limit.enforced")); + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_2_->decodeHeaders(headers, false)); + EXPECT_EQ(2U, findCounter("test.http_local_rate_limit.enabled")); + EXPECT_EQ(1U, findCounter("test.http_local_rate_limit.enforced")); EXPECT_EQ(1U, findCounter("test.http_local_rate_limit.ok")); + EXPECT_EQ(1U, findCounter("test.http_local_rate_limit.rate_limited")); +} + +TEST_F(FilterTest, RequestOkPerConnection) { + setup(fmt::format(config_yaml, "1", "true")); + auto headers = Http::TestRequestHeaderMapImpl(); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, false)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_2_->decodeHeaders(headers, false)); + EXPECT_EQ(2U, findCounter("test.http_local_rate_limit.enabled")); + EXPECT_EQ(0U, findCounter("test.http_local_rate_limit.enforced")); + EXPECT_EQ(2U, findCounter("test.http_local_rate_limit.ok")); + EXPECT_EQ(0U, findCounter("test.http_local_rate_limit.rate_limited")); } TEST_F(FilterTest, RequestRateLimited) { - setup(fmt::format(config_yaml, "0")); + setup(fmt::format(config_yaml, "1", "false")); - EXPECT_CALL(decoder_callbacks_, sendLocalReply(Http::Code::TooManyRequests, _, _, _, _)) + EXPECT_CALL(decoder_callbacks_2_, sendLocalReply(Http::Code::TooManyRequests, _, _, _, _)) .WillOnce(Invoke([](Http::Code code, absl::string_view body, std::function modify_headers, const absl::optional grpc_status, @@ -136,16 +162,61 @@ TEST_F(FilterTest, RequestRateLimited) { auto request_headers = Http::TestRequestHeaderMapImpl(); auto expected_headers = Http::TestRequestHeaderMapImpl(); - EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, - filter_->decodeHeaders(request_headers, false)); EXPECT_EQ(request_headers, expected_headers); - EXPECT_EQ(1U, findCounter("test.http_local_rate_limit.enabled")); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_2_->decodeHeaders(request_headers, false)); + EXPECT_EQ(2U, findCounter("test.http_local_rate_limit.enabled")); EXPECT_EQ(1U, findCounter("test.http_local_rate_limit.enforced")); + EXPECT_EQ(1U, findCounter("test.http_local_rate_limit.ok")); EXPECT_EQ(1U, findCounter("test.http_local_rate_limit.rate_limited")); } +/* +This test sets 'local_rate_limit_per_downstream_connection' to true. Doing this enables per +connection rate limiting and even though 'max_token' is set to 1, it allows 2 requests to go through +- one on each connection. This is in contrast to the 'RequestOk' test above where only 1 request is +allowed (across the process) for the same configuration. +*/ +TEST_F(FilterTest, RequestRateLimitedPerConnection) { + setup(fmt::format(config_yaml, "1", "true")); + + EXPECT_CALL(decoder_callbacks_, sendLocalReply(Http::Code::TooManyRequests, _, _, _, _)) + .WillOnce(Invoke([](Http::Code code, absl::string_view body, + std::function modify_headers, + const absl::optional grpc_status, + absl::string_view details) { + EXPECT_EQ(Http::Code::TooManyRequests, code); + EXPECT_EQ("local_rate_limited", body); + + Http::TestResponseHeaderMapImpl response_headers{{":status", "200"}}; + modify_headers(response_headers); + EXPECT_EQ("true", response_headers.get(Http::LowerCaseString("x-test-rate-limit"))[0] + ->value() + .getStringView()); + + EXPECT_EQ(grpc_status, absl::nullopt); + EXPECT_EQ(details, "local_rate_limited"); + })); + + auto request_headers = Http::TestRequestHeaderMapImpl(); + auto expected_headers = Http::TestRequestHeaderMapImpl(); + + EXPECT_EQ(request_headers, expected_headers); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_2_->decodeHeaders(request_headers, false)); + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_2_->decodeHeaders(request_headers, false)); + EXPECT_EQ(4U, findCounter("test.http_local_rate_limit.enabled")); + EXPECT_EQ(2U, findCounter("test.http_local_rate_limit.enforced")); + EXPECT_EQ(2U, findCounter("test.http_local_rate_limit.ok")); + EXPECT_EQ(2U, findCounter("test.http_local_rate_limit.rate_limited")); +} + TEST_F(FilterTest, RequestRateLimitedButNotEnforced) { - setup(fmt::format(config_yaml, "0"), true, false); + setup(fmt::format(config_yaml, "0", "false"), true, false); EXPECT_CALL(decoder_callbacks_, sendLocalReply(Http::Code::TooManyRequests, _, _, _, _)).Times(0); @@ -181,6 +252,7 @@ stat_prefix: test header: key: x-test-rate-limit value: 'true' +local_rate_limit_per_downstream_connection: true descriptors: - entries: - key: hello diff --git a/test/extensions/filters/http/local_ratelimit/local_ratelimit_integration_test.cc b/test/extensions/filters/http/local_ratelimit/local_ratelimit_integration_test.cc new file mode 100644 index 0000000000000..06a8b0ca1b3f3 --- /dev/null +++ b/test/extensions/filters/http/local_ratelimit/local_ratelimit_integration_test.cc @@ -0,0 +1,136 @@ +#include "test/integration/http_protocol_integration.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace { + +class LocalRateLimitFilterIntegrationTest : public Event::TestUsingSimulatedTime, + public HttpProtocolIntegrationTest { +protected: + void initializeFilter(const std::string& filter_config) { + config_helper_.addFilter(filter_config); + initialize(); + } + + const std::string filter_config_ = + R"EOF( +name: envoy.filters.http.local_ratelimit +typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.local_ratelimit.v3.LocalRateLimit + stat_prefix: http_local_rate_limiter + token_bucket: + max_tokens: 1 + tokens_per_fill: 1 + fill_interval: 1000s + filter_enabled: + runtime_key: local_rate_limit_enabled + default_value: + numerator: 100 + denominator: HUNDRED + filter_enforced: + runtime_key: local_rate_limit_enforced + default_value: + numerator: 100 + denominator: HUNDRED + response_headers_to_add: + - append: false + header: + key: x-local-rate-limit + value: 'true' + local_rate_limit_per_downstream_connection: {} +)EOF"; +}; + +INSTANTIATE_TEST_SUITE_P(Protocols, LocalRateLimitFilterIntegrationTest, + testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams()), + HttpProtocolIntegrationTest::protocolTestParamsToString); + +TEST_P(LocalRateLimitFilterIntegrationTest, DenyRequestPerProcess) { + initializeFilter(fmt::format(filter_config_, "false")); + + codec_client_ = makeHttpConnection(lookupPort("http")); + auto response = codec_client_->makeRequestWithBody(default_request_headers_, 0); + + waitForNextUpstreamRequest(); + upstream_request_->encodeHeaders(default_response_headers_, 1); + + ASSERT_TRUE(response->waitForEndStream()); + + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_EQ(0U, upstream_request_->bodyLength()); + EXPECT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); + EXPECT_EQ(0, response->body().size()); + + cleanupUpstreamAndDownstream(); + + codec_client_ = makeHttpConnection(lookupPort("http")); + response = codec_client_->makeRequestWithBody(default_request_headers_, 0); + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_TRUE(response->complete()); + EXPECT_EQ("429", response->headers().getStatusValue()); + EXPECT_EQ(18, response->body().size()); +} + +TEST_P(LocalRateLimitFilterIntegrationTest, DenyRequestWithinSameConnection) { + initializeFilter(fmt::format(filter_config_, "true")); + + codec_client_ = makeHttpConnection(lookupPort("http")); + auto response = codec_client_->makeRequestWithBody(default_request_headers_, 0); + + waitForNextUpstreamRequest(); + upstream_request_->encodeHeaders(default_response_headers_, 1); + + ASSERT_TRUE(response->waitForEndStream()); + + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_EQ(0U, upstream_request_->bodyLength()); + EXPECT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); + EXPECT_EQ(0, response->body().size()); + + response = codec_client_->makeRequestWithBody(default_request_headers_, 0); + + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_TRUE(response->complete()); + EXPECT_EQ("429", response->headers().getStatusValue()); + EXPECT_EQ(18, response->body().size()); +} + +TEST_P(LocalRateLimitFilterIntegrationTest, PermitRequestAcrossDifferentConnections) { + initializeFilter(fmt::format(filter_config_, "true")); + + codec_client_ = makeHttpConnection(lookupPort("http")); + auto response = codec_client_->makeRequestWithBody(default_request_headers_, 0); + + waitForNextUpstreamRequest(); + upstream_request_->encodeHeaders(default_response_headers_, 1); + + ASSERT_TRUE(response->waitForEndStream()); + + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_EQ(0U, upstream_request_->bodyLength()); + EXPECT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); + EXPECT_EQ(0, response->body().size()); + + cleanupUpstreamAndDownstream(); + + codec_client_ = makeHttpConnection(lookupPort("http")); + response = codec_client_->makeRequestWithBody(default_request_headers_, 0); + + waitForNextUpstreamRequest(); + upstream_request_->encodeHeaders(default_response_headers_, 1); + + ASSERT_TRUE(response->waitForEndStream()); + + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_EQ(0U, upstream_request_->bodyLength()); + EXPECT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); + EXPECT_EQ(0, response->body().size()); +} + +} // namespace +} // namespace Envoy diff --git a/test/extensions/filters/http/lua/lua_integration_test.cc b/test/extensions/filters/http/lua/lua_integration_test.cc index f02f903307c50..d7197868d842e 100644 --- a/test/extensions/filters/http/lua/lua_integration_test.cc +++ b/test/extensions/filters/http/lua/lua_integration_test.cc @@ -16,14 +16,14 @@ namespace { class LuaIntegrationTest : public testing::TestWithParam, public HttpIntegrationTest { public: - LuaIntegrationTest() : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, GetParam()) {} + LuaIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam()) {} void createUpstreams() override { HttpIntegrationTest::createUpstreams(); - addFakeUpstream(FakeHttpConnection::Type::HTTP1); - addFakeUpstream(FakeHttpConnection::Type::HTTP1); + addFakeUpstream(Http::CodecType::HTTP1); + addFakeUpstream(Http::CodecType::HTTP1); // Create the xDS upstream. - addFakeUpstream(FakeHttpConnection::Type::HTTP2); + addFakeUpstream(Http::CodecType::HTTP2); } void initializeFilter(const std::string& filter_config, const std::string& domain = "*") { diff --git a/test/extensions/filters/http/oauth2/oauth_integration_test.cc b/test/extensions/filters/http/oauth2/oauth_integration_test.cc index e12091ef503ef..fe2cc0649334a 100644 --- a/test/extensions/filters/http/oauth2/oauth_integration_test.cc +++ b/test/extensions/filters/http/oauth2/oauth_integration_test.cc @@ -20,7 +20,7 @@ namespace { class OauthIntegrationTest : public testing::Test, public HttpIntegrationTest { public: OauthIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, Network::Address::IpVersion::v4) { + : HttpIntegrationTest(Http::CodecType::HTTP2, Network::Address::IpVersion::v4) { enableHalfClose(true); } @@ -38,7 +38,7 @@ class OauthIntegrationTest : public testing::Test, public HttpIntegrationTest { } void initialize() override { - setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); + setUpstreamProtocol(Http::CodecType::HTTP2); TestEnvironment::writeStringToFileForTest("token_secret.yaml", R"EOF( resources: diff --git a/test/extensions/filters/http/ratelimit/ratelimit_integration_test.cc b/test/extensions/filters/http/ratelimit/ratelimit_integration_test.cc index fbcaf61a12222..aab0ff2a93e32 100644 --- a/test/extensions/filters/http/ratelimit/ratelimit_integration_test.cc +++ b/test/extensions/filters/http/ratelimit/ratelimit_integration_test.cc @@ -25,7 +25,7 @@ namespace { class RatelimitIntegrationTest : public Grpc::VersionedGrpcClientIntegrationParamTest, public HttpIntegrationTest { public: - RatelimitIntegrationTest() : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, ipVersion()) {} + RatelimitIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, ipVersion()) {} void SetUp() override { XDS_DEPRECATED_FEATURE_TEST_SKIP; @@ -34,7 +34,7 @@ class RatelimitIntegrationTest : public Grpc::VersionedGrpcClientIntegrationPara void createUpstreams() override { HttpIntegrationTest::createUpstreams(); - addFakeUpstream(FakeHttpConnection::Type::HTTP2); + addFakeUpstream(Http::CodecType::HTTP2); } void initialize() override { diff --git a/test/extensions/filters/http/router/auto_sni_integration_test.cc b/test/extensions/filters/http/router/auto_sni_integration_test.cc index 4c16326c2e518..6515e1aff0007 100644 --- a/test/extensions/filters/http/router/auto_sni_integration_test.cc +++ b/test/extensions/filters/http/router/auto_sni_integration_test.cc @@ -16,10 +16,10 @@ class AutoSniIntegrationTest : public testing::TestWithParammutable_clusters()->at(0); @@ -41,7 +41,7 @@ class AutoSniIntegrationTest : public testing::TestWithParam + +#include "envoy/extensions/filters/http/set_metadata/v3/set_metadata.pb.h" +#include "envoy/extensions/filters/http/set_metadata/v3/set_metadata.pb.validate.h" + +#include "extensions/filters/http/set_metadata/config.h" +#include "extensions/filters/http/set_metadata/set_metadata_filter.h" + +#include "test/mocks/server/factory_context.h" +#include "test/mocks/server/instance.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace SetMetadataFilter { + +using SetMetadataProtoConfig = envoy::extensions::filters::http::set_metadata::v3::Config; + +TEST(SetMetadataFilterConfigTest, SimpleConfig) { + const std::string yaml = R"EOF( +metadata_namespace: thenamespace +value: + mynumber: 20 + mylist: ["b"] + tags: + mytag1: 1 + )EOF"; + + SetMetadataProtoConfig proto_config; + TestUtility::loadFromYamlAndValidate(yaml, proto_config); + + testing::NiceMock context; + SetMetadataConfig factory; + + Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context); + Http::MockFilterChainFactoryCallbacks filter_callbacks; + EXPECT_CALL(filter_callbacks, addStreamDecoderFilter(_)); + cb(filter_callbacks); +} + +} // namespace SetMetadataFilter +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/http/set_metadata/set_metadata_filter_test.cc b/test/extensions/filters/http/set_metadata/set_metadata_filter_test.cc new file mode 100644 index 0000000000000..ad644a71e419b --- /dev/null +++ b/test/extensions/filters/http/set_metadata/set_metadata_filter_test.cc @@ -0,0 +1,143 @@ +#include + +#include "common/protobuf/protobuf.h" + +#include "extensions/filters/http/set_metadata/set_metadata_filter.h" +#include "extensions/filters/http/well_known_names.h" + +#include "test/mocks/http/mocks.h" +#include "test/mocks/stream_info/mocks.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::NiceMock; +using testing::ReturnRef; + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace SetMetadataFilter { + +class SetMetadataIntegrationTest : public testing::Test { + +public: + SetMetadataIntegrationTest() = default; + + void runFilter(envoy::config::core::v3::Metadata& metadata, const std::string& yaml_config) { + envoy::extensions::filters::http::set_metadata::v3::Config ext_config; + TestUtility::loadFromYaml(yaml_config, ext_config); + auto config = std::make_shared(ext_config); + auto filter = std::make_shared(config); + + Http::TestRequestHeaderMapImpl headers; + + NiceMock decoder_callbacks; + NiceMock req_info; + filter->setDecoderFilterCallbacks(decoder_callbacks); + + EXPECT_CALL(decoder_callbacks, streamInfo()).WillRepeatedly(ReturnRef(req_info)); + EXPECT_CALL(req_info, dynamicMetadata()).WillRepeatedly(ReturnRef(metadata)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter->decodeHeaders(headers, true)); + Buffer::OwnedImpl buffer; + EXPECT_EQ(Http::FilterDataStatus::Continue, filter->decodeData(buffer, true)); + filter->onDestroy(); + } + + void checkKeyInt(const ProtobufWkt::Struct& s, const char* key, int val) { + const auto& fields = s.fields(); + const auto it = fields.find(key); + ASSERT_TRUE(it != fields.end()); + const auto& pbval = it->second; + ASSERT_EQ(pbval.kind_case(), ProtobufWkt::Value::kNumberValue); + EXPECT_EQ(pbval.number_value(), val); + } +}; + +TEST_F(SetMetadataIntegrationTest, TestTagsHeaders) { + const std::string yaml_config = R"EOF( + metadata_namespace: thenamespace + value: + tags: + mytag0: 1 + )EOF"; + + envoy::config::core::v3::Metadata metadata; + runFilter(metadata, yaml_config); + + // Verify that `metadata` contains `{"thenamespace": {"tags": {"mytag0": 1}}}` + const auto& filter_metadata = metadata.filter_metadata(); + const auto it_namespace = filter_metadata.find("thenamespace"); + ASSERT_TRUE(it_namespace != filter_metadata.end()); + const auto& fields = it_namespace->second.fields(); + const auto it_tags = fields.find("tags"); + ASSERT_TRUE(it_tags != fields.end()); + const auto& tags = it_tags->second; + ASSERT_EQ(tags.kind_case(), ProtobufWkt::Value::kStructValue); + checkKeyInt(tags.struct_value(), "mytag0", 1); +} + +TEST_F(SetMetadataIntegrationTest, TestTagsHeadersUpdate) { + envoy::config::core::v3::Metadata metadata; + + { + const std::string yaml_config = R"EOF( + metadata_namespace: thenamespace + value: + mynumber: 10 + mylist: ["a"] + tags: + mytag0: 1 + )EOF"; + + runFilter(metadata, yaml_config); + } + { + const std::string yaml_config = R"EOF( + metadata_namespace: thenamespace + value: + mynumber: 20 + mylist: ["b"] + tags: + mytag1: 1 + )EOF"; + + runFilter(metadata, yaml_config); + } + + // Verify that `metadata` contains: + // ``{"thenamespace": {number: 20, mylist: ["a","b"], "tags": {"mytag0": 1, "mytag1": 1}}}`` + const auto& filter_metadata = metadata.filter_metadata(); + const auto it_namespace = filter_metadata.find("thenamespace"); + ASSERT_TRUE(it_namespace != filter_metadata.end()); + const auto& namespace_ = it_namespace->second; + + checkKeyInt(namespace_, "mynumber", 20); + + const auto& fields = namespace_.fields(); + const auto it_mylist = fields.find("mylist"); + ASSERT_TRUE(it_mylist != fields.end()); + const auto& mylist = it_mylist->second; + ASSERT_EQ(mylist.kind_case(), ProtobufWkt::Value::kListValue); + const auto& vals = mylist.list_value().values(); + ASSERT_EQ(vals.size(), 2); + ASSERT_EQ(vals[0].kind_case(), ProtobufWkt::Value::kStringValue); + EXPECT_EQ(vals[0].string_value(), "a"); + ASSERT_EQ(vals[1].kind_case(), ProtobufWkt::Value::kStringValue); + EXPECT_EQ(vals[1].string_value(), "b"); + + const auto it_tags = fields.find("tags"); + ASSERT_TRUE(it_tags != fields.end()); + const auto& tags = it_tags->second; + ASSERT_EQ(tags.kind_case(), ProtobufWkt::Value::kStructValue); + const auto& tags_struct = tags.struct_value(); + + checkKeyInt(tags_struct, "mytag0", 1); + checkKeyInt(tags_struct, "mytag1", 1); +} + +} // namespace SetMetadataFilter +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/http/squash/squash_filter_integration_test.cc b/test/extensions/filters/http/squash/squash_filter_integration_test.cc index 280aa5d63d519..06c87e4e5e314 100644 --- a/test/extensions/filters/http/squash/squash_filter_integration_test.cc +++ b/test/extensions/filters/http/squash/squash_filter_integration_test.cc @@ -19,7 +19,7 @@ namespace Envoy { class SquashFilterIntegrationTest : public testing::TestWithParam, public HttpIntegrationTest { public: - SquashFilterIntegrationTest() : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, GetParam()) {} + SquashFilterIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam()) {} ~SquashFilterIntegrationTest() override { if (fake_squash_connection_) { @@ -71,7 +71,7 @@ class SquashFilterIntegrationTest : public testing::TestWithParam("source", "grpc_call_proto")); root()->handler_ = new MyGrpcCallHandler(); if (root()->grpcCallHandler( "bogus grpc_service", "service", "method", initial_metadata, value, 1000, @@ -97,6 +98,7 @@ FilterHeadersStatus GrpcCallContext::onRequestHeaders(uint32_t, bool end_of_stre google::protobuf::Value value; value.set_string_value("request"); HeaderStringPairs initial_metadata; + initial_metadata.push_back(std::make_pair("source", "grpc_call")); root()->handler_ = new MyGrpcCallHandler(); if (root()->grpcCallHandler( "bogus grpc_service", "service", "method", initial_metadata, value, 1000, diff --git a/test/extensions/filters/http/wasm/test_data/test_grpc_stream_cpp.cc b/test/extensions/filters/http/wasm/test_data/test_grpc_stream_cpp.cc index d4c375cb52572..30d85abc4a2dc 100644 --- a/test/extensions/filters/http/wasm/test_data/test_grpc_stream_cpp.cc +++ b/test/extensions/filters/http/wasm/test_data/test_grpc_stream_cpp.cc @@ -77,6 +77,8 @@ FilterHeadersStatus GrpcStreamContextProto::onRequestHeaders(uint32_t, bool) { std::string grpc_service_string; grpc_service.SerializeToString(&grpc_service_string); HeaderStringPairs initial_metadata; + initial_metadata.push_back( + std::make_pair("source", "grpc_stream_proto")); if (root()->grpcStreamHandler("bogus service string", "service", "method", initial_metadata, std::unique_ptr( new MyGrpcStreamHandler())) != WasmResult::ParseFailure) { @@ -105,6 +107,7 @@ static RegisterContextFactory register_GrpcStreamContext(CONTEXT_FACTORY(GrpcStr FilterHeadersStatus GrpcStreamContext::onRequestHeaders(uint32_t, bool) { HeaderStringPairs initial_metadata; + initial_metadata.push_back(std::make_pair("source", "grpc_stream")); if (root()->grpcStreamHandler("bogus service string", "service", "method", initial_metadata, std::unique_ptr( new MyGrpcStreamHandler())) == WasmResult::ParseFailure) { diff --git a/test/extensions/filters/http/wasm/wasm_filter_test.cc b/test/extensions/filters/http/wasm/wasm_filter_test.cc index 74f6c0ce9dbe1..57c62919f32b9 100644 --- a/test/extensions/filters/http/wasm/wasm_filter_test.cc +++ b/test/extensions/filters/http/wasm/wasm_filter_test.cc @@ -947,6 +947,9 @@ TEST_P(WasmHttpFilterTest, GrpcCall) { NiceMock span; if (callbacks) { callbacks->onCreateInitialMetadata(request_headers); + const auto source = request_headers.get(Http::LowerCaseString{"source"}); + EXPECT_EQ(source.size(), 1); + EXPECT_EQ(source[0]->value().getStringView(), id); callbacks->onSuccessRaw(std::move(response), span); } } @@ -1369,8 +1372,10 @@ TEST_P(WasmHttpFilterTest, GrpcStream) { auto response = std::make_unique(response_string); EXPECT_NE(callbacks, nullptr); if (callbacks) { - Http::TestRequestHeaderMapImpl create_initial_metadata{{"test", "create_initial_metadata"}}; - callbacks->onCreateInitialMetadata(create_initial_metadata); + callbacks->onCreateInitialMetadata(request_headers); + const auto source = request_headers.get(Http::LowerCaseString{"source"}); + EXPECT_EQ(source.size(), 1); + EXPECT_EQ(source[0]->value().getStringView(), id); callbacks->onReceiveInitialMetadata(std::make_unique()); callbacks->onReceiveMessageRaw(std::move(response)); callbacks->onReceiveTrailingMetadata(std::make_unique()); @@ -1417,8 +1422,10 @@ TEST_P(WasmHttpFilterTest, GrpcStreamCloseLocal) { auto response = std::make_unique(response_string); EXPECT_NE(callbacks, nullptr); if (callbacks) { - Http::TestRequestHeaderMapImpl create_initial_metadata{{"test", "create_initial_metadata"}}; - callbacks->onCreateInitialMetadata(create_initial_metadata); + callbacks->onCreateInitialMetadata(request_headers); + const auto source = request_headers.get(Http::LowerCaseString{"source"}); + EXPECT_EQ(source.size(), 1); + EXPECT_EQ(source[0]->value().getStringView(), id); callbacks->onReceiveInitialMetadata(std::make_unique()); callbacks->onReceiveMessageRaw(std::move(response)); callbacks->onRemoteClose(Grpc::Status::WellKnownGrpcStatus::Ok, "ok"); @@ -1465,8 +1472,10 @@ TEST_P(WasmHttpFilterTest, GrpcStreamCloseRemote) { auto response = std::make_unique(response_string); EXPECT_NE(callbacks, nullptr); if (callbacks) { - Http::TestRequestHeaderMapImpl create_initial_metadata{{"test", "create_initial_metadata"}}; - callbacks->onCreateInitialMetadata(create_initial_metadata); + callbacks->onCreateInitialMetadata(request_headers); + const auto source = request_headers.get(Http::LowerCaseString{"source"}); + EXPECT_EQ(source.size(), 1); + EXPECT_EQ(source[0]->value().getStringView(), id); callbacks->onReceiveInitialMetadata(std::make_unique()); callbacks->onReceiveMessageRaw(std::move(response)); callbacks->onRemoteClose(Grpc::Status::WellKnownGrpcStatus::Ok, "close"); @@ -1511,8 +1520,10 @@ TEST_P(WasmHttpFilterTest, GrpcStreamCancel) { EXPECT_NE(callbacks, nullptr); NiceMock span; if (callbacks) { - Http::TestRequestHeaderMapImpl create_initial_metadata{{"test", "create_initial_metadata"}}; - callbacks->onCreateInitialMetadata(create_initial_metadata); + callbacks->onCreateInitialMetadata(request_headers); + const auto source = request_headers.get(Http::LowerCaseString{"source"}); + EXPECT_EQ(source.size(), 1); + EXPECT_EQ(source[0]->value().getStringView(), id); callbacks->onReceiveInitialMetadata(std::make_unique( Http::TestResponseHeaderMapImpl{{"test", "reset"}})); } @@ -1557,8 +1568,10 @@ TEST_P(WasmHttpFilterTest, GrpcStreamOpenAtShutdown) { EXPECT_NE(callbacks, nullptr); NiceMock span; if (callbacks) { - Http::TestRequestHeaderMapImpl create_initial_metadata{{"test", "create_initial_metadata"}}; - callbacks->onCreateInitialMetadata(create_initial_metadata); + callbacks->onCreateInitialMetadata(request_headers); + const auto source = request_headers.get(Http::LowerCaseString{"source"}); + EXPECT_EQ(source.size(), 1); + EXPECT_EQ(source[0]->value().getStringView(), id); callbacks->onReceiveInitialMetadata(std::make_unique()); callbacks->onReceiveMessageRaw(std::move(response)); callbacks->onReceiveTrailingMetadata(std::make_unique()); diff --git a/test/extensions/filters/network/common/fuzz/BUILD b/test/extensions/filters/network/common/fuzz/BUILD index 41b2869c3d84e..c27cea334c5a2 100644 --- a/test/extensions/filters/network/common/fuzz/BUILD +++ b/test/extensions/filters/network/common/fuzz/BUILD @@ -12,6 +12,10 @@ load( licenses(["notice"]) # Apache 2 +exports_files([ + "uber_per_readfilter.cc", +]) + envoy_package() envoy_proto_library( diff --git a/test/extensions/filters/network/http_connection_manager/config_test.cc b/test/extensions/filters/network/http_connection_manager/config_test.cc index 7d2a11ceb7f0e..6137c357640e7 100644 --- a/test/extensions/filters/network/http_connection_manager/config_test.cc +++ b/test/extensions/filters/network/http_connection_manager/config_test.cc @@ -1152,6 +1152,59 @@ TEST_F(HttpConnectionManagerConfigTest, RemoveAnyPortFalse) { EXPECT_EQ(Http::StripPortType::None, config.stripPortType()); } +// Validated that by default we don't remove host's trailing dot. +TEST_F(HttpConnectionManagerConfigTest, RemoveTrailingDotDefault) { + const std::string yaml_string = R"EOF( + stat_prefix: ingress_http + route_config: + name: local_route + http_filters: + - name: envoy.filters.http.router + )EOF"; + + HttpConnectionManagerConfig config(parseHttpConnectionManagerFromYaml(yaml_string), context_, + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_, http_tracer_manager_, + filter_config_provider_manager_); + EXPECT_EQ(false, config.shouldStripTrailingHostDot()); +} + +// Validated that when configured, we remove host's trailing dot. +TEST_F(HttpConnectionManagerConfigTest, RemoveTrailingDotTrue) { + const std::string yaml_string = R"EOF( + stat_prefix: ingress_http + route_config: + name: local_route + strip_trailing_host_dot: true + http_filters: + - name: envoy.filters.http.router + )EOF"; + + HttpConnectionManagerConfig config(parseHttpConnectionManagerFromYaml(yaml_string), context_, + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_, http_tracer_manager_, + filter_config_provider_manager_); + EXPECT_EQ(true, config.shouldStripTrailingHostDot()); +} + +// Validated that when explicitly set false, then we don't remove trailing host dot. +TEST_F(HttpConnectionManagerConfigTest, RemoveTrailingDotFalse) { + const std::string yaml_string = R"EOF( + stat_prefix: ingress_http + route_config: + name: local_route + strip_trailing_host_dot: false + http_filters: + - name: envoy.filters.http.router + )EOF"; + + HttpConnectionManagerConfig config(parseHttpConnectionManagerFromYaml(yaml_string), context_, + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_, http_tracer_manager_, + filter_config_provider_manager_); + EXPECT_EQ(false, config.shouldStripTrailingHostDot()); +} + // Validated that by default we allow requests with header names containing underscores. TEST_F(HttpConnectionManagerConfigTest, HeadersWithUnderscoresAllowedByDefault) { const std::string yaml_string = R"EOF( diff --git a/test/extensions/filters/network/sni_dynamic_forward_proxy/proxy_filter_integration_test.cc b/test/extensions/filters/network/sni_dynamic_forward_proxy/proxy_filter_integration_test.cc index 93a58f03bf5e8..ef4d92e831372 100644 --- a/test/extensions/filters/network/sni_dynamic_forward_proxy/proxy_filter_integration_test.cc +++ b/test/extensions/filters/network/sni_dynamic_forward_proxy/proxy_filter_integration_test.cc @@ -20,11 +20,10 @@ class SniDynamicProxyFilterIntegrationTest // This test is using HTTP integration test to use the utilities to pass SNI from downstream // to upstream. The config being tested is tcp_proxy. SniDynamicProxyFilterIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, GetParam(), - ConfigHelper::tcpProxyConfig()) {} + : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam(), ConfigHelper::tcpProxyConfig()) {} void setup(uint64_t max_hosts = 1024, uint32_t max_pending_requests = 1024) { - setUpstreamProtocol(FakeHttpConnection::Type::HTTP1); + setUpstreamProtocol(Http::CodecType::HTTP1); config_helper_.addListenerFilter(ConfigHelper::tlsInspectorFilter()); @@ -86,7 +85,7 @@ name: envoy.clusters.dynamic_forward_proxy void createUpstreams() override { addFakeUpstream( Ssl::createFakeUpstreamSslContext(upstream_cert_name_, context_manager_, factory_context_), - FakeHttpConnection::Type::HTTP1); + Http::CodecType::HTTP1); } Network::ClientConnectionPtr diff --git a/test/extensions/filters/network/thrift_proxy/router_test.cc b/test/extensions/filters/network/thrift_proxy/router_test.cc index 03448be78b93a..d17758c682bd5 100644 --- a/test/extensions/filters/network/thrift_proxy/router_test.cc +++ b/test/extensions/filters/network/thrift_proxy/router_test.cc @@ -740,6 +740,14 @@ TEST_F(ThriftRouterTest, UnexpectedRouterDestroy) { } TEST_F(ThriftRouterTest, ProtocolUpgrade) { + Stats::MockStore cluster_scope; + ON_CALL(*context_.cluster_manager_.thread_local_cluster_.cluster_.info_, statsScope()) + .WillByDefault(ReturnRef(cluster_scope)); + + EXPECT_CALL(cluster_scope, counter("thrift.upstream_rq_call")); + EXPECT_CALL(cluster_scope, counter("thrift.upstream_resp_reply")); + EXPECT_CALL(cluster_scope, counter("thrift.upstream_resp_success")); + initializeRouter(); startRequest(MessageType::Call); @@ -760,6 +768,21 @@ TEST_F(ThriftRouterTest, ProtocolUpgrade) { EXPECT_CALL(*protocol_, supportsUpgrade()).WillOnce(Return(true)); + EXPECT_CALL(cluster_scope, + histogram("thrift.upstream_rq_time", Stats::Histogram::Unit::Milliseconds)); + EXPECT_CALL(cluster_scope, + deliverHistogramToSinks( + testing::Property(&Stats::Metric::name, "thrift.upstream_rq_time"), _)); + + EXPECT_CALL(cluster_scope, histogram("thrift.upstream_rq_size", Stats::Histogram::Unit::Bytes)); + EXPECT_CALL(cluster_scope, + deliverHistogramToSinks( + testing::Property(&Stats::Metric::name, "thrift.upstream_rq_size"), _)); + EXPECT_CALL(cluster_scope, histogram("thrift.upstream_resp_size", Stats::Histogram::Unit::Bytes)); + EXPECT_CALL(cluster_scope, + deliverHistogramToSinks( + testing::Property(&Stats::Metric::name, "thrift.upstream_resp_size"), _)); + MockThriftObject* upgrade_response = new NiceMock(); EXPECT_CALL(*protocol_, attemptUpgrade(_, _, _)) @@ -796,16 +819,6 @@ TEST_F(ThriftRouterTest, ProtocolUpgrade) { completeRequest(); returnResponse(); destroyRouter(); - - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_rq_call") - .value()); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_resp_reply") - .value()); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_resp_success") - .value()); } // Test the case where an upgrade will occur, but the conn pool @@ -1022,6 +1035,15 @@ TEST_P(ThriftRouterFieldTypeTest, CallWithUpstreamRqTime) { EXPECT_CALL(cluster_scope, counter("thrift.upstream_resp_reply")); EXPECT_CALL(cluster_scope, counter("thrift.upstream_resp_success")); + EXPECT_CALL(cluster_scope, histogram("thrift.upstream_rq_size", Stats::Histogram::Unit::Bytes)); + EXPECT_CALL(cluster_scope, + deliverHistogramToSinks( + testing::Property(&Stats::Metric::name, "thrift.upstream_rq_size"), _)); + EXPECT_CALL(cluster_scope, histogram("thrift.upstream_resp_size", Stats::Histogram::Unit::Bytes)); + EXPECT_CALL(cluster_scope, + deliverHistogramToSinks( + testing::Property(&Stats::Metric::name, "thrift.upstream_resp_size"), _)); + startRequest(MessageType::Call); connectUpstream(); sendTrivialStruct(field_type); @@ -1297,6 +1319,39 @@ TEST_P(ThriftRouterPassthroughTest, PassthroughEnable) { ConnectionPool::PoolFailureReason::RemoteConnectionFailure); } +TEST_F(ThriftRouterTest, RequestResponseSize) { + initializeRouter(); + + Stats::MockStore cluster_scope; + ON_CALL(*context_.cluster_manager_.thread_local_cluster_.cluster_.info_, statsScope()) + .WillByDefault(ReturnRef(cluster_scope)); + + EXPECT_CALL(cluster_scope, counter("thrift.upstream_rq_call")).Times(AtLeast(1)); + EXPECT_CALL(cluster_scope, counter("thrift.upstream_resp_reply")).Times(AtLeast(1)); + EXPECT_CALL(cluster_scope, counter("thrift.upstream_resp_success")).Times(AtLeast(1)); + + EXPECT_CALL(cluster_scope, + histogram("thrift.upstream_rq_time", Stats::Histogram::Unit::Milliseconds)); + EXPECT_CALL(cluster_scope, + deliverHistogramToSinks( + testing::Property(&Stats::Metric::name, "thrift.upstream_rq_time"), _)); + + EXPECT_CALL(cluster_scope, histogram("thrift.upstream_rq_size", Stats::Histogram::Unit::Bytes)); + EXPECT_CALL(cluster_scope, + deliverHistogramToSinks( + testing::Property(&Stats::Metric::name, "thrift.upstream_rq_size"), _)); + EXPECT_CALL(cluster_scope, histogram("thrift.upstream_resp_size", Stats::Histogram::Unit::Bytes)); + EXPECT_CALL(cluster_scope, + deliverHistogramToSinks( + testing::Property(&Stats::Metric::name, "thrift.upstream_resp_size"), _)); + + startRequestWithExistingConnection(MessageType::Call); + sendTrivialStruct(FieldType::I32); + completeRequest(); + returnResponse(); + destroyRouter(); +} + } // namespace Router } // namespace ThriftProxy } // namespace NetworkFilters diff --git a/test/extensions/filters/network/wasm/test_data/BUILD b/test/extensions/filters/network/wasm/test_data/BUILD index 02fbb6028c224..fa78e6e3ce841 100644 --- a/test/extensions/filters/network/wasm/test_data/BUILD +++ b/test/extensions/filters/network/wasm/test_data/BUILD @@ -42,7 +42,6 @@ envoy_cc_library( "//source/common/common:c_smart_ptr_lib", "//source/extensions/common/wasm:wasm_hdr", "//source/extensions/common/wasm:wasm_lib", - "//source/extensions/common/wasm:well_known_names", ], ) diff --git a/test/extensions/filters/udp/udp_proxy/udp_proxy_filter_test.cc b/test/extensions/filters/udp/udp_proxy/udp_proxy_filter_test.cc index ddcb11b891572..c0f78a1b9b744 100644 --- a/test/extensions/filters/udp/udp_proxy/udp_proxy_filter_test.cc +++ b/test/extensions/filters/udp/udp_proxy/udp_proxy_filter_test.cc @@ -105,7 +105,11 @@ class UdpProxyFilterTest : public testing::Test { if (parent_.expect_gro_) { EXPECT_CALL(*socket_->io_handle_, supportsUdpGro()); } - EXPECT_CALL(*socket_->io_handle_, supportsMmsg()); + EXPECT_CALL(*socket_->io_handle_, supportsMmsg()) + .Times(Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.udp_per_event_loop_read_limit") + ? 2u + : 1u); // Return the datagram. EXPECT_CALL(*socket_->io_handle_, recvmsg(_, 1, _, _)) .WillOnce( @@ -136,9 +140,6 @@ class UdpProxyFilterTest : public testing::Test { } })); // Return an EAGAIN result. - if (parent_.expect_gro_) { - EXPECT_CALL(*socket_->io_handle_, supportsUdpGro()); - } EXPECT_CALL(*socket_->io_handle_, supportsMmsg()); EXPECT_CALL(*socket_->io_handle_, recvmsg(_, 1, _, _)) .WillOnce(Return(ByMove(Api::IoCallUint64Result( diff --git a/test/extensions/grpc_credentials/aws_iam/BUILD b/test/extensions/grpc_credentials/aws_iam/BUILD index 4c9b6e7f22bf8..392ffda32ac97 100644 --- a/test/extensions/grpc_credentials/aws_iam/BUILD +++ b/test/extensions/grpc_credentials/aws_iam/BUILD @@ -14,7 +14,6 @@ envoy_cc_test( srcs = envoy_select_google_grpc(["aws_iam_grpc_credentials_test.cc"]), data = ["//test/config/integration/certs"], deps = [ - "//source/extensions/grpc_credentials:well_known_names", "//source/extensions/grpc_credentials/aws_iam:config", "//test/common/grpc:grpc_client_integration_test_harness_lib", "//test/integration:integration_lib", diff --git a/test/extensions/grpc_credentials/aws_iam/aws_iam_grpc_credentials_test.cc b/test/extensions/grpc_credentials/aws_iam/aws_iam_grpc_credentials_test.cc index 6a5852bb9a1a3..09bfe76927176 100644 --- a/test/extensions/grpc_credentials/aws_iam/aws_iam_grpc_credentials_test.cc +++ b/test/extensions/grpc_credentials/aws_iam/aws_iam_grpc_credentials_test.cc @@ -5,8 +5,6 @@ #include "common/common/utility.h" #include "common/grpc/google_async_client_impl.h" -#include "extensions/grpc_credentials/well_known_names.h" - #include "test/common/grpc/grpc_client_integration_test_harness.h" #include "test/integration/fake_upstream.h" #include "test/test_common/environment.h" @@ -122,7 +120,7 @@ TEST_P(GrpcAwsIamClientIntegrationTest, AwsIamGrpcAuth_ConfigRegion) { service_name_ = "test_service"; region_name_ = "test_region_static"; region_location_ = RegionLocation::InConfig; - credentials_factory_name_ = Extensions::GrpcCredentials::GrpcCredentialsNames::get().AwsIam; + credentials_factory_name_ = "envoy.grpc_credentials.aws_iam"; initialize(); auto request = createRequest(empty_metadata_); request->sendReply(); @@ -134,7 +132,7 @@ TEST_P(GrpcAwsIamClientIntegrationTest, AwsIamGrpcAuth_EnvRegion) { service_name_ = "test_service"; region_name_ = "test_region_env"; region_location_ = RegionLocation::InEnvironment; - credentials_factory_name_ = Extensions::GrpcCredentials::GrpcCredentialsNames::get().AwsIam; + credentials_factory_name_ = "envoy.grpc_credentials.aws_iam"; initialize(); auto request = createRequest(empty_metadata_); request->sendReply(); @@ -146,14 +144,14 @@ TEST_P(GrpcAwsIamClientIntegrationTest, AwsIamGrpcAuth_NoRegion) { service_name_ = "test_service"; region_name_ = "test_region_env"; region_location_ = RegionLocation::NotProvided; - credentials_factory_name_ = Extensions::GrpcCredentials::GrpcCredentialsNames::get().AwsIam; + credentials_factory_name_ = "envoy.grpc_credentials.aws_iam"; EXPECT_THROW_WITH_REGEX(initialize();, EnvoyException, "AWS region"); } TEST_P(GrpcAwsIamClientIntegrationTest, AwsIamGrpcAuth_UnexpectedCallCredentials) { SKIP_IF_GRPC_CLIENT(ClientType::EnvoyGrpc); call_credentials_ = CallCredentials::AccessToken; - credentials_factory_name_ = Extensions::GrpcCredentials::GrpcCredentialsNames::get().AwsIam; + credentials_factory_name_ = "envoy.grpc_credentials.aws_iam"; initialize(); auto request = createRequest(empty_metadata_); request->sendReply(); diff --git a/test/extensions/grpc_credentials/file_based_metadata/BUILD b/test/extensions/grpc_credentials/file_based_metadata/BUILD index 8ccb3fe5b848a..2c711ca2dda71 100644 --- a/test/extensions/grpc_credentials/file_based_metadata/BUILD +++ b/test/extensions/grpc_credentials/file_based_metadata/BUILD @@ -14,7 +14,6 @@ envoy_cc_test( srcs = ["integration_test.cc"], data = ["//test/config/integration/certs"], deps = [ - "//source/extensions/grpc_credentials:well_known_names", "//source/extensions/grpc_credentials/file_based_metadata:config", "//test/common/grpc:grpc_client_integration_test_harness_lib", "//test/integration:integration_lib", diff --git a/test/extensions/grpc_credentials/file_based_metadata/integration_test.cc b/test/extensions/grpc_credentials/file_based_metadata/integration_test.cc index 416059bdb7e67..6016c8c6df31f 100644 --- a/test/extensions/grpc_credentials/file_based_metadata/integration_test.cc +++ b/test/extensions/grpc_credentials/file_based_metadata/integration_test.cc @@ -7,7 +7,6 @@ #include "common/grpc/google_async_client_impl.h" #include "extensions/grpc_credentials/file_based_metadata/config.h" -#include "extensions/grpc_credentials/well_known_names.h" #include "test/common/grpc/grpc_client_integration_test_harness.h" #include "test/integration/fake_upstream.h" @@ -90,8 +89,7 @@ TEST_P(GrpcFileBasedMetadataClientIntegrationTest, FileBasedMetadataGrpcAuthRequ header_key_1_ = "header1"; header_prefix_1_ = "prefix1"; header_value_1_ = "secretvalue"; - credentials_factory_name_ = - Extensions::GrpcCredentials::GrpcCredentialsNames::get().FileBasedMetadata; + credentials_factory_name_ = "envoy.grpc_credentials.file_based_metadata"; initialize(); auto request = createRequest(empty_metadata_); request->sendReply(); @@ -105,8 +103,7 @@ TEST_P(GrpcFileBasedMetadataClientIntegrationTest, DoubleFileBasedMetadataGrpcAu header_prefix_1_ = "prefix1"; header_value_1_ = "secretvalue"; header_value_2_ = "secret2"; - credentials_factory_name_ = - Extensions::GrpcCredentials::GrpcCredentialsNames::get().FileBasedMetadata; + credentials_factory_name_ = "envoy.grpc_credentials.file_based_metadata"; initialize(); auto request = createRequest(empty_metadata_); request->sendReply(); @@ -116,8 +113,7 @@ TEST_P(GrpcFileBasedMetadataClientIntegrationTest, DoubleFileBasedMetadataGrpcAu // Validate that FileBasedMetadata auth plugin works without a config loaded TEST_P(GrpcFileBasedMetadataClientIntegrationTest, EmptyFileBasedMetadataGrpcAuthRequest) { SKIP_IF_GRPC_CLIENT(ClientType::EnvoyGrpc); - credentials_factory_name_ = - Extensions::GrpcCredentials::GrpcCredentialsNames::get().FileBasedMetadata; + credentials_factory_name_ = "envoy.grpc_credentials.file_based_metadata"; initialize(); auto request = createRequest(empty_metadata_); request->sendReply(); @@ -131,8 +127,7 @@ TEST_P(GrpcFileBasedMetadataClientIntegrationTest, ExtraConfigFileBasedMetadataG header_key_1_ = "header1"; header_prefix_1_ = "prefix1"; header_value_1_ = "secretvalue"; - credentials_factory_name_ = - Extensions::GrpcCredentials::GrpcCredentialsNames::get().FileBasedMetadata; + credentials_factory_name_ = "envoy.grpc_credentials.file_based_metadata"; initialize(); auto request = createRequest(empty_metadata_); request->sendReply(); diff --git a/test/extensions/http/header_formatters/preserve_case/preserve_case_formatter_integration_test.cc b/test/extensions/http/header_formatters/preserve_case/preserve_case_formatter_integration_test.cc index 8e285da69488f..92c72f43bb660 100644 --- a/test/extensions/http/header_formatters/preserve_case/preserve_case_formatter_integration_test.cc +++ b/test/extensions/http/header_formatters/preserve_case/preserve_case_formatter_integration_test.cc @@ -28,7 +28,7 @@ class PreserveCaseIntegrationTest : public testing::TestWithParam + +#include + +#include "envoy/config/bootstrap/v3/bootstrap.pb.h" +#include "envoy/config/overload/v3/overload.pb.h" +#include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.h" + +#include "test/config/utility.h" +#include "test/integration/http_integration.h" +#include "test/test_common/utility.h" + +namespace Envoy { + +void updateResource(AtomicFileUpdater& updater, double pressure) { + updater.update(absl::StrCat(pressure)); +} + +// A test that sets up its own client connection with customized quic version and connection ID. +class OverloadIntegrationTest : public HttpIntegrationTest { +public: + OverloadIntegrationTest() + : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, GetParam().first), + injected_resource_filename_1_(TestEnvironment::temporaryPath("injected_resource_1")), + injected_resource_filename_2_(TestEnvironment::temporaryPath("injected_resource_2")), + file_updater_1_(injected_resource_filename_1_), + file_updater_2_(injected_resource_filename_2_) {} + + ~OverloadIntegrationTest() override { + cleanupUpstreamAndDownstream(); + // Release the client before destroying |conn_helper_|. No such need once |conn_helper_| is + // moved into a client connection factory in the base test class. + codec_client_.reset(); + } + + IntegrationCodecClientPtr makeRawHttpConnection( + Network::ClientConnectionPtr&& conn, + absl::optional http2_options) override { + IntegrationCodecClientPtr codec = + HttpIntegrationTest::makeRawHttpConnection(std::move(conn), http2_options); + if (!codec->disconnected()) { + codec->setCodecClientCallbacks(client_codec_callback_); + EXPECT_EQ(transport_socket_factory_->clientContextConfig().serverNameIndication(), + codec->connection()->requestedServerName()); + } + return codec; + } + + void initialize() override { + config_helper_.addConfigModifier([this](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + const std::string overload_config = + fmt::format(R"EOF( + refresh_interval: + seconds: 0 + nanos: 1000000 + resource_monitors: + - name: "envoy.resource_monitors.injected_resource_1" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.injected_resource.v3.InjectedResourceConfig + filename: "{}" + - name: "envoy.resource_monitors.injected_resource_2" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.injected_resource.v3.InjectedResourceConfig + filename: "{}" + actions: + - name: "envoy.overload_actions.stop_accepting_requests" + triggers: + - name: "envoy.resource_monitors.injected_resource_1" + threshold: + value: 0.95 + - name: "envoy.overload_actions.stop_accepting_connections" + triggers: + - name: "envoy.resource_monitors.injected_resource_1" + threshold: + value: 0.9 + - name: "envoy.overload_actions.disable_http_keepalive" + triggers: + - name: "envoy.resource_monitors.injected_resource_2" + threshold: + value: 0.8 + )EOF", + injected_resource_filename_1_, injected_resource_filename_2_); + *bootstrap.mutable_overload_manager() = + TestUtility::parseYaml(overload_config); + }); + config_helper_.addConfigModifier( + [](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& + hcm) { + hcm.mutable_drain_timeout()->clear_seconds(); + hcm.mutable_drain_timeout()->set_nanos(500 * 1000 * 1000); + EXPECT_EQ(hcm.codec_type(), envoy::extensions::filters::network::http_connection_manager:: + v3::HttpConnectionManager::HTTP3); + }); + + updateResource(file_updater_1_, 0); + updateResource(file_updater_2_, 0); + HttpIntegrationTest::initialize(); + // registerTestServerPorts({"http"}); + } + +protected: + CodecClientCallbacksForTest client_codec_callback_; + Network::Address::InstanceConstSharedPtr server_addr_; + const std::string injected_resource_filename_1_; + const std::string injected_resource_filename_2_; + AtomicFileUpdater file_updater_1_; + AtomicFileUpdater file_updater_2_; +}; + +INSTANTIATE_TEST_SUITE_P(OverloadIntegrationTests, OverloadIntegrationTest, + testing::ValuesIn(generateTestParam()), testParamsToString); + +TEST_P(OverloadIntegrationTest, StopAcceptingConnectionsWhenOverloaded) { + initialize(); + + // Put envoy in overloaded state and check that it doesn't accept the new client connection. + updateResource(file_updater_1_, 0.9); + test_server_->waitForGaugeEq("overload.envoy.overload_actions.stop_accepting_connections.active", + 1); + codec_client_ = makeRawHttpConnection(makeClientConnection((lookupPort("http"))), absl::nullopt); + EXPECT_TRUE(codec_client_->disconnected()); + + // Reduce load a little to allow the connection to be accepted connection. + updateResource(file_updater_1_, 0.8); + test_server_->waitForGaugeEq("overload.envoy.overload_actions.stop_accepting_connections.active", + 0); + codec_client_ = makeHttpConnection(makeClientConnection((lookupPort("http")))); + auto response = codec_client_->makeHeaderOnlyRequest(default_request_headers_); + waitForNextUpstreamRequest(0); + // Send response headers, but hold response body for now. + upstream_request_->encodeHeaders(default_response_headers_, /*end_stream=*/false); + + updateResource(file_updater_1_, 0.95); + test_server_->waitForGaugeEq("overload.envoy.overload_actions.stop_accepting_requests.active", 1); + // Existing request should be able to finish. + upstream_request_->encodeData(10, true); + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); + + // New request should be rejected. + auto response2 = codec_client_->makeHeaderOnlyRequest(default_request_headers_); + ASSERT_TRUE(response2->waitForEndStream()); + EXPECT_EQ("503", response2->headers().getStatusValue()); + EXPECT_EQ("envoy overloaded", response2->body()); + codec_client_->close(); + + EXPECT_TRUE(makeRawHttpConnection(makeClientConnection((lookupPort("http"))), absl::nullopt) + ->disconnected()); +} + +TEST_P(OverloadIntegrationTest, NoNewStreamsWhenOverloaded) { + initialize(); + updateResource(file_updater_1_, 0.7); + + codec_client_ = makeHttpConnection(makeClientConnection((lookupPort("http")))); + + // Send a complete request and start a second. + auto response = codec_client_->makeHeaderOnlyRequest(default_request_headers_); + waitForNextUpstreamRequest(0); + upstream_request_->encodeHeaders(default_response_headers_, true); + ASSERT_TRUE(response->waitForEndStream()); + + auto response2 = codec_client_->makeHeaderOnlyRequest(default_request_headers_); + waitForNextUpstreamRequest(0); + + // Enable the disable-keepalive overload action. This should send a shutdown notice before + // encoding the headers. + updateResource(file_updater_2_, 0.9); + test_server_->waitForGaugeEq("overload.envoy.overload_actions.disable_http_keepalive.active", 1); + + upstream_request_->encodeHeaders(default_response_headers_, /*end_stream=*/false); + upstream_request_->encodeData(10, true); + + response2->waitForHeaders(); + EXPECT_TRUE(codec_client_->waitForDisconnect()); + + EXPECT_TRUE(codec_client_->sawGoAway()); + codec_client_->close(); +} + +} // namespace Envoy diff --git a/test/extensions/retry/host/previous_hosts/integration_test.cc b/test/extensions/retry/host/previous_hosts/integration_test.cc index 5897e01377e99..12925d0906d7b 100644 --- a/test/extensions/retry/host/previous_hosts/integration_test.cc +++ b/test/extensions/retry/host/previous_hosts/integration_test.cc @@ -14,7 +14,7 @@ namespace { class PrevioustHostsIntegrationTest : public testing::Test, public HttpIntegrationTest { public: PrevioustHostsIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, Network::Address::IpVersion::v4) {} + : HttpIntegrationTest(Http::CodecType::HTTP2, Network::Address::IpVersion::v4) {} void initialize() override { setDeterministic(); diff --git a/test/extensions/retry/priority/previous_priorities/BUILD b/test/extensions/retry/priority/previous_priorities/BUILD index cc5af8f0a7f82..663245db705e7 100644 --- a/test/extensions/retry/priority/previous_priorities/BUILD +++ b/test/extensions/retry/priority/previous_priorities/BUILD @@ -17,7 +17,6 @@ envoy_extension_cc_test( extension_name = "envoy.retry_priorities.previous_priorities", deps = [ "//source/common/protobuf:message_validator_lib", - "//source/extensions/retry/priority:well_known_names", "//source/extensions/retry/priority/previous_priorities:config", "//test/mocks/upstream:host_mocks", "//test/mocks/upstream:host_set_mocks", diff --git a/test/extensions/retry/priority/previous_priorities/config_test.cc b/test/extensions/retry/priority/previous_priorities/config_test.cc index 3d46262e926b5..2b7e88d8cacd3 100644 --- a/test/extensions/retry/priority/previous_priorities/config_test.cc +++ b/test/extensions/retry/priority/previous_priorities/config_test.cc @@ -5,7 +5,6 @@ #include "common/protobuf/message_validator_impl.h" #include "extensions/retry/priority/previous_priorities/config.h" -#include "extensions/retry/priority/well_known_names.h" #include "test/mocks/upstream/host.h" #include "test/mocks/upstream/host_set.h" @@ -28,7 +27,7 @@ class RetryPriorityTest : public testing::Test { void initialize(const Upstream::HealthyLoad& original_healthy_priority_load, const Upstream::DegradedLoad& original_degraded_priority_load) { auto factory = Registry::FactoryRegistry::getFactory( - RetryPriorityValues::get().PreviousPrioritiesRetryPriority); + "envoy.retry_priorities.previous_priorities"); envoy::extensions::retry::priority::previous_priorities::v3::PreviousPrioritiesConfig config; config.set_update_frequency(update_frequency_); diff --git a/test/extensions/stats_sinks/hystrix/hystrix_integration_test.cc b/test/extensions/stats_sinks/hystrix/hystrix_integration_test.cc index 221201bff5c8b..758c9edacf10b 100644 --- a/test/extensions/stats_sinks/hystrix/hystrix_integration_test.cc +++ b/test/extensions/stats_sinks/hystrix/hystrix_integration_test.cc @@ -10,8 +10,8 @@ class HystrixIntegrationTest : public HttpProtocolIntegrationTest {}; INSTANTIATE_TEST_SUITE_P(Protocols, HystrixIntegrationTest, testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams( - {Http::CodecClient::Type::HTTP1, Http::CodecClient::Type::HTTP2}, - {FakeHttpConnection::Type::HTTP1})), + {Http::CodecType::HTTP1, Http::CodecType::HTTP2}, + {Http::CodecType::HTTP1})), HttpProtocolIntegrationTest::protocolTestParamsToString); TEST_P(HystrixIntegrationTest, NoChunkEncoding) { @@ -23,7 +23,7 @@ TEST_P(HystrixIntegrationTest, NoChunkEncoding) { }); initialize(); - if (downstreamProtocol() == Http::CodecClient::Type::HTTP1) { + if (downstreamProtocol() == Http::CodecType::HTTP1) { // For HTTP/1.1 we use a raw client to make absolutely sure there is no chunk encoding. std::string response; auto connection = createConnectionDriver( diff --git a/test/extensions/stats_sinks/metrics_service/metrics_service_integration_test.cc b/test/extensions/stats_sinks/metrics_service/metrics_service_integration_test.cc index e325e04d50372..a66de95bf8573 100644 --- a/test/extensions/stats_sinks/metrics_service/metrics_service_integration_test.cc +++ b/test/extensions/stats_sinks/metrics_service/metrics_service_integration_test.cc @@ -21,12 +21,11 @@ namespace { class MetricsServiceIntegrationTest : public Grpc::VersionedGrpcClientIntegrationParamTest, public HttpIntegrationTest { public: - MetricsServiceIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, ipVersion()) {} + MetricsServiceIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, ipVersion()) {} void createUpstreams() override { HttpIntegrationTest::createUpstreams(); - addFakeUpstream(FakeHttpConnection::Type::HTTP2); + addFakeUpstream(Http::CodecType::HTTP2); } void initialize() override { diff --git a/test/extensions/stats_sinks/wasm/test_data/BUILD b/test/extensions/stats_sinks/wasm/test_data/BUILD index d3458434aec87..f2dca30447bc7 100644 --- a/test/extensions/stats_sinks/wasm/test_data/BUILD +++ b/test/extensions/stats_sinks/wasm/test_data/BUILD @@ -19,7 +19,6 @@ envoy_cc_library( deps = [ "//source/extensions/common/wasm:wasm_hdr", "//source/extensions/common/wasm:wasm_lib", - "//source/extensions/common/wasm:well_known_names", "//source/extensions/common/wasm/ext:envoy_null_plugin", ], ) diff --git a/test/extensions/transport_sockets/alts/alts_integration_test.cc b/test/extensions/transport_sockets/alts/alts_integration_test.cc index 283ce7cf835f1..92624f9181271 100644 --- a/test/extensions/transport_sockets/alts/alts_integration_test.cc +++ b/test/extensions/transport_sockets/alts/alts_integration_test.cc @@ -94,7 +94,7 @@ class AltsIntegrationTestBase : public Event::TestUsingSimulatedTime, AltsIntegrationTestBase(const std::string& server_peer_identity, const std::string& client_peer_identity, bool server_connect_handshaker, bool client_connect_handshaker, bool capturing_handshaker = false) - : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, GetParam()), + : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam()), server_peer_identity_(server_peer_identity), client_peer_identity_(client_peer_identity), server_connect_handshaker_(server_connect_handshaker), client_connect_handshaker_(client_connect_handshaker), diff --git a/test/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator_integration_test.h b/test/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator_integration_test.h index 26a08fbbae20a..b1e57169ea18f 100644 --- a/test/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator_integration_test.h +++ b/test/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator_integration_test.h @@ -18,7 +18,7 @@ class SslSPIFFECertValidatorIntegrationTest public HttpIntegrationTest { public: SslSPIFFECertValidatorIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, std::get<0>(GetParam())) {} + : HttpIntegrationTest(Http::CodecType::HTTP1, std::get<0>(GetParam())) {} void initialize() override; void TearDown() override; diff --git a/test/extensions/transport_sockets/tls/integration/ssl_integration_test.cc b/test/extensions/transport_sockets/tls/integration/ssl_integration_test.cc index 25ca940ceec49..ff3a3212ef033 100644 --- a/test/extensions/transport_sockets/tls/integration/ssl_integration_test.cc +++ b/test/extensions/transport_sockets/tls/integration/ssl_integration_test.cc @@ -139,7 +139,7 @@ TEST_P(SslIntegrationTest, RouterRequestAndResponseWithBodyNoBuffer) { } TEST_P(SslIntegrationTest, RouterRequestAndResponseWithBodyNoBufferHttp2) { - setDownstreamProtocol(Http::CodecClient::Type::HTTP2); + setDownstreamProtocol(Http::CodecType::HTTP2); config_helper_.setClientCodec(envoy::extensions::filters::network::http_connection_manager::v3:: HttpConnectionManager::AUTO); ConnectionCreationFunction creator = [&]() -> Network::ClientConnectionPtr { @@ -158,7 +158,7 @@ TEST_P(SslIntegrationTest, RouterRequestAndResponseWithBodyNoBufferVerifySAN) { } TEST_P(SslIntegrationTest, RouterRequestAndResponseWithBodyNoBufferHttp2VerifySAN) { - setDownstreamProtocol(Http::CodecClient::Type::HTTP2); + setDownstreamProtocol(Http::CodecType::HTTP2); ConnectionCreationFunction creator = [&]() -> Network::ClientConnectionPtr { return makeSslClientConnection(ClientSslTransportOptions().setAlpn(true).setSan(san_to_match_)); }; @@ -194,7 +194,7 @@ TEST_P(SslIntegrationTest, RouterDownstreamDisconnectBeforeResponseComplete) { #if defined(__APPLE__) || defined(WIN32) // Skip this test on OS X + Windows: we can't detect the early close on non-Linux, and we // won't clean up the upstream connection until it times out. See #4294. - if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + if (downstream_protocol_ == Http::CodecType::HTTP1) { return; } #endif diff --git a/test/extensions/transport_sockets/tls/integration/ssl_integration_test.h b/test/extensions/transport_sockets/tls/integration/ssl_integration_test.h index e7f615c54476c..930cc38697f87 100644 --- a/test/extensions/transport_sockets/tls/integration/ssl_integration_test.h +++ b/test/extensions/transport_sockets/tls/integration/ssl_integration_test.h @@ -17,7 +17,7 @@ namespace Ssl { class SslIntegrationTestBase : public HttpIntegrationTest { public: SslIntegrationTestBase(Network::Address::IpVersion ip_version) - : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, ip_version) {} + : HttpIntegrationTest(Http::CodecType::HTTP1, ip_version) {} void initialize() override; diff --git a/test/extensions/transport_sockets/tls/utility_test.cc b/test/extensions/transport_sockets/tls/utility_test.cc index 5b416aaa369a2..32077060c04fc 100644 --- a/test/extensions/transport_sockets/tls/utility_test.cc +++ b/test/extensions/transport_sockets/tls/utility_test.cc @@ -12,6 +12,7 @@ #include "absl/time/time.h" #include "gtest/gtest.h" +#include "openssl/ssl.h" #include "openssl/x509v3.h" namespace Envoy { @@ -148,31 +149,18 @@ TEST(UtilityTest, TestGetCertificationExtensionValue) { TEST(UtilityTest, SslErrorDescriptionTest) { const std::vector> test_set = { - {0, "NONE"}, - {1, "SSL"}, - {2, "WANT_READ"}, - {3, "WANT_WRITE"}, - {4, "WANT_X509_LOOKUP"}, - {5, "SYSCALL"}, - {6, "ZERO_RETURN"}, - {7, "WANT_CONNECT"}, - {8, "WANT_ACCEPT"}, - {9, "WANT_CHANNEL_ID_LOOKUP"}, - {11, "PENDING_SESSION"}, - {12, "PENDING_CERTIFICATE"}, - {13, "WANT_PRIVATE_KEY_OPERATION"}, - {14, "PENDING_TICKET"}, - {15, "EARLY_DATA_REJECTED"}, - {16, "WANT_CERTIFICATE_VERIFY"}, - {17, "HANDOFF"}, - {18, "HANDBACK"}, + {SSL_ERROR_NONE, "NONE"}, + {SSL_ERROR_SSL, "SSL"}, + {SSL_ERROR_WANT_READ, "WANT_READ"}, + {SSL_ERROR_WANT_WRITE, "WANT_WRITE"}, + {SSL_ERROR_WANT_PRIVATE_KEY_OPERATION, "WANT_PRIVATE_KEY_OPERATION"}, }; for (const auto& test_data : test_set) { EXPECT_EQ(test_data.second, Utility::getErrorDescription(test_data.first)); } - EXPECT_ENVOY_BUG(EXPECT_EQ(Utility::getErrorDescription(19), "UNKNOWN_ERROR"), + EXPECT_ENVOY_BUG(EXPECT_EQ(Utility::getErrorDescription(-1), "UNKNOWN_ERROR"), "Unknown BoringSSL error had occurred"); } diff --git a/test/integration/BUILD b/test/integration/BUILD index 41c7e0d44e2e6..1dfbc527eae75 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -320,12 +320,12 @@ envoy_cc_test( deps = [ ":autonomous_upstream_lib", ":http_integration_lib", + ":socket_interface_swap_lib", ":tracked_watermark_buffer_lib", "//test/common/http/http2:http2_frame", "//test/integration/filters:backpressure_filter_config_lib", "//test/integration/filters:set_response_code_filter_config_proto_cc_proto", "//test/integration/filters:set_response_code_filter_lib", - "//test/integration/filters:test_socket_interface_lib", "//test/mocks/http:http_mocks", "//test/test_common:utility_lib", "@com_google_absl//absl/synchronization", @@ -366,6 +366,37 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "buffer_accounting_integration_test", + srcs = [ + "buffer_accounting_integration_test.cc", + ], + deps = [ + ":http_integration_lib", + ":socket_interface_swap_lib", + ":tracked_watermark_buffer_lib", + "//test/mocks/http:http_mocks", + "//test/test_common:utility_lib", + "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", + "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto", + ], +) + +envoy_cc_test_library( + name = "socket_interface_swap_lib", + srcs = [ + "socket_interface_swap.cc", + ], + hdrs = [ + "socket_interface_swap.h", + ], + deps = [ + "//test/integration/filters:test_socket_interface_lib", + "@com_google_absl//absl/synchronization", + ], +) + envoy_cc_test( name = "http_subset_lb_integration_test", srcs = [ @@ -1295,6 +1326,7 @@ envoy_cc_test_library( ], deps = [ "//source/common/buffer:watermark_buffer_lib", + "//test/test_common:utility_lib", ], ) diff --git a/test/integration/README.md b/test/integration/README.md index d16031807a9e4..6e13c958aa2c1 100644 --- a/test/integration/README.md +++ b/test/integration/README.md @@ -50,7 +50,7 @@ The [`ConfigHelper`](../config/utility.h) has utilities for common alterations s ```c++ // Set the default protocol to HTTP2 -setDownstreamProtocol(Http::CodecClient::Type::HTTP2); +setDownstreamProtocol(Http::CodecType::HTTP2); ``` or diff --git a/test/integration/ads_integration.cc b/test/integration/ads_integration.cc index cf0fbeb710467..5d19303690533 100644 --- a/test/integration/ads_integration.cc +++ b/test/integration/ads_integration.cc @@ -22,7 +22,7 @@ namespace Envoy { AdsIntegrationTest::AdsIntegrationTest(envoy::config::core::v3::ApiVersion resource_api_version, envoy::config::core::v3::ApiVersion transport_api_version) - : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), + : HttpIntegrationTest(Http::CodecType::HTTP2, ipVersion(), ConfigHelper::adsBootstrap( sotwOrDelta() == Grpc::SotwOrDelta::Sotw ? "GRPC" : "DELTA_GRPC", resource_api_version, transport_api_version)) { @@ -31,7 +31,7 @@ AdsIntegrationTest::AdsIntegrationTest(envoy::config::core::v3::ApiVersion resou tls_xds_upstream_ = true; sotw_or_delta_ = sotwOrDelta(); api_version_ = resource_api_version; - setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); + setUpstreamProtocol(Http::CodecType::HTTP2); } void AdsIntegrationTest::TearDown() { cleanUpXdsConnection(); } diff --git a/test/integration/ads_integration_test.cc b/test/integration/ads_integration_test.cc index 08e6d8661a9b8..739b6d9f896b6 100644 --- a/test/integration/ads_integration_test.cc +++ b/test/integration/ads_integration_test.cc @@ -89,7 +89,7 @@ TEST_P(AdsIntegrationTest, TestPrimaryClusterWarmClusterInitialization) { const auto cds_type_url = Config::getTypeUrl( envoy::config::core::v3::ApiVersion::V3); auto loopback = Network::Test::getLoopbackAddressString(ipVersion()); - addFakeUpstream(FakeHttpConnection::Type::HTTP2); + addFakeUpstream(Http::CodecType::HTTP2); auto port = fake_upstreams_.back()->localAddress()->ip()->port(); // This cluster will be blocked since endpoint name cannot be resolved. @@ -1055,7 +1055,7 @@ class AdsFailIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public HttpIntegrationTest { public: AdsFailIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), + : HttpIntegrationTest(Http::CodecType::HTTP2, ipVersion(), ConfigHelper::adsBootstrap( sotwOrDelta() == Grpc::SotwOrDelta::Sotw ? "GRPC" : "DELTA_GRPC", envoy::config::core::v3::ApiVersion::V3)) { @@ -1075,7 +1075,7 @@ class AdsFailIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, ads_cluster->MergeFrom(bootstrap.static_resources().clusters()[0]); ads_cluster->set_name("ads_cluster"); }); - setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); + setUpstreamProtocol(Http::CodecType::HTTP2); HttpIntegrationTest::initialize(); } }; @@ -1096,7 +1096,7 @@ class AdsConfigIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public HttpIntegrationTest { public: AdsConfigIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), + : HttpIntegrationTest(Http::CodecType::HTTP2, ipVersion(), ConfigHelper::adsBootstrap( sotwOrDelta() == Grpc::SotwOrDelta::Sotw ? "GRPC" : "DELTA_GRPC", envoy::config::core::v3::ApiVersion::V3)) { @@ -1125,7 +1125,7 @@ class AdsConfigIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, eds_config->set_resource_api_version(envoy::config::core::v3::ApiVersion::V3); eds_config->mutable_ads(); }); - setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); + setUpstreamProtocol(Http::CodecType::HTTP2); HttpIntegrationTest::initialize(); } }; @@ -1282,7 +1282,7 @@ class AdsClusterFromFileIntegrationTest : public Grpc::DeltaSotwIntegrationParam public HttpIntegrationTest { public: AdsClusterFromFileIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), + : HttpIntegrationTest(Http::CodecType::HTTP2, ipVersion(), ConfigHelper::adsBootstrap( sotwOrDelta() == Grpc::SotwOrDelta::Sotw ? "GRPC" : "DELTA_GRPC", envoy::config::core::v3::ApiVersion::V3)) { @@ -1323,7 +1323,7 @@ class AdsClusterFromFileIntegrationTest : public Grpc::DeltaSotwIntegrationParam eds_config->set_resource_api_version(envoy::config::core::v3::ApiVersion::V3); eds_config->mutable_ads(); }); - setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); + setUpstreamProtocol(Http::CodecType::HTTP2); HttpIntegrationTest::initialize(); } diff --git a/test/integration/alpn_integration_test.cc b/test/integration/alpn_integration_test.cc index 264036a4bc2a8..fa1e33fcee360 100644 --- a/test/integration/alpn_integration_test.cc +++ b/test/integration/alpn_integration_test.cc @@ -9,12 +9,12 @@ namespace { class AlpnIntegrationTest : public testing::TestWithParam, public HttpIntegrationTest { public: - AlpnIntegrationTest() : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, GetParam()) {} + AlpnIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP2, GetParam()) {} void SetUp() override { autonomous_upstream_ = true; setUpstreamCount(2); - setDownstreamProtocol(Http::CodecClient::Type::HTTP2); + setDownstreamProtocol(Http::CodecType::HTTP2); upstream_tls_ = true; config_helper_.configureUpstreamTls(true); @@ -40,7 +40,7 @@ class AlpnIntegrationTest : public testing::TestWithParam protocols_; + std::vector protocols_; }; INSTANTIATE_TEST_SUITE_P(IpVersions, AlpnIntegrationTest, @@ -48,8 +48,8 @@ INSTANTIATE_TEST_SUITE_P(IpVersions, AlpnIntegrationTest, TestUtility::ipTestParamsToString); TEST_P(AlpnIntegrationTest, Http2) { - setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); - protocols_ = {FakeHttpConnection::Type::HTTP2, FakeHttpConnection::Type::HTTP2}; + setUpstreamProtocol(Http::CodecType::HTTP2); + protocols_ = {Http::CodecType::HTTP2, Http::CodecType::HTTP2}; initialize(); codec_client_ = makeHttpConnection(makeClientConnection((lookupPort("http")))); @@ -60,8 +60,8 @@ TEST_P(AlpnIntegrationTest, Http2) { } TEST_P(AlpnIntegrationTest, Http1) { - setUpstreamProtocol(FakeHttpConnection::Type::HTTP1); - protocols_ = {FakeHttpConnection::Type::HTTP1, FakeHttpConnection::Type::HTTP1}; + setUpstreamProtocol(Http::CodecType::HTTP1); + protocols_ = {Http::CodecType::HTTP1, Http::CodecType::HTTP1}; initialize(); codec_client_ = makeHttpConnection(makeClientConnection((lookupPort("http")))); @@ -72,7 +72,7 @@ TEST_P(AlpnIntegrationTest, Http1) { } TEST_P(AlpnIntegrationTest, Mixed) { - protocols_ = {FakeHttpConnection::Type::HTTP1, FakeHttpConnection::Type::HTTP2}; + protocols_ = {Http::CodecType::HTTP1, Http::CodecType::HTTP2}; initialize(); codec_client_ = makeHttpConnection(makeClientConnection((lookupPort("http")))); diff --git a/test/integration/alpn_selection_integration_test.cc b/test/integration/alpn_selection_integration_test.cc index b28c42fbffc03..052bc5e14f752 100644 --- a/test/integration/alpn_selection_integration_test.cc +++ b/test/integration/alpn_selection_integration_test.cc @@ -18,14 +18,12 @@ namespace Envoy { class AlpnSelectionIntegrationTest : public testing::Test, public HttpIntegrationTest { public: AlpnSelectionIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, - TestEnvironment::getIpVersionsForTest().front(), + : HttpIntegrationTest(Http::CodecType::HTTP1, TestEnvironment::getIpVersionsForTest().front(), ConfigHelper::httpProxyConfig()) {} void initialize() override { - setDownstreamProtocol(Http::CodecClient::Type::HTTP1); - setUpstreamProtocol(use_h2_ ? FakeHttpConnection::Type::HTTP2 - : FakeHttpConnection::Type::HTTP1); + setDownstreamProtocol(Http::CodecType::HTTP1); + setUpstreamProtocol(use_h2_ ? Http::CodecType::HTTP2 : Http::CodecType::HTTP1); config_helper_.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { auto* static_resources = bootstrap.mutable_static_resources(); auto* cluster = static_resources->mutable_clusters(0); @@ -81,8 +79,7 @@ require_client_certificate: true void createUpstreams() override { auto endpoint = upstream_address_fn_(0); FakeUpstreamConfig config = upstreamConfig(); - config.upstream_protocol_ = - use_h2_ ? FakeHttpConnection::Type::HTTP2 : FakeHttpConnection::Type::HTTP1; + config.upstream_protocol_ = use_h2_ ? Http::CodecType::HTTP2 : Http::CodecType::HTTP1; fake_upstreams_.emplace_back( new FakeUpstream(createUpstreamSslContext(), endpoint->ip()->port(), version_, config)); } diff --git a/test/integration/autonomous_upstream.cc b/test/integration/autonomous_upstream.cc index 762da275b31d1..65d16dd25fd7d 100644 --- a/test/integration/autonomous_upstream.cc +++ b/test/integration/autonomous_upstream.cc @@ -94,7 +94,8 @@ void AutonomousStream::sendResponse() { AutonomousHttpConnection::AutonomousHttpConnection(AutonomousUpstream& autonomous_upstream, SharedConnectionWrapper& shared_connection, - Type type, AutonomousUpstream& upstream) + Http::CodecType type, + AutonomousUpstream& upstream) : FakeHttpConnection(autonomous_upstream, shared_connection, type, upstream.timeSystem(), Http::DEFAULT_MAX_REQUEST_HEADERS_KB, Http::DEFAULT_MAX_HEADERS_COUNT, envoy::config::core::v3::HttpProtocolOptions::ALLOW), diff --git a/test/integration/autonomous_upstream.h b/test/integration/autonomous_upstream.h index d5ae283fec568..aa0e15fb1baa1 100644 --- a/test/integration/autonomous_upstream.h +++ b/test/integration/autonomous_upstream.h @@ -45,7 +45,7 @@ class AutonomousStream : public FakeStream { class AutonomousHttpConnection : public FakeHttpConnection { public: AutonomousHttpConnection(AutonomousUpstream& autonomous_upstream, - SharedConnectionWrapper& shared_connection, Type type, + SharedConnectionWrapper& shared_connection, Http::CodecType type, AutonomousUpstream& upstream); Http::RequestDecoder& newStream(Http::ResponseEncoder& response_encoder, bool) override; diff --git a/test/integration/base_integration_test.cc b/test/integration/base_integration_test.cc index ac4cf88eb7030..012197a98f1fd 100644 --- a/test/integration/base_integration_test.cc +++ b/test/integration/base_integration_test.cc @@ -121,12 +121,12 @@ BaseIntegrationTest::createUpstreamTlsContext(const FakeUpstreamConfig& upstream TestEnvironment::runfilesPath("test/config/integration/certs/upstreamkey.pem"), TestEnvironment::runfilesPath("test/config/integration/certs/cacert.pem")); TestUtility::loadFromYaml(yaml, tls_context); - if (upstream_config.upstream_protocol_ == FakeHttpConnection::Type::HTTP2) { + if (upstream_config.upstream_protocol_ == Http::CodecType::HTTP2) { tls_context.mutable_common_tls_context()->add_alpn_protocols("h2"); - } else if (upstream_config.upstream_protocol_ == FakeHttpConnection::Type::HTTP1) { + } else if (upstream_config.upstream_protocol_ == Http::CodecType::HTTP1) { tls_context.mutable_common_tls_context()->add_alpn_protocols("http/1.1"); } - if (upstream_config.upstream_protocol_ != FakeHttpConnection::Type::HTTP3) { + if (upstream_config.upstream_protocol_ != Http::CodecType::HTTP3) { auto cfg = std::make_unique( tls_context, factory_context_); static Stats::Scope* upstream_stats_store = new Stats::IsolatedStoreImpl(); @@ -217,9 +217,9 @@ void BaseIntegrationTest::createEnvoy() { createGeneratedApiTestServer(bootstrap_path, named_ports, {false, true, false}, false); } -void BaseIntegrationTest::setUpstreamProtocol(FakeHttpConnection::Type protocol) { +void BaseIntegrationTest::setUpstreamProtocol(Http::CodecType protocol) { upstream_config_.upstream_protocol_ = protocol; - if (upstream_config_.upstream_protocol_ == FakeHttpConnection::Type::HTTP2) { + if (upstream_config_.upstream_protocol_ == Http::CodecType::HTTP2) { config_helper_.addConfigModifier( [&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { RELEASE_ASSERT(bootstrap.mutable_static_resources()->clusters_size() >= 1, ""); @@ -228,7 +228,7 @@ void BaseIntegrationTest::setUpstreamProtocol(FakeHttpConnection::Type protocol) ConfigHelper::setProtocolOptions( *bootstrap.mutable_static_resources()->mutable_clusters(0), protocol_options); }); - } else if (upstream_config_.upstream_protocol_ == FakeHttpConnection::Type::HTTP1) { + } else if (upstream_config_.upstream_protocol_ == Http::CodecType::HTTP1) { config_helper_.addConfigModifier( [&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { RELEASE_ASSERT(bootstrap.mutable_static_resources()->clusters_size() >= 1, ""); @@ -238,7 +238,7 @@ void BaseIntegrationTest::setUpstreamProtocol(FakeHttpConnection::Type protocol) *bootstrap.mutable_static_resources()->mutable_clusters(0), protocol_options); }); } else { - RELEASE_ASSERT(protocol == FakeHttpConnection::Type::HTTP3, ""); + RELEASE_ASSERT(protocol == Http::CodecType::HTTP3, ""); setUdpFakeUpstream(FakeUpstreamConfig::UdpConfig()); upstream_tls_ = true; config_helper_.configureUpstreamTls(false, true); @@ -445,7 +445,7 @@ void BaseIntegrationTest::createXdsUpstream() { return; } if (tls_xds_upstream_ == false) { - addFakeUpstream(FakeHttpConnection::Type::HTTP2); + addFakeUpstream(Http::CodecType::HTTP2); } else { envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext tls_context; auto* common_tls_context = tls_context.mutable_common_tls_context(); @@ -461,7 +461,7 @@ void BaseIntegrationTest::createXdsUpstream() { upstream_stats_store_ = std::make_unique(); auto context = std::make_unique( std::move(cfg), context_manager_, *upstream_stats_store_, std::vector{}); - addFakeUpstream(std::move(context), FakeHttpConnection::Type::HTTP2); + addFakeUpstream(std::move(context), Http::CodecType::HTTP2); } xds_upstream_ = fake_upstreams_.back().get(); } diff --git a/test/integration/base_integration_test.h b/test/integration/base_integration_test.h index 9f902f958d2dc..37230364cd60a 100644 --- a/test/integration/base_integration_test.h +++ b/test/integration/base_integration_test.h @@ -64,7 +64,7 @@ class BaseIntegrationTest : protected Logger::Loggable { // Finalize the config and spin up an Envoy instance. virtual void createEnvoy(); // Sets upstream_protocol_ and alters the upstream protocol in the config_helper_ - void setUpstreamProtocol(FakeHttpConnection::Type protocol); + void setUpstreamProtocol(Http::CodecType protocol); // Sets fake_upstreams_count_ void setUpstreamCount(uint32_t count) { fake_upstreams_count_ = count; } // Skip validation that ensures that all upstream ports are referenced by the @@ -73,7 +73,7 @@ class BaseIntegrationTest : protected Logger::Loggable { // Make test more deterministic by using a fixed RNG value. void setDeterministic() { deterministic_ = true; } - FakeHttpConnection::Type upstreamProtocol() const { return upstream_config_.upstream_protocol_; } + Http::CodecType upstreamProtocol() const { return upstream_config_.upstream_protocol_; } IntegrationTcpClientPtr makeTcpConnection(uint32_t port, @@ -307,23 +307,23 @@ class BaseIntegrationTest : protected Logger::Loggable { *dispatcher_, std::move(transport_socket)); } - FakeUpstreamConfig configWithType(FakeHttpConnection::Type type) const { + FakeUpstreamConfig configWithType(Http::CodecType type) const { FakeUpstreamConfig config = upstream_config_; config.upstream_protocol_ = type; - if (type != FakeHttpConnection::Type::HTTP3) { + if (type != Http::CodecType::HTTP3) { config.udp_fake_upstream_ = absl::nullopt; } return config; } - FakeUpstream& addFakeUpstream(FakeHttpConnection::Type type) { + FakeUpstream& addFakeUpstream(Http::CodecType type) { auto config = configWithType(type); fake_upstreams_.emplace_back(std::make_unique(0, version_, config)); return *fake_upstreams_.back(); } FakeUpstream& addFakeUpstream(Network::TransportSocketFactoryPtr&& transport_socket_factory, - FakeHttpConnection::Type type) { + Http::CodecType type) { auto config = configWithType(type); fake_upstreams_.emplace_back( std::make_unique(std::move(transport_socket_factory), 0, version_, config)); diff --git a/test/integration/buffer_accounting_integration_test.cc b/test/integration/buffer_accounting_integration_test.cc new file mode 100644 index 0000000000000..1ccfdb1121336 --- /dev/null +++ b/test/integration/buffer_accounting_integration_test.cc @@ -0,0 +1,227 @@ +#include + +#include "envoy/config/bootstrap/v3/bootstrap.pb.h" +#include "envoy/config/cluster/v3/cluster.pb.h" +#include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.h" +#include "envoy/network/address.h" + +#include "common/buffer/buffer_impl.h" + +#include "test/integration/autonomous_upstream.h" +#include "test/integration/tracked_watermark_buffer.h" +#include "test/integration/utility.h" +#include "test/mocks/http/mocks.h" + +#include "fake_upstream.h" +#include "gtest/gtest.h" +#include "http_integration.h" +#include "integration_stream_decoder.h" +#include "socket_interface_swap.h" + +namespace Envoy { +namespace { +std::string ipVersionAndBufferAccountingTestParamsToString( + const ::testing::TestParamInfo>& params) { + return fmt::format( + "{}_{}", + TestUtility::ipTestParamsToString(::testing::TestParamInfo( + std::get<0>(params.param), params.index)), + std::get<1>(params.param) ? "with_per_stream_buffer_accounting" + : "without_per_stream_buffer_accounting"); +} +} // namespace + +class HttpBufferWatermarksTest + : public SocketInterfaceSwap, + public testing::TestWithParam>, + public HttpIntegrationTest { +public: + std::vector + sendRequests(uint32_t num_responses, uint32_t request_body_size, uint32_t response_body_size) { + std::vector responses; + + Http::TestRequestHeaderMapImpl header_map{ + {"response_data_blocks", absl::StrCat(1)}, + {"response_size_bytes", absl::StrCat(response_body_size)}, + {"no_trailers", "0"}}; + header_map.copyFrom(default_request_headers_); + header_map.setContentLength(request_body_size); + + for (uint32_t idx = 0; idx < num_responses; ++idx) { + responses.emplace_back(codec_client_->makeRequestWithBody(header_map, request_body_size)); + } + + return responses; + } + + // TODO(kbaichoo): Parameterize on the client codec type when other protocols + // (H1, H3) support buffer accounting. + HttpBufferWatermarksTest() + : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, std::get<0>(GetParam())) { + config_helper_.addRuntimeOverride("envoy.test_only.per_stream_buffer_accounting", + streamBufferAccounting() ? "true" : "false"); + setServerBufferFactory(buffer_factory_); + setDownstreamProtocol(Http::CodecClient::Type::HTTP2); + setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); + } + +protected: + std::shared_ptr buffer_factory_ = + std::make_shared(); + + bool streamBufferAccounting() { return std::get<1>(GetParam()); } + + std::string printAccounts() { + std::stringstream stream; + auto print_map = + [&stream](Buffer::TrackedWatermarkBufferFactory::AccountToBoundBuffersMap& map) { + stream << "Printing Account map. Size: " << map.size() << '\n'; + for (auto& entry : map) { + // This runs in the context of the worker thread, so we can access + // the balance. + stream << " Account: " << entry.first << '\n'; + stream << " Balance:" + << static_cast(entry.first.get())->balance() + << '\n'; + stream << " Number of associated buffers: " << entry.second.size() << '\n'; + } + }; + buffer_factory_->inspectAccounts(print_map, test_server_->server()); + return stream.str(); + } +}; + +INSTANTIATE_TEST_SUITE_P( + IpVersions, HttpBufferWatermarksTest, + testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), testing::Bool()), + ipVersionAndBufferAccountingTestParamsToString); + +// We should create four buffers each billing the same downstream request's +// account which originated the chain. +TEST_P(HttpBufferWatermarksTest, ShouldCreateFourBuffersPerAccount) { + FakeStreamPtr upstream_request1; + FakeStreamPtr upstream_request2; + default_request_headers_.setContentLength(1000); + + initialize(); + codec_client_ = makeHttpConnection(lookupPort("http")); + + // Sends the first request. + auto response1 = codec_client_->makeRequestWithBody(default_request_headers_, 1000); + waitForNextUpstreamRequest(); + upstream_request1 = std::move(upstream_request_); + + // Check the expected number of buffers per account + if (streamBufferAccounting()) { + EXPECT_TRUE(buffer_factory_->waitUntilExpectedNumberOfAccountsAndBoundBuffers(1, 4)); + } else { + EXPECT_TRUE(buffer_factory_->waitUntilExpectedNumberOfAccountsAndBoundBuffers(0, 0)); + } + + // Send the second request. + auto response2 = codec_client_->makeRequestWithBody(default_request_headers_, 1000); + waitForNextUpstreamRequest(); + upstream_request2 = std::move(upstream_request_); + + // Check the expected number of buffers per account + if (streamBufferAccounting()) { + EXPECT_TRUE(buffer_factory_->waitUntilExpectedNumberOfAccountsAndBoundBuffers(2, 8)); + } else { + EXPECT_TRUE(buffer_factory_->waitUntilExpectedNumberOfAccountsAndBoundBuffers(0, 0)); + } + + // Respond to the first request and wait for complete + upstream_request1->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, false); + upstream_request1->encodeData(1000, true); + ASSERT_TRUE(response1->waitForEndStream()); + ASSERT_TRUE(upstream_request1->complete()); + + // Check the expected number of buffers per account + if (streamBufferAccounting()) { + EXPECT_TRUE(buffer_factory_->waitUntilExpectedNumberOfAccountsAndBoundBuffers(1, 4)); + } else { + EXPECT_TRUE(buffer_factory_->waitUntilExpectedNumberOfAccountsAndBoundBuffers(0, 0)); + } + + // Respond to the second request and wait for complete + upstream_request2->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, false); + upstream_request2->encodeData(1000, true); + ASSERT_TRUE(response2->waitForEndStream()); + ASSERT_TRUE(upstream_request2->complete()); + + // Check the expected number of buffers per account + EXPECT_TRUE(buffer_factory_->waitUntilExpectedNumberOfAccountsAndBoundBuffers(0, 0)); +} + +TEST_P(HttpBufferWatermarksTest, ShouldTrackAllocatedBytesToUpstream) { + const int num_requests = 5; + const uint32_t request_body_size = 4096; + const uint32_t response_body_size = 4096; + + autonomous_upstream_ = true; + autonomous_allow_incomplete_streams_ = true; + initialize(); + + buffer_factory_->setExpectedAccountBalance(request_body_size, num_requests); + + // Makes us have Envoy's writes to upstream return EAGAIN + writev_matcher_->setDestinationPort(fake_upstreams_[0]->localAddress()->ip()->port()); + writev_matcher_->setWritevReturnsEgain(); + + codec_client_ = makeHttpConnection(lookupPort("http")); + + auto responses = sendRequests(num_requests, request_body_size, response_body_size); + + // Wait for all requests to have accounted for the requests we've sent. + if (streamBufferAccounting()) { + EXPECT_TRUE( + buffer_factory_->waitForExpectedAccountBalanceWithTimeout(TestUtility::DefaultTimeout)) + << "buffer total: " << buffer_factory_->totalBufferSize() + << " buffer max: " << buffer_factory_->maxBufferSize() << printAccounts(); + } + + writev_matcher_->setResumeWrites(); + + for (auto& response : responses) { + ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->complete()); + } +} + +TEST_P(HttpBufferWatermarksTest, ShouldTrackAllocatedBytesToDownstream) { + const int num_requests = 5; + const uint32_t request_body_size = 4096; + const uint32_t response_body_size = 16384; + + autonomous_upstream_ = true; + autonomous_allow_incomplete_streams_ = true; + initialize(); + + buffer_factory_->setExpectedAccountBalance(response_body_size, num_requests); + writev_matcher_->setSourcePort(lookupPort("http")); + codec_client_ = makeHttpConnection(lookupPort("http")); + + // Simulate TCP push back on the Envoy's downstream network socket, so that outbound frames + // start to accumulate in the transport socket buffer. + writev_matcher_->setWritevReturnsEgain(); + + auto responses = sendRequests(num_requests, request_body_size, response_body_size); + + // Wait for all requests to buffered the response from upstream. + if (streamBufferAccounting()) { + EXPECT_TRUE( + buffer_factory_->waitForExpectedAccountBalanceWithTimeout(TestUtility::DefaultTimeout)) + << "buffer total: " << buffer_factory_->totalBufferSize() + << " buffer max: " << buffer_factory_->maxBufferSize() << printAccounts(); + } + + writev_matcher_->setResumeWrites(); + + // Wait for streams to terminate. + for (auto& response : responses) { + ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->complete()); + } +} + +} // namespace Envoy diff --git a/test/integration/cds_integration_test.cc b/test/integration/cds_integration_test.cc index aecabc64bee69..0377cdcb1da2d 100644 --- a/test/integration/cds_integration_test.cc +++ b/test/integration/cds_integration_test.cc @@ -31,7 +31,7 @@ const int UpstreamIndex2 = 2; class CdsIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public HttpIntegrationTest { public: CdsIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), + : HttpIntegrationTest(Http::CodecType::HTTP2, ipVersion(), ConfigHelper::discoveredClustersBootstrap( sotwOrDelta() == Grpc::SotwOrDelta::Sotw ? "GRPC" : "DELTA_GRPC")) { use_lds_ = false; @@ -53,8 +53,8 @@ class CdsIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public Ht // BaseIntegrationTest::createUpstreams() (which is part of initialize()). // Make sure this number matches the size of the 'clusters' repeated field in the bootstrap // config that you use! - setUpstreamCount(1); // the CDS cluster - setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); // CDS uses gRPC uses HTTP2. + setUpstreamCount(1); // the CDS cluster + setUpstreamProtocol(Http::CodecType::HTTP2); // CDS uses gRPC uses HTTP2. // HttpIntegrationTest::initialize() does many things: // 1) It appends to fake_upstreams_ as many as you asked for via setUpstreamCount(). @@ -71,8 +71,8 @@ class CdsIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public Ht // Create the regular (i.e. not an xDS server) upstreams. We create them manually here after // initialize() because finalize() expects all fake_upstreams_ to correspond to a static // cluster in the bootstrap config - which we don't want since we're testing dynamic CDS! - addFakeUpstream(FakeHttpConnection::Type::HTTP2); - addFakeUpstream(FakeHttpConnection::Type::HTTP2); + addFakeUpstream(Http::CodecType::HTTP2); + addFakeUpstream(Http::CodecType::HTTP2); cluster1_ = ConfigHelper::buildStaticCluster( ClusterName1, fake_upstreams_[UpstreamIndex1]->localAddress()->ip()->port(), Network::Test::getLoopbackAddressString(ipVersion())); diff --git a/test/integration/cluster_upstream_extension_integration_test.cc b/test/integration/cluster_upstream_extension_integration_test.cc index e4ac4ee0ba76d..dfbc519933ddc 100644 --- a/test/integration/cluster_upstream_extension_integration_test.cc +++ b/test/integration/cluster_upstream_extension_integration_test.cc @@ -20,7 +20,7 @@ class ClusterUpstreamExtensionIntegrationTest public HttpIntegrationTest { public: ClusterUpstreamExtensionIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, GetParam()) {} + : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam()) {} void populateMetadataTestData(envoy::config::core::v3::Metadata& metadata, const std::string& key1, const std::string& key2, diff --git a/test/integration/command_formatter_extension_integration_test.cc b/test/integration/command_formatter_extension_integration_test.cc index 3cc1a21527dc3..fe0643b39a906 100644 --- a/test/integration/command_formatter_extension_integration_test.cc +++ b/test/integration/command_formatter_extension_integration_test.cc @@ -13,7 +13,7 @@ namespace Formatter { class CommandFormatterExtensionIntegrationTest : public testing::Test, public HttpIntegrationTest { public: CommandFormatterExtensionIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, Network::Address::IpVersion::v4) {} + : HttpIntegrationTest(Http::CodecType::HTTP1, Network::Address::IpVersion::v4) {} }; TEST_F(CommandFormatterExtensionIntegrationTest, BasicExtension) { diff --git a/test/integration/custom_cluster_integration_test.cc b/test/integration/custom_cluster_integration_test.cc index e5553e9a87ef7..ecd21ccddf05d 100644 --- a/test/integration/custom_cluster_integration_test.cc +++ b/test/integration/custom_cluster_integration_test.cc @@ -18,8 +18,7 @@ const int UpstreamIndex = 0; class CustomClusterIntegrationTest : public testing::TestWithParam, public HttpIntegrationTest { public: - CustomClusterIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, GetParam()) {} + CustomClusterIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam()) {} void initialize() override { setUpstreamCount(1); diff --git a/test/integration/direct_response_integration_test.cc b/test/integration/direct_response_integration_test.cc index da246a27f74c3..bc79926acaff7 100644 --- a/test/integration/direct_response_integration_test.cc +++ b/test/integration/direct_response_integration_test.cc @@ -6,8 +6,7 @@ namespace Envoy { class DirectResponseIntegrationTest : public testing::TestWithParam, public HttpIntegrationTest { public: - DirectResponseIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, GetParam()) {} + DirectResponseIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam()) {} void TearDown() override { cleanupUpstreamAndDownstream(); } diff --git a/test/integration/drain_close_integration_test.cc b/test/integration/drain_close_integration_test.cc index 22f410e1ce69a..de57d7fcba64c 100644 --- a/test/integration/drain_close_integration_test.cc +++ b/test/integration/drain_close_integration_test.cc @@ -35,7 +35,7 @@ TEST_P(DrainCloseIntegrationTest, DrainCloseGradual) { EXPECT_TRUE(response->complete()); EXPECT_EQ("200", response->headers().getStatusValue()); - if (downstream_protocol_ == Http::CodecClient::Type::HTTP2) { + if (downstream_protocol_ == Http::CodecType::HTTP2) { EXPECT_TRUE(codec_client_->sawGoAway()); } else { EXPECT_EQ("close", response->headers().getConnectionValue()); @@ -66,7 +66,7 @@ TEST_P(DrainCloseIntegrationTest, DrainCloseImmediate) { EXPECT_TRUE(response->complete()); EXPECT_EQ("200", response->headers().getStatusValue()); - if (downstream_protocol_ == Http::CodecClient::Type::HTTP2) { + if (downstream_protocol_ == Http::CodecType::HTTP2) { EXPECT_TRUE(codec_client_->sawGoAway()); } else { EXPECT_EQ("close", response->headers().getConnectionValue()); @@ -108,7 +108,7 @@ TEST_P(DrainCloseIntegrationTest, AdminGracefulDrain) { // Connections will terminate on request complete ASSERT_TRUE(codec_client_->waitForDisconnect()); - if (downstream_protocol_ == Http::CodecClient::Type::HTTP2) { + if (downstream_protocol_ == Http::CodecType::HTTP2) { EXPECT_TRUE(codec_client_->sawGoAway()); } else { EXPECT_EQ("close", response->headers().getConnectionValue()); @@ -169,8 +169,8 @@ TEST_P(DrainCloseIntegrationTest, RepeatedAdminGracefulDrain) { INSTANTIATE_TEST_SUITE_P(Protocols, DrainCloseIntegrationTest, testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams( - {Http::CodecClient::Type::HTTP1, Http::CodecClient::Type::HTTP2}, - {FakeHttpConnection::Type::HTTP1})), + {Http::CodecType::HTTP1, Http::CodecType::HTTP2}, + {Http::CodecType::HTTP1})), HttpProtocolIntegrationTest::protocolTestParamsToString); } // namespace diff --git a/test/integration/dynamic_validation_integration_test.cc b/test/integration/dynamic_validation_integration_test.cc index aab7833a53724..d9cde92eeffec 100644 --- a/test/integration/dynamic_validation_integration_test.cc +++ b/test/integration/dynamic_validation_integration_test.cc @@ -62,10 +62,10 @@ class DynamicValidationIntegrationTest public HttpIntegrationTest { public: DynamicValidationIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, std::get<0>(GetParam())), + : HttpIntegrationTest(Http::CodecType::HTTP2, std::get<0>(GetParam())), reject_unknown_dynamic_fields_(std::get<1>(GetParam())), ignore_unknown_dynamic_fields_(std::get<2>(GetParam())) { - setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); + setUpstreamProtocol(Http::CodecType::HTTP2); } void createEnvoy() override { diff --git a/test/integration/eds_integration_test.cc b/test/integration/eds_integration_test.cc index 5b742158d6998..ddef6576eab4d 100644 --- a/test/integration/eds_integration_test.cc +++ b/test/integration/eds_integration_test.cc @@ -22,7 +22,7 @@ class EdsIntegrationTest : public testing::TestWithParam cb) { - parent_.connection().dispatcher().post(cb); + parent_.postToConnectionThread(cb); } void FakeStream::encode100ContinueHeaders(const Http::ResponseHeaderMap& headers) { std::shared_ptr headers_copy( Http::createHeaderMap(headers)); - parent_.connection().dispatcher().post([this, headers_copy]() -> void { + postToConnectionThread([this, headers_copy]() -> void { { absl::MutexLock lock(&lock_); - if (saw_reset_) { + if (!parent_.connected() || saw_reset_) { // Encoded already deleted. return; } @@ -95,10 +95,10 @@ void FakeStream::encodeHeaders(const Http::HeaderMap& headers, bool end_stream) parent_.connection().addressProvider().localAddress()->asString()); } - parent_.connection().dispatcher().post([this, headers_copy, end_stream]() -> void { + postToConnectionThread([this, headers_copy, end_stream]() -> void { { absl::MutexLock lock(&lock_); - if (saw_reset_) { + if (!parent_.connected() || saw_reset_) { // Encoded already deleted. return; } @@ -108,10 +108,10 @@ void FakeStream::encodeHeaders(const Http::HeaderMap& headers, bool end_stream) } void FakeStream::encodeData(absl::string_view data, bool end_stream) { - parent_.connection().dispatcher().post([this, data, end_stream]() -> void { + postToConnectionThread([this, data, end_stream]() -> void { { absl::MutexLock lock(&lock_); - if (saw_reset_) { + if (!parent_.connected() || saw_reset_) { // Encoded already deleted. return; } @@ -122,10 +122,10 @@ void FakeStream::encodeData(absl::string_view data, bool end_stream) { } void FakeStream::encodeData(uint64_t size, bool end_stream) { - parent_.connection().dispatcher().post([this, size, end_stream]() -> void { + postToConnectionThread([this, size, end_stream]() -> void { { absl::MutexLock lock(&lock_); - if (saw_reset_) { + if (!parent_.connected() || saw_reset_) { // Encoded already deleted. return; } @@ -137,10 +137,10 @@ void FakeStream::encodeData(uint64_t size, bool end_stream) { void FakeStream::encodeData(Buffer::Instance& data, bool end_stream) { std::shared_ptr data_copy = std::make_shared(data); - parent_.connection().dispatcher().post([this, data_copy, end_stream]() -> void { + postToConnectionThread([this, data_copy, end_stream]() -> void { { absl::MutexLock lock(&lock_); - if (saw_reset_) { + if (!parent_.connected() || saw_reset_) { // Encoded already deleted. return; } @@ -152,10 +152,10 @@ void FakeStream::encodeData(Buffer::Instance& data, bool end_stream) { void FakeStream::encodeTrailers(const Http::HeaderMap& trailers) { std::shared_ptr trailers_copy( Http::createHeaderMap(trailers)); - parent_.connection().dispatcher().post([this, trailers_copy]() -> void { + postToConnectionThread([this, trailers_copy]() -> void { { absl::MutexLock lock(&lock_); - if (saw_reset_) { + if (!parent_.connected() || saw_reset_) { // Encoded already deleted. return; } @@ -165,10 +165,10 @@ void FakeStream::encodeTrailers(const Http::HeaderMap& trailers) { } void FakeStream::encodeResetStream() { - parent_.connection().dispatcher().post([this]() -> void { + postToConnectionThread([this]() -> void { { absl::MutexLock lock(&lock_); - if (saw_reset_) { + if (!parent_.connected() || saw_reset_) { // Encoded already deleted. return; } @@ -178,10 +178,10 @@ void FakeStream::encodeResetStream() { } void FakeStream::encodeMetadata(const Http::MetadataMapVector& metadata_map_vector) { - parent_.connection().dispatcher().post([this, &metadata_map_vector]() -> void { + postToConnectionThread([this, &metadata_map_vector]() -> void { { absl::MutexLock lock(&lock_); - if (saw_reset_) { + if (!parent_.connected() || saw_reset_) { // Encoded already deleted. return; } @@ -191,10 +191,10 @@ void FakeStream::encodeMetadata(const Http::MetadataMapVector& metadata_map_vect } void FakeStream::readDisable(bool disable) { - parent_.connection().dispatcher().post([this, disable]() -> void { + postToConnectionThread([this, disable]() -> void { { absl::MutexLock lock(&lock_); - if (saw_reset_) { + if (!parent_.connected() || saw_reset_) { // Encoded already deleted. return; } @@ -319,14 +319,14 @@ class TestHttp1ServerConnectionImpl : public Http::Http1::ServerConnectionImpl { }; FakeHttpConnection::FakeHttpConnection( - FakeUpstream& fake_upstream, SharedConnectionWrapper& shared_connection, Type type, + FakeUpstream& fake_upstream, SharedConnectionWrapper& shared_connection, Http::CodecType type, Event::TestTimeSystem& time_system, uint32_t max_request_headers_kb, uint32_t max_request_headers_count, envoy::config::core::v3::HttpProtocolOptions::HeadersWithUnderscoresAction headers_with_underscores_action) : FakeConnectionBase(shared_connection, time_system), type_(type) { ASSERT(max_request_headers_count != 0); - if (type == Type::HTTP1) { + if (type == Http::CodecType::HTTP1) { Http::Http1Settings http1_settings; // For the purpose of testing, we always have the upstream encode the trailers if any http1_settings.enable_trailers_ = true; @@ -334,14 +334,14 @@ FakeHttpConnection::FakeHttpConnection( codec_ = std::make_unique( shared_connection_.connection(), stats, *this, http1_settings, max_request_headers_kb, max_request_headers_count, headers_with_underscores_action); - } else if (type == Type::HTTP2) { + } else if (type == Http::CodecType::HTTP2) { envoy::config::core::v3::Http2ProtocolOptions http2_options = fake_upstream.http2Options(); Http::Http2::CodecStats& stats = fake_upstream.http2CodecStats(); codec_ = std::make_unique( shared_connection_.connection(), *this, stats, random_, http2_options, max_request_headers_kb, max_request_headers_count, headers_with_underscores_action); } else { - ASSERT(type == Type::HTTP3); + ASSERT(type == Http::CodecType::HTTP3); #ifdef ENVOY_ENABLE_QUIC Http::Http3::CodecStats& stats = fake_upstream.http3CodecStats(); codec_ = std::make_unique( @@ -380,25 +380,25 @@ Http::RequestDecoder& FakeHttpConnection::newStream(Http::ResponseEncoder& encod } void FakeHttpConnection::onGoAway(Http::GoAwayErrorCode code) { - ASSERT(type_ >= Type::HTTP2); + ASSERT(type_ >= Http::CodecType::HTTP2); // Usually indicates connection level errors, no operations are needed since // the connection will be closed soon. ENVOY_LOG(info, "FakeHttpConnection receives GOAWAY: ", code); } void FakeHttpConnection::encodeGoAway() { - ASSERT(type_ >= Type::HTTP2); + ASSERT(type_ >= Http::CodecType::HTTP2); - shared_connection_.connection().dispatcher().post([this]() { codec_->goAway(); }); + postToConnectionThread([this]() { codec_->goAway(); }); } void FakeHttpConnection::encodeProtocolError() { - ASSERT(type_ >= Type::HTTP2); + ASSERT(type_ >= Http::CodecType::HTTP2); Http::Http2::ServerConnectionImpl* codec = dynamic_cast(codec_.get()); ASSERT(codec != nullptr); - shared_connection_.connection().dispatcher().post([codec]() { + postToConnectionThread([codec]() { Http::Status status = codec->protocolErrorForTest(); ASSERT(Http::getStatusCode(status) == Http::StatusCode::CodecProtocolError); }); @@ -429,6 +429,14 @@ AssertionResult FakeConnectionBase::waitForHalfClose(milliseconds timeout) { return AssertionSuccess(); } +void FakeConnectionBase::postToConnectionThread(std::function cb) { + ++pending_cbs_; + dispatcher_.post([this, cb]() { + cb(); + --pending_cbs_; + }); +} + AssertionResult FakeHttpConnection::waitForNewStream(Event::Dispatcher& client_dispatcher, FakeStreamPtr& stream, std::chrono::milliseconds timeout) { @@ -505,8 +513,9 @@ FakeUpstream::FakeUpstream(Network::TransportSocketFactoryPtr&& transport_socket dispatcher_(api_->allocateDispatcher("fake_upstream")), handler_(new Server::ConnectionHandlerImpl(*dispatcher_, 0)), config_(config), read_disable_on_new_connection_(true), enable_half_close_(config.enable_half_close_), - listener_(*this, http_type_ == FakeHttpConnection::Type::HTTP3), - filter_chain_(Network::Test::createEmptyFilterChain(std::move(transport_socket_factory))) { + listener_(*this, http_type_ == Http::CodecType::HTTP3), + filter_chain_(Network::Test::createEmptyFilterChain(std::move(transport_socket_factory))), + stats_scope_(stats_store_.createScope("test_server_scope")) { ENVOY_LOG(info, "starting fake server at {}. UDP={} codec={}", localAddress()->asString(), config.udp_fake_upstream_.has_value(), FakeHttpConnection::typeToString(http_type_)); if (config.udp_fake_upstream_.has_value() && @@ -532,7 +541,7 @@ void FakeUpstream::cleanUp() { bool FakeUpstream::createNetworkFilterChain(Network::Connection& connection, const std::vector&) { absl::MutexLock lock(&lock_); - if (read_disable_on_new_connection_ && http_type_ != FakeHttpConnection::Type::HTTP3) { + if (read_disable_on_new_connection_ && http_type_ != Http::CodecType::HTTP3) { // Disable early close detection to avoid closing the network connection before full // initialization is complete. connection.detectEarlyCloseWhenReadDisabled(false); @@ -545,7 +554,7 @@ bool FakeUpstream::createNetworkFilterChain(Network::Connection& connection, // Normally we don't associate a logical network connection with a FakeHttpConnection until // waitForHttpConnection is called, but QUIC needs to be set up as packets come in, so we do // not lazily create for HTTP/3 - if (http_type_ == FakeHttpConnection::Type::HTTP3) { + if (http_type_ == Http::CodecType::HTTP3) { quic_connections_.push_back(std::make_unique( *this, consumeConnection(), http_type_, time_system_, config_.max_request_headers_kb_, config_.max_request_headers_count_, config_.headers_with_underscores_action_)); @@ -582,7 +591,7 @@ AssertionResult FakeUpstream::waitForHttpConnection(Event::Dispatcher& client_di // As noted in createNetworkFilterChain, HTTP3 FakeHttpConnections are not // lazily created, so HTTP3 needs a different wait path here. - if (http_type_ == FakeHttpConnection::Type::HTTP3) { + if (http_type_ == Http::CodecType::HTTP3) { if (quic_connections_.empty() && !waitForWithDispatcherRun( time_system_, lock_, @@ -686,12 +695,11 @@ SharedConnectionWrapper& FakeUpstream::consumeConnection() { auto* const connection_wrapper = new_connections_.front().get(); // Skip the thread safety check if the network connection has already been freed since there's no // alternate way to get access to the dispatcher. - ASSERT(!connection_wrapper->connected() || - connection_wrapper->connection().dispatcher().isThreadSafe()); + ASSERT(!connection_wrapper->connected() || connection_wrapper->dispatcher().isThreadSafe()); connection_wrapper->setParented(); connection_wrapper->moveBetweenLists(new_connections_, consumed_connections_); if (read_disable_on_new_connection_ && connection_wrapper->connected() && - http_type_ != FakeHttpConnection::Type::HTTP3) { + http_type_ != Http::CodecType::HTTP3) { // Re-enable read and early close detection. auto& connection = connection_wrapper->connection(); connection.detectEarlyCloseWhenReadDisabled(true); @@ -778,7 +786,7 @@ void FakeRawConnection::initialize() { ENVOY_LOG(warn, "FakeRawConnection::initialize: network connection is already disconnected"); return; } - ASSERT(shared_connection_.connection().dispatcher().isThreadSafe()); + ASSERT(shared_connection_.dispatcher().isThreadSafe()); shared_connection_.connection().addReadFilter(filter); } diff --git a/test/integration/fake_upstream.h b/test/integration/fake_upstream.h index eb3e9ac757095..184c2a7d96591 100644 --- a/test/integration/fake_upstream.h +++ b/test/integration/fake_upstream.h @@ -265,7 +265,8 @@ class SharedConnectionWrapper : public Network::ConnectionCallbacks, public: using DisconnectCallback = std::function; - SharedConnectionWrapper(Network::Connection& connection) : connection_(connection) { + SharedConnectionWrapper(Network::Connection& connection) + : connection_(connection), dispatcher_(connection_.dispatcher()) { connection_.addConnectionCallbacks(*this); } @@ -285,6 +286,8 @@ class SharedConnectionWrapper : public Network::ConnectionCallbacks, void onAboveWriteBufferHighWatermark() override {} void onBelowWriteBufferLowWatermark() override {} + Event::Dispatcher& dispatcher() { return dispatcher_; } + bool connected() { absl::MutexLock lock(&lock_); return connectedLockHeld(); @@ -355,6 +358,7 @@ class SharedConnectionWrapper : public Network::ConnectionCallbacks, private: Network::Connection& connection_; + Event::Dispatcher& dispatcher_; absl::Mutex lock_; bool parented_ ABSL_GUARDED_BY(lock_){}; bool disconnected_ ABSL_GUARDED_BY(lock_){}; @@ -370,6 +374,7 @@ class FakeConnectionBase : public Logger::Loggable { virtual ~FakeConnectionBase() { absl::MutexLock lock(&lock_); ASSERT(initialized_); + ASSERT(pending_cbs_ == 0); } ABSL_MUST_USE_RESULT @@ -395,16 +400,20 @@ class FakeConnectionBase : public Logger::Loggable { Network::Connection& connection() const { return shared_connection_.connection(); } bool connected() const { return shared_connection_.connected(); } + void postToConnectionThread(std::function cb); + protected: FakeConnectionBase(SharedConnectionWrapper& shared_connection, Event::TestTimeSystem& time_system) : shared_connection_(shared_connection), lock_(shared_connection.lock()), - time_system_(time_system) {} + dispatcher_(shared_connection_.dispatcher()), time_system_(time_system) {} SharedConnectionWrapper& shared_connection_; absl::Mutex& lock_; // TODO(mattklein123): Use the shared connection lock and figure out better // guarded by annotations. + Event::Dispatcher& dispatcher_; bool initialized_ ABSL_GUARDED_BY(lock_){}; bool half_closed_ ABSL_GUARDED_BY(lock_){}; + std::atomic pending_cbs_{}; Event::TestTimeSystem& time_system_; }; @@ -413,22 +422,23 @@ class FakeConnectionBase : public Logger::Loggable { */ class FakeHttpConnection : public Http::ServerConnectionCallbacks, public FakeConnectionBase { public: - enum class Type { HTTP1, HTTP2, HTTP3 }; - static absl::string_view typeToString(Type type) { + // This is a legacy alias. + using Type = Envoy::Http::CodecType; + static absl::string_view typeToString(Http::CodecType type) { switch (type) { - case Type::HTTP1: + case Http::CodecType::HTTP1: return "http1"; - case Type::HTTP2: + case Http::CodecType::HTTP2: return "http2"; - case Type::HTTP3: + case Http::CodecType::HTTP3: return "http3"; } return "invalid"; } FakeHttpConnection(FakeUpstream& fake_upstream, SharedConnectionWrapper& shared_connection, - Type type, Event::TestTimeSystem& time_system, uint32_t max_request_headers_kb, - uint32_t max_request_headers_count, + Http::CodecType type, Event::TestTimeSystem& time_system, + uint32_t max_request_headers_kb, uint32_t max_request_headers_count, envoy::config::core::v3::HttpProtocolOptions::HeadersWithUnderscoresAction headers_with_underscores_action); @@ -475,7 +485,7 @@ class FakeHttpConnection : public Http::ServerConnectionCallbacks, public FakeCo FakeHttpConnection& parent_; }; - const Type type_; + const Http::CodecType type_; Http::ServerConnectionPtr codec_; std::list new_streams_ ABSL_GUARDED_BY(lock_); testing::NiceMock random_; @@ -559,7 +569,7 @@ struct FakeUpstreamConfig { } Event::TestTimeSystem& time_system_; - FakeHttpConnection::Type upstream_protocol_{FakeHttpConnection::Type::HTTP1}; + Http::CodecType upstream_protocol_{Http::CodecType::HTTP1}; bool enable_half_close_{}; absl::optional udp_fake_upstream_; envoy::config::core::v3::Http2ProtocolOptions http2_options_; @@ -593,7 +603,7 @@ class FakeUpstream : Logger::Loggable, Network::Address::IpVersion version, const FakeUpstreamConfig& config); ~FakeUpstream() override; - FakeHttpConnection::Type httpType() { return http_type_; } + Http::CodecType httpType() { return http_type_; } // Returns the new connection via the connection argument. ABSL_MUST_USE_RESULT @@ -653,15 +663,15 @@ class FakeUpstream : Logger::Loggable, void cleanUp(); Http::Http1::CodecStats& http1CodecStats() { - return Http::Http1::CodecStats::atomicGet(http1_codec_stats_, stats_store_); + return Http::Http1::CodecStats::atomicGet(http1_codec_stats_, *stats_scope_); } Http::Http2::CodecStats& http2CodecStats() { - return Http::Http2::CodecStats::atomicGet(http2_codec_stats_, stats_store_); + return Http::Http2::CodecStats::atomicGet(http2_codec_stats_, *stats_scope_); } Http::Http3::CodecStats& http3CodecStats() { - return Http::Http3::CodecStats::atomicGet(http3_codec_stats_, stats_store_); + return Http::Http3::CodecStats::atomicGet(http3_codec_stats_, *stats_scope_); } // Write into the outbound buffer of the network connection at the specified index. @@ -678,7 +688,7 @@ class FakeUpstream : Logger::Loggable, protected: Stats::IsolatedStoreImpl stats_store_; - const FakeHttpConnection::Type http_type_; + const Http::CodecType http_type_; private: FakeUpstream(Network::TransportSocketFactoryPtr&& transport_socket_factory, @@ -826,6 +836,7 @@ class FakeUpstream : Logger::Loggable, FakeListener listener_; const Network::FilterChainSharedPtr filter_chain_; std::list received_datagrams_ ABSL_GUARDED_BY(lock_); + Stats::ScopePtr stats_scope_; Http::Http1::CodecStats::AtomicPtr http1_codec_stats_; Http::Http2::CodecStats::AtomicPtr http2_codec_stats_; Http::Http3::CodecStats::AtomicPtr http3_codec_stats_; diff --git a/test/integration/filter_manager_integration_test.cc b/test/integration/filter_manager_integration_test.cc index 6d4ec55cf34bf..2640f5a17472f 100644 --- a/test/integration/filter_manager_integration_test.cc +++ b/test/integration/filter_manager_integration_test.cc @@ -582,7 +582,7 @@ TEST_P(FilterChainAccessLogTest, FilterChainName) { */ class InjectDataWithHttpConnectionManagerIntegrationTest : public testing::TestWithParam< - std::tuple>, + std::tuple>, public HttpIntegrationTest, public TestWithAuxiliaryFilter { public: @@ -590,12 +590,12 @@ class InjectDataWithHttpConnectionManagerIntegrationTest // FooTestCase.BarInstance/IPv4_Http_no_inject_data static std::string testParamsToString( const testing::TestParamInfo< - std::tuple>& params) { + std::tuple>& params) { return fmt::format( "{}_{}_{}", TestUtility::ipTestParamsToString(testing::TestParamInfo( std::get<0>(params.param), params.index)), - (std::get<1>(params.param) == Http::CodecClient::Type::HTTP2 ? "Http2" : "Http"), + (std::get<1>(params.param) == Http::CodecType::HTTP2 ? "Http2" : "Http"), std::regex_replace(std::get<2>(params.param), invalid_param_name_regex(), "_")); } @@ -625,8 +625,7 @@ class InjectDataWithHttpConnectionManagerIntegrationTest INSTANTIATE_TEST_SUITE_P( Params, InjectDataWithHttpConnectionManagerIntegrationTest, testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), - testing::Values(Http::CodecClient::Type::HTTP1, - Http::CodecClient::Type::HTTP2), + testing::Values(Http::CodecType::HTTP1, Http::CodecType::HTTP2), testing::ValuesIn(auxiliary_filters())), InjectDataWithHttpConnectionManagerIntegrationTest::testParamsToString); diff --git a/test/integration/filters/BUILD b/test/integration/filters/BUILD index 3c5a4fa79d660..7285345d40549 100644 --- a/test/integration/filters/BUILD +++ b/test/integration/filters/BUILD @@ -235,6 +235,7 @@ envoy_cc_test_library( srcs = [ "pause_filter_for_quic.cc", ], + tags = ["nofips"], deps = [ "//include/envoy/http:filter_interface", "//include/envoy/registry", diff --git a/test/integration/filters/test_socket_interface.h b/test/integration/filters/test_socket_interface.h index e32f8fe4a773d..07980b5a9e59a 100644 --- a/test/integration/filters/test_socket_interface.h +++ b/test/integration/filters/test_socket_interface.h @@ -27,12 +27,30 @@ class TestIoSocketHandle : public IoSocketHandleImpl { bool socket_v6only = false, absl::optional domain = absl::nullopt) : IoSocketHandleImpl(fd, socket_v6only, domain), writev_override_(writev_override_proc) {} + void initializeFileEvent(Event::Dispatcher& dispatcher, Event::FileReadyCb cb, + Event::FileTriggerType trigger, uint32_t events) override { + absl::MutexLock lock(&mutex_); + dispatcher_ = &dispatcher; + IoSocketHandleImpl::initializeFileEvent(dispatcher, cb, trigger, events); + } + + // Schedule resumption on the IoHandle by posting a callback to the IoHandle's dispatcher. Note + // that this operation is inherently racy, nothing guarantees that the TestIoSocketHandle is not + // deleted before the posted callback executes. + void activateInDispatcherThread(uint32_t events) { + absl::MutexLock lock(&mutex_); + RELEASE_ASSERT(dispatcher_ != nullptr, "null dispatcher"); + dispatcher_->post([this, events]() { activateFileEvents(events); }); + } + private: IoHandlePtr accept(struct sockaddr* addr, socklen_t* addrlen) override; Api::IoCallUint64Result writev(const Buffer::RawSlice* slices, uint64_t num_slice) override; IoHandlePtr duplicate() override; const WritevOverrideProc writev_override_; + absl::Mutex mutex_; + Event::Dispatcher* dispatcher_ ABSL_GUARDED_BY(mutex_) = nullptr; }; /** diff --git a/test/integration/h1_fuzz.h b/test/integration/h1_fuzz.h index f1179598e0e75..2d993f6b49022 100644 --- a/test/integration/h1_fuzz.h +++ b/test/integration/h1_fuzz.h @@ -12,7 +12,7 @@ namespace Envoy { class H1FuzzIntegrationTest : public HttpIntegrationTest { public: H1FuzzIntegrationTest(Network::Address::IpVersion version) - : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, version) {} + : HttpIntegrationTest(Http::CodecType::HTTP1, version) {} void initialize() override; void replay(const test::integration::CaptureFuzzTestCase&, bool ignore_response); diff --git a/test/integration/h2_capture_direct_response_fuzz_test.cc b/test/integration/h2_capture_direct_response_fuzz_test.cc index fd7c2e9ef0cc0..1c9ec4c27804c 100644 --- a/test/integration/h2_capture_direct_response_fuzz_test.cc +++ b/test/integration/h2_capture_direct_response_fuzz_test.cc @@ -9,8 +9,8 @@ void H2FuzzIntegrationTest::initialize() { const std::string prefix("/"); const Http::Code status(Http::Code::OK); - setDownstreamProtocol(Http::CodecClient::Type::HTTP2); - setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); + setDownstreamProtocol(Http::CodecType::HTTP2); + setUpstreamProtocol(Http::CodecType::HTTP2); config_helper_.addConfigModifier( [&body, &prefix]( diff --git a/test/integration/h2_capture_fuzz_test.cc b/test/integration/h2_capture_fuzz_test.cc index f07c927fd8380..51db5f87a6e7a 100644 --- a/test/integration/h2_capture_fuzz_test.cc +++ b/test/integration/h2_capture_fuzz_test.cc @@ -14,8 +14,8 @@ void H2FuzzIntegrationTest::initialize() { config_helper_.addConfigModifier( [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& hcm) -> void { hcm.mutable_http2_protocol_options()->set_allow_metadata(true); }); - setDownstreamProtocol(Http::CodecClient::Type::HTTP2); - setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); + setDownstreamProtocol(Http::CodecType::HTTP2); + setUpstreamProtocol(Http::CodecType::HTTP2); HttpIntegrationTest::initialize(); } diff --git a/test/integration/h2_fuzz.h b/test/integration/h2_fuzz.h index 48498d7854324..97a19b9ebc272 100644 --- a/test/integration/h2_fuzz.h +++ b/test/integration/h2_fuzz.h @@ -14,7 +14,7 @@ namespace Envoy { class H2FuzzIntegrationTest : public HttpIntegrationTest { public: H2FuzzIntegrationTest(Network::Address::IpVersion version) - : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, version) {} + : HttpIntegrationTest(Http::CodecType::HTTP2, version) {} void initialize() override; void replay(const test::integration::H2CaptureFuzzTestCase&, bool ignore_response); diff --git a/test/integration/hds_integration_test.cc b/test/integration/hds_integration_test.cc index 63ae64a0b500c..15ef2c8f36d1e 100644 --- a/test/integration/hds_integration_test.cc +++ b/test/integration/hds_integration_test.cc @@ -30,10 +30,10 @@ namespace { class HdsIntegrationTest : public Grpc::VersionedGrpcClientIntegrationParamTest, public HttpIntegrationTest { public: - HdsIntegrationTest() : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, ipVersion()) {} + HdsIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, ipVersion()) {} void createUpstreams() override { - addFakeUpstream(FakeHttpConnection::Type::HTTP2); + addFakeUpstream(Http::CodecType::HTTP2); hds_upstream_ = fake_upstreams_.back().get(); HttpIntegrationTest::createUpstreams(); } @@ -370,7 +370,7 @@ class HdsIntegrationTest : public Grpc::VersionedGrpcClientIntegrationParamTest, FakeHttpConnectionPtr host_fake_connection_; FakeHttpConnectionPtr host2_fake_connection_; FakeRawConnectionPtr host_fake_raw_connection_; - FakeHttpConnection::Type http_conn_type_{FakeHttpConnection::Type::HTTP1}; + Http::CodecType http_conn_type_{Http::CodecType::HTTP1}; bool tls_hosts_{false}; static constexpr int MaxTimeout = 100; @@ -959,7 +959,7 @@ TEST_P(HdsIntegrationTest, SingleEndpointHealthyTlsHttp2) { tls_hosts_ = true; // Change hosts to operate over HTTP/2 instead of default HTTP. - http_conn_type_ = FakeHttpConnection::Type::HTTP2; + http_conn_type_ = Http::CodecType::HTTP2; initialize(); diff --git a/test/integration/header_casing_integration_test.cc b/test/integration/header_casing_integration_test.cc index c5707660ec826..f16994578c98b 100644 --- a/test/integration/header_casing_integration_test.cc +++ b/test/integration/header_casing_integration_test.cc @@ -13,11 +13,11 @@ namespace Envoy { class HeaderCasingIntegrationTest : public testing::TestWithParam, public HttpIntegrationTest { public: - HeaderCasingIntegrationTest() : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, GetParam()) {} + HeaderCasingIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam()) {} void SetUp() override { - setDownstreamProtocol(Http::CodecClient::Type::HTTP1); - setUpstreamProtocol(FakeHttpConnection::Type::HTTP1); + setDownstreamProtocol(Http::CodecType::HTTP1); + setUpstreamProtocol(Http::CodecType::HTTP1); } void initialize() override { diff --git a/test/integration/header_integration_test.cc b/test/integration/header_integration_test.cc index 073c05bf097b2..0d6c37098d2cb 100644 --- a/test/integration/header_integration_test.cc +++ b/test/integration/header_integration_test.cc @@ -180,8 +180,7 @@ class HeaderIntegrationTest : public testing::TestWithParam>, public HttpIntegrationTest { public: - HeaderIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, std::get<0>(GetParam())) {} + HeaderIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, std::get<0>(GetParam())) {} bool routerSuppressEnvoyHeaders() const { return std::get<1>(GetParam()); } @@ -370,7 +369,7 @@ class HeaderIntegrationTest HttpIntegrationTest::createUpstreams(); if (use_eds_) { - addFakeUpstream(FakeHttpConnection::Type::HTTP2); + addFakeUpstream(Http::CodecType::HTTP2); } } diff --git a/test/integration/header_prefix_integration_test.cc b/test/integration/header_prefix_integration_test.cc index 52ca839c76072..99db2e41393c6 100644 --- a/test/integration/header_prefix_integration_test.cc +++ b/test/integration/header_prefix_integration_test.cc @@ -60,7 +60,7 @@ TEST_P(HeaderPrefixIntegrationTest, FailedCustomHeaderPrefix) { INSTANTIATE_TEST_SUITE_P(Protocols, HeaderPrefixIntegrationTest, testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams( - {Http::CodecClient::Type::HTTP1, Http::CodecClient::Type::HTTP2}, - {FakeHttpConnection::Type::HTTP1})), + {Http::CodecType::HTTP1, Http::CodecType::HTTP2}, + {Http::CodecType::HTTP1})), HttpProtocolIntegrationTest::protocolTestParamsToString); } // namespace Envoy diff --git a/test/integration/health_check_integration_test.cc b/test/integration/health_check_integration_test.cc index e280b3d2b4cf2..5870929ce3e9b 100644 --- a/test/integration/health_check_integration_test.cc +++ b/test/integration/health_check_integration_test.cc @@ -17,10 +17,9 @@ namespace { // checking after Envoy and the hosts are initialized. class HealthCheckIntegrationTestBase : public HttpIntegrationTest { public: - HealthCheckIntegrationTestBase( - Network::Address::IpVersion ip_version, - FakeHttpConnection::Type upstream_protocol = FakeHttpConnection::Type::HTTP2) - : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ip_version, + HealthCheckIntegrationTestBase(Network::Address::IpVersion ip_version, + Http::CodecType upstream_protocol = Http::CodecType::HTTP2) + : HttpIntegrationTest(Http::CodecType::HTTP2, ip_version, ConfigHelper::discoveredClustersBootstrap("GRPC")), ip_version_(ip_version), upstream_protocol_(upstream_protocol) {} @@ -48,8 +47,8 @@ class HealthCheckIntegrationTestBase : public HttpIntegrationTest { // BaseIntegrationTest::createUpstreams() (which is part of initialize()). // Make sure this number matches the size of the 'clusters' repeated field in the bootstrap // config that you use! - setUpstreamCount(1); // the CDS cluster - setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); // CDS uses gRPC uses HTTP2. + setUpstreamCount(1); // the CDS cluster + setUpstreamProtocol(Http::CodecType::HTTP2); // CDS uses gRPC uses HTTP2. // HttpIntegrationTest::initialize() does many things: // 1) It appends to fake_upstreams_ as many as you asked for via setUpstreamCount(). @@ -126,12 +125,12 @@ class HealthCheckIntegrationTestBase : public HttpIntegrationTest { static constexpr size_t clusters_num_ = 2; std::array clusters_{{{"cluster_1"}, {"cluster_2"}}}; Network::Address::IpVersion ip_version_; - FakeHttpConnection::Type upstream_protocol_; + Http::CodecType upstream_protocol_; }; struct HttpHealthCheckIntegrationTestParams { Network::Address::IpVersion ip_version; - FakeHttpConnection::Type upstream_protocol; + Http::CodecType upstream_protocol; }; class HttpHealthCheckIntegrationTestBase @@ -148,8 +147,7 @@ class HttpHealthCheckIntegrationTestBase std::vector ret; for (auto ip_version : TestEnvironment::getIpVersionsForTest()) { - for (auto upstream_protocol : - {FakeHttpConnection::Type::HTTP1, FakeHttpConnection::Type::HTTP2}) { + for (auto upstream_protocol : {Http::CodecType::HTTP1, Http::CodecType::HTTP2}) { ret.push_back(HttpHealthCheckIntegrationTestParams{ip_version, upstream_protocol}); } } @@ -160,8 +158,8 @@ class HttpHealthCheckIntegrationTestBase const ::testing::TestParamInfo& params) { return absl::StrCat( (params.param.ip_version == Network::Address::IpVersion::v4 ? "IPv4_" : "IPv6_"), - (params.param.upstream_protocol == FakeHttpConnection::Type::HTTP2 ? "Http2Upstream" - : "HttpUpstream")); + (params.param.upstream_protocol == Http::CodecType::HTTP2 ? "Http2Upstream" + : "HttpUpstream")); } void TearDown() override { @@ -173,9 +171,8 @@ class HttpHealthCheckIntegrationTestBase // check probe to be received. void initHttpHealthCheck(uint32_t cluster_idx) { const envoy::type::v3::CodecClientType codec_client_type = - (FakeHttpConnection::Type::HTTP1 == upstream_protocol_) - ? envoy::type::v3::CodecClientType::HTTP1 - : envoy::type::v3::CodecClientType::HTTP2; + (Http::CodecType::HTTP1 == upstream_protocol_) ? envoy::type::v3::CodecClientType::HTTP1 + : envoy::type::v3::CodecClientType::HTTP2; auto& cluster_data = clusters_[cluster_idx]; auto* health_check = addHealthCheck(cluster_data.cluster_); @@ -315,7 +312,7 @@ TEST_P(HttpHealthCheckIntegrationTest, SingleEndpointGoAway) { initialize(); // GOAWAY doesn't exist in HTTP1. - if (upstream_protocol_ == FakeHttpConnection::Type::HTTP1) { + if (upstream_protocol_ == Http::CodecType::HTTP1) { return; } @@ -377,7 +374,7 @@ TEST_P(RealTimeHttpHealthCheckIntegrationTest, SingleEndpointGoAwayErroSingleEnd initialize(); // GOAWAY doesn't exist in HTTP1. - if (upstream_protocol_ == FakeHttpConnection::Type::HTTP1) { + if (upstream_protocol_ == Http::CodecType::HTTP1) { return; } diff --git a/test/integration/http2_flood_integration_test.cc b/test/integration/http2_flood_integration_test.cc index 9d900353cb615..31ceac494591f 100644 --- a/test/integration/http2_flood_integration_test.cc +++ b/test/integration/http2_flood_integration_test.cc @@ -11,8 +11,8 @@ #include "common/network/socket_option_impl.h" #include "test/integration/autonomous_upstream.h" -#include "test/integration/filters/test_socket_interface.h" #include "test/integration/http_integration.h" +#include "test/integration/socket_interface_swap.h" #include "test/integration/tracked_watermark_buffer.h" #include "test/integration/utility.h" #include "test/mocks/http/mocks.h" @@ -31,70 +31,6 @@ const uint32_t ControlFrameFloodLimit = 100; const uint32_t AllFrameFloodLimit = 1000; } // namespace -class SocketInterfaceSwap { -public: - // Object of this class hold the state determining the IoHandle which - // should return EAGAIN from the `writev` call. - struct IoHandleMatcher { - bool shouldReturnEgain(uint32_t src_port, uint32_t dst_port) const { - absl::ReaderMutexLock lock(&mutex_); - return writev_returns_egain_ && (src_port == src_port_ || dst_port == dst_port_); - } - - // Source port to match. The port specified should be associated with a listener. - void setSourcePort(uint32_t port) { - absl::WriterMutexLock lock(&mutex_); - src_port_ = port; - } - - // Destination port to match. The port specified should be associated with a listener. - void setDestinationPort(uint32_t port) { - absl::WriterMutexLock lock(&mutex_); - dst_port_ = port; - } - - void setWritevReturnsEgain() { - absl::WriterMutexLock lock(&mutex_); - ASSERT(src_port_ != 0 || dst_port_ != 0); - writev_returns_egain_ = true; - } - - private: - mutable absl::Mutex mutex_; - uint32_t src_port_ ABSL_GUARDED_BY(mutex_) = 0; - uint32_t dst_port_ ABSL_GUARDED_BY(mutex_) = 0; - bool writev_returns_egain_ ABSL_GUARDED_BY(mutex_) = false; - }; - - SocketInterfaceSwap() { - Envoy::Network::SocketInterfaceSingleton::clear(); - test_socket_interface_loader_ = std::make_unique( - std::make_unique( - [writev_matcher = writev_matcher_]( - Envoy::Network::TestIoSocketHandle* io_handle, const Buffer::RawSlice*, - uint64_t) -> absl::optional { - if (writev_matcher->shouldReturnEgain(io_handle->localAddress()->ip()->port(), - io_handle->peerAddress()->ip()->port())) { - return Api::IoCallUint64Result( - 0, Api::IoErrorPtr(Network::IoSocketError::getIoSocketEagainInstance(), - Network::IoSocketError::deleteIoError)); - } - return absl::nullopt; - })); - } - - ~SocketInterfaceSwap() { - test_socket_interface_loader_.reset(); - Envoy::Network::SocketInterfaceSingleton::initialize(previous_socket_interface_); - } - -protected: - Envoy::Network::SocketInterface* const previous_socket_interface_{ - Envoy::Network::SocketInterfaceSingleton::getExisting()}; - std::shared_ptr writev_matcher_{std::make_shared()}; - std::unique_ptr test_socket_interface_loader_; -}; - // It is important that the new socket interface is installed before any I/O activity starts and // the previous one is restored after all I/O activity stops. Since the HttpIntegrationTest // destructor stops Envoy the SocketInterfaceSwap destructor needs to run after it. This order of @@ -131,8 +67,8 @@ INSTANTIATE_TEST_SUITE_P(IpVersions, Http2FloodMitigationTest, TestUtility::ipTestParamsToString); bool Http2FloodMitigationTest::initializeUpstreamFloodTest() { - setDownstreamProtocol(Http::CodecClient::Type::HTTP2); - setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); + setDownstreamProtocol(Http::CodecType::HTTP2); + setUpstreamProtocol(Http::CodecType::HTTP2); // set lower upstream outbound frame limits to make tests run faster config_helper_.setUpstreamOutboundFramesLimits(AllFrameFloodLimit, ControlFrameFloodLimit); initialize(); @@ -158,8 +94,8 @@ void Http2FloodMitigationTest::setNetworkConnectionBufferSize() { } void Http2FloodMitigationTest::beginSession() { - setDownstreamProtocol(Http::CodecClient::Type::HTTP2); - setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); + setDownstreamProtocol(Http::CodecType::HTTP2); + setUpstreamProtocol(Http::CodecType::HTTP2); // set lower outbound frame limits to make tests run faster config_helper_.setDownstreamOutboundFramesLimits(AllFrameFloodLimit, ControlFrameFloodLimit); initialize(); @@ -396,7 +332,7 @@ TEST_P(Http2FloodMitigationTest, Data) { // The factory will be used to create 4 buffers: the input and output buffers for request and // response pipelines. - EXPECT_EQ(4, buffer_factory->numBuffersCreated()); + EXPECT_EQ(8, buffer_factory->numBuffersCreated()); // Expect at least 1000 1 byte data frames in the output buffer. Each data frame comes with a // 9-byte frame header; 10 bytes per data frame, 10000 bytes total. The output buffer should also @@ -411,7 +347,7 @@ TEST_P(Http2FloodMitigationTest, Data) { EXPECT_GE(22000, buffer_factory->sumMaxBufferSizes()); // Verify that all buffers have watermarks set. EXPECT_THAT(buffer_factory->highWatermarkRange(), - testing::Pair(1024 * 1024 * 1024, 1024 * 1024 * 1024)); + testing::Pair(256 * 1024 * 1024, 1024 * 1024 * 1024)); } // Verify that the server can detect flood triggered by a DATA frame from a decoder filter call @@ -1554,8 +1490,8 @@ TEST_P(Http2FloodMitigationTest, RequestMetadata) { // Validate that the default configuration has flood protection enabled. TEST_P(Http2FloodMitigationTest, UpstreamFloodDetectionIsOnByDefault) { - setDownstreamProtocol(Http::CodecClient::Type::HTTP2); - setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); + setDownstreamProtocol(Http::CodecType::HTTP2); + setUpstreamProtocol(Http::CodecType::HTTP2); initialize(); floodClient(Http2Frame::makePingFrame(), diff --git a/test/integration/http_integration.cc b/test/integration/http_integration.cc index 2c225369cb8f4..96bec817e661e 100644 --- a/test/integration/http_integration.cc +++ b/test/integration/http_integration.cc @@ -55,15 +55,15 @@ namespace { using testing::HasSubstr; envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager::CodecType -typeToCodecType(Http::CodecClient::Type type) { +typeToCodecType(Http::CodecType type) { switch (type) { - case Http::CodecClient::Type::HTTP1: + case Http::CodecType::HTTP1: return envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager:: HTTP1; - case Http::CodecClient::Type::HTTP2: + case Http::CodecType::HTTP2: return envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager:: HTTP2; - case Http::CodecClient::Type::HTTP3: + case Http::CodecType::HTTP3: return envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager:: HTTP3; default: @@ -76,7 +76,7 @@ typeToCodecType(Http::CodecClient::Type type) { IntegrationCodecClient::IntegrationCodecClient( Event::Dispatcher& dispatcher, Random::RandomGenerator& random, Network::ClientConnectionPtr&& conn, Upstream::HostDescriptionConstSharedPtr host_description, - CodecClient::Type type) + Http::CodecType type) : CodecClientProd(type, std::move(conn), host_description, dispatcher, random), dispatcher_(dispatcher), callbacks_(*this), codec_callbacks_(*this) { connection_->addConnectionCallbacks(callbacks_); @@ -208,7 +208,7 @@ void IntegrationCodecClient::ConnectionCallbacks::onEvent(Network::ConnectionEve parent_.disconnected_ = true; parent_.connection_->dispatcher().exit(); } else { - if (parent_.type() == CodecClient::Type::HTTP3 && !parent_.connected_) { + if (parent_.type() == Http::CodecType::HTTP3 && !parent_.connected_) { // Before handshake gets established, any connection failure should exit the loop. I.e. a QUIC // connection may fail of INVALID_VERSION if both this client doesn't support any of the // versions the server advertised before handshake established. In this case the connection is @@ -221,7 +221,7 @@ void IntegrationCodecClient::ConnectionCallbacks::onEvent(Network::ConnectionEve Network::ClientConnectionPtr HttpIntegrationTest::makeClientConnectionWithOptions( uint32_t port, const Network::ConnectionSocket::OptionsSharedPtr& options) { - if (downstream_protocol_ <= Http::CodecClient::Type::HTTP2) { + if (downstream_protocol_ <= Http::CodecType::HTTP2) { return BaseIntegrationTest::makeClientConnectionWithOptions(port, options); } #ifdef ENVOY_ENABLE_QUIC @@ -270,7 +270,7 @@ IntegrationCodecClientPtr HttpIntegrationTest::makeRawHttpConnection( // in-connection version negotiation. auto codec = std::make_unique(*dispatcher_, random_, std::move(conn), host_description, downstream_protocol_); - if (downstream_protocol_ == Http::CodecClient::Type::HTTP3 && codec->disconnected()) { + if (downstream_protocol_ == Http::CodecType::HTTP3 && codec->disconnected()) { // Connection may get closed during version negotiation or handshake. // TODO(#8479) QUIC connection doesn't support in-connection version negotiationPropagate // INVALID_VERSION error to caller and let caller to use server advertised version list to @@ -288,7 +288,7 @@ HttpIntegrationTest::makeHttpConnection(Network::ClientConnectionPtr&& conn) { return codec; } -HttpIntegrationTest::HttpIntegrationTest(Http::CodecClient::Type downstream_protocol, +HttpIntegrationTest::HttpIntegrationTest(Http::CodecType downstream_protocol, Network::Address::IpVersion version, const std::string& config) : HttpIntegrationTest::HttpIntegrationTest( @@ -299,7 +299,7 @@ HttpIntegrationTest::HttpIntegrationTest(Http::CodecClient::Type downstream_prot }, version, config) {} -HttpIntegrationTest::HttpIntegrationTest(Http::CodecClient::Type downstream_protocol, +HttpIntegrationTest::HttpIntegrationTest(Http::CodecType downstream_protocol, const InstanceConstSharedPtrFn& upstream_address_fn, Network::Address::IpVersion version, const std::string& config) @@ -321,7 +321,7 @@ void HttpIntegrationTest::useAccessLog( HttpIntegrationTest::~HttpIntegrationTest() { cleanupUpstreamAndDownstream(); } void HttpIntegrationTest::initialize() { - if (downstream_protocol_ != Http::CodecClient::Type::HTTP3) { + if (downstream_protocol_ != Http::CodecType::HTTP3) { return BaseIntegrationTest::initialize(); } #ifdef ENVOY_ENABLE_QUIC @@ -341,7 +341,7 @@ void HttpIntegrationTest::initialize() { "udp://{}:{}", Network::Test::getLoopbackAddressUrlString(version_), lookupPort("http"))); // Needs to outlive all QUIC connections. auto quic_connection_persistent_info = std::make_unique( - *dispatcher_, *quic_transport_socket_factory_, timeSystem(), server_addr); + *dispatcher_, *quic_transport_socket_factory_, timeSystem(), server_addr, 0); // Config IETF QUIC flow control window. quic_connection_persistent_info->quic_config_ .SetInitialMaxStreamDataBytesIncomingBidirectionalToSend( @@ -356,7 +356,7 @@ void HttpIntegrationTest::initialize() { #endif } -void HttpIntegrationTest::setDownstreamProtocol(Http::CodecClient::Type downstream_protocol) { +void HttpIntegrationTest::setDownstreamProtocol(Http::CodecType downstream_protocol) { downstream_protocol_ = downstream_protocol; config_helper_.setClientCodec(typeToCodecType(downstream_protocol_)); } @@ -369,7 +369,7 @@ ConfigHelper::HttpModifierFunction HttpIntegrationTest::setEnableDownstreamTrail ConfigHelper::ConfigModifierFunction HttpIntegrationTest::setEnableUpstreamTrailersHttp1() { return [&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { RELEASE_ASSERT(bootstrap.mutable_static_resources()->clusters_size() == 1, ""); - if (fake_upstreams_[0]->httpType() == FakeHttpConnection::Type::HTTP1) { + if (fake_upstreams_[0]->httpType() == Http::CodecType::HTTP1) { ConfigHelper::HttpProtocolOptions protocol_options; protocol_options.mutable_explicit_http_config() ->mutable_http_protocol_options() @@ -671,7 +671,7 @@ void HttpIntegrationTest::testRouterUpstreamDisconnectBeforeRequestComplete() { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); ASSERT_TRUE(response->waitForEndStream()); - if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + if (downstream_protocol_ == Http::CodecType::HTTP1) { ASSERT_TRUE(codec_client_->waitForDisconnect()); } else { codec_client_->close(); @@ -699,7 +699,7 @@ void HttpIntegrationTest::testRouterUpstreamDisconnectBeforeResponseComplete( ASSERT_TRUE(fake_upstream_connection_->close()); ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); - if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + if (downstream_protocol_ == Http::CodecType::HTTP1) { ASSERT_TRUE(codec_client_->waitForDisconnect()); } else { ASSERT_TRUE(response->waitForReset()); @@ -727,7 +727,7 @@ void HttpIntegrationTest::testRouterDownstreamDisconnectBeforeRequestComplete( ASSERT_TRUE(upstream_request_->waitForHeadersComplete()); codec_client_->close(); - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() == Http::CodecType::HTTP1) { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); } else { ASSERT_TRUE(upstream_request_->waitForReset()); @@ -746,7 +746,7 @@ void HttpIntegrationTest::testRouterDownstreamDisconnectBeforeResponseComplete( #if defined(__APPLE__) || defined(WIN32) // Skip this test on OS/X + Windows: we can't detect the early close, and we // won't clean up the upstream connection until it times out. See #4294. - if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + if (downstream_protocol_ == Http::CodecType::HTTP1) { return; } #endif @@ -760,7 +760,7 @@ void HttpIntegrationTest::testRouterDownstreamDisconnectBeforeResponseComplete( response->waitForBodyData(512); codec_client_->close(); - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() == Http::CodecType::HTTP1) { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); } else { ASSERT_TRUE(upstream_request_->waitForReset()); @@ -788,7 +788,7 @@ void HttpIntegrationTest::testRouterUpstreamResponseBeforeRequestComplete() { upstream_request_->encodeData(512, true); ASSERT_TRUE(response->waitForEndStream()); - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() == Http::CodecType::HTTP1) { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); } else { ASSERT_TRUE(upstream_request_->waitForReset()); @@ -796,7 +796,7 @@ void HttpIntegrationTest::testRouterUpstreamResponseBeforeRequestComplete() { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); } - if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + if (downstream_protocol_ == Http::CodecType::HTTP1) { ASSERT_TRUE(codec_client_->waitForDisconnect()); } else { codec_client_->close(); @@ -824,7 +824,7 @@ void HttpIntegrationTest::testRetry() { waitForNextUpstreamRequest(); upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "503"}}, false); - if (fake_upstreams_[0]->httpType() == FakeHttpConnection::Type::HTTP1) { + if (fake_upstreams_[0]->httpType() == Http::CodecType::HTTP1) { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_)); } else { @@ -866,7 +866,7 @@ void HttpIntegrationTest::testRetryAttemptCountHeader() { EXPECT_EQ(atoi(std::string(upstream_request_->headers().getEnvoyAttemptCountValue()).c_str()), 1); - if (fake_upstreams_[0]->httpType() == FakeHttpConnection::Type::HTTP1) { + if (fake_upstreams_[0]->httpType() == Http::CodecType::HTTP1) { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_)); } else { @@ -905,7 +905,7 @@ void HttpIntegrationTest::testGrpcRetry() { waitForNextUpstreamRequest(); upstream_request_->encodeHeaders( Http::TestResponseHeaderMapImpl{{":status", "200"}, {"grpc-status", "1"}}, false); - if (fake_upstreams_[0]->httpType() == FakeHttpConnection::Type::HTTP1) { + if (fake_upstreams_[0]->httpType() == Http::CodecType::HTTP1) { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_)); } else { @@ -914,9 +914,8 @@ void HttpIntegrationTest::testGrpcRetry() { waitForNextUpstreamRequest(); upstream_request_->encodeHeaders(default_response_headers_, false); - upstream_request_->encodeData(512, - fake_upstreams_[0]->httpType() != FakeHttpConnection::Type::HTTP2); - if (fake_upstreams_[0]->httpType() == FakeHttpConnection::Type::HTTP2) { + upstream_request_->encodeData(512, fake_upstreams_[0]->httpType() != Http::CodecType::HTTP2); + if (fake_upstreams_[0]->httpType() == Http::CodecType::HTTP2) { upstream_request_->encodeTrailers(response_trailers); } @@ -927,7 +926,7 @@ void HttpIntegrationTest::testGrpcRetry() { EXPECT_TRUE(response->complete()); EXPECT_EQ("200", response->headers().getStatusValue()); EXPECT_EQ(512U, response->body().size()); - if (fake_upstreams_[0]->httpType() == FakeHttpConnection::Type::HTTP2) { + if (fake_upstreams_[0]->httpType() == Http::CodecType::HTTP2) { EXPECT_THAT(*response->trailers(), HeaderMapEqualRef(&response_trailers)); } } @@ -1091,7 +1090,7 @@ void HttpIntegrationTest::testTwoRequests(bool network_backup) { typed_config: "@type": type.googleapis.com/google.protobuf.Empty )EOF", - downstreamProtocol() == Http::CodecClient::Type::HTTP3 ? "-for-quic" : "")); + downstreamProtocol() == Http::CodecType::HTTP3 ? "-for-quic" : "")); } initialize(); @@ -1148,9 +1147,9 @@ void HttpIntegrationTest::testLargeRequestUrl(uint32_t url_size, uint32_t max_he auto encoder_decoder = codec_client_->startRequest(big_headers); auto response = std::move(encoder_decoder.second); - if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + if (downstream_protocol_ == Http::CodecType::HTTP1) { ASSERT_TRUE(codec_client_->waitForDisconnect()); - EXPECT_TRUE(response->complete()); + ASSERT_TRUE(response->complete()); EXPECT_EQ("431", response->headers().Status()->value().getStringView()); } else { ASSERT_TRUE(response->waitForReset()); @@ -1197,9 +1196,9 @@ void HttpIntegrationTest::testLargeRequestHeaders(uint32_t size, uint32_t count, auto encoder_decoder = codec_client_->startRequest(big_headers); auto response = std::move(encoder_decoder.second); - if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + if (downstream_protocol_ == Http::CodecType::HTTP1) { ASSERT_TRUE(codec_client_->waitForDisconnect()); - EXPECT_TRUE(response->complete()); + ASSERT_TRUE(response->complete()); EXPECT_EQ("431", response->headers().getStatusValue()); } else { ASSERT_TRUE(response->waitForReset()); @@ -1239,7 +1238,7 @@ void HttpIntegrationTest::testLargeRequestTrailers(uint32_t size, uint32_t max_s codec_client_->sendTrailers(*request_encoder_, request_trailers); if (size >= max_size) { - if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + if (downstream_protocol_ == Http::CodecType::HTTP1) { ASSERT_TRUE(codec_client_->waitForDisconnect()); EXPECT_TRUE(response->complete()); EXPECT_EQ("431", response->headers().getStatusValue()); @@ -1316,7 +1315,7 @@ void HttpIntegrationTest::testDownstreamResetBeforeResponseComplete() { response->waitForBodyData(512); codec_client_->sendReset(*request_encoder_); - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() == Http::CodecType::HTTP1) { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); } else { ASSERT_TRUE(upstream_request_->waitForReset()); @@ -1376,7 +1375,7 @@ void HttpIntegrationTest::testTrailers(uint64_t request_size, uint64_t response_ } } -void HttpIntegrationTest::testAdminDrain(Http::CodecClient::Type admin_request_type) { +void HttpIntegrationTest::testAdminDrain(Http::CodecType admin_request_type) { initialize(); uint32_t http_port = lookupPort("http"); @@ -1413,7 +1412,7 @@ void HttpIntegrationTest::testAdminDrain(Http::CodecClient::Type admin_request_t // Validate that port is closed and can be bound by other sockets. // This does not work for HTTP/3 because the port is not closed until the listener is completely // destroyed. TODO(danzh) Match TCP behavior as much as possible. - if (downstreamProtocol() != Http::CodecClient::Type::HTTP3) { + if (downstreamProtocol() != Http::CodecType::HTTP3) { ASSERT_TRUE(waitForPortAvailable(http_port)); } } @@ -1440,7 +1439,7 @@ void HttpIntegrationTest::testMaxStreamDuration() { test_server_->waitForCounterGe("cluster.cluster_0.upstream_rq_max_duration_reached", 1); - if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + if (downstream_protocol_ == Http::CodecType::HTTP1) { ASSERT_TRUE(codec_client_->waitForDisconnect()); } else { ASSERT_TRUE(response->waitForEndStream()); @@ -1472,7 +1471,7 @@ void HttpIntegrationTest::testMaxStreamDurationWithRetry(bool invoke_retry_upstr ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_)); ASSERT_TRUE(upstream_request_->waitForHeadersComplete()); - if (fake_upstreams_[0]->httpType() == FakeHttpConnection::Type::HTTP1) { + if (fake_upstreams_[0]->httpType() == Http::CodecType::HTTP1) { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_)); } else { @@ -1485,7 +1484,7 @@ void HttpIntegrationTest::testMaxStreamDurationWithRetry(bool invoke_retry_upstr if (invoke_retry_upstream_disconnect) { test_server_->waitForCounterGe("cluster.cluster_0.upstream_rq_max_duration_reached", 2); - if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + if (downstream_protocol_ == Http::CodecType::HTTP1) { ASSERT_TRUE(codec_client_->waitForDisconnect()); } else { ASSERT_TRUE(response->waitForEndStream()); @@ -1505,6 +1504,30 @@ void HttpIntegrationTest::testMaxStreamDurationWithRetry(bool invoke_retry_upstr } } +std::string HttpIntegrationTest::downstreamProtocolStatsRoot() const { + switch (downstreamProtocol()) { + case Http::CodecClient::Type::HTTP1: + return "http1"; + case Http::CodecClient::Type::HTTP2: + return "http2"; + case Http::CodecClient::Type::HTTP3: + return "http3"; + } + return "invalid"; +} + +std::string HttpIntegrationTest::upstreamProtocolStatsRoot() const { + switch (upstreamProtocol()) { + case FakeHttpConnection::Type::HTTP1: + return "http1"; + case FakeHttpConnection::Type::HTTP2: + return "http2"; + case FakeHttpConnection::Type::HTTP3: + return "http3"; + } + return "invalid"; +} + std::string HttpIntegrationTest::listenerStatPrefix(const std::string& stat_name) { if (version_ == Network::Address::IpVersion::v4) { return "listener.127.0.0.1_0." + stat_name; @@ -1532,8 +1555,8 @@ void Http2RawFrameIntegrationTest::startHttp2Session() { } void Http2RawFrameIntegrationTest::beginSession() { - setDownstreamProtocol(Http::CodecClient::Type::HTTP2); - setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); + setDownstreamProtocol(Http::CodecType::HTTP2); + setUpstreamProtocol(Http::CodecType::HTTP2); // set lower outbound frame limits to make tests run faster config_helper_.setDownstreamOutboundFramesLimits(1000, 100); initialize(); diff --git a/test/integration/http_integration.h b/test/integration/http_integration.h index 23227ce7f2fbe..c6f709aa189b5 100644 --- a/test/integration/http_integration.h +++ b/test/integration/http_integration.h @@ -25,7 +25,7 @@ class IntegrationCodecClient : public Http::CodecClientProd { IntegrationCodecClient(Event::Dispatcher& dispatcher, Random::RandomGenerator& random, Network::ClientConnectionPtr&& conn, Upstream::HostDescriptionConstSharedPtr host_description, - Http::CodecClient::Type type); + Http::CodecType type); IntegrationStreamDecoderPtr makeHeaderOnlyRequest(const Http::RequestHeaderMap& headers); IntegrationStreamDecoderPtr makeRequestWithBody(const Http::RequestHeaderMap& headers, @@ -89,11 +89,10 @@ using IntegrationCodecClientPtr = std::unique_ptr; */ class HttpIntegrationTest : public BaseIntegrationTest { public: - HttpIntegrationTest(Http::CodecClient::Type downstream_protocol, - Network::Address::IpVersion version, + HttpIntegrationTest(Http::CodecType downstream_protocol, Network::Address::IpVersion version, const std::string& config = ConfigHelper::httpProxyConfig()); - HttpIntegrationTest(Http::CodecClient::Type downstream_protocol, + HttpIntegrationTest(Http::CodecType downstream_protocol, const InstanceConstSharedPtrFn& upstream_address_fn, Network::Address::IpVersion version, const std::string& config = ConfigHelper::httpProxyConfig()); @@ -118,7 +117,7 @@ class HttpIntegrationTest : public BaseIntegrationTest { // Sets downstream_protocol_ and alters the HTTP connection manager codec type in the // config_helper_. - void setDownstreamProtocol(Http::CodecClient::Type type); + void setDownstreamProtocol(Http::CodecType type); // Enable the encoding/decoding of Http1 trailers downstream ConfigHelper::HttpModifierFunction setEnableDownstreamTrailersHttp1(); @@ -238,11 +237,14 @@ class HttpIntegrationTest : public BaseIntegrationTest { void testTrailers(uint64_t request_size, uint64_t response_size, bool request_trailers_present, bool response_trailers_present); // Test /drain_listener from admin portal. - void testAdminDrain(Http::CodecClient::Type admin_request_type); + void testAdminDrain(Http::CodecType admin_request_type); // Test max stream duration. void testMaxStreamDuration(); void testMaxStreamDurationWithRetry(bool invoke_retry_upstream_disconnect); - Http::CodecClient::Type downstreamProtocol() const { return downstream_protocol_; } + Http::CodecType downstreamProtocol() const { return downstream_protocol_; } + std::string downstreamProtocolStatsRoot() const; + // Return the upstream protocol part of the stats root. + std::string upstreamProtocolStatsRoot() const; // Prefix listener stat with IP:port, including IP version dependent loopback address. std::string listenerStatPrefix(const std::string& stat_name); @@ -263,7 +265,7 @@ class HttpIntegrationTest : public BaseIntegrationTest { Http::TestRequestHeaderMapImpl default_request_headers_{ {":method", "GET"}, {":path", "/test/long/url"}, {":scheme", "http"}, {":authority", "host"}}; // The codec type for the client-to-Envoy connection - Http::CodecClient::Type downstream_protocol_{Http::CodecClient::Type::HTTP1}; + Http::CodecType downstream_protocol_{Http::CodecType::HTTP1}; std::string access_log_name_; testing::NiceMock random_; @@ -275,7 +277,7 @@ class HttpIntegrationTest : public BaseIntegrationTest { class Http2RawFrameIntegrationTest : public HttpIntegrationTest { public: Http2RawFrameIntegrationTest(Network::Address::IpVersion version) - : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, version) {} + : HttpIntegrationTest(Http::CodecType::HTTP2, version) {} protected: void startHttp2Session(); diff --git a/test/integration/http_protocol_integration.cc b/test/integration/http_protocol_integration.cc index 5bdcd4aa9c5ff..0538a024d2541 100644 --- a/test/integration/http_protocol_integration.cc +++ b/test/integration/http_protocol_integration.cc @@ -4,8 +4,8 @@ namespace Envoy { std::vector HttpProtocolIntegrationTest::getProtocolTestParams( - const std::vector& downstream_protocols, - const std::vector& upstream_protocols) { + const std::vector& downstream_protocols, + const std::vector& upstream_protocols) { std::vector ret; for (auto ip_version : TestEnvironment::getIpVersionsForTest()) { @@ -14,8 +14,8 @@ std::vector HttpProtocolIntegrationTest::getProtocolTest #ifdef ENVOY_ENABLE_QUIC ret.push_back(HttpProtocolTestParams{ip_version, downstream_protocol, upstream_protocol}); #else - if (downstream_protocol == Http::CodecClient::Type::HTTP3 || - upstream_protocol == FakeHttpConnection::Type::HTTP3) { + if (downstream_protocol == Http::CodecType::HTTP3 || + upstream_protocol == Http::CodecType::HTTP3) { ENVOY_LOG_MISC(warn, "Skipping HTTP/3 as support is compiled out"); } else { ret.push_back(HttpProtocolTestParams{ip_version, downstream_protocol, upstream_protocol}); @@ -27,25 +27,25 @@ std::vector HttpProtocolIntegrationTest::getProtocolTest return ret; } -absl::string_view upstreamToString(FakeHttpConnection::Type type) { +absl::string_view upstreamToString(Http::CodecType type) { switch (type) { - case FakeHttpConnection::Type::HTTP1: + case Http::CodecType::HTTP1: return "HttpUpstream"; - case FakeHttpConnection::Type::HTTP2: + case Http::CodecType::HTTP2: return "Http2Upstream"; - case FakeHttpConnection::Type::HTTP3: + case Http::CodecType::HTTP3: return "Http3Upstream"; } return "UnknownUpstream"; } -absl::string_view downstreamToString(Http::CodecClient::Type type) { +absl::string_view downstreamToString(Http::CodecType type) { switch (type) { - case Http::CodecClient::Type::HTTP1: + case Http::CodecType::HTTP1: return "HttpDownstream_"; - case Http::CodecClient::Type::HTTP2: + case Http::CodecType::HTTP2: return "Http2Downstream_"; - case Http::CodecClient::Type::HTTP3: + case Http::CodecType::HTTP3: return "Http3Downstream_"; } return "UnknownDownstream"; diff --git a/test/integration/http_protocol_integration.h b/test/integration/http_protocol_integration.h index 05a91eed1bb1d..d66f53692919d 100644 --- a/test/integration/http_protocol_integration.h +++ b/test/integration/http_protocol_integration.h @@ -8,8 +8,8 @@ namespace Envoy { struct HttpProtocolTestParams { Network::Address::IpVersion version; - Http::CodecClient::Type downstream_protocol; - FakeHttpConnection::Type upstream_protocol; + Http::CodecType downstream_protocol; + Http::CodecType upstream_protocol; }; // Allows easy testing of Envoy code for HTTP/HTTP2 upstream/downstream. @@ -37,11 +37,11 @@ class HttpProtocolIntegrationTest : public testing::TestWithParam - getProtocolTestParams(const std::vector& downstream_protocols = - {Http::CodecClient::Type::HTTP1, Http::CodecClient::Type::HTTP2}, - const std::vector& upstream_protocols = { - FakeHttpConnection::Type::HTTP1, FakeHttpConnection::Type::HTTP2}); + static std::vector getProtocolTestParams( + const std::vector& downstream_protocols = {Http::CodecType::HTTP1, + Http::CodecType::HTTP2}, + const std::vector& upstream_protocols = {Http::CodecType::HTTP1, + Http::CodecType::HTTP2}); // Allows pretty printed test names of the form // FooTestCase.BarInstance/IPv4_Http2Downstream_HttpUpstream @@ -52,7 +52,7 @@ class HttpProtocolIntegrationTest : public testing::TestWithParamAdd(); filter->set_name("filter.unknown"); - filter->set_is_optional("true"); + filter->set_is_optional(true); // keep router the last auto size = hcm.http_filters_size(); hcm.mutable_http_filters()->SwapElements(size - 2, size - 1); diff --git a/test/integration/idle_timeout_integration_test.cc b/test/integration/idle_timeout_integration_test.cc index 40328d9a1971d..d640d54c5da92 100644 --- a/test/integration/idle_timeout_integration_test.cc +++ b/test/integration/idle_timeout_integration_test.cc @@ -68,7 +68,7 @@ class IdleTimeoutIntegrationTest : public HttpProtocolIntegrationTest { void waitForTimeout(IntegrationStreamDecoder& response, absl::string_view stat_name = "", absl::string_view stat_prefix = "http.config_test") { - if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + if (downstream_protocol_ == Http::CodecType::HTTP1) { ASSERT_TRUE(codec_client_->waitForDisconnect()); } else { ASSERT_TRUE(response.waitForReset()); @@ -357,7 +357,7 @@ TEST_P(IdleTimeoutIntegrationTest, RequestTimeoutUnconfiguredDoesNotTriggerOnBod TEST_P(IdleTimeoutIntegrationTest, RequestTimeoutTriggersOnRawIncompleteRequestWithHeaders) { // Omitting \r\n\r\n does not indicate incomplete request in HTTP2 - if (downstreamProtocol() == Envoy::Http::CodecClient::Type::HTTP2) { + if (downstreamProtocol() == Envoy::Http::CodecType::HTTP2) { return; } enable_request_timeout_ = true; @@ -370,7 +370,7 @@ TEST_P(IdleTimeoutIntegrationTest, RequestTimeoutTriggersOnRawIncompleteRequestW } TEST_P(IdleTimeoutIntegrationTest, RequestTimeoutDoesNotTriggerOnRawCompleteRequestWithHeaders) { - if (downstreamProtocol() == Envoy::Http::CodecClient::Type::HTTP2) { + if (downstreamProtocol() == Envoy::Http::CodecType::HTTP2) { return; } enable_request_timeout_ = true; diff --git a/test/integration/instantiate_protocol_integration_test.cc b/test/integration/instantiate_protocol_integration_test.cc index d789995f91701..b5c219a8c521a 100644 --- a/test/integration/instantiate_protocol_integration_test.cc +++ b/test/integration/instantiate_protocol_integration_test.cc @@ -6,8 +6,8 @@ namespace Envoy { // run with both HTTP/1 and HTTP/2 upstreams. INSTANTIATE_TEST_SUITE_P(Protocols, DownstreamProtocolIntegrationTest, testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams( - {Http::CodecClient::Type::HTTP1, Http::CodecClient::Type::HTTP2}, - {FakeHttpConnection::Type::HTTP1})), + {Http::CodecType::HTTP1, Http::CodecType::HTTP2}, + {Http::CodecType::HTTP1})), HttpProtocolIntegrationTest::protocolTestParamsToString); INSTANTIATE_TEST_SUITE_P(Protocols, ProtocolIntegrationTest, diff --git a/test/integration/integration_admin_test.cc b/test/integration/integration_admin_test.cc index 84622c5c92378..dd6e62d2bdfa9 100644 --- a/test/integration/integration_admin_test.cc +++ b/test/integration/integration_admin_test.cc @@ -30,8 +30,8 @@ namespace Envoy { INSTANTIATE_TEST_SUITE_P(Protocols, IntegrationAdminTest, testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams( - {Http::CodecClient::Type::HTTP1, Http::CodecClient::Type::HTTP2}, - {FakeHttpConnection::Type::HTTP1})), + {Http::CodecType::HTTP1, Http::CodecType::HTTP2}, + {Http::CodecType::HTTP1})), HttpProtocolIntegrationTest::protocolTestParamsToString); TEST_P(IntegrationAdminTest, HealthCheck) { @@ -280,19 +280,19 @@ TEST_P(IntegrationAdminTest, Admin) { EXPECT_EQ("text/plain; charset=UTF-8", ContentType(response)); switch (GetParam().downstream_protocol) { - case Http::CodecClient::Type::HTTP1: + case Http::CodecType::HTTP1: EXPECT_EQ(" Count Lookup\n" "\n" "total: 0\n", response->body()); break; - case Http::CodecClient::Type::HTTP2: + case Http::CodecType::HTTP2: EXPECT_EQ(" Count Lookup\n" "\n" "total: 0\n", response->body()); break; - case Http::CodecClient::Type::HTTP3: + case Http::CodecType::HTTP3: NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } @@ -490,7 +490,7 @@ TEST_P(IntegrationAdminTest, AdminCpuProfilerStart) { class IntegrationAdminIpv4Ipv6Test : public testing::Test, public HttpIntegrationTest { public: IntegrationAdminIpv4Ipv6Test() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, Network::Address::IpVersion::v4) {} + : HttpIntegrationTest(Http::CodecType::HTTP1, Network::Address::IpVersion::v4) {} void initialize() override { config_helper_.addConfigModifier( @@ -530,7 +530,7 @@ class StatsMatcherIntegrationTest public HttpIntegrationTest, public testing::WithParamInterface { public: - StatsMatcherIntegrationTest() : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, GetParam()) {} + StatsMatcherIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam()) {} void initialize() override { config_helper_.addConfigModifier( diff --git a/test/integration/integration_test.cc b/test/integration/integration_test.cc index a1ace5f25b9bf..c2406ddb79ac5 100644 --- a/test/integration/integration_test.cc +++ b/test/integration/integration_test.cc @@ -638,7 +638,7 @@ TEST_P(IntegrationTest, TestServerAllowChunkedLength) { TEST_P(IntegrationTest, TestClientAllowChunkedLength) { config_helper_.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { RELEASE_ASSERT(bootstrap.mutable_static_resources()->clusters_size() == 1, ""); - if (fake_upstreams_[0]->httpType() == FakeHttpConnection::Type::HTTP1) { + if (fake_upstreams_[0]->httpType() == Http::CodecType::HTTP1) { ConfigHelper::HttpProtocolOptions protocol_options; protocol_options.mutable_explicit_http_config() ->mutable_http_protocol_options() @@ -1399,40 +1399,6 @@ TEST_P(IntegrationTest, TestDelayedConnectionTeardownConfig) { EXPECT_EQ(codec_client_->lastConnectionEvent(), Network::ConnectionEvent::RemoteClose); } -// Test that delay closed connections are eventually force closed when the timeout triggers. -TEST_P(IntegrationTest, TestDelayedConnectionTeardownTimeoutTrigger) { - config_helper_.addFilter("{ name: encoder-decoder-buffer-filter, typed_config: { \"@type\": " - "type.googleapis.com/google.protobuf.Empty } }"); - config_helper_.setBufferLimits(1024, 1024); - config_helper_.addConfigModifier( - [](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& - hcm) { - // 200ms. - hcm.mutable_delayed_close_timeout()->set_nanos(200000000); - }); - - initialize(); - - codec_client_ = makeHttpConnection(lookupPort("http")); - - auto encoder_decoder = - codec_client_->startRequest(Http::TestRequestHeaderMapImpl{{":method", "POST"}, - {":path", "/test/long/url"}, - {":scheme", "http"}, - {":authority", "host"}}); - request_encoder_ = &encoder_decoder.first; - auto response = std::move(encoder_decoder.second); - - codec_client_->sendData(*request_encoder_, 1024 * 65, false); - - ASSERT_TRUE(response->waitForEndStream()); - // The delayed close timeout should trigger since client is not closing the connection. - EXPECT_TRUE(codec_client_->waitForDisconnect(std::chrono::milliseconds(2000))); - EXPECT_EQ(codec_client_->lastConnectionEvent(), Network::ConnectionEvent::RemoteClose); - EXPECT_EQ(test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value(), - 1); -} - // Test that if the route cache is cleared, it doesn't cause problems. TEST_P(IntegrationTest, TestClearingRouteCacheFilter) { config_helper_.addFilter("{ name: clear-route-cache, typed_config: { \"@type\": " diff --git a/test/integration/integration_test.h b/test/integration/integration_test.h index 2af85a268ec9c..e36f2e0b6301b 100644 --- a/test/integration/integration_test.h +++ b/test/integration/integration_test.h @@ -9,7 +9,7 @@ namespace Envoy { class IntegrationTest : public testing::TestWithParam, public HttpIntegrationTest { public: - IntegrationTest() : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, GetParam()) {} + IntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam()) {} }; class UpstreamEndpointIntegrationTest : public testing::TestWithParam, @@ -17,7 +17,7 @@ class UpstreamEndpointIntegrationTest : public testing::TestWithParam stream_by_resource_name_; }; - ListenerIntegrationTest() : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, ipVersion()) {} + ListenerIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, ipVersion()) {} ~ListenerIntegrationTest() override { resetConnections(); } @@ -98,9 +98,9 @@ class ListenerIntegrationTest : public HttpIntegrationTest, void createUpstreams() override { HttpIntegrationTest::createUpstreams(); // Create the LDS upstream (fake_upstreams_[1]). - addFakeUpstream(FakeHttpConnection::Type::HTTP2); + addFakeUpstream(Http::CodecType::HTTP2); // Create the RDS upstream (fake_upstreams_[2]). - addFakeUpstream(FakeHttpConnection::Type::HTTP2); + addFakeUpstream(Http::CodecType::HTTP2); } void resetFakeUpstreamInfo(FakeUpstreamInfo* upstream_info) { diff --git a/test/integration/load_stats_integration_test.cc b/test/integration/load_stats_integration_test.cc index bda71d612a88c..630db22619042 100644 --- a/test/integration/load_stats_integration_test.cc +++ b/test/integration/load_stats_integration_test.cc @@ -20,7 +20,7 @@ namespace { class LoadStatsIntegrationTest : public Grpc::VersionedGrpcClientIntegrationParamTest, public HttpIntegrationTest { public: - LoadStatsIntegrationTest() : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, ipVersion()) { + LoadStatsIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, ipVersion()) { // We rely on some fairly specific load balancing picks in this test, so // determinize the schedule. setDeterministic(); @@ -99,7 +99,7 @@ class LoadStatsIntegrationTest : public Grpc::VersionedGrpcClientIntegrationPara } void createUpstreams() override { - addFakeUpstream(FakeHttpConnection::Type::HTTP2); + addFakeUpstream(Http::CodecType::HTTP2); load_report_upstream_ = fake_upstreams_.back().get(); HttpIntegrationTest::createUpstreams(); } diff --git a/test/integration/local_reply_integration_test.cc b/test/integration/local_reply_integration_test.cc index 778158f01bd4b..fb9225a88bad3 100644 --- a/test/integration/local_reply_integration_test.cc +++ b/test/integration/local_reply_integration_test.cc @@ -66,7 +66,7 @@ TEST_P(LocalReplyIntegrationTest, MapStatusCodeAndFormatToJson) { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); ASSERT_TRUE(response->waitForEndStream()); - if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + if (downstream_protocol_ == Http::CodecType::HTTP1) { ASSERT_TRUE(codec_client_->waitForDisconnect()); } else { codec_client_->close(); @@ -125,7 +125,7 @@ TEST_P(LocalReplyIntegrationTest, EmptyStructFormatter) { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); ASSERT_TRUE(response->waitForEndStream()); - if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + if (downstream_protocol_ == Http::CodecType::HTTP1) { ASSERT_TRUE(codec_client_->waitForDisconnect()); } else { codec_client_->close(); @@ -177,7 +177,7 @@ TEST_P(LocalReplyIntegrationTest, MapStatusCodeAndFormatToJson4Grpc) { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); ASSERT_TRUE(response->waitForEndStream()); - if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + if (downstream_protocol_ == Http::CodecType::HTTP1) { ASSERT_TRUE(codec_client_->waitForDisconnect()); } else { codec_client_->close(); @@ -256,7 +256,7 @@ TEST_P(LocalReplyIntegrationTest, MapStatusCodeAndFormatToJsonForFirstMatchingFi ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); ASSERT_TRUE(response->waitForEndStream()); - if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + if (downstream_protocol_ == Http::CodecType::HTTP1) { ASSERT_TRUE(codec_client_->waitForDisconnect()); } else { codec_client_->close(); @@ -330,7 +330,7 @@ TEST_P(LocalReplyIntegrationTest, ShouldNotMatchAnyFilter) { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); ASSERT_TRUE(response->waitForEndStream()); - if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + if (downstream_protocol_ == Http::CodecType::HTTP1) { ASSERT_TRUE(codec_client_->waitForDisconnect()); } else { codec_client_->close(); @@ -391,7 +391,7 @@ TEST_P(LocalReplyIntegrationTest, ShouldMapResponseCodeAndMapToDefaultTextRespon ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); ASSERT_TRUE(response->waitForEndStream()); - if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + if (downstream_protocol_ == Http::CodecType::HTTP1) { ASSERT_TRUE(codec_client_->waitForDisconnect()); } else { codec_client_->close(); @@ -449,7 +449,7 @@ TEST_P(LocalReplyIntegrationTest, ShouldFormatResponseToCustomString) { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); ASSERT_TRUE(response->waitForEndStream()); - if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + if (downstream_protocol_ == Http::CodecType::HTTP1) { ASSERT_TRUE(codec_client_->waitForDisconnect()); } else { codec_client_->close(); @@ -507,7 +507,7 @@ TEST_P(LocalReplyIntegrationTest, ShouldFormatResponseToEmptyBody) { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); ASSERT_TRUE(response->waitForEndStream()); - if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + if (downstream_protocol_ == Http::CodecType::HTTP1) { ASSERT_TRUE(codec_client_->waitForDisconnect()); } else { codec_client_->close(); diff --git a/test/integration/multiplexed_integration_test.cc b/test/integration/multiplexed_integration_test.cc index 63a976a738e03..fbd7588c8c4bf 100644 --- a/test/integration/multiplexed_integration_test.cc +++ b/test/integration/multiplexed_integration_test.cc @@ -27,14 +27,14 @@ namespace Envoy { // TODO(#2557) fix all the failures. #define EXCLUDE_DOWNSTREAM_HTTP3 \ - if (downstreamProtocol() == Http::CodecClient::Type::HTTP3) { \ + if (downstreamProtocol() == Http::CodecType::HTTP3) { \ return; \ } INSTANTIATE_TEST_SUITE_P(IpVersions, Http2IntegrationTest, testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams( - {Http::CodecClient::Type::HTTP2, Http::CodecClient::Type::HTTP3}, - {FakeHttpConnection::Type::HTTP1})), + {Http::CodecType::HTTP2, Http::CodecType::HTTP3}, + {Http::CodecType::HTTP1})), HttpProtocolIntegrationTest::protocolTestParamsToString); TEST_P(Http2IntegrationTest, RouterRequestAndResponseWithBodyNoBuffer) { @@ -80,8 +80,6 @@ TEST_P(Http2IntegrationTest, FlowControlOnAndGiantBodyWithContentLength) { } TEST_P(Http2IntegrationTest, LargeFlowControlOnAndGiantBodyWithContentLength) { - // https://github.com/envoyproxy/envoy/issues/16335 - EXCLUDE_DOWNSTREAM_HTTP3; config_helper_.addConfigModifier(ConfigHelper::adjustUpstreamTimeoutForTsan); config_helper_.setBufferLimits(128 * 1024, 128 * 1024); // Set buffer limits upstream and downstream. @@ -309,6 +307,12 @@ TEST_P(Http2MetadataIntegrationTest, ProxyMetadataInResponse) { // Verifies stream is reset. ASSERT_TRUE(response->waitForReset()); ASSERT_FALSE(response->complete()); + + // The cluster should have received the reset. + // The downstream codec should send one. + std::string counter = + absl::StrCat("cluster.cluster_0.", upstreamProtocolStatsRoot(), ".rx_reset"); + test_server_->waitForCounterEq(counter, 1); } TEST_P(Http2MetadataIntegrationTest, ProxyMultipleMetadata) { @@ -902,7 +906,7 @@ TEST_P(Http2IntegrationTest, CodecErrorAfterStreamStart) { } TEST_P(Http2IntegrationTest, Http2BadMagic) { - if (downstreamProtocol() == Http::CodecClient::Type::HTTP3) { + if (downstreamProtocol() == Http::CodecType::HTTP3) { // The "magic" payload is an HTTP/2 specific thing. return; } @@ -1401,18 +1405,18 @@ Http2RingHashIntegrationTest::~Http2RingHashIntegrationTest() { void Http2RingHashIntegrationTest::createUpstreams() { for (int i = 0; i < num_upstreams_; i++) { - addFakeUpstream(FakeHttpConnection::Type::HTTP1); + addFakeUpstream(Http::CodecType::HTTP1); } } INSTANTIATE_TEST_SUITE_P(IpVersions, Http2RingHashIntegrationTest, testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams( - {Http::CodecClient::Type::HTTP2}, {FakeHttpConnection::Type::HTTP1})), + {Http::CodecType::HTTP2}, {Http::CodecType::HTTP1})), HttpProtocolIntegrationTest::protocolTestParamsToString); INSTANTIATE_TEST_SUITE_P(IpVersions, Http2MetadataIntegrationTest, testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams( - {Http::CodecClient::Type::HTTP2}, {FakeHttpConnection::Type::HTTP2})), + {Http::CodecType::HTTP2}, {Http::CodecType::HTTP2})), HttpProtocolIntegrationTest::protocolTestParamsToString); void Http2RingHashIntegrationTest::sendMultipleRequests( diff --git a/test/integration/multiplexed_upstream_integration_test.cc b/test/integration/multiplexed_upstream_integration_test.cc index 2c86cba32fb0a..8118928ad1cca 100644 --- a/test/integration/multiplexed_upstream_integration_test.cc +++ b/test/integration/multiplexed_upstream_integration_test.cc @@ -17,14 +17,14 @@ namespace Envoy { INSTANTIATE_TEST_SUITE_P(Protocols, Http2UpstreamIntegrationTest, testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams( - {Http::CodecClient::Type::HTTP2}, {FakeHttpConnection::Type::HTTP2})), + {Http::CodecType::HTTP2}, {Http::CodecType::HTTP2})), HttpProtocolIntegrationTest::protocolTestParamsToString); // TODO(alyssawilk) move #defines into getProtocolTestParams in a follow-up #ifdef ENVOY_ENABLE_QUIC INSTANTIATE_TEST_SUITE_P(ProtocolsWithQuic, Http2UpstreamIntegrationTest, testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams( - {Http::CodecClient::Type::HTTP2}, {FakeHttpConnection::Type::HTTP3})), + {Http::CodecType::HTTP2}, {Http::CodecType::HTTP3})), HttpProtocolIntegrationTest::protocolTestParamsToString); #endif @@ -50,6 +50,10 @@ TEST_P(Http2UpstreamIntegrationTest, RouterUpstreamDisconnectBeforeResponseCompl TEST_P(Http2UpstreamIntegrationTest, RouterDownstreamDisconnectBeforeRequestComplete) { testRouterDownstreamDisconnectBeforeRequestComplete(); + + // Given the downstream disconnect, Envoy will reset the upstream stream. + EXPECT_EQ(1, upstreamTxResetCounterValue()); + EXPECT_EQ(0, upstreamRxResetCounterValue()); } TEST_P(Http2UpstreamIntegrationTest, RouterDownstreamDisconnectBeforeResponseComplete) { @@ -136,6 +140,24 @@ TEST_P(Http2UpstreamIntegrationTest, LargeBidirectionalStreamingWithBufferLimits bidirectionalStreaming(1024 * 32); } +uint64_t Http2UpstreamIntegrationTest::upstreamRxResetCounterValue() { + return test_server_ + ->counter(absl::StrCat("cluster.cluster_0.", upstreamProtocolStatsRoot(), ".rx_reset")) + ->value(); +} + +uint64_t Http2UpstreamIntegrationTest::upstreamTxResetCounterValue() { + return test_server_ + ->counter(absl::StrCat("cluster.cluster_0.", upstreamProtocolStatsRoot(), ".tx_reset")) + ->value(); +} +uint64_t Http2UpstreamIntegrationTest::downstreamRxResetCounterValue() { + return test_server_->counter(absl::StrCat(downstreamProtocolStatsRoot(), ".rx_reset"))->value(); +} +uint64_t Http2UpstreamIntegrationTest::downstreamTxResetCounterValue() { + return test_server_->counter(absl::StrCat(downstreamProtocolStatsRoot(), ".tx_reset"))->value(); +} + TEST_P(Http2UpstreamIntegrationTest, BidirectionalStreamingReset) { initialize(); codec_client_ = makeHttpConnection(lookupPort("http")); @@ -169,6 +191,13 @@ TEST_P(Http2UpstreamIntegrationTest, BidirectionalStreamingReset) { upstream_request_->encodeResetStream(); ASSERT_TRUE(response->waitForReset()); EXPECT_FALSE(response->complete()); + + // The upstream stats should reflect receiving the reset, and downstream + // reflect sending it on. + EXPECT_EQ(1, upstreamRxResetCounterValue()); + EXPECT_EQ(0, upstreamTxResetCounterValue()); + EXPECT_EQ(0, downstreamRxResetCounterValue()); + EXPECT_EQ(1, downstreamTxResetCounterValue()); } void Http2UpstreamIntegrationTest::simultaneousRequest(uint32_t request1_bytes, @@ -309,7 +338,7 @@ TEST_P(Http2UpstreamIntegrationTest, ManyLargeSimultaneousRequestWithRandomBacku name: pause-filter{} typed_config: "@type": type.googleapis.com/google.protobuf.Empty)EOF", - downstreamProtocol() == Http::CodecClient::Type::HTTP3 ? "-for-quic" : "")); + downstreamProtocol() == Http::CodecType::HTTP3 ? "-for-quic" : "")); manySimultaneousRequests(1024 * 20, 1024 * 20); } @@ -371,6 +400,8 @@ TEST_P(Http2UpstreamIntegrationTest, UpstreamConnectionCloseWithManyStreams) { for (uint32_t i = 1; i < num_requests; ++i) { ASSERT_TRUE(responses[i]->waitForReset()); } + + EXPECT_NE(0, downstreamRxResetCounterValue()); } // Regression test for https://github.com/envoyproxy/envoy/issues/6744 @@ -431,6 +462,13 @@ name: router ASSERT_TRUE(response->waitForEndStream()); EXPECT_TRUE(response->complete()); + + // As the error was internal, Envoy should reset the upstream connection. + // Downstream gets an error, so no resets there. + EXPECT_EQ(1, upstreamTxResetCounterValue()); + EXPECT_EQ(0, downstreamTxResetCounterValue()); + EXPECT_EQ(0, upstreamRxResetCounterValue()); + EXPECT_EQ(0, downstreamRxResetCounterValue()); } // Tests the default limit for the number of response headers is 100. Results in a stream reset if @@ -609,18 +647,18 @@ class MixedUpstreamIntegrationTest : public Http2UpstreamIntegrationTest { Http2UpstreamIntegrationTest::initialize(); } void createUpstreams() override { - ASSERT_EQ(upstreamProtocol(), FakeHttpConnection::Type::HTTP3); + ASSERT_EQ(upstreamProtocol(), Http::CodecType::HTTP3); ASSERT_EQ(fake_upstreams_count_, 1); ASSERT_FALSE(autonomous_upstream_); if (use_http2_) { - auto config = configWithType(FakeHttpConnection::Type::HTTP2); + auto config = configWithType(Http::CodecType::HTTP2); Network::TransportSocketFactoryPtr factory = createUpstreamTlsContext(config); - addFakeUpstream(std::move(factory), FakeHttpConnection::Type::HTTP2); + addFakeUpstream(std::move(factory), Http::CodecType::HTTP2); } else { - auto config = configWithType(FakeHttpConnection::Type::HTTP3); + auto config = configWithType(Http::CodecType::HTTP3); Network::TransportSocketFactoryPtr factory = createUpstreamTlsContext(config); - addFakeUpstream(std::move(factory), FakeHttpConnection::Type::HTTP3); + addFakeUpstream(std::move(factory), Http::CodecType::HTTP3); } } @@ -638,7 +676,7 @@ TEST_P(MixedUpstreamIntegrationTest, SimultaneousRequestAutoWithHttp2) { INSTANTIATE_TEST_SUITE_P(Protocols, MixedUpstreamIntegrationTest, testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams( - {Http::CodecClient::Type::HTTP2}, {FakeHttpConnection::Type::HTTP3})), + {Http::CodecType::HTTP2}, {Http::CodecType::HTTP3})), HttpProtocolIntegrationTest::protocolTestParamsToString); #endif diff --git a/test/integration/multiplexed_upstream_integration_test.h b/test/integration/multiplexed_upstream_integration_test.h index 45e35a83b42e6..a0903c717d2ec 100644 --- a/test/integration/multiplexed_upstream_integration_test.h +++ b/test/integration/multiplexed_upstream_integration_test.h @@ -9,8 +9,7 @@ class Http2UpstreamIntegrationTest : public HttpProtocolIntegrationTest { public: void initialize() override { upstream_tls_ = true; - config_helper_.configureUpstreamTls(use_alpn_, - upstreamProtocol() == FakeHttpConnection::Type::HTTP3); + config_helper_.configureUpstreamTls(use_alpn_, upstreamProtocol() == Http::CodecType::HTTP3); HttpProtocolIntegrationTest::initialize(); } @@ -20,5 +19,10 @@ class Http2UpstreamIntegrationTest : public HttpProtocolIntegrationTest { void manySimultaneousRequests(uint32_t request_bytes, uint32_t response_bytes); bool use_alpn_{false}; + + uint64_t upstreamRxResetCounterValue(); + uint64_t upstreamTxResetCounterValue(); + uint64_t downstreamRxResetCounterValue(); + uint64_t downstreamTxResetCounterValue(); }; } // namespace Envoy diff --git a/test/integration/original_ip_detection_integration_test.cc b/test/integration/original_ip_detection_integration_test.cc index 9fa25edce17e2..8bc353e0da494 100644 --- a/test/integration/original_ip_detection_integration_test.cc +++ b/test/integration/original_ip_detection_integration_test.cc @@ -15,8 +15,7 @@ class OriginalIPDetectionIntegrationTest : public testing::TestWithParam, public HttpIntegrationTest { public: - OriginalIPDetectionIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, GetParam()) {} + OriginalIPDetectionIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam()) {} void runTest(const std::string& ip) { useAccessLog("%DOWNSTREAM_REMOTE_ADDRESS_WITHOUT_PORT%"); diff --git a/test/integration/overload_integration_test.cc b/test/integration/overload_integration_test.cc index 055cc6a3d4144..e284321697bea 100644 --- a/test/integration/overload_integration_test.cc +++ b/test/integration/overload_integration_test.cc @@ -168,7 +168,7 @@ TEST_P(OverloadIntegrationTest, CloseStreamsWhenOverloaded) { } TEST_P(OverloadIntegrationTest, DisableKeepaliveWhenOverloaded) { - if (downstreamProtocol() != Http::CodecClient::Type::HTTP1) { + if (downstreamProtocol() != Http::CodecType::HTTP1) { return; // only relevant for downstream HTTP1.x connections } @@ -305,7 +305,7 @@ TEST_P(OverloadScaledTimerIntegrationTest, CloseIdleHttpConnections) { test_server_->waitForCounterGe("http.config_test.downstream_cx_idle_timeout", 1); dispatcher_->run(Event::Dispatcher::RunType::NonBlock); - if (GetParam().downstream_protocol == Http::CodecClient::Type::HTTP1) { + if (GetParam().downstream_protocol == Http::CodecType::HTTP1) { // For HTTP1, Envoy will start draining but will wait to close the // connection. If a new stream comes in, it will set the connection header // to "close" on the response and close the connection after. diff --git a/test/integration/protocol_integration_test.cc b/test/integration/protocol_integration_test.cc index 3c799a0566a55..13798f7509f0b 100644 --- a/test/integration/protocol_integration_test.cc +++ b/test/integration/protocol_integration_test.cc @@ -55,7 +55,7 @@ void setDoNotValidateRouteConfig( // TODO(#2557) fix all the failures. #define EXCLUDE_DOWNSTREAM_HTTP3 \ - if (downstreamProtocol() == Http::CodecClient::Type::HTTP3) { \ + if (downstreamProtocol() == Http::CodecType::HTTP3) { \ return; \ } @@ -301,7 +301,7 @@ name: add-trailers-filter upstream_request_->encodeData(128, true); ASSERT_TRUE(response->waitForEndStream()); - if (upstreamProtocol() >= FakeHttpConnection::Type::HTTP2) { + if (upstreamProtocol() >= Http::CodecType::HTTP2) { EXPECT_EQ("decode", upstream_request_->trailers() ->get(Http::LowerCaseString("grpc-message"))[0] ->value() @@ -309,7 +309,7 @@ name: add-trailers-filter } EXPECT_TRUE(response->complete()); EXPECT_EQ("503", response->headers().getStatusValue()); - if (downstream_protocol_ >= Http::CodecClient::Type::HTTP2) { + if (downstream_protocol_ >= Http::CodecType::HTTP2) { EXPECT_EQ("encode", response->trailers()->getGrpcMessageValue()); } } @@ -335,7 +335,7 @@ TEST_P(ProtocolIntegrationTest, ResponseWithHostHeader) { // Tests missing headers needed for H/1 codec first line. TEST_P(DownstreamProtocolIntegrationTest, DownstreamRequestWithFaultyFilter) { - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP3) { + if (upstreamProtocol() == Http::CodecType::HTTP3) { // For QUIC, even through the headers are not sent upstream, the stream will // be created. Use the autonomous upstream and allow incomplete streams. autonomous_allow_incomplete_streams_ = true; @@ -375,7 +375,7 @@ TEST_P(DownstreamProtocolIntegrationTest, DownstreamRequestWithFaultyFilter) { TEST_P(DownstreamProtocolIntegrationTest, FaultyFilterWithConnect) { // TODO(danzh) re-enable after adding http3 option "allow_connect". EXCLUDE_DOWNSTREAM_HTTP3; - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP3) { + if (upstreamProtocol() == Http::CodecType::HTTP3) { // For QUIC, even through the headers are not sent upstream, the stream will // be created. Use the autonomous upstream and allow incomplete streams. autonomous_allow_incomplete_streams_ = true; @@ -403,7 +403,7 @@ TEST_P(DownstreamProtocolIntegrationTest, FaultyFilterWithConnect) { auto headers = Http::TestRequestHeaderMapImpl{ {":method", "CONNECT"}, {":scheme", "http"}, {":authority", "www.host.com:80"}}; - auto response = (downstream_protocol_ == Http::CodecClient::Type::HTTP1) + auto response = (downstream_protocol_ == Http::CodecType::HTTP1) ? std::move((codec_client_->startRequest(headers)).second) : codec_client_->makeHeaderOnlyRequest(headers); @@ -507,7 +507,7 @@ TEST_P(ProtocolIntegrationTest, Retry) { waitForNextUpstreamRequest(); upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "503"}}, false); - if (fake_upstreams_[0]->httpType() == FakeHttpConnection::Type::HTTP1) { + if (fake_upstreams_[0]->httpType() == Http::CodecType::HTTP1) { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_, std::chrono::milliseconds(500))); @@ -526,22 +526,15 @@ TEST_P(ProtocolIntegrationTest, Retry) { EXPECT_EQ("200", response->headers().getStatusValue()); EXPECT_EQ(512U, response->body().size()); Stats::Store& stats = test_server_->server().stats(); - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP2) { - Stats::CounterSharedPtr counter = - TestUtility::findCounter(stats, "cluster.cluster_0.http2.tx_reset"); + if (upstreamProtocol() == Http::CodecType::HTTP2) { + Stats::CounterSharedPtr counter = TestUtility::findCounter( + stats, absl::StrCat("cluster.cluster_0.", upstreamProtocolStatsRoot(), ".tx_reset")); ASSERT_NE(nullptr, counter); EXPECT_EQ(1L, counter->value()); - } else if (upstreamProtocol() == FakeHttpConnection::Type::HTTP3) { - // TODO(alyssawilk) http3 stats. - Stats::CounterSharedPtr counter = - TestUtility::findCounter(stats, "cluster.cluster_0.upstream_rq_tx_reset"); - ASSERT_NE(nullptr, counter); - EXPECT_EQ(1L, counter->value()); - } else { - Stats::CounterSharedPtr counter = - TestUtility::findCounter(stats, "cluster.cluster_0.http1.dropped_headers_with_underscores"); - EXPECT_NE(nullptr, counter); } + EXPECT_NE(nullptr, + test_server_->counter(absl::StrCat("cluster.cluster_0.", upstreamProtocolStatsRoot(), + ".dropped_headers_with_underscores"))); } TEST_P(ProtocolIntegrationTest, RetryStreaming) { @@ -568,7 +561,7 @@ TEST_P(ProtocolIntegrationTest, RetryStreaming) { // Send back an upstream failure. upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "503"}}, false); - if (fake_upstreams_[0]->httpType() == FakeHttpConnection::Type::HTTP1) { + if (fake_upstreams_[0]->httpType() == Http::CodecType::HTTP1) { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_)); } else { @@ -631,7 +624,7 @@ TEST_P(ProtocolIntegrationTest, RetryStreamingReset) { }); // Make sure the fake stream is reset. - if (fake_upstreams_[0]->httpType() == FakeHttpConnection::Type::HTTP1) { + if (fake_upstreams_[0]->httpType() == Http::CodecType::HTTP1) { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_)); } else { @@ -698,7 +691,7 @@ TEST_P(ProtocolIntegrationTest, RetryStreamingCancelDueToBufferOverflow) { // Send back an upstream failure. upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "503"}}, false); - if (fake_upstreams_[0]->httpType() == FakeHttpConnection::Type::HTTP1) { + if (fake_upstreams_[0]->httpType() == Http::CodecType::HTTP1) { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); } else { ASSERT_TRUE(upstream_request_->waitForReset()); @@ -740,7 +733,7 @@ TEST_P(DownstreamProtocolIntegrationTest, RetryAttemptCountHeader) { EXPECT_EQ(atoi(std::string(upstream_request_->headers().getEnvoyAttemptCountValue()).c_str()), 1); - if (fake_upstreams_[0]->httpType() == FakeHttpConnection::Type::HTTP1) { + if (fake_upstreams_[0]->httpType() == Http::CodecType::HTTP1) { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_)); } else { @@ -811,14 +804,14 @@ TEST_P(DownstreamProtocolIntegrationTest, RetryPriority) { waitForNextUpstreamRequest(0); upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "503"}}, false); - if (fake_upstreams_[0]->httpType() == FakeHttpConnection::Type::HTTP1) { + if (fake_upstreams_[0]->httpType() == Http::CodecType::HTTP1) { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); ASSERT_TRUE(fake_upstreams_[1]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_)); } else { ASSERT_TRUE(upstream_request_->waitForReset()); } - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP3) { + if (upstreamProtocol() == Http::CodecType::HTTP3) { // Make sure waitForNextUpstreamRequest waits for a new connection. fake_upstream_connection_.reset(); } @@ -878,7 +871,7 @@ TEST_P(DownstreamProtocolIntegrationTest, RetryHostPredicateFilter) { ASSERT_TRUE(upstream_idx.has_value()); upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "503"}}, false); - if (fake_upstreams_[*upstream_idx]->httpType() == FakeHttpConnection::Type::HTTP1) { + if (fake_upstreams_[*upstream_idx]->httpType() == Http::CodecType::HTTP1) { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); ASSERT_TRUE(fake_upstreams_[*upstream_idx]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_)); @@ -978,7 +971,7 @@ TEST_P(DownstreamProtocolIntegrationTest, HittingDecoderFilterLimit) { // the 413-and-connection-close may be sent while the body is still being // sent, resulting in a write error and the connection being closed before the // response is read. - if (downstream_protocol_ >= Http::CodecClient::Type::HTTP2) { + if (downstream_protocol_ >= Http::CodecType::HTTP2) { ASSERT_TRUE(response->complete()); } if (response->complete()) { @@ -1020,7 +1013,7 @@ TEST_P(ProtocolIntegrationTest, HittingEncoderFilterLimit) { // Now send an overly large response body. At some point, too much data will // be buffered, the stream will be reset, and the connection will disconnect. upstream_request_->encodeData(1024 * 65, false); - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() == Http::CodecType::HTTP1) { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); } else { ASSERT_TRUE(upstream_request_->waitForReset()); @@ -1116,13 +1109,13 @@ TEST_P(ProtocolIntegrationTest, HeadersWithUnderscoresDropped) { Stats::Store& stats = test_server_->server().stats(); std::string stat_name; switch (downstreamProtocol()) { - case Http::CodecClient::Type::HTTP1: + case Http::CodecType::HTTP1: stat_name = "http1.dropped_headers_with_underscores"; break; - case Http::CodecClient::Type::HTTP2: + case Http::CodecType::HTTP2: stat_name = "http2.dropped_headers_with_underscores"; break; - case Http::CodecClient::Type::HTTP3: + case Http::CodecType::HTTP3: stat_name = "http3.dropped_headers_with_underscores"; break; default: @@ -1171,7 +1164,7 @@ TEST_P(DownstreamProtocolIntegrationTest, HeadersWithUnderscoresCauseRequestReje {":authority", "host"}, {"foo_bar", "baz"}}); - if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + if (downstream_protocol_ == Http::CodecType::HTTP1) { ASSERT_TRUE(codec_client_->waitForDisconnect()); ASSERT_TRUE(response->complete()); EXPECT_EQ("400", response->headers().getStatusValue()); @@ -1218,15 +1211,15 @@ TEST_P(ProtocolIntegrationTest, 304WithBody) { // For HTTP/1.1 http_parser is explicitly told that 304s are header-only // requests. - if (downstream_protocol_ == Http::CodecClient::Type::HTTP1 || - upstreamProtocol() == FakeHttpConnection::Type::HTTP1) { + if (downstream_protocol_ == Http::CodecType::HTTP1 || + upstreamProtocol() == Http::CodecType::HTTP1) { ASSERT_TRUE(response->complete()); } else { ASSERT_FALSE(response->complete()); } upstream_request_->encodeData(2, true); - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() == Http::CodecType::HTTP1) { // Any body sent after the request is considered complete will not be handled as part of the // active request, but will be flagged as a protocol error for the no-longer-associated // connection. @@ -1239,8 +1232,8 @@ TEST_P(ProtocolIntegrationTest, 304WithBody) { // Only for HTTP/2 and Http/3, where streams are ended with an explicit end-stream so we // can differentiate between 304-with-advertised-but-absent-body and // 304-with-body, is there a protocol error on the active stream. - if (downstream_protocol_ >= Http::CodecClient::Type::HTTP2 && - upstreamProtocol() >= FakeHttpConnection::Type::HTTP2) { + if (downstream_protocol_ >= Http::CodecType::HTTP2 && + upstreamProtocol() >= Http::CodecType::HTTP2) { ASSERT_TRUE(response->waitForReset()); } } @@ -1251,7 +1244,7 @@ TEST_P(ProtocolIntegrationTest, OverflowingResponseCode) { codec_client_ = makeHttpConnection(lookupPort("http")); auto response = codec_client_->makeHeaderOnlyRequest(default_request_headers_); - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() == Http::CodecType::HTTP1) { // The http1 codec won't send illegal status codes so send raw HTTP/1.1 FakeRawConnectionPtr fake_upstream_connection; ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); @@ -1280,13 +1273,13 @@ TEST_P(ProtocolIntegrationTest, MissingStatus) { codec_client_ = makeHttpConnection(lookupPort("http")); auto response = codec_client_->makeHeaderOnlyRequest(default_request_headers_); - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() == Http::CodecType::HTTP1) { FakeRawConnectionPtr fake_upstream_connection; ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); ASSERT(fake_upstream_connection != nullptr); ASSERT_TRUE(fake_upstream_connection->write("HTTP/1.1 OK\r\n", false)); ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); - } else if (upstreamProtocol() == FakeHttpConnection::Type::HTTP2) { + } else if (upstreamProtocol() == Http::CodecType::HTTP2) { FakeRawConnectionPtr fake_upstream_connection; ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); Http::Http2::Http2Frame::SettingsFlags settings_flags = @@ -1314,7 +1307,7 @@ TEST_P(ProtocolIntegrationTest, MissingStatus) { // Validate that lots of tiny cookies doesn't cause a DoS (single cookie header). TEST_P(DownstreamProtocolIntegrationTest, LargeCookieParsingConcatenated) { - if (downstreamProtocol() == Http::CodecClient::Type::HTTP3) { + if (downstreamProtocol() == Http::CodecType::HTTP3) { // QUICHE Qpack splits concatenated cookies into crumbs to increase // compression ratio. On the receiver side, the total size of these crumbs // may be larger than coalesced cookie headers. Increase the max request @@ -1326,7 +1319,7 @@ TEST_P(DownstreamProtocolIntegrationTest, LargeCookieParsingConcatenated) { hcm.mutable_common_http_protocol_options()->mutable_max_headers_count()->set_value(8000); }); } - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP3) { + if (upstreamProtocol() == Http::CodecType::HTTP3) { setMaxRequestHeadersKb(96); setMaxRequestHeadersCount(8000); } @@ -1393,7 +1386,7 @@ TEST_P(DownstreamProtocolIntegrationTest, InvalidContentLength) { ASSERT_TRUE(codec_client_->waitForDisconnect()); - if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + if (downstream_protocol_ == Http::CodecType::HTTP1) { ASSERT_TRUE(response->complete()); EXPECT_EQ("400", response->headers().getStatusValue()); test_server_->waitForCounterGe("http.config_test.downstream_rq_4xx", 1); @@ -1429,14 +1422,14 @@ TEST_P(DownstreamProtocolIntegrationTest, InvalidContentLengthAllowed) { {"content-length", "-1"}}); auto response = std::move(encoder_decoder.second); - if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + if (downstream_protocol_ == Http::CodecType::HTTP1) { ASSERT_TRUE(codec_client_->waitForDisconnect()); } else { ASSERT_TRUE(response->waitForReset()); codec_client_->close(); } - if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + if (downstream_protocol_ == Http::CodecType::HTTP1) { ASSERT_TRUE(response->complete()); EXPECT_EQ("400", response->headers().getStatusValue()); } else { @@ -1457,7 +1450,7 @@ TEST_P(DownstreamProtocolIntegrationTest, MultipleContentLengths) { ASSERT_TRUE(codec_client_->waitForDisconnect()); - if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + if (downstream_protocol_ == Http::CodecType::HTTP1) { ASSERT_TRUE(response->complete()); EXPECT_EQ("400", response->headers().getStatusValue()); } else { @@ -1488,14 +1481,14 @@ TEST_P(DownstreamProtocolIntegrationTest, MultipleContentLengthsAllowed) { {"content-length", "3,2"}}); auto response = std::move(encoder_decoder.second); - if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + if (downstream_protocol_ == Http::CodecType::HTTP1) { ASSERT_TRUE(codec_client_->waitForDisconnect()); } else { ASSERT_TRUE(response->waitForReset()); codec_client_->close(); } - if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + if (downstream_protocol_ == Http::CodecType::HTTP1) { ASSERT_TRUE(response->complete()); EXPECT_EQ("400", response->headers().getStatusValue()); } else { @@ -1570,14 +1563,11 @@ TEST_P(DownstreamProtocolIntegrationTest, LargeRequestHeadersRejected) { } TEST_P(DownstreamProtocolIntegrationTest, VeryLargeRequestHeadersRejected) { -#ifdef WIN32 - // TODO(alyssawilk, wrowe) debug. - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP3) { - return; - } -#endif - // Send one very large 2048 kB (2 MB) header with limit 1024 kB (1 MB) and 100 headers. - testLargeRequestHeaders(2048, 1, 1024, 100); + // Send one very large 600 kB header with limit 500 kB and 100 headers. + // The limit and the header size are set in such a way to accommodate for flow control limits. + // If the headers are too large and the flow control blocks the response is truncated and the test + // flakes. + testLargeRequestHeaders(600, 1, 500, 100); } TEST_P(DownstreamProtocolIntegrationTest, LargeRequestHeadersAccepted) { @@ -1586,10 +1576,6 @@ TEST_P(DownstreamProtocolIntegrationTest, LargeRequestHeadersAccepted) { } TEST_P(DownstreamProtocolIntegrationTest, ManyLargeRequestHeadersAccepted) { - // Fail under TSAN. Quic blackhole detection fired and closed the connection with - // QUIC_TOO_MANY_RTOS while waiting for upstream finishing transferring the large header. Observed - // long event loop. - EXCLUDE_DOWNSTREAM_HTTP3; // Send 70 headers each of size 100 kB with limit 8192 kB (8 MB) and 100 headers. testLargeRequestHeaders(100, 70, 8192, 100, TSAN_TIMEOUT_FACTOR * TestUtility::DefaultTimeout); } @@ -1626,7 +1612,7 @@ TEST_P(DownstreamProtocolIntegrationTest, ManyRequestTrailersRejected) { codec_client_->sendData(*request_encoder_, 1, false); codec_client_->sendTrailers(*request_encoder_, request_trailers); - if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + if (downstream_protocol_ == Http::CodecType::HTTP1) { ASSERT_TRUE(codec_client_->waitForDisconnect()); EXPECT_TRUE(response->complete()); EXPECT_EQ("431", response->headers().getStatusValue()); @@ -1749,7 +1735,7 @@ TEST_P(ProtocolIntegrationTest, LargeRequestMethod) { initialize(); codec_client_ = makeHttpConnection(lookupPort("http")); - if (downstreamProtocol() == Http::CodecClient::Type::HTTP1) { + if (downstreamProtocol() == Http::CodecType::HTTP1) { auto encoder_decoder = codec_client_->startRequest(request_headers); request_encoder_ = &encoder_decoder.first; auto response = std::move(encoder_decoder.second); @@ -1757,8 +1743,8 @@ TEST_P(ProtocolIntegrationTest, LargeRequestMethod) { EXPECT_TRUE(response->complete()); EXPECT_EQ("400", response->headers().getStatusValue()); } else { - ASSERT(downstreamProtocol() >= Http::CodecClient::Type::HTTP2); - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP1) { + ASSERT(downstreamProtocol() >= Http::CodecType::HTTP2); + if (upstreamProtocol() == Http::CodecType::HTTP1) { auto response = codec_client_->makeHeaderOnlyRequest(request_headers); ASSERT_TRUE( fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_)); @@ -1766,7 +1752,7 @@ TEST_P(ProtocolIntegrationTest, LargeRequestMethod) { EXPECT_TRUE(response->complete()); EXPECT_EQ("400", response->headers().getStatusValue()); } else { - ASSERT(upstreamProtocol() >= FakeHttpConnection::Type::HTTP2); + ASSERT(upstreamProtocol() >= Http::CodecType::HTTP2); auto response = sendRequestAndWaitForResponse(request_headers, 0, default_response_headers_, 0); EXPECT_TRUE(response->complete()); @@ -1964,15 +1950,7 @@ name: encode-headers-return-stop-all-filter ASSERT_TRUE(response->waitForEndStream()); ASSERT_TRUE(response->complete()); // Data is added in encodeData for all protocols, and encodeTrailers for HTTP/2 and above. - int times_added = (upstreamProtocol() == FakeHttpConnection::Type::HTTP1 || - downstreamProtocol() == Http::CodecClient::Type::HTTP1) - ? 1 - : 2; - if (downstreamProtocol() == Http::CodecClient::Type::HTTP1 && - upstreamProtocol() == FakeHttpConnection::Type::HTTP3) { - // TODO(alyssawilk) Figure out why the bytes mismatch with the test expectation below. - return; - } + int times_added = upstreamProtocol() == Http::CodecType::HTTP1 ? 1 : 2; EXPECT_EQ(count_ * size_ + added_decoded_data_size_ * times_added, response->body().size()); } @@ -2016,29 +1994,28 @@ name: encode-headers-return-stop-all-filter ASSERT_TRUE(response->waitForEndStream()); ASSERT_TRUE(response->complete()); // Data is added in encodeData for all protocols, and encodeTrailers for HTTP/2 and above. - int times_added = (upstreamProtocol() == FakeHttpConnection::Type::HTTP1 || - downstreamProtocol() == Http::CodecClient::Type::HTTP1) - ? 1 - : 2; - if (downstreamProtocol() == Http::CodecClient::Type::HTTP1 && - upstreamProtocol() == FakeHttpConnection::Type::HTTP3) { - // TODO(alyssawilk) Figure out why the bytes mismatch with the test expectation below. - return; - } + int times_added = upstreamProtocol() == Http::CodecType::HTTP1 ? 1 : 2; EXPECT_EQ(count_ * size_ + added_decoded_data_size_ * times_added, response->body().size()); } // Per https://github.com/envoyproxy/envoy/issues/7488 make sure we don't // combine set-cookie headers -TEST_P(ProtocolIntegrationTest, MultipleSetCookies) { +TEST_P(ProtocolIntegrationTest, MultipleCookiesAndSetCookies) { initialize(); codec_client_ = makeHttpConnection(lookupPort("http")); + Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, {":path", "/dynamo/url"}, + {":scheme", "http"}, {":authority", "host"}, + {"cookie", "a=b"}, {"cookie", "c=d"}}; Http::TestResponseHeaderMapImpl response_headers{ {":status", "200"}, {"set-cookie", "foo"}, {"set-cookie", "bar"}}; - auto response = sendRequestAndWaitForResponse(default_request_headers_, 0, response_headers, 0); + auto response = sendRequestAndWaitForResponse(request_headers, 0, response_headers, 0); + if (downstreamProtocol() == Http::CodecClient::Type::HTTP3) { + EXPECT_EQ(upstream_request_->headers().get(Http::Headers::get().Cookie)[0]->value(), + "a=b; c=d"); + } ASSERT_TRUE(response->complete()); EXPECT_EQ("200", response->headers().getStatusValue()); @@ -2049,6 +2026,42 @@ TEST_P(ProtocolIntegrationTest, MultipleSetCookies) { ASSERT_EQ(out[1]->value().getStringView(), "bar"); } +// Test that delay closed connections are eventually force closed when the timeout triggers. +TEST_P(DownstreamProtocolIntegrationTest, TestDelayedConnectionTeardownTimeoutTrigger) { + config_helper_.addFilter("{ name: encoder-decoder-buffer-filter, typed_config: { \"@type\": " + "type.googleapis.com/google.protobuf.Empty } }"); + config_helper_.setBufferLimits(1024, 1024); + config_helper_.addConfigModifier( + [](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& + hcm) { + // 200ms. + hcm.mutable_delayed_close_timeout()->set_nanos(200000000); + hcm.mutable_drain_timeout()->set_seconds(1); + hcm.mutable_common_http_protocol_options()->mutable_idle_timeout()->set_seconds(1); + }); + + initialize(); + + codec_client_ = makeHttpConnection(lookupPort("http")); + + auto encoder_decoder = + codec_client_->startRequest(Http::TestRequestHeaderMapImpl{{":method", "POST"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", "host"}}); + request_encoder_ = &encoder_decoder.first; + auto response = std::move(encoder_decoder.second); + + codec_client_->sendData(*request_encoder_, 1024 * 65, false); + + ASSERT_TRUE(response->waitForEndStream()); + // The delayed close timeout should trigger since client is not closing the connection. + EXPECT_TRUE(codec_client_->waitForDisconnect(std::chrono::milliseconds(5000))); + EXPECT_EQ(codec_client_->lastConnectionEvent(), Network::ConnectionEvent::RemoteClose); + EXPECT_EQ(test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value(), + 1); +} + // Resets the downstream stream immediately and verifies that we clean up everything. TEST_P(ProtocolIntegrationTest, TestDownstreamResetIdleTimeout) { useAccessLog("%RESPONSE_FLAGS% %RESPONSE_CODE_DETAILS%"); @@ -2063,13 +2076,13 @@ TEST_P(ProtocolIntegrationTest, TestDownstreamResetIdleTimeout) { EXPECT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_)); - if (downstreamProtocol() == Http::CodecClient::Type::HTTP1) { + if (downstreamProtocol() == Http::CodecType::HTTP1) { codec_client_->close(); } else { codec_client_->sendReset(encoder_decoder.first); } - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() == Http::CodecType::HTTP1) { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); } else { ASSERT_TRUE(upstream_request_->waitForReset()); @@ -2157,7 +2170,7 @@ TEST_P(DownstreamProtocolIntegrationTest, TestPreconnect) { auto response = sendRequestAndWaitForResponse(default_request_headers_, 0, default_response_headers_, 0); FakeHttpConnectionPtr fake_upstream_connection_two; - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() == Http::CodecType::HTTP1) { // For HTTP/1.1 there should be a preconnected connection. ASSERT_TRUE( fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_two)); @@ -2219,7 +2232,7 @@ TEST_P(DownstreamProtocolIntegrationTest, InvalidAuthority) { {":authority", "ho|st|"}}; auto response = codec_client_->makeHeaderOnlyRequest(request_headers); - if (downstreamProtocol() == Http::CodecClient::Type::HTTP1) { + if (downstreamProtocol() == Http::CodecType::HTTP1) { // For HTTP/1 this is handled by the HCM, which sends a full 400 response. ASSERT_TRUE(response->waitForEndStream()); ASSERT_TRUE(response->complete()); @@ -2240,7 +2253,7 @@ TEST_P(DownstreamProtocolIntegrationTest, ConnectIsBlocked) { request_encoder_ = &encoder_decoder.first; auto response = std::move(encoder_decoder.second); - if (downstreamProtocol() == Http::CodecClient::Type::HTTP1) { + if (downstreamProtocol() == Http::CodecType::HTTP1) { // Because CONNECT requests for HTTP/1 do not include a path, they will fail // to find a route match and return a 404. ASSERT_TRUE(response->waitForEndStream()); @@ -2257,7 +2270,7 @@ TEST_P(DownstreamProtocolIntegrationTest, ConnectIsBlocked) { TEST_P(DownstreamProtocolIntegrationTest, ConnectStreamRejection) { // TODO(danzh) add "allow_connect" to http3 options. EXCLUDE_DOWNSTREAM_HTTP3; - if (downstreamProtocol() == Http::CodecClient::Type::HTTP1) { + if (downstreamProtocol() == Http::CodecType::HTTP1) { return; } config_helper_.addConfigModifier( @@ -2332,4 +2345,59 @@ TEST_P(DownstreamProtocolIntegrationTest, LocalReplyWithMetadata) { ASSERT_EQ("200", response->headers().getStatusValue()); } +// Verify that host's trailing dot is removed and matches the domain for routing request. +TEST_P(ProtocolIntegrationTest, EnableStripTrailingHostDot) { + config_helper_.addConfigModifier( + [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& + hcm) -> void { + hcm.set_strip_trailing_host_dot(true); + // clear existing domains and add new domain. + auto* route_config = hcm.mutable_route_config(); + auto* virtual_host = route_config->mutable_virtual_hosts(0); + virtual_host->clear_domains(); + virtual_host->add_domains("host"); + }); + + initialize(); + codec_client_ = makeHttpConnection(lookupPort("http")); + auto response = codec_client_->makeHeaderOnlyRequest( + Http::TestRequestHeaderMapImpl{{":method", "GET"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", "host."}}); + waitForNextUpstreamRequest(); + + upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, true); + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); +} + +// Verify that host's trailing dot is not removed and thus fails to match configured domains for +// routing request. +TEST_P(DownstreamProtocolIntegrationTest, DisableStripTrailingHostDot) { + config_helper_.addConfigModifier( + [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& + hcm) -> void { + hcm.set_strip_trailing_host_dot(false); + // clear existing domains and add new domain. + auto* route_config = hcm.mutable_route_config(); + auto* virtual_host = route_config->mutable_virtual_hosts(0); + virtual_host->clear_domains(); + virtual_host->add_domains("host"); + }); + + initialize(); + codec_client_ = makeHttpConnection(lookupPort("http")); + auto response = codec_client_->makeHeaderOnlyRequest( + Http::TestRequestHeaderMapImpl{{":method", "GET"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", "host."}}); + // Expect local reply as request host fails to match configured domains. + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_TRUE(response->complete()); + EXPECT_EQ("404", response->headers().getStatusValue()); +} + } // namespace Envoy diff --git a/test/integration/protocol_integration_test.h b/test/integration/protocol_integration_test.h index 953111451954b..fdebe2034f8d7 100644 --- a/test/integration/protocol_integration_test.h +++ b/test/integration/protocol_integration_test.h @@ -42,7 +42,7 @@ class DownstreamProtocolIntegrationTest : public HttpProtocolIntegrationTest { } void verifyUpStreamRequestAfterStopAllFilter() { - if (downstreamProtocol() >= Http::CodecClient::Type::HTTP2) { + if (downstreamProtocol() >= Http::CodecType::HTTP2) { // decode-headers-return-stop-all-filter calls addDecodedData in decodeData and // decodeTrailers. 2 decoded data were added. EXPECT_EQ(count_ * size_ + added_decoded_data_size_ * 2, upstream_request_->bodyLength()); diff --git a/test/integration/proxy_proto_integration_test.cc b/test/integration/proxy_proto_integration_test.cc index b678c4d2977b5..6a5f060303858 100644 --- a/test/integration/proxy_proto_integration_test.cc +++ b/test/integration/proxy_proto_integration_test.cc @@ -30,7 +30,7 @@ insertProxyProtocolFilterConfigModifier(envoy::config::bootstrap::v3::Bootstrap& } ProxyProtoIntegrationTest::ProxyProtoIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, GetParam()) { + : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam()) { config_helper_.addConfigModifier(insertProxyProtocolFilterConfigModifier); } diff --git a/test/integration/quic_http_integration_test.cc b/test/integration/quic_http_integration_test.cc index 8f6ce0c4bc69b..3790873bc89a6 100644 --- a/test/integration/quic_http_integration_test.cc +++ b/test/integration/quic_http_integration_test.cc @@ -40,10 +40,10 @@ #include "test/config/integration/certs/clientcert_hash.h" #include "extensions/transport_sockets/tls/context_config_impl.h" -#if (defined(__has_feature) && __has_feature(thread_sanitizer)) || defined(ENVOY_CONFIG_COVERAGE) -#define DISABLE_UNDER_TSAN_OR_COVERAGE return +#if defined(ENVOY_CONFIG_COVERAGE) +#define DISABLE_UNDER_COVERAGE return #else -#define DISABLE_UNDER_TSAN_OR_COVERAGE \ +#define DISABLE_UNDER_COVERAGE \ do { \ } while (0) #endif @@ -70,7 +70,7 @@ void updateResource(AtomicFileUpdater& updater, double pressure) { class QuicHttpIntegrationTest : public HttpIntegrationTest, public QuicMultiVersionTest { public: QuicHttpIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP3, GetParam().first, + : HttpIntegrationTest(Http::CodecType::HTTP3, GetParam().first, ConfigHelper::quicHttpProxyConfig()), supported_versions_([]() { if (GetParam().second == QuicVersionType::GquicQuicCrypto) { @@ -315,107 +315,6 @@ TEST_P(QuicHttpIntegrationTest, ZeroRtt) { codec_client_->close(); } -TEST_P(QuicHttpIntegrationTest, GetRequestAndResponseWithBody) { - initialize(); - sendRequestAndVerifyResponse(default_request_headers_, /*request_size=*/0, - default_response_headers_, /*response_size=*/1024, - /*backend_index*/ 0); -} - -TEST_P(QuicHttpIntegrationTest, PostRequestAndResponseWithBody) { - testRouterRequestAndResponseWithBody(1024, 512, false); -} - -TEST_P(QuicHttpIntegrationTest, PostRequestWithBigHeadersAndResponseWithBody) { - testRouterRequestAndResponseWithBody(1024, 512, true); -} - -TEST_P(QuicHttpIntegrationTest, RouterUpstreamDisconnectBeforeRequestcomplete) { - testRouterUpstreamDisconnectBeforeRequestComplete(); -} - -TEST_P(QuicHttpIntegrationTest, RouterUpstreamDisconnectBeforeResponseComplete) { - testRouterUpstreamDisconnectBeforeResponseComplete(); - EXPECT_EQ(Http::StreamResetReason::RemoteReset, client_codec_callback_.last_stream_reset_reason_); -} - -TEST_P(QuicHttpIntegrationTest, RouterDownstreamDisconnectBeforeRequestComplete) { - testRouterDownstreamDisconnectBeforeRequestComplete(); -} - -TEST_P(QuicHttpIntegrationTest, RouterDownstreamDisconnectBeforeResponseComplete) { - testRouterDownstreamDisconnectBeforeResponseComplete(); -} - -TEST_P(QuicHttpIntegrationTest, RouterUpstreamResponseBeforeRequestComplete) { - testRouterUpstreamResponseBeforeRequestComplete(); -} - -TEST_P(QuicHttpIntegrationTest, Retry) { testRetry(); } - -TEST_P(QuicHttpIntegrationTest, UpstreamReadDisabledOnGiantResponseBody) { - DISABLE_UNDER_TSAN_OR_COVERAGE; - config_helper_.addConfigModifier(ConfigHelper::adjustUpstreamTimeoutForTsan); - config_helper_.setBufferLimits(/*upstream_buffer_limit=*/1024, /*downstream_buffer_limit=*/1024); - testRouterRequestAndResponseWithBody(/*request_size=*/512, /*response_size=*/10 * 1024 * 1024, - false, false, nullptr, - TSAN_TIMEOUT_FACTOR * TestUtility::DefaultTimeout); -} - -TEST_P(QuicHttpIntegrationTest, DownstreamReadDisabledOnGiantPost) { - DISABLE_UNDER_TSAN_OR_COVERAGE; - config_helper_.addConfigModifier(ConfigHelper::adjustUpstreamTimeoutForTsan); - config_helper_.setBufferLimits(/*upstream_buffer_limit=*/1024, /*downstream_buffer_limit=*/1024); - testRouterRequestAndResponseWithBody(/*request_size=*/10 * 1024 * 1024, /*response_size=*/1024, - false); -} - -TEST_P(QuicHttpIntegrationTest, LargeFlowControlOnAndGiantBody) { - DISABLE_UNDER_TSAN_OR_COVERAGE; - config_helper_.addConfigModifier(ConfigHelper::adjustUpstreamTimeoutForTsan); - config_helper_.setBufferLimits(/*upstream_buffer_limit=*/128 * 1024, - /*downstream_buffer_limit=*/128 * 1024); - testRouterRequestAndResponseWithBody(/*request_size=*/10 * 1024 * 1024, - /*response_size=*/10 * 1024 * 1024, false, false, nullptr, - TSAN_TIMEOUT_FACTOR * TestUtility::DefaultTimeout); -} - -// Tests that a connection idle times out after 1s and starts delayed close. -TEST_P(QuicHttpIntegrationTest, TestDelayedConnectionTeardownTimeoutTrigger) { - config_helper_.addFilter("{ name: encoder-decoder-buffer-filter, typed_config: { \"@type\": " - "type.googleapis.com/google.protobuf.Empty } }"); - config_helper_.setBufferLimits(1024, 1024); - config_helper_.addConfigModifier( - [](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& - hcm) { - // 200ms. - hcm.mutable_delayed_close_timeout()->set_nanos(200000000); - hcm.mutable_drain_timeout()->set_seconds(1); - hcm.mutable_common_http_protocol_options()->mutable_idle_timeout()->set_seconds(1); - }); - - initialize(); - - codec_client_ = makeHttpConnection(lookupPort("http")); - - auto encoder_decoder = - codec_client_->startRequest(Http::TestRequestHeaderMapImpl{{":method", "POST"}, - {":path", "/test/long/url"}, - {":scheme", "http"}, - {":authority", "host"}}); - request_encoder_ = &encoder_decoder.first; - auto response = std::move(encoder_decoder.second); - - codec_client_->sendData(*request_encoder_, 1024 * 65, false); - - ASSERT_TRUE(response->waitForEndStream()); - // The delayed close timeout should trigger since client is not closing the connection. - EXPECT_TRUE(codec_client_->waitForDisconnect(std::chrono::milliseconds(5000))); - EXPECT_EQ(codec_client_->lastConnectionEvent(), Network::ConnectionEvent::RemoteClose); - EXPECT_EQ(test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value(), - 1); -} - // Ensure multiple quic connections work, regardless of platform BPF support TEST_P(QuicHttpIntegrationTest, MultipleQuicConnectionsDefaultMode) { testMultipleQuicConnections(); @@ -548,7 +447,7 @@ TEST_P(QuicHttpIntegrationTest, NoNewStreamsWhenOverloaded) { } TEST_P(QuicHttpIntegrationTest, AdminDrainDrainsListeners) { - testAdminDrain(Http::CodecClient::Type::HTTP1); + testAdminDrain(Http::CodecType::HTTP1); } TEST_P(QuicHttpIntegrationTest, CertVerificationFailure) { @@ -566,18 +465,6 @@ TEST_P(QuicHttpIntegrationTest, CertVerificationFailure) { EXPECT_EQ(failure_reason, codec_client_->connection()->transportFailureReason()); } -TEST_P(QuicHttpIntegrationTest, RequestResponseWithTrailers) { - config_helper_.addConfigModifier(setEnableUpstreamTrailersHttp1()); - testTrailers(/*request_size=*/10, /*response_size=*/10, /*request_trailers_present=*/true, - /*response_trailers_present=*/true); -} - -// Multiple 1xx before the request completes. -TEST_P(QuicHttpIntegrationTest, EnvoyProxyingEarlyMultiple1xx) { - testEnvoyProxying1xx(/*continue_before_upstream_complete=*/true, /*with_encoder_filter=*/false, - /*with_multiple_1xx_headers=*/true); -} - // HTTP3 doesn't support 101 SwitchProtocol response code, the client should // reset the request. TEST_P(QuicHttpIntegrationTest, Reset101SwitchProtocolResponse) { @@ -621,38 +508,5 @@ TEST_P(QuicHttpIntegrationTest, ResetRequestWithoutAuthorityHeader) { EXPECT_EQ("400", response->headers().getStatusValue()); } -TEST_P(QuicHttpIntegrationTest, MultipleSetCookieAndCookieHeaders) { - initialize(); - - codec_client_ = makeHttpConnection(lookupPort("http")); - auto encoder_decoder = - codec_client_->startRequest(Http::TestRequestHeaderMapImpl{{":method", "GET"}, - {":path", "/dynamo/url"}, - {":scheme", "http"}, - {":authority", "host"}, - {"cookie", "a=b"}, - {"cookie", "c=d"}}); - request_encoder_ = &encoder_decoder.first; - auto response = std::move(encoder_decoder.second); - codec_client_->sendData(*request_encoder_, 0, true); - waitForNextUpstreamRequest(); - if (Runtime::runtimeFeatureEnabled( - "envoy.reloadable_features.header_map_correctly_coalesce_cookies")) { - EXPECT_EQ(upstream_request_->headers().get(Http::Headers::get().Cookie)[0]->value(), - "a=b; c=d"); - } - - upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}, - {"set-cookie", "foo"}, - {"set-cookie", "bar"}}, - true); - ASSERT_TRUE(response->waitForEndStream()); - EXPECT_TRUE(response->complete()); - const auto out = response->headers().get(Http::LowerCaseString("set-cookie")); - ASSERT_EQ(out.size(), 2); - ASSERT_EQ(out[0]->value().getStringView(), "foo"); - ASSERT_EQ(out[1]->value().getStringView(), "bar"); -} - } // namespace Quic } // namespace Envoy diff --git a/test/integration/quic_protocol_integration_test.cc b/test/integration/quic_protocol_integration_test.cc index 197eb3761c7f2..94601c9356b79 100644 --- a/test/integration/quic_protocol_integration_test.cc +++ b/test/integration/quic_protocol_integration_test.cc @@ -5,27 +5,27 @@ namespace Envoy { // These will run with HTTP/3 downstream, and Http upstream. INSTANTIATE_TEST_SUITE_P(DownstreamProtocols, DownstreamProtocolIntegrationTest, testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams( - {Http::CodecClient::Type::HTTP3}, {FakeHttpConnection::Type::HTTP1})), + {Http::CodecType::HTTP3}, {Http::CodecType::HTTP1})), HttpProtocolIntegrationTest::protocolTestParamsToString); // These will run with HTTP/3 downstream, and Http and HTTP/2 upstream. INSTANTIATE_TEST_SUITE_P(DownstreamProtocols, ProtocolIntegrationTest, testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams( - {Http::CodecClient::Type::HTTP3}, - {FakeHttpConnection::Type::HTTP1, FakeHttpConnection::Type::HTTP2})), + {Http::CodecType::HTTP3}, + {Http::CodecType::HTTP1, Http::CodecType::HTTP2})), HttpProtocolIntegrationTest::protocolTestParamsToString); // These will run with HTTP/1 and HTTP/2 downstream, and HTTP/3 upstream. INSTANTIATE_TEST_SUITE_P(UpstreamProtocols, DownstreamProtocolIntegrationTest, testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams( - {Http::CodecClient::Type::HTTP1, Http::CodecClient::Type::HTTP2}, - {FakeHttpConnection::Type::HTTP3})), + {Http::CodecType::HTTP1, Http::CodecType::HTTP2}, + {Http::CodecType::HTTP3})), HttpProtocolIntegrationTest::protocolTestParamsToString); INSTANTIATE_TEST_SUITE_P(UpstreamProtocols, ProtocolIntegrationTest, testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams( - {Http::CodecClient::Type::HTTP1, Http::CodecClient::Type::HTTP2}, - {FakeHttpConnection::Type::HTTP3})), + {Http::CodecType::HTTP1, Http::CodecType::HTTP2}, + {Http::CodecType::HTTP3})), HttpProtocolIntegrationTest::protocolTestParamsToString); } // namespace Envoy diff --git a/test/integration/redirect_integration_test.cc b/test/integration/redirect_integration_test.cc index 4fac5f20765dc..68f0193948a56 100644 --- a/test/integration/redirect_integration_test.cc +++ b/test/integration/redirect_integration_test.cc @@ -397,7 +397,7 @@ TEST_P(RedirectIntegrationTest, InternalRedirectCancelledDueToEarlyResponse) { upstream_request_->encodeHeaders(redirect_response_, true); ASSERT_TRUE(response->waitForEndStream()); - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() == Http::CodecType::HTTP1) { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); } else { ASSERT_TRUE(upstream_request_->waitForReset()); @@ -405,7 +405,7 @@ TEST_P(RedirectIntegrationTest, InternalRedirectCancelledDueToEarlyResponse) { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); } - if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + if (downstream_protocol_ == Http::CodecType::HTTP1) { ASSERT_TRUE(codec_client_->waitForDisconnect()); } else { codec_client_->close(); diff --git a/test/integration/rtds_integration_test.cc b/test/integration/rtds_integration_test.cc index 5114d1078849e..114b24a67957f 100644 --- a/test/integration/rtds_integration_test.cc +++ b/test/integration/rtds_integration_test.cc @@ -83,7 +83,7 @@ class RtdsIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public H public: RtdsIntegrationTest() : HttpIntegrationTest( - Http::CodecClient::Type::HTTP2, ipVersion(), + Http::CodecType::HTTP2, ipVersion(), tdsBootstrapConfig(sotwOrDelta() == Grpc::SotwOrDelta::Sotw ? "GRPC" : "DELTA_GRPC")) { use_lds_ = false; create_xds_upstream_ = true; @@ -96,7 +96,7 @@ class RtdsIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public H // The tests infra expects the xDS server to be the second fake upstream, so // we need a dummy data plane cluster. setUpstreamCount(1); - setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); + setUpstreamProtocol(Http::CodecType::HTTP2); HttpIntegrationTest::initialize(); // Register admin port. registerTestServerPorts({}); diff --git a/test/integration/scoped_rds_integration_test.cc b/test/integration/scoped_rds_integration_test.cc index 0de02704c054c..7a90c3e003df1 100644 --- a/test/integration/scoped_rds_integration_test.cc +++ b/test/integration/scoped_rds_integration_test.cc @@ -29,7 +29,7 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, absl::flat_hash_map stream_by_resource_name_; }; - ScopedRdsIntegrationTest() : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, ipVersion()) {} + ScopedRdsIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, ipVersion()) {} ~ScopedRdsIntegrationTest() override { resetConnections(); } @@ -125,9 +125,9 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, void createUpstreams() override { HttpIntegrationTest::createUpstreams(); // Create the SRDS upstream. - addFakeUpstream(FakeHttpConnection::Type::HTTP2); + addFakeUpstream(Http::CodecType::HTTP2); // Create the RDS upstream. - addFakeUpstream(FakeHttpConnection::Type::HTTP2); + addFakeUpstream(Http::CodecType::HTTP2); } void resetFakeUpstreamInfo(FakeUpstreamInfo* upstream_info) { @@ -552,7 +552,7 @@ route_configuration_name: {} http_connection_manager) { auto* filter = http_connection_manager.mutable_http_filters()->Add(); filter->set_name("filter.unknown"); - filter->set_is_optional("true"); + filter->set_is_optional(true); // keep router the last auto size = http_connection_manager.http_filters_size(); http_connection_manager.mutable_http_filters()->SwapElements(size - 2, size - 1); diff --git a/test/integration/sds_dynamic_integration_test.cc b/test/integration/sds_dynamic_integration_test.cc index 19b2636fcb5fb..f4af19bc4f97e 100644 --- a/test/integration/sds_dynamic_integration_test.cc +++ b/test/integration/sds_dynamic_integration_test.cc @@ -82,14 +82,12 @@ class SdsDynamicIntegrationBaseTest : public Grpc::BaseGrpcClientIntegrationPara public testing::TestWithParam { public: SdsDynamicIntegrationBaseTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, GetParam().ip_version), - server_cert_("server_cert"), validation_secret_("validation_secret"), - client_cert_("client_cert"), test_quic_(GetParam().test_quic) {} + : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam().ip_version), + test_quic_(GetParam().test_quic) {} - SdsDynamicIntegrationBaseTest(Http::CodecClient::Type downstream_protocol, + SdsDynamicIntegrationBaseTest(Http::CodecType downstream_protocol, Network::Address::IpVersion version, const std::string& config) - : HttpIntegrationTest(downstream_protocol, version, config), server_cert_("server_cert"), - validation_secret_("validation_secret"), client_cert_("client_cert"), + : HttpIntegrationTest(downstream_protocol, version, config), test_quic_(GetParam().test_quic) {} Network::Address::IpVersion ipVersion() const override { return GetParam().ip_version; } @@ -116,9 +114,9 @@ class SdsDynamicIntegrationBaseTest : public Grpc::BaseGrpcClientIntegrationPara setGrpcService(*grpc_service, "sds_cluster", fake_upstreams_.back()->localAddress()); } - envoy::extensions::transport_sockets::tls::v3::Secret getServerSecret() { + envoy::extensions::transport_sockets::tls::v3::Secret getServerSecretRsa() { envoy::extensions::transport_sockets::tls::v3::Secret secret; - secret.set_name(server_cert_); + secret.set_name(server_cert_rsa_); auto* tls_certificate = secret.mutable_tls_certificate(); tls_certificate->mutable_certificate_chain()->set_filename( TestEnvironment::runfilesPath("test/config/integration/certs/servercert.pem")); @@ -181,9 +179,10 @@ class SdsDynamicIntegrationBaseTest : public Grpc::BaseGrpcClientIntegrationPara } } - const std::string server_cert_; - const std::string validation_secret_; - const std::string client_cert_; + const std::string server_cert_rsa_{"server_cert_rsa"}; + const std::string server_cert_ecdsa_{"server_cert_ecdsa"}; + const std::string validation_secret_{"validation_secret"}; + const std::string client_cert_{"client_cert"}; bool v3_resource_api_{false}; bool test_quic_; }; @@ -192,14 +191,13 @@ class SdsDynamicIntegrationBaseTest : public Grpc::BaseGrpcClientIntegrationPara class SdsDynamicDownstreamIntegrationTest : public SdsDynamicIntegrationBaseTest { public: SdsDynamicDownstreamIntegrationTest() - : SdsDynamicIntegrationBaseTest((GetParam().test_quic ? Http::CodecClient::Type::HTTP3 - : Http::CodecClient::Type::HTTP1), - GetParam().ip_version, - ConfigHelper::httpProxyConfig(GetParam().test_quic)) {} + : SdsDynamicIntegrationBaseTest( + (GetParam().test_quic ? Http::CodecType::HTTP3 : Http::CodecType::HTTP1), + GetParam().ip_version, ConfigHelper::httpProxyConfig(GetParam().test_quic)) {} void initialize() override { - ASSERT(test_quic_ ? downstream_protocol_ == Http::CodecClient::Type::HTTP3 - : downstream_protocol_ == Http::CodecClient::Type::HTTP1); + ASSERT(test_quic_ ? downstream_protocol_ == Http::CodecType::HTTP3 + : downstream_protocol_ == Http::CodecType::HTTP1); config_helper_.addConfigModifier([this](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { config_helper_.configDownstreamTransportSocketWithTls( bootstrap, @@ -232,8 +230,40 @@ class SdsDynamicDownstreamIntegrationTest : public SdsDynamicIntegrationBaseTest validation_context->add_verify_certificate_hash(TEST_CLIENT_CERT_HASH); // Modify the listener ssl cert to use SDS from sds_cluster - auto* secret_config = common_tls_context.add_tls_certificate_sds_secret_configs(); - setUpSdsConfig(secret_config, "server_cert"); + auto* secret_config_rsa = common_tls_context.add_tls_certificate_sds_secret_configs(); + setUpSdsConfig(secret_config_rsa, server_cert_rsa_); + + // Add an additional SDS config for an EC cert (the base test has SDS config for an RSA cert). + // This is done via the filesystem instead of gRPC to simplify the test setup. + if (dual_cert_) { + auto* secret_config_ecdsa = common_tls_context.add_tls_certificate_sds_secret_configs(); + + secret_config_ecdsa->set_name(server_cert_ecdsa_); + auto* config_source = secret_config_ecdsa->mutable_sds_config(); + constexpr absl::string_view sds_template = + R"EOF( +--- +version_info: "0" +resources: +- "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret + name: "{}" + tls_certificate: + certificate_chain: + filename: "{}" + private_key: + filename: "{}" +)EOF"; + + const std::string sds_content = fmt::format( + sds_template, server_cert_ecdsa_, + TestEnvironment::runfilesPath("test/config/integration/certs/server_ecdsacert.pem"), + TestEnvironment::runfilesPath("test/config/integration/certs/server_ecdsakey.pem")); + + auto sds_path = + TestEnvironment::writeStringToFileForTest("server_cert_ecdsa.sds.yaml", sds_content); + config_source->set_path(sds_path); + config_source->set_resource_api_version(envoy::config::core::v3::ApiVersion::V3); + } } void createUpstreams() override { @@ -256,7 +286,7 @@ class SdsDynamicDownstreamIntegrationTest : public SdsDynamicIntegrationBaseTest Network::ClientConnectionPtr makeSslClientConnection() { int port = lookupPort("http"); - if (downstream_protocol_ <= Http::CodecClient::Type::HTTP2) { + if (downstream_protocol_ <= Http::CodecType::HTTP2) { Network::Address::InstanceConstSharedPtr address = getSslAddress(version_, port); return dispatcher_->createClientConnection( address, Network::Address::InstanceConstSharedPtr(), @@ -281,6 +311,7 @@ class SdsDynamicDownstreamIntegrationTest : public SdsDynamicIntegrationBaseTest protected: Network::TransportSocketFactoryPtr client_ssl_ctx_; + bool dual_cert_{false}; }; INSTANTIATE_TEST_SUITE_P(IpVersionsClientType, SdsDynamicDownstreamIntegrationTest, @@ -290,7 +321,7 @@ class SdsDynamicKeyRotationIntegrationTest : public SdsDynamicDownstreamIntegrat protected: envoy::extensions::transport_sockets::tls::v3::Secret getCurrentServerSecret() { envoy::extensions::transport_sockets::tls::v3::Secret secret; - secret.set_name(server_cert_); + secret.set_name(server_cert_rsa_); auto* tls_certificate = secret.mutable_tls_certificate(); tls_certificate->mutable_certificate_chain()->set_filename( TestEnvironment::temporaryPath("root/current/servercert.pem")); @@ -333,8 +364,8 @@ TEST_P(SdsDynamicKeyRotationIntegrationTest, BasicRotation) { TestEnvironment::temporaryPath("root/current")); waitForSdsUpdateStats(2); // The rotation is not a SDS attempt, so no change to these stats. - EXPECT_EQ(1, test_server_->counter("sds.server_cert.update_success")->value()); - EXPECT_EQ(0, test_server_->counter("sds.server_cert.update_rejected")->value()); + EXPECT_EQ(1, test_server_->counter("sds.server_cert_rsa.update_success")->value()); + EXPECT_EQ(0, test_server_->counter("sds.server_cert_rsa.update_rejected")->value()); // First request with server_ecdsa{cert,key}.pem. testRouterHeaderOnlyRequestAndResponse(&creator); @@ -365,11 +396,11 @@ TEST_P(SdsDynamicKeyRotationIntegrationTest, EmptyRotation) { // Rotate to an empty directory, this should fail. TestEnvironment::renameFile(TestEnvironment::temporaryPath("root/empty"), TestEnvironment::temporaryPath("root/current")); - test_server_->waitForCounterEq("sds.server_cert.key_rotation_failed", 1); + test_server_->waitForCounterEq("sds.server_cert_rsa.key_rotation_failed", 1); waitForSdsUpdateStats(1); // The rotation is not a SDS attempt, so no change to these stats. - EXPECT_EQ(1, test_server_->counter("sds.server_cert.update_success")->value()); - EXPECT_EQ(0, test_server_->counter("sds.server_cert.update_rejected")->value()); + EXPECT_EQ(1, test_server_->counter("sds.server_cert_rsa.update_success")->value()); + EXPECT_EQ(0, test_server_->counter("sds.server_cert_rsa.update_rejected")->value()); // Requests continue to work with key/cert pair. testRouterHeaderOnlyRequestAndResponse(&creator); @@ -380,18 +411,64 @@ TEST_P(SdsDynamicKeyRotationIntegrationTest, EmptyRotation) { TEST_P(SdsDynamicDownstreamIntegrationTest, BasicSuccess) { on_server_init_function_ = [this]() { createSdsStream(*(fake_upstreams_[1])); - sendSdsResponse(getServerSecret()); + sendSdsResponse(getServerSecretRsa()); + }; + initialize(); + + ConnectionCreationFunction creator = [&]() -> Network::ClientConnectionPtr { + return makeSslClientConnection(); + }; + testRouterHeaderOnlyRequestAndResponse(&creator); + + // Success + EXPECT_EQ(1, test_server_->counter("sds.server_cert_rsa.update_success")->value()); + EXPECT_EQ(0, test_server_->counter("sds.server_cert_rsa.update_rejected")->value()); +} + +TEST_P(SdsDynamicDownstreamIntegrationTest, DualCert) { + on_server_init_function_ = [this]() { + createSdsStream(*(fake_upstreams_[1])); + sendSdsResponse(getServerSecretRsa()); }; + + dual_cert_ = true; initialize(); ConnectionCreationFunction creator = [&]() -> Network::ClientConnectionPtr { return makeSslClientConnection(); }; + + client_ssl_ctx_ = createClientSslTransportSocketFactory( + ClientSslTransportOptions() + .setTlsVersion(envoy::extensions::transport_sockets::tls::v3::TlsParameters::TLSv1_2) + .setCipherSuites({"ECDHE-ECDSA-AES128-GCM-SHA256"}), + context_manager_, *api_); + testRouterHeaderOnlyRequestAndResponse(&creator); + + cleanupUpstreamAndDownstream(); + client_ssl_ctx_ = createClientSslTransportSocketFactory( + ClientSslTransportOptions() + .setTlsVersion(envoy::extensions::transport_sockets::tls::v3::TlsParameters::TLSv1_2) + .setCipherSuites({"ECDHE-RSA-AES128-GCM-SHA256"}), + context_manager_, *api_); testRouterHeaderOnlyRequestAndResponse(&creator); // Success - EXPECT_EQ(1, test_server_->counter("sds.server_cert.update_success")->value()); - EXPECT_EQ(0, test_server_->counter("sds.server_cert.update_rejected")->value()); + EXPECT_EQ(1, test_server_->counter("sds.server_cert_rsa.update_success")->value()); + EXPECT_EQ(0, test_server_->counter("sds.server_cert_rsa.update_rejected")->value()); + EXPECT_EQ(1, test_server_->counter("sds.server_cert_ecdsa.update_success")->value()); + EXPECT_EQ(0, test_server_->counter("sds.server_cert_ecdsa.update_rejected")->value()); + + // QUIC ignores the `setTlsVersion` set above and always uses TLS 1.3, and TLS 1.3 ignores the + // `setCipherSuites`, so in the QUIC config, this test only uses one of the certs. + if (!test_quic_) { + EXPECT_EQ(1, + test_server_->counter(listenerStatPrefix("ssl.ciphers.ECDHE-RSA-AES128-GCM-SHA256")) + ->value()); + EXPECT_EQ(1, + test_server_->counter(listenerStatPrefix("ssl.ciphers.ECDHE-ECDSA-AES128-GCM-SHA256")) + ->value()); + } } // A test that SDS server send a bad secret for a static listener, @@ -400,7 +477,7 @@ TEST_P(SdsDynamicDownstreamIntegrationTest, BasicSuccess) { TEST_P(SdsDynamicDownstreamIntegrationTest, WrongSecretFirst) { on_server_init_function_ = [this]() { createSdsStream(*(fake_upstreams_[1])); - sendSdsResponse(getWrongSecret(server_cert_)); + sendSdsResponse(getWrongSecret(server_cert_rsa_)); }; initialize(); @@ -410,10 +487,10 @@ TEST_P(SdsDynamicDownstreamIntegrationTest, WrongSecretFirst) { codec_client_->connection()->close(Network::ConnectionCloseType::NoFlush); // Failure - EXPECT_EQ(0, test_server_->counter("sds.server_cert.update_success")->value()); - EXPECT_EQ(1, test_server_->counter("sds.server_cert.update_rejected")->value()); + EXPECT_EQ(0, test_server_->counter("sds.server_cert_rsa.update_success")->value()); + EXPECT_EQ(1, test_server_->counter("sds.server_cert_rsa.update_rejected")->value()); - sendSdsResponse(getServerSecret()); + sendSdsResponse(getServerSecretRsa()); // Wait for sds update counter. waitForSdsUpdateStats(1); @@ -424,8 +501,8 @@ TEST_P(SdsDynamicDownstreamIntegrationTest, WrongSecretFirst) { testRouterHeaderOnlyRequestAndResponse(&creator); // Success - EXPECT_EQ(1, test_server_->counter("sds.server_cert.update_success")->value()); - EXPECT_EQ(1, test_server_->counter("sds.server_cert.update_rejected")->value()); + EXPECT_EQ(1, test_server_->counter("sds.server_cert_rsa.update_success")->value()); + EXPECT_EQ(1, test_server_->counter("sds.server_cert_rsa.update_rejected")->value()); } class SdsDynamicDownstreamCertValidationContextTest : public SdsDynamicDownstreamIntegrationTest { @@ -643,7 +720,7 @@ class SdsDynamicUpstreamIntegrationTest : public SdsDynamicIntegrationBaseTest { void initialize() override { if (test_quic_) { upstream_tls_ = true; - setUpstreamProtocol(FakeHttpConnection::Type::HTTP3); + setUpstreamProtocol(Http::CodecType::HTTP3); } config_helper_.addConfigModifier([this](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { // add sds cluster first. @@ -746,7 +823,7 @@ TEST_P(SdsDynamicUpstreamIntegrationTest, WrongSecretFirst) { EXPECT_EQ("503", response->headers().getStatusValue()); // Wait for the raw TCP connection with bad credentials and close it. - if (upstreamProtocol() != FakeHttpConnection::Type::HTTP3) { + if (upstreamProtocol() != Http::CodecType::HTTP3) { FakeRawConnectionPtr fake_upstream_connection; ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); @@ -831,11 +908,11 @@ class SdsCdsIntegrationTest : public SdsDynamicIntegrationBaseTest { void createUpstreams() override { // Static cluster. - addFakeUpstream(FakeHttpConnection::Type::HTTP1); + addFakeUpstream(Http::CodecType::HTTP1); // Cds Cluster. - addFakeUpstream(FakeHttpConnection::Type::HTTP2); + addFakeUpstream(Http::CodecType::HTTP2); // Sds Cluster. - addFakeUpstream(FakeHttpConnection::Type::HTTP2); + addFakeUpstream(Http::CodecType::HTTP2); } void sendCdsResponse() { diff --git a/test/integration/sds_generic_secret_integration_test.cc b/test/integration/sds_generic_secret_integration_test.cc index 9cf1019fc6dc1..e830dce4a2344 100644 --- a/test/integration/sds_generic_secret_integration_test.cc +++ b/test/integration/sds_generic_secret_integration_test.cc @@ -90,7 +90,7 @@ class SdsGenericSecretIntegrationTest : public Grpc::GrpcClientIntegrationParamT public HttpIntegrationTest { public: SdsGenericSecretIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, ipVersion()), registration_(factory_) {} + : HttpIntegrationTest(Http::CodecType::HTTP1, ipVersion()), registration_(factory_) {} void initialize() override { config_helper_.addConfigModifier([](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { diff --git a/test/integration/sds_static_integration_test.cc b/test/integration/sds_static_integration_test.cc index 8e0be765e98fa..97636f2516201 100644 --- a/test/integration/sds_static_integration_test.cc +++ b/test/integration/sds_static_integration_test.cc @@ -35,8 +35,7 @@ class SdsStaticDownstreamIntegrationTest : public testing::TestWithParam, public HttpIntegrationTest { public: - SdsStaticDownstreamIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, GetParam()) {} + SdsStaticDownstreamIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam()) {} void initialize() override { config_helper_.addConfigModifier([](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { @@ -110,8 +109,7 @@ TEST_P(SdsStaticDownstreamIntegrationTest, RouterRequestAndResponseWithGiantBody class SdsStaticUpstreamIntegrationTest : public testing::TestWithParam, public HttpIntegrationTest { public: - SdsStaticUpstreamIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, GetParam()) {} + SdsStaticUpstreamIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam()) {} void initialize() override { config_helper_.addConfigModifier([](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { @@ -146,8 +144,7 @@ class SdsStaticUpstreamIntegrationTest : public testing::TestWithParamcomplete()); EXPECT_EQ("200", response->headers().getStatusValue()); server_gone_.WaitForNotification(); diff --git a/test/integration/server.h b/test/integration/server.h index d62e3f7dd5f48..b28b08aaac0e8 100644 --- a/test/integration/server.h +++ b/test/integration/server.h @@ -442,27 +442,24 @@ class IntegrationTestServer : public Logger::Loggable, Buffer::WatermarkFactorySharedPtr watermark_factory, bool v2_bootstrap); void waitForCounterEq(const std::string& name, uint64_t value, - std::chrono::milliseconds timeout = std::chrono::milliseconds::zero(), + std::chrono::milliseconds timeout = TestUtility::DefaultTimeout, Event::Dispatcher* dispatcher = nullptr) override { ASSERT_TRUE( TestUtility::waitForCounterEq(statStore(), name, value, time_system_, timeout, dispatcher)); } - void - waitForCounterGe(const std::string& name, uint64_t value, - std::chrono::milliseconds timeout = std::chrono::milliseconds::zero()) override { + void waitForCounterGe(const std::string& name, uint64_t value, + std::chrono::milliseconds timeout = TestUtility::DefaultTimeout) override { ASSERT_TRUE(TestUtility::waitForCounterGe(statStore(), name, value, time_system_, timeout)); } - void - waitForGaugeEq(const std::string& name, uint64_t value, - std::chrono::milliseconds timeout = std::chrono::milliseconds::zero()) override { + void waitForGaugeEq(const std::string& name, uint64_t value, + std::chrono::milliseconds timeout = TestUtility::DefaultTimeout) override { ASSERT_TRUE(TestUtility::waitForGaugeEq(statStore(), name, value, time_system_, timeout)); } - void - waitForGaugeGe(const std::string& name, uint64_t value, - std::chrono::milliseconds timeout = std::chrono::milliseconds::zero()) override { + void waitForGaugeGe(const std::string& name, uint64_t value, + std::chrono::milliseconds timeout = TestUtility::DefaultTimeout) override { ASSERT_TRUE(TestUtility::waitForGaugeGe(statStore(), name, value, time_system_, timeout)); } diff --git a/test/integration/socket_interface_swap.cc b/test/integration/socket_interface_swap.cc new file mode 100644 index 0000000000000..521153817a256 --- /dev/null +++ b/test/integration/socket_interface_swap.cc @@ -0,0 +1,30 @@ +#include "test/integration/socket_interface_swap.h" + +namespace Envoy { + +SocketInterfaceSwap::SocketInterfaceSwap() { + Envoy::Network::SocketInterfaceSingleton::clear(); + test_socket_interface_loader_ = std::make_unique( + std::make_unique( + [writev_matcher = writev_matcher_](Envoy::Network::TestIoSocketHandle* io_handle, + const Buffer::RawSlice*, + uint64_t) -> absl::optional { + if (writev_matcher->shouldReturnEgain(io_handle)) { + return Api::IoCallUint64Result( + 0, Api::IoErrorPtr(Network::IoSocketError::getIoSocketEagainInstance(), + Network::IoSocketError::deleteIoError)); + } + return absl::nullopt; + })); +} + +void SocketInterfaceSwap::IoHandleMatcher::setResumeWrites() { + absl::MutexLock lock(&mutex_); + mutex_.Await(absl::Condition( + +[](Network::TestIoSocketHandle** matched_iohandle) { return *matched_iohandle != nullptr; }, + &matched_iohandle_)); + writev_returns_egain_ = false; + matched_iohandle_->activateInDispatcherThread(Event::FileReadyType::Write); +} + +} // namespace Envoy diff --git a/test/integration/socket_interface_swap.h b/test/integration/socket_interface_swap.h new file mode 100644 index 0000000000000..77e4fbca342dd --- /dev/null +++ b/test/integration/socket_interface_swap.h @@ -0,0 +1,73 @@ +#pragma once + +#include "common/network/socket_interface.h" + +#include "test/integration/filters/test_socket_interface.h" + +namespace Envoy { + +// Enables control at the socket level to stop and resume writes. +// Useful for tests want to temporarily stop Envoy from draining data. + +class SocketInterfaceSwap { +public: + // Object of this class hold the state determining the IoHandle which + // should return EAGAIN from the `writev` call. + struct IoHandleMatcher { + bool shouldReturnEgain(Envoy::Network::TestIoSocketHandle* io_handle) { + absl::MutexLock lock(&mutex_); + if (writev_returns_egain_ && (io_handle->localAddress()->ip()->port() == src_port_ || + io_handle->peerAddress()->ip()->port() == dst_port_)) { + ASSERT(matched_iohandle_ == nullptr || matched_iohandle_ == io_handle, + "Matched multiple io_handles, expected at most one to match."); + matched_iohandle_ = io_handle; + return true; + } + return false; + } + + // Source port to match. The port specified should be associated with a listener. + void setSourcePort(uint32_t port) { + absl::WriterMutexLock lock(&mutex_); + dst_port_ = 0; + src_port_ = port; + } + + // Destination port to match. The port specified should be associated with a listener. + void setDestinationPort(uint32_t port) { + absl::WriterMutexLock lock(&mutex_); + src_port_ = 0; + dst_port_ = port; + } + + void setWritevReturnsEgain() { + absl::WriterMutexLock lock(&mutex_); + ASSERT(src_port_ != 0 || dst_port_ != 0); + writev_returns_egain_ = true; + } + + void setResumeWrites(); + + private: + mutable absl::Mutex mutex_; + uint32_t src_port_ ABSL_GUARDED_BY(mutex_) = 0; + uint32_t dst_port_ ABSL_GUARDED_BY(mutex_) = 0; + bool writev_returns_egain_ ABSL_GUARDED_BY(mutex_) = false; + Network::TestIoSocketHandle* matched_iohandle_{}; + }; + + SocketInterfaceSwap(); + + ~SocketInterfaceSwap() { + test_socket_interface_loader_.reset(); + Envoy::Network::SocketInterfaceSingleton::initialize(previous_socket_interface_); + } + +protected: + Envoy::Network::SocketInterface* const previous_socket_interface_{ + Envoy::Network::SocketInterfaceSingleton::getExisting()}; + std::shared_ptr writev_matcher_{std::make_shared()}; + std::unique_ptr test_socket_interface_loader_; +}; + +} // namespace Envoy diff --git a/test/integration/tcp_proxy_integration_test.cc b/test/integration/tcp_proxy_integration_test.cc index 8cb605c65ba3a..532337a848dc7 100644 --- a/test/integration/tcp_proxy_integration_test.cc +++ b/test/integration/tcp_proxy_integration_test.cc @@ -114,7 +114,7 @@ TEST_P(TcpProxyIntegrationTest, TcpProxyUpstreamWritesFirst) { // Test TLS upstream. TEST_P(TcpProxyIntegrationTest, TcpProxyUpstreamTls) { upstream_tls_ = true; - setUpstreamProtocol(FakeHttpConnection::Type::HTTP1); + setUpstreamProtocol(Http::CodecType::HTTP1); config_helper_.configureUpstreamTls(); initialize(); IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); @@ -1469,7 +1469,7 @@ TEST_P(MysqlIntegrationTest, Preconnect) { testPreconnect(); } TEST_P(MysqlIntegrationTest, PreconnectWithTls) { upstream_tls_ = true; - setUpstreamProtocol(FakeHttpConnection::Type::HTTP1); + setUpstreamProtocol(Http::CodecType::HTTP1); config_helper_.configureUpstreamTls(); testPreconnect(); } diff --git a/test/integration/tcp_tunneling_integration_test.cc b/test/integration/tcp_tunneling_integration_test.cc index 42d70385524ef..c6aae7bef4ba0 100644 --- a/test/integration/tcp_tunneling_integration_test.cc +++ b/test/integration/tcp_tunneling_integration_test.cc @@ -16,8 +16,7 @@ class ConnectTerminationIntegrationTest : public testing::TestWithParam, public HttpIntegrationTest { public: - ConnectTerminationIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, GetParam()) { + ConnectTerminationIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP2, GetParam()) { enableHalfClose(true); } @@ -226,7 +225,7 @@ TEST_P(ConnectTerminationIntegrationTest, BasicMaxStreamDuration) { test_server_->waitForCounterGe("cluster.cluster_0.upstream_rq_max_duration_reached", 1); - if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + if (downstream_protocol_ == Http::CodecType::HTTP1) { ASSERT_TRUE(codec_client_->waitForDisconnect()); } else { ASSERT_TRUE(response_->waitForReset()); @@ -274,7 +273,7 @@ TEST_P(ProxyingConnectIntegrationTest, ProxyConnect) { RELEASE_ASSERT(result, result.message()); ASSERT_TRUE(upstream_request_->waitForHeadersComplete()); EXPECT_EQ(upstream_request_->headers().get(Http::Headers::get().Method)[0]->value(), "CONNECT"); - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() == Http::CodecType::HTTP1) { EXPECT_TRUE(upstream_request_->headers().get(Http::Headers::get().Protocol).empty()); } else { EXPECT_EQ(upstream_request_->headers().get(Http::Headers::get().Protocol)[0]->value(), @@ -322,7 +321,7 @@ TEST_P(ProxyingConnectIntegrationTest, ProxyConnectWithPortStrippingLegacy) { ASSERT_TRUE(upstream_request_->waitForHeadersComplete()); EXPECT_EQ(upstream_request_->headers().getMethodValue(), "CONNECT"); EXPECT_EQ(upstream_request_->headers().getHostValue(), "host:80"); - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() == Http::CodecType::HTTP1) { EXPECT_TRUE(upstream_request_->headers().get(Http::Headers::get().Protocol).empty()); } else { EXPECT_EQ(upstream_request_->headers().get(Http::Headers::get().Protocol)[0]->value(), @@ -376,7 +375,7 @@ TEST_P(ProxyingConnectIntegrationTest, ProxyConnectWithPortStripping) { ASSERT_TRUE(upstream_request_->waitForHeadersComplete()); EXPECT_EQ(upstream_request_->headers().getMethodValue(), "CONNECT"); EXPECT_EQ(upstream_request_->headers().getHostValue(), "host:80"); - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() == Http::CodecType::HTTP1) { EXPECT_TRUE(upstream_request_->headers().getProtocolValue().empty()); } else { EXPECT_EQ(upstream_request_->headers().getProtocolValue(), "bytestream"); @@ -421,7 +420,7 @@ TEST_P(ProxyingConnectIntegrationTest, ProxyConnectWithIP) { RELEASE_ASSERT(result, result.message()); ASSERT_TRUE(upstream_request_->waitForHeadersComplete()); EXPECT_EQ(upstream_request_->headers().get(Http::Headers::get().Method)[0]->value(), "CONNECT"); - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() == Http::CodecType::HTTP1) { EXPECT_TRUE(upstream_request_->headers().get(Http::Headers::get().Protocol).empty()); } else { EXPECT_EQ(upstream_request_->headers().get(Http::Headers::get().Protocol)[0]->value(), @@ -442,26 +441,26 @@ INSTANTIATE_TEST_SUITE_P(IpVersions, ConnectTerminationIntegrationTest, testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), TestUtility::ipTestParamsToString); -using Params = std::tuple; +using Params = std::tuple; // Tunneling downstream TCP over an upstream HTTP CONNECT tunnel. class TcpTunnelingIntegrationTest : public testing::TestWithParam, public HttpIntegrationTest { public: TcpTunnelingIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, std::get<0>(GetParam())) {} + : HttpIntegrationTest(Http::CodecType::HTTP2, std::get<0>(GetParam())) {} static std::string paramsToString(const testing::TestParamInfo& p) { return fmt::format( "{}_{}_{}", std::get<0>(p.param) == Network::Address::IpVersion::v4 ? "IPv4" : "IPv6", - std::get<1>(p.param) == FakeHttpConnection::Type::HTTP1 ? "HTTP1Upstream" : "HTTP2Upstream", + std::get<1>(p.param) == Http::CodecType::HTTP1 ? "HTTP1Upstream" : "HTTP2Upstream", std::get<2>(p.param) ? "WaitConnectResponse" : "DoNotWaitConnectResponse"); } void SetUp() override { wait_for_connect_response_ = std::get<2>(GetParam()); enableHalfClose(true); - setDownstreamProtocol(Http::CodecClient::Type::HTTP2); + setDownstreamProtocol(Http::CodecType::HTTP2); setUpstreamProtocol(std::get<1>(GetParam())); if (wait_for_connect_response_) { @@ -519,7 +518,7 @@ TEST_P(TcpTunnelingIntegrationTest, Basic) { ASSERT_TRUE(tcp_client->write("hello", false)); tcp_client->close(); ASSERT_TRUE(upstream_request_->waitForData(*dispatcher_, 5)); - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() == Http::CodecType::HTTP1) { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); } else { ASSERT_TRUE(upstream_request_->waitForEndStream(*dispatcher_)); @@ -574,7 +573,7 @@ TEST_P(TcpTunnelingIntegrationTest, BasicUsePost) { ASSERT_TRUE(tcp_client->write("hello", false)); tcp_client->close(); ASSERT_TRUE(upstream_request_->waitForData(*dispatcher_, 5)); - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() == Http::CodecType::HTTP1) { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); } else { ASSERT_TRUE(upstream_request_->waitForEndStream(*dispatcher_)); @@ -596,7 +595,7 @@ TEST_P(TcpTunnelingIntegrationTest, InvalidResponseHeaders) { // upstream gets a stream reset. default_response_headers_.setStatus(enumToInt(Http::Code::ServiceUnavailable)); upstream_request_->encodeHeaders(default_response_headers_, false); - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() == Http::CodecType::HTTP1) { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); } else { ASSERT_TRUE(upstream_request_->waitForReset()); @@ -625,13 +624,13 @@ TEST_P(TcpTunnelingIntegrationTest, CloseUpstreamFirst) { // Send data from upstream to downstream with an end stream and make sure the data is received // before the connection is half-closed. upstream_request_->encodeData(12, true); - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() == Http::CodecType::HTTP1) { ASSERT_TRUE(fake_upstream_connection_->close()); } ASSERT_TRUE(tcp_client->waitForData(12)); tcp_client->waitForHalfClose(); - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() == Http::CodecType::HTTP1) { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); tcp_client->close(); } else { @@ -647,7 +646,7 @@ TEST_P(TcpTunnelingIntegrationTest, CloseUpstreamFirst) { } TEST_P(TcpTunnelingIntegrationTest, ResetStreamTest) { - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() == Http::CodecType::HTTP1) { return; } enableHalfClose(false); @@ -696,7 +695,7 @@ TEST_P(TcpTunnelingIntegrationTest, TestIdletimeoutWithLargeOutstandingData) { upstream_request_->encodeData(data, false); tcp_client->waitForDisconnect(); - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() == Http::CodecType::HTTP1) { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); tcp_client->close(); } else { @@ -719,7 +718,7 @@ TEST_P(TcpTunnelingIntegrationTest, TcpProxyDownstreamFlush) { upstream_request_->encodeHeaders(default_response_headers_, false); tcp_client->readDisable(true); - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() == Http::CodecType::HTTP1) { ASSERT_TRUE(tcp_client->write("hello", false)); ASSERT_TRUE(upstream_request_->waitForData(*dispatcher_, 5)); @@ -739,7 +738,7 @@ TEST_P(TcpTunnelingIntegrationTest, TcpProxyDownstreamFlush) { tcp_client->readDisable(false); tcp_client->waitForData(data); tcp_client->waitForHalfClose(); - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() == Http::CodecType::HTTP1) { tcp_client->close(); } } @@ -765,7 +764,7 @@ TEST_P(TcpTunnelingIntegrationTest, TcpProxyUpstreamFlush) { ASSERT_TRUE(tcp_client->waitForData(5)); ASSERT_TRUE(tcp_client->write(data, true)); - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() == Http::CodecType::HTTP1) { tcp_client->close(); upstream_request_->readDisable(false); @@ -785,7 +784,7 @@ TEST_P(TcpTunnelingIntegrationTest, TcpProxyUpstreamFlush) { // Test that h2 connection is reused. TEST_P(TcpTunnelingIntegrationTest, H2ConnectionReuse) { - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() == Http::CodecType::HTTP1) { return; } initialize(); @@ -831,7 +830,7 @@ TEST_P(TcpTunnelingIntegrationTest, H2ConnectionReuse) { // Test that with HTTP1 we have no connection reuse with downstream close. TEST_P(TcpTunnelingIntegrationTest, H1NoConnectionReuse) { - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP2) { + if (upstreamProtocol() == Http::CodecType::HTTP2) { return; } initialize(); @@ -872,7 +871,7 @@ TEST_P(TcpTunnelingIntegrationTest, H1NoConnectionReuse) { // Test that with HTTP1 we have no connection with upstream close. TEST_P(TcpTunnelingIntegrationTest, H1UpstreamCloseNoConnectionReuse) { - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP2) { + if (upstreamProtocol() == Http::CodecType::HTTP2) { return; } initialize(); @@ -916,7 +915,7 @@ TEST_P(TcpTunnelingIntegrationTest, H1UpstreamCloseNoConnectionReuse) { } TEST_P(TcpTunnelingIntegrationTest, 2xxStatusCodeValidHttp1) { - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP2) { + if (upstreamProtocol() == Http::CodecType::HTTP2) { return; } initialize(); @@ -946,7 +945,7 @@ TEST_P(TcpTunnelingIntegrationTest, 2xxStatusCodeValidHttp1) { } TEST_P(TcpTunnelingIntegrationTest, ContentLengthHeaderIgnoredHttp1) { - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP2) { + if (upstreamProtocol() == Http::CodecType::HTTP2) { return; } initialize(); @@ -975,7 +974,7 @@ TEST_P(TcpTunnelingIntegrationTest, ContentLengthHeaderIgnoredHttp1) { } TEST_P(TcpTunnelingIntegrationTest, TransferEncodingHeaderIgnoredHttp1) { - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP2) { + if (upstreamProtocol() == Http::CodecType::HTTP2) { return; } initialize(); @@ -1029,7 +1028,7 @@ TEST_P(TcpTunnelingIntegrationTest, DeferTransmitDataUntilSuccessConnectResponse ASSERT_TRUE(upstream_request_->waitForData(*dispatcher_, 5)); tcp_client->close(); - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() == Http::CodecType::HTTP1) { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); } else { ASSERT_TRUE(upstream_request_->waitForEndStream(*dispatcher_)); @@ -1061,7 +1060,7 @@ TEST_P(TcpTunnelingIntegrationTest, NoDataTransmittedIfConnectFailureResponseIsR ASSERT_FALSE(upstream_request_->waitForData(*dispatcher_, 1, std::chrono::milliseconds(100))); tcp_client->close(); - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() == Http::CodecType::HTTP1) { ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); } else { ASSERT_TRUE(upstream_request_->waitForReset()); @@ -1086,8 +1085,7 @@ TEST_P(TcpTunnelingIntegrationTest, UpstreamDisconnectBeforeResponseReceived) { INSTANTIATE_TEST_SUITE_P( IpAndHttpVersions, TcpTunnelingIntegrationTest, ::testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), - testing::Values(FakeHttpConnection::Type::HTTP1, - FakeHttpConnection::Type::HTTP2), + testing::Values(Http::CodecType::HTTP1, Http::CodecType::HTTP2), testing::Values(false, true)), TcpTunnelingIntegrationTest::paramsToString); diff --git a/test/integration/tracked_watermark_buffer.cc b/test/integration/tracked_watermark_buffer.cc index e67acf088f3ae..3bc07cb7a9252 100644 --- a/test/integration/tracked_watermark_buffer.cc +++ b/test/integration/tracked_watermark_buffer.cc @@ -1,5 +1,11 @@ #include "test/integration/tracked_watermark_buffer.h" +#include "envoy/thread/thread.h" +#include "envoy/thread_local/thread_local.h" +#include "envoy/thread_local/thread_local_object.h" + +#include "common/common/assert.h" + namespace Envoy { namespace Buffer { @@ -18,18 +24,52 @@ TrackedWatermarkBufferFactory::create(std::function below_low_watermark, return std::make_unique( [this, &buffer_info](uint64_t current_size) { absl::MutexLock lock(&mutex_); + total_buffer_size_ = total_buffer_size_ + current_size - buffer_info.current_size_; if (buffer_info.max_size_ < current_size) { buffer_info.max_size_ = current_size; } + buffer_info.current_size_ = current_size; + + checkIfExpectedBalancesMet(); }, [this, &buffer_info](uint32_t watermark) { absl::MutexLock lock(&mutex_); buffer_info.watermark_ = watermark; }, - [this]() { + [this, &buffer_info](TrackedWatermarkBuffer* buffer) { absl::MutexLock lock(&mutex_); ASSERT(active_buffer_count_ > 0); --active_buffer_count_; + total_buffer_size_ -= buffer_info.current_size_; + buffer_info.current_size_ = 0; + + // Remove bound account tracking. + auto account = buffer->getAccountForTest(); + if (account) { + auto& set = account_infos_[account]; + RELEASE_ASSERT(set.erase(buffer) == 1, "Expected to remove buffer from account_infos."); + RELEASE_ASSERT(actively_bound_buffers_.erase(buffer) == 1, + "Did not find buffer in actively_bound_buffers_."); + // Erase account entry if there are no active bound buffers, and + // there's no other pointers to the account besides the local account + // pointer and within the map. + // + // It's possible for an account to no longer be bound to a buffer in + // the case that the H2 stream completes, but the data hasn't flushed + // at TCP. + if (set.empty() && account.use_count() == 2) { + RELEASE_ASSERT(account_infos_.erase(account) == 1, + "Expected to remove account from account_infos."); + } + } + }, + [this](BufferMemoryAccountSharedPtr& account, TrackedWatermarkBuffer* buffer) { + absl::MutexLock lock(&mutex_); + // Only track non-null accounts. + if (account) { + account_infos_[account].emplace(buffer); + actively_bound_buffers_.emplace(buffer); + } }, below_low_watermark, above_high_watermark, above_overflow_watermark); } @@ -44,6 +84,11 @@ uint64_t TrackedWatermarkBufferFactory::numBuffersActive() const { return active_buffer_count_; } +uint64_t TrackedWatermarkBufferFactory::totalBufferSize() const { + absl::MutexLock lock(&mutex_); + return total_buffer_size_; +} + uint64_t TrackedWatermarkBufferFactory::maxBufferSize() const { absl::MutexLock lock(&mutex_); uint64_t val = 0; @@ -94,5 +139,107 @@ std::pair TrackedWatermarkBufferFactory::highWatermarkRange( return std::make_pair(min_watermark, max_watermark); } +bool TrackedWatermarkBufferFactory::waitUntilTotalBufferedExceeds( + uint64_t byte_size, std::chrono::milliseconds timeout) { + absl::MutexLock lock(&mutex_); + auto predicate = [this, byte_size]() ABSL_SHARED_LOCKS_REQUIRED(mutex_) { + mutex_.AssertHeld(); + return total_buffer_size_ >= byte_size; + }; + return mutex_.AwaitWithTimeout(absl::Condition(&predicate), absl::Milliseconds(timeout.count())); +} + +void TrackedWatermarkBufferFactory::removeDanglingAccounts() { + auto accounts_it = account_infos_.begin(); + while (accounts_it != account_infos_.end()) { + auto next = std::next(accounts_it); + + // Remove all "dangling" accounts. + if (accounts_it->first.use_count() == 1) { + ASSERT(accounts_it->second.empty()); + account_infos_.erase(accounts_it); + } + + accounts_it = next; + } +} + +void TrackedWatermarkBufferFactory::inspectAccounts( + std::function func, Server::Instance& server) { + absl::Notification done_notification; + ThreadLocal::TypedSlotPtr<> slot; + Envoy::Thread::ThreadId main_tid; + + server.dispatcher().post([&] { + slot = ThreadLocal::TypedSlot<>::makeUnique(server.threadLocal()); + slot->set( + [](Envoy::Event::Dispatcher&) -> std::shared_ptr { + return nullptr; + }); + + main_tid = server.api().threadFactory().currentThreadId(); + + slot->runOnAllThreads( + [main_tid, &server, &func, this](OptRef) { + // Run on the worker thread. + if (server.api().threadFactory().currentThreadId() != main_tid) { + absl::MutexLock lock(&(this->mutex_)); + func(this->account_infos_); + } + }, + [&slot, &done_notification] { + slot.reset(nullptr); + done_notification.Notify(); + }); + }); + + done_notification.WaitForNotification(); +} + +void TrackedWatermarkBufferFactory::setExpectedAccountBalance(uint64_t byte_size_per_account, + uint32_t num_accounts) { + absl::MutexLock lock(&mutex_); + ASSERT(!expected_balances_.has_value()); + expected_balances_.emplace(byte_size_per_account, num_accounts); +} + +bool TrackedWatermarkBufferFactory::waitForExpectedAccountBalanceWithTimeout( + std::chrono::milliseconds timeout) { + return expected_balances_met_.WaitForNotificationWithTimeout(absl::FromChrono(timeout)); +} + +bool TrackedWatermarkBufferFactory::waitUntilExpectedNumberOfAccountsAndBoundBuffers( + uint32_t num_accounts, uint32_t num_bound_buffers, std::chrono::milliseconds timeout) { + absl::MutexLock lock(&mutex_); + auto predicate = [this, num_accounts, num_bound_buffers]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_) { + mutex_.AssertHeld(); + removeDanglingAccounts(); + return num_bound_buffers == actively_bound_buffers_.size() && + num_accounts == account_infos_.size(); + }; + return mutex_.AwaitWithTimeout(absl::Condition(&predicate), absl::FromChrono(timeout)); +} + +void TrackedWatermarkBufferFactory::checkIfExpectedBalancesMet() { + if (!expected_balances_ || expected_balances_met_.HasBeenNotified()) { + return; + } + + removeDanglingAccounts(); + + if (account_infos_.size() == expected_balances_->num_accounts_) { + // This is thread safe since this function should run on the only Envoy worker + // thread. + for (auto& acc : account_infos_) { + if (static_cast(acc.first.get())->balance() < + expected_balances_->balance_per_account_) { + return; + } + } + + expected_balances_met_.Notify(); + } +} + } // namespace Buffer } // namespace Envoy diff --git a/test/integration/tracked_watermark_buffer.h b/test/integration/tracked_watermark_buffer.h index 64d8473dbf104..bc05db5d4ca03 100644 --- a/test/integration/tracked_watermark_buffer.h +++ b/test/integration/tracked_watermark_buffer.h @@ -1,9 +1,16 @@ #pragma once +#include "envoy/buffer/buffer.h" +#include "envoy/server/instance.h" + +#include "common/buffer/buffer_impl.h" #include "common/buffer/watermark_buffer.h" +#include "test/test_common/utility.h" + #include "absl/container/node_hash_map.h" #include "absl/synchronization/mutex.h" +#include "absl/synchronization/notification.h" namespace Envoy { namespace Buffer { @@ -11,31 +18,44 @@ namespace Buffer { // WatermarkBuffer subclass that hooks into updates to buffer size and buffer high watermark config. class TrackedWatermarkBuffer : public Buffer::WatermarkBuffer { public: - TrackedWatermarkBuffer(std::function update_max_size, - std::function update_high_watermark, - std::function on_delete, std::function below_low_watermark, - std::function above_high_watermark, - std::function above_overflow_watermark) + TrackedWatermarkBuffer( + std::function update_size, + std::function update_high_watermark, + std::function on_delete, + std::function on_bind, + std::function below_low_watermark, std::function above_high_watermark, + std::function above_overflow_watermark) : WatermarkBuffer(below_low_watermark, above_high_watermark, above_overflow_watermark), - update_max_size_(update_max_size), update_high_watermark_(update_high_watermark), - on_delete_(on_delete) {} - ~TrackedWatermarkBuffer() override { on_delete_(); } + update_size_(update_size), update_high_watermark_(update_high_watermark), + on_delete_(on_delete), on_bind_(on_bind) {} + ~TrackedWatermarkBuffer() override { on_delete_(this); } void setWatermarks(uint32_t watermark) override { update_high_watermark_(watermark); WatermarkBuffer::setWatermarks(watermark); } + void bindAccount(BufferMemoryAccountSharedPtr account) override { + on_bind_(account, this); + WatermarkBuffer::bindAccount(account); + } + protected: void checkHighAndOverflowWatermarks() override { - update_max_size_(length()); + update_size_(length()); WatermarkBuffer::checkHighAndOverflowWatermarks(); } + void checkLowWatermark() override { + update_size_(length()); + WatermarkBuffer::checkLowWatermark(); + } + private: - std::function update_max_size_; + std::function update_size_; std::function update_high_watermark_; - std::function on_delete_; + std::function on_delete_; + std::function on_bind_; }; // Factory that tracks how the created buffers are used. @@ -52,6 +72,8 @@ class TrackedWatermarkBufferFactory : public Buffer::WatermarkFactory { uint64_t numBuffersCreated() const; // Number of buffers still in use. uint64_t numBuffersActive() const; + // Total bytes buffered. + uint64_t totalBufferSize() const; // Size of the largest buffer. uint64_t maxBufferSize() const; // Sum of the max size of all known buffers. @@ -60,19 +82,84 @@ class TrackedWatermarkBufferFactory : public Buffer::WatermarkFactory { // functionality is disabled. std::pair highWatermarkRange() const; + // Total bytes currently buffered across all known buffers. + uint64_t totalBytesBuffered() const { + absl::MutexLock lock(&mutex_); + return total_buffer_size_; + } + + // Wait until total bytes buffered exceeds the a given size. + bool + waitUntilTotalBufferedExceeds(uint64_t byte_size, + std::chrono::milliseconds timeout = TestUtility::DefaultTimeout); + + // Set the expected account balance, prior to sending requests. + // The test thread can then wait for this condition to be true. + // This is separated so that the test thread can spin up requests however it + // desires in between. + // + // The Envoy worker thread will notify the test thread once the condition is + // met. + void setExpectedAccountBalance(uint64_t byte_size_per_account, uint32_t num_accounts); + bool waitForExpectedAccountBalanceWithTimeout( + std::chrono::milliseconds timeout = TestUtility::DefaultTimeout); + + // Wait for the expected number of accounts and number of bound buffers. + // + // Due to deferred deletion, it possible that the Envoy hasn't cleaned up on + // its end, but the stream has been completed. This avoids that by awaiting + // for the side effect of the deletion to have occurred. + bool waitUntilExpectedNumberOfAccountsAndBoundBuffers( + uint32_t num_accounts, uint32_t num_bound_buffers, + std::chrono::milliseconds timeout = TestUtility::DefaultTimeout); + + using AccountToBoundBuffersMap = + absl::flat_hash_map>; + void inspectAccounts(std::function func, + Server::Instance& server); + private: + // Remove "dangling" accounts; accounts where the account_info map is the only + // entity still pointing to the account. + void removeDanglingAccounts() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + // Should only be executed on the Envoy's worker thread. Otherwise, we have a + // possible race condition. + void checkIfExpectedBalancesMet() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + struct BufferInfo { uint32_t watermark_ = 0; + uint64_t current_size_ = 0; uint64_t max_size_ = 0; }; + struct AccountBalanceExpectations { + AccountBalanceExpectations(uint64_t balance_per_account, uint32_t num_accounts) + : balance_per_account_(balance_per_account), num_accounts_(num_accounts) {} + + uint64_t balance_per_account_ = 0; + uint32_t num_accounts_ = 0; + }; + mutable absl::Mutex mutex_; // Id of the next buffer to create. uint64_t next_idx_ ABSL_GUARDED_BY(mutex_) = 0; // Number of buffers currently in existence. uint64_t active_buffer_count_ ABSL_GUARDED_BY(mutex_) = 0; + // total bytes buffered across all buffers. + uint64_t total_buffer_size_ ABSL_GUARDED_BY(mutex_) = 0; // Info about the buffer, by buffer idx. absl::node_hash_map buffer_infos_ ABSL_GUARDED_BY(mutex_); + // The expected balances for the accounts. If set, when a buffer updates its + // size, it also checks whether the expected_balances has been satisfied, and + // notifies waiters of expected_balances_met_. + absl::optional expected_balances_ ABSL_GUARDED_BY(mutex_); + absl::Notification expected_balances_met_; + // Map from accounts to buffers bound to that account. + AccountToBoundBuffersMap account_infos_ ABSL_GUARDED_BY(mutex_); + // Set of actively bound buffers. Used for asserting that buffers are bound + // only once. + absl::flat_hash_set actively_bound_buffers_ ABSL_GUARDED_BY(mutex_); }; } // namespace Buffer diff --git a/test/integration/tracked_watermark_buffer_test.cc b/test/integration/tracked_watermark_buffer_test.cc index 9d4eb6110d9ea..f38a2968dcc37 100644 --- a/test/integration/tracked_watermark_buffer_test.cc +++ b/test/integration/tracked_watermark_buffer_test.cc @@ -1,6 +1,14 @@ +#include +#include + +#include "envoy/buffer/buffer.h" + +#include "common/buffer/buffer_impl.h" + #include "test/integration/tracked_watermark_buffer.h" #include "test/mocks/common.h" #include "test/test_common/test_runtime.h" +#include "test/test_common/thread_factory_for_test.h" #include "gtest/gtest.h" @@ -59,29 +67,31 @@ TEST_F(TrackedWatermarkBufferTest, BufferSizes) { auto buffer = factory_.create([]() {}, []() {}, []() {}); buffer->setWatermarks(100); auto buffer2 = factory_.create([]() {}, []() {}, []() {}); - EXPECT_EQ(2, factory_.numBuffersCreated()); EXPECT_EQ(2, factory_.numBuffersActive()); - // Add some bytes to the buffers, and verify max and sum(max). buffer->add("abcde"); buffer2->add("a"); EXPECT_EQ(5, factory_.maxBufferSize()); EXPECT_EQ(6, factory_.sumMaxBufferSizes()); + EXPECT_EQ(6, factory_.totalBytesBuffered()); // Add more bytes and drain the buffer. Verify that max is latched. buffer->add(std::string(1000, 'a')); EXPECT_TRUE(buffer->highWatermarkTriggered()); + EXPECT_EQ(1006, factory_.totalBytesBuffered()); buffer->drain(1005); EXPECT_EQ(0, buffer->length()); EXPECT_FALSE(buffer->highWatermarkTriggered()); EXPECT_EQ(1005, factory_.maxBufferSize()); EXPECT_EQ(1006, factory_.sumMaxBufferSizes()); + EXPECT_EQ(1, factory_.totalBytesBuffered()); buffer2->add("a"); EXPECT_EQ(1005, factory_.maxBufferSize()); EXPECT_EQ(1007, factory_.sumMaxBufferSizes()); + EXPECT_EQ(2, factory_.totalBytesBuffered()); // Verify cleanup tracking. buffer.reset(); @@ -90,12 +100,101 @@ TEST_F(TrackedWatermarkBufferTest, BufferSizes) { buffer2.reset(); EXPECT_EQ(2, factory_.numBuffersCreated()); EXPECT_EQ(0, factory_.numBuffersActive()); + // Bytes in deleted buffers are removed from the total. + EXPECT_EQ(0, factory_.totalBytesBuffered()); // Max sizes are remembered even after buffers are deleted. EXPECT_EQ(1005, factory_.maxBufferSize()); EXPECT_EQ(1007, factory_.sumMaxBufferSizes()); } +TEST_F(TrackedWatermarkBufferTest, WaitUntilTotalBufferedExceeds) { + auto buffer1 = factory_.create([]() {}, []() {}, []() {}); + auto buffer2 = factory_.create([]() {}, []() {}, []() {}); + auto buffer3 = factory_.create([]() {}, []() {}, []() {}); + + auto thread1 = Thread::threadFactoryForTest().createThread([&]() { buffer1->add("a"); }); + auto thread2 = Thread::threadFactoryForTest().createThread([&]() { buffer2->add("b"); }); + auto thread3 = Thread::threadFactoryForTest().createThread([&]() { buffer3->add("c"); }); + + factory_.waitUntilTotalBufferedExceeds(2, std::chrono::milliseconds(10000)); + thread1->join(); + thread2->join(); + thread3->join(); + + EXPECT_EQ(3, factory_.totalBytesBuffered()); + EXPECT_EQ(1, factory_.maxBufferSize()); +} + +TEST_F(TrackedWatermarkBufferTest, TracksNumberOfBuffersActivelyBound) { + auto buffer1 = factory_.create([]() {}, []() {}, []() {}); + auto buffer2 = factory_.create([]() {}, []() {}, []() {}); + auto buffer3 = factory_.create([]() {}, []() {}, []() {}); + BufferMemoryAccountSharedPtr account = std::make_shared(); + ASSERT_TRUE(factory_.waitUntilExpectedNumberOfAccountsAndBoundBuffers(0, 0)); + + buffer1->bindAccount(account); + EXPECT_TRUE(factory_.waitUntilExpectedNumberOfAccountsAndBoundBuffers(1, 1)); + buffer2->bindAccount(account); + EXPECT_TRUE(factory_.waitUntilExpectedNumberOfAccountsAndBoundBuffers(1, 2)); + buffer3->bindAccount(account); + EXPECT_TRUE(factory_.waitUntilExpectedNumberOfAccountsAndBoundBuffers(1, 3)); + + // Release test access to the account. + account.reset(); + + buffer3.reset(); + EXPECT_TRUE(factory_.waitUntilExpectedNumberOfAccountsAndBoundBuffers(1, 2)); + buffer2.reset(); + EXPECT_TRUE(factory_.waitUntilExpectedNumberOfAccountsAndBoundBuffers(1, 1)); + buffer1.reset(); + EXPECT_TRUE(factory_.waitUntilExpectedNumberOfAccountsAndBoundBuffers(0, 0)); +} + +TEST_F(TrackedWatermarkBufferTest, TracksNumberOfAccountsActive) { + auto buffer1 = factory_.create([]() {}, []() {}, []() {}); + auto buffer2 = factory_.create([]() {}, []() {}, []() {}); + auto buffer3 = factory_.create([]() {}, []() {}, []() {}); + BufferMemoryAccountSharedPtr account1 = std::make_shared(); + ASSERT_TRUE(factory_.waitUntilExpectedNumberOfAccountsAndBoundBuffers(0, 0)); + + buffer1->bindAccount(account1); + EXPECT_TRUE(factory_.waitUntilExpectedNumberOfAccountsAndBoundBuffers(1, 1)); + buffer2->bindAccount(account1); + EXPECT_TRUE(factory_.waitUntilExpectedNumberOfAccountsAndBoundBuffers(1, 2)); + + // Release test access to the account. + account1.reset(); + + buffer3->bindAccount(std::make_shared()); + EXPECT_TRUE(factory_.waitUntilExpectedNumberOfAccountsAndBoundBuffers(2, 3)); + + buffer2.reset(); + EXPECT_TRUE(factory_.waitUntilExpectedNumberOfAccountsAndBoundBuffers(2, 2)); + buffer1.reset(); + EXPECT_TRUE(factory_.waitUntilExpectedNumberOfAccountsAndBoundBuffers(1, 1)); + + buffer3.reset(); + EXPECT_TRUE(factory_.waitUntilExpectedNumberOfAccountsAndBoundBuffers(0, 0)); +} + +TEST_F(TrackedWatermarkBufferTest, WaitForExpectedAccountBalanceShouldReturnTrueWhenConditionsMet) { + auto buffer1 = factory_.create([]() {}, []() {}, []() {}); + auto buffer2 = factory_.create([]() {}, []() {}, []() {}); + BufferMemoryAccountSharedPtr account1 = std::make_shared(); + BufferMemoryAccountSharedPtr account2 = std::make_shared(); + buffer1->bindAccount(account1); + buffer2->bindAccount(account2); + + factory_.setExpectedAccountBalance(4096, 2); + + buffer1->add("Need to wait on the other buffer to get data."); + EXPECT_FALSE(factory_.waitForExpectedAccountBalanceWithTimeout(std::chrono::seconds(0))); + + buffer2->add("Now we have expected balances!"); + EXPECT_TRUE(factory_.waitForExpectedAccountBalanceWithTimeout(std::chrono::seconds(0))); +} + } // namespace } // namespace Buffer } // namespace Envoy diff --git a/test/integration/transport_socket_match_integration_test.cc b/test/integration/transport_socket_match_integration_test.cc index 1bc92bae21f8d..fe5fe47026df6 100644 --- a/test/integration/transport_socket_match_integration_test.cc +++ b/test/integration/transport_socket_match_integration_test.cc @@ -13,8 +13,7 @@ namespace Envoy { class TransportSockeMatchIntegrationTest : public testing::Test, public HttpIntegrationTest { public: TransportSockeMatchIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, - TestEnvironment::getIpVersionsForTest().front(), + : HttpIntegrationTest(Http::CodecType::HTTP1, TestEnvironment::getIpVersionsForTest().front(), ConfigHelper::httpProxyConfig()) { autonomous_upstream_ = true; setUpstreamCount(num_hosts_); @@ -130,8 +129,8 @@ name: "tls_socket" } void SetUp() override { - setDownstreamProtocol(Http::CodecClient::Type::HTTP1); - setUpstreamProtocol(FakeHttpConnection::Type::HTTP1); + setDownstreamProtocol(Http::CodecType::HTTP1); + setUpstreamProtocol(Http::CodecType::HTTP1); } const uint32_t num_hosts_{2}; diff --git a/test/integration/uds_integration_test.h b/test/integration/uds_integration_test.h index 8d5675e168053..f2961b62eb141 100644 --- a/test/integration/uds_integration_test.h +++ b/test/integration/uds_integration_test.h @@ -21,12 +21,12 @@ class UdsUpstreamIntegrationTest public HttpIntegrationTest { public: UdsUpstreamIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, std::get<0>(GetParam())), + : HttpIntegrationTest(Http::CodecType::HTTP1, std::get<0>(GetParam())), abstract_namespace_(std::get<1>(GetParam())) {} void createUpstreams() override { FakeUpstreamConfig config = upstreamConfig(); - config.upstream_protocol_ = FakeHttpConnection::Type::HTTP1; + config.upstream_protocol_ = Http::CodecType::HTTP1; auto uds_path = TestEnvironment::unixDomainSocketPath("udstest.1.sock", abstract_namespace_); fake_upstreams_.emplace_back(std::make_unique(uds_path, config)); @@ -60,7 +60,7 @@ class UdsListenerIntegrationTest public HttpIntegrationTest { public: UdsListenerIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, std::get<0>(GetParam())), + : HttpIntegrationTest(Http::CodecType::HTTP1, std::get<0>(GetParam())), abstract_namespace_(std::get<1>(GetParam())), mode_(std::get<2>(GetParam())) {} void initialize() override; diff --git a/test/integration/utility.cc b/test/integration/utility.cc index 7d99f7e4dcc8b..3837236fe2568 100644 --- a/test/integration/utility.cc +++ b/test/integration/utility.cc @@ -179,7 +179,7 @@ sendRequestAndWaitForResponse(Event::Dispatcher& dispatcher, const std::string& BufferingStreamDecoderPtr IntegrationUtil::makeSingleRequest(const Network::Address::InstanceConstSharedPtr& addr, const std::string& method, const std::string& url, - const std::string& body, Http::CodecClient::Type type, + const std::string& body, Http::CodecType type, const std::string& host, const std::string& content_type) { NiceMock mock_stats_store; NiceMock random; @@ -191,11 +191,10 @@ IntegrationUtil::makeSingleRequest(const Network::Address::InstanceConstSharedPt TestConnectionCallbacks connection_callbacks(*dispatcher); std::shared_ptr cluster{new NiceMock()}; Upstream::HostDescriptionConstSharedPtr host_description{Upstream::makeTestHostDescription( - cluster, - fmt::format("{}://127.0.0.1:80", (type == Http::CodecClient::Type::HTTP3 ? "udp" : "tcp")), + cluster, fmt::format("{}://127.0.0.1:80", (type == Http::CodecType::HTTP3 ? "udp" : "tcp")), time_system)}; - if (type <= Http::CodecClient::Type::HTTP2) { + if (type <= Http::CodecType::HTTP2) { Http::CodecClientProd client( type, dispatcher->createClientConnection(addr, Network::Address::InstanceConstSharedPtr(), @@ -212,7 +211,7 @@ IntegrationUtil::makeSingleRequest(const Network::Address::InstanceConstSharedPt "spiffe://lyft.com/backend-team"); std::unique_ptr persistent_info; persistent_info = std::make_unique( - *dispatcher, *transport_socket_factory, time_system, addr); + *dispatcher, *transport_socket_factory, time_system, addr, 0); Network::Address::InstanceConstSharedPtr local_address; if (addr->ip()->version() == Network::Address::IpVersion::v4) { @@ -236,7 +235,7 @@ IntegrationUtil::makeSingleRequest(const Network::Address::InstanceConstSharedPt BufferingStreamDecoderPtr IntegrationUtil::makeSingleRequest(uint32_t port, const std::string& method, const std::string& url, - const std::string& body, Http::CodecClient::Type type, + const std::string& body, Http::CodecType type, Network::Address::IpVersion ip_version, const std::string& host, const std::string& content_type) { auto addr = Network::Utility::resolveUrl( diff --git a/test/integration/utility.h b/test/integration/utility.h index 2deaf89203701..6fb283dcb1a0e 100644 --- a/test/integration/utility.h +++ b/test/integration/utility.h @@ -168,7 +168,7 @@ class IntegrationUtil { */ static BufferingStreamDecoderPtr makeSingleRequest(const Network::Address::InstanceConstSharedPtr& addr, const std::string& method, - const std::string& url, const std::string& body, Http::CodecClient::Type type, + const std::string& url, const std::string& body, Http::CodecType type, const std::string& host = "host", const std::string& content_type = ""); /** @@ -184,11 +184,12 @@ class IntegrationUtil { * @return BufferingStreamDecoderPtr the complete request or a partial request if there was * remote early disconnection. */ - static BufferingStreamDecoderPtr - makeSingleRequest(uint32_t port, const std::string& method, const std::string& url, - const std::string& body, Http::CodecClient::Type type, - Network::Address::IpVersion ip_version, const std::string& host = "host", - const std::string& content_type = ""); + static BufferingStreamDecoderPtr makeSingleRequest(uint32_t port, const std::string& method, + const std::string& url, + const std::string& body, Http::CodecType type, + Network::Address::IpVersion ip_version, + const std::string& host = "host", + const std::string& content_type = ""); /** * Create transport socket factory for Quic upstream transport socket. diff --git a/test/integration/version_integration_test.cc b/test/integration/version_integration_test.cc index 4888f5c7ceefb..67b376a83e527 100644 --- a/test/integration/version_integration_test.cc +++ b/test/integration/version_integration_test.cc @@ -9,7 +9,7 @@ namespace { class VersionIntegrationTest : public testing::TestWithParam, public HttpIntegrationTest { public: - VersionIntegrationTest() : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, GetParam()) {} + VersionIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam()) {} }; INSTANTIATE_TEST_SUITE_P(IpVersions, VersionIntegrationTest, diff --git a/test/integration/vhds_integration_test.cc b/test/integration/vhds_integration_test.cc index f310ded250f42..d2798fe6e2fdc 100644 --- a/test/integration/vhds_integration_test.cc +++ b/test/integration/vhds_integration_test.cc @@ -155,8 +155,7 @@ domains: [{}] class VhdsInitializationTest : public HttpIntegrationTest, public Grpc::GrpcClientIntegrationParamTest { public: - VhdsInitializationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), config()) { + VhdsInitializationTest() : HttpIntegrationTest(Http::CodecType::HTTP2, ipVersion(), config()) { use_lds_ = false; } @@ -169,8 +168,8 @@ class VhdsInitializationTest : public HttpIntegrationTest, // BaseIntegrationTest::createUpstreams() (which is part of initialize()). // Make sure this number matches the size of the 'clusters' repeated field in the bootstrap // config that you use! - setUpstreamCount(2); // the CDS cluster - setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); // CDS uses gRPC uses HTTP2. + setUpstreamCount(2); // the CDS cluster + setUpstreamProtocol(Http::CodecType::HTTP2); // CDS uses gRPC uses HTTP2. // BaseIntegrationTest::initialize() does many things: // 1) It appends to fake_upstreams_ as many as you asked for via setUpstreamCount(). @@ -252,8 +251,7 @@ TEST_P(VhdsInitializationTest, InitializeVhdsAfterRdsHasBeenInitialized) { class VhdsIntegrationTest : public HttpIntegrationTest, public Grpc::GrpcClientIntegrationParamTest { public: - VhdsIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), config()) { + VhdsIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP2, ipVersion(), config()) { use_lds_ = false; } @@ -291,8 +289,8 @@ class VhdsIntegrationTest : public HttpIntegrationTest, // BaseIntegrationTest::createUpstreams() (which is part of initialize()). // Make sure this number matches the size of the 'clusters' repeated field in the bootstrap // config that you use! - setUpstreamCount(2); // the CDS cluster - setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); // CDS uses gRPC uses HTTP2. + setUpstreamCount(2); // the CDS cluster + setUpstreamProtocol(Http::CodecType::HTTP2); // CDS uses gRPC uses HTTP2. // BaseIntegrationTest::initialize() does many things: // 1) It appends to fake_upstreams_ as many as you asked for via setUpstreamCount(). diff --git a/test/integration/websocket_integration_test.cc b/test/integration/websocket_integration_test.cc index ef6cab3ed57ce..bf96e27d49683 100644 --- a/test/integration/websocket_integration_test.cc +++ b/test/integration/websocket_integration_test.cc @@ -117,7 +117,7 @@ ConfigHelper::HttpModifierFunction setRouteUsingWebsocket() { } void WebsocketIntegrationTest::initialize() { - if (upstreamProtocol() != FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() != Http::CodecType::HTTP1) { config_helper_.addConfigModifier( [&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { ConfigHelper::HttpProtocolOptions protocol_options; @@ -128,7 +128,7 @@ void WebsocketIntegrationTest::initialize() { *bootstrap.mutable_static_resources()->mutable_clusters(0), protocol_options); }); } - if (downstreamProtocol() != Http::CodecClient::Type::HTTP1) { + if (downstreamProtocol() != Http::CodecType::HTTP1) { config_helper_.addConfigModifier( [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& hcm) -> void { hcm.mutable_http2_protocol_options()->set_allow_connect(true); }); @@ -195,7 +195,7 @@ TEST_P(WebsocketIntegrationTest, WebSocketConnectionDownstreamDisconnect) { } TEST_P(WebsocketIntegrationTest, PortStrippingForHttp2) { - if (downstreamProtocol() != Http::CodecClient::Type::HTTP2) { + if (downstreamProtocol() != Http::CodecType::HTTP2) { return; } @@ -217,8 +217,8 @@ TEST_P(WebsocketIntegrationTest, PortStrippingForHttp2) { } TEST_P(WebsocketIntegrationTest, EarlyData) { - if (downstreamProtocol() == Http::CodecClient::Type::HTTP2 || - upstreamProtocol() == FakeHttpConnection::Type::HTTP2) { + if (downstreamProtocol() == Http::CodecType::HTTP2 || + upstreamProtocol() == Http::CodecType::HTTP2) { return; } config_helper_.addConfigModifier(setRouteUsingWebsocket()); @@ -255,7 +255,7 @@ TEST_P(WebsocketIntegrationTest, EarlyData) { auto upgrade_response_headers(upgradeResponseHeaders()); validateUpgradeResponseHeaders(response_->headers(), upgrade_response_headers); - if (downstreamProtocol() == Http::CodecClient::Type::HTTP1) { + if (downstreamProtocol() == Http::CodecType::HTTP1) { // For H2, the disconnect may result in the terminal data not being proxied. response_->waitForBodyData(5); } @@ -301,7 +301,7 @@ TEST_P(WebsocketIntegrationTest, NonWebsocketUpgrade) { performUpgrade(upgradeRequestHeaders("foo", 0), upgradeResponseHeaders("foo")); sendBidirectionalData(); codec_client_->sendData(*request_encoder_, "bye!", false); - if (downstreamProtocol() == Http::CodecClient::Type::HTTP1) { + if (downstreamProtocol() == Http::CodecType::HTTP1) { codec_client_->close(); } else { codec_client_->sendReset(*request_encoder_); @@ -331,7 +331,7 @@ TEST_P(WebsocketIntegrationTest, RouteSpecificUpgrade) { performUpgrade(upgradeRequestHeaders("foo", 0), upgradeResponseHeaders("foo")); sendBidirectionalData(); codec_client_->sendData(*request_encoder_, "bye!", false); - if (downstreamProtocol() == Http::CodecClient::Type::HTTP1) { + if (downstreamProtocol() == Http::CodecType::HTTP1) { codec_client_->close(); } else { codec_client_->sendReset(*request_encoder_); @@ -395,7 +395,7 @@ TEST_P(WebsocketIntegrationTest, WebsocketCustomFilterChain) { // Foo upgrades are configured without the buffer filter, so should explicitly // allow large payload. - if (downstreamProtocol() != Http::CodecClient::Type::HTTP2) { + if (downstreamProtocol() != Http::CodecType::HTTP2) { performUpgrade(upgradeRequestHeaders("foo"), upgradeResponseHeaders("foo")); codec_client_->sendData(*request_encoder_, large_req_str, false); ASSERT_TRUE(upstream_request_->waitForData(*dispatcher_, large_req_str)); @@ -407,8 +407,8 @@ TEST_P(WebsocketIntegrationTest, WebsocketCustomFilterChain) { } TEST_P(WebsocketIntegrationTest, BidirectionalChunkedData) { - if (downstreamProtocol() == Http::CodecClient::Type::HTTP2 || - upstreamProtocol() == FakeHttpConnection::Type::HTTP2) { + if (downstreamProtocol() == Http::CodecType::HTTP2 || + upstreamProtocol() == Http::CodecType::HTTP2) { return; } @@ -423,7 +423,7 @@ TEST_P(WebsocketIntegrationTest, BidirectionalChunkedData) { // With content-length not present, the HTTP codec will send the request with // transfer-encoding: chunked. - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() == Http::CodecType::HTTP1) { ASSERT_TRUE(upstream_request_->headers().TransferEncoding() != nullptr); } diff --git a/test/integration/websocket_integration_test.h b/test/integration/websocket_integration_test.h index 1f3aa362ebf20..4e03ca44bd87a 100644 --- a/test/integration/websocket_integration_test.h +++ b/test/integration/websocket_integration_test.h @@ -10,8 +10,8 @@ namespace Envoy { struct WebsocketProtocolTestParams { Network::Address::IpVersion version; - Http::CodecClient::Type downstream_protocol; - FakeHttpConnection::Type upstream_protocol; + Http::CodecType downstream_protocol; + Http::CodecType upstream_protocol; }; class WebsocketIntegrationTest : public HttpProtocolIntegrationTest { @@ -30,7 +30,7 @@ class WebsocketIntegrationTest : public HttpProtocolIntegrationTest { ABSL_MUST_USE_RESULT testing::AssertionResult waitForUpstreamDisconnectOrReset() { - if (upstreamProtocol() != FakeHttpConnection::Type::HTTP1) { + if (upstreamProtocol() != Http::CodecType::HTTP1) { return upstream_request_->waitForReset(); } else { return fake_upstream_connection_->waitForDisconnect(); @@ -39,7 +39,7 @@ class WebsocketIntegrationTest : public HttpProtocolIntegrationTest { void waitForClientDisconnectOrReset( Http::StreamResetReason reason = Http::StreamResetReason::RemoteReset) { - if (downstreamProtocol() != Http::CodecClient::Type::HTTP1) { + if (downstreamProtocol() != Http::CodecType::HTTP1) { ASSERT_TRUE(response_->waitForReset()); ASSERT_EQ(reason, response_->resetReason()); } else { diff --git a/test/integration/xds_integration_test.cc b/test/integration/xds_integration_test.cc index c5d3167719665..c4dd65445a156 100644 --- a/test/integration/xds_integration_test.cc +++ b/test/integration/xds_integration_test.cc @@ -21,8 +21,8 @@ using testing::HasSubstr; class XdsIntegrationTest : public testing::TestWithParam, public HttpIntegrationTest { public: - XdsIntegrationTest() : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, GetParam()) { - setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); + XdsIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP2, GetParam()) { + setUpstreamProtocol(Http::CodecType::HTTP2); } void createEnvoy() override { @@ -291,8 +291,7 @@ class LdsInplaceUpdateHttpIntegrationTest : public testing::TestWithParam, public HttpIntegrationTest { public: - LdsInplaceUpdateHttpIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, GetParam()) {} + LdsInplaceUpdateHttpIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam()) {} void inplaceInitialize(bool add_default_filter_chain = false) { autonomous_upstream_ = true; @@ -533,7 +532,7 @@ using LdsIntegrationTest = HttpProtocolIntegrationTest; INSTANTIATE_TEST_SUITE_P(Protocols, LdsIntegrationTest, testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams( - {Http::CodecClient::Type::HTTP1}, {FakeHttpConnection::Type::HTTP1})), + {Http::CodecType::HTTP1}, {Http::CodecType::HTTP1})), HttpProtocolIntegrationTest::protocolTestParamsToString); // Sample test making sure our config framework correctly reloads listeners. diff --git a/test/integration/xfcc_integration_test.cc b/test/integration/xfcc_integration_test.cc index 0df12253e728c..820fb494bea60 100644 --- a/test/integration/xfcc_integration_test.cc +++ b/test/integration/xfcc_integration_test.cc @@ -118,7 +118,7 @@ Network::ClientConnectionPtr XfccIntegrationTest::makeMtlsClientConnection() { } void XfccIntegrationTest::createUpstreams() { - addFakeUpstream(createUpstreamSslContext(), FakeHttpConnection::Type::HTTP1); + addFakeUpstream(createUpstreamSslContext(), Http::CodecType::HTTP1); } void XfccIntegrationTest::initialize() { diff --git a/test/integration/xfcc_integration_test.h b/test/integration/xfcc_integration_test.h index 6e4997f8b0dc8..1057051b5685f 100644 --- a/test/integration/xfcc_integration_test.h +++ b/test/integration/xfcc_integration_test.h @@ -37,7 +37,7 @@ class XfccIntegrationTest : public testing::TestWithParam modify_headers, const absl::optional grpc_status, absl::string_view details)); + MOCK_METHOD(Buffer::BufferMemoryAccountSharedPtr, account, (), (const)); Buffer::InstancePtr buffer_; std::list callbacks_{}; testing::NiceMock active_span_; testing::NiceMock tracing_config_; - testing::NiceMock scope_; + testing::NiceMock scope_; bool is_grpc_request_{}; bool is_head_request_{false}; bool stream_destroyed_{}; @@ -327,7 +328,7 @@ class MockStreamEncoderFilterCallbacks : public StreamEncoderFilterCallbacks, Buffer::InstancePtr buffer_; testing::NiceMock active_span_; testing::NiceMock tracing_config_; - testing::NiceMock scope_; + testing::NiceMock scope_; }; class MockStreamDecoderFilter : public StreamDecoderFilter { @@ -585,6 +586,7 @@ class MockConnectionManagerConfig : public ConnectionManagerConfig { MOCK_METHOD(const Http::Http1Settings&, http1Settings, (), (const)); MOCK_METHOD(bool, shouldNormalizePath, (), (const)); MOCK_METHOD(bool, shouldMergeSlashes, (), (const)); + MOCK_METHOD(bool, shouldStripTrailingHostDot, (), (const)); MOCK_METHOD(Http::StripPortType, stripPortType, (), (const)); MOCK_METHOD(envoy::config::core::v3::HttpProtocolOptions::HeadersWithUnderscoresAction, headersWithUnderscoresAction, (), (const)); diff --git a/test/mocks/http/stream.cc b/test/mocks/http/stream.cc index 25e8e14b25441..1a3d4e8bcae67 100644 --- a/test/mocks/http/stream.cc +++ b/test/mocks/http/stream.cc @@ -23,6 +23,10 @@ MockStream::MockStream() { })); ON_CALL(*this, connectionLocalAddress()).WillByDefault(ReturnRef(connection_local_address_)); + + ON_CALL(*this, setAccount(_)) + .WillByDefault(Invoke( + [this](Buffer::BufferMemoryAccountSharedPtr account) -> void { account_ = account; })); } MockStream::~MockStream() = default; diff --git a/test/mocks/http/stream.h b/test/mocks/http/stream.h index ce6579f13ad4e..f456ade6fc96b 100644 --- a/test/mocks/http/stream.h +++ b/test/mocks/http/stream.h @@ -21,9 +21,11 @@ class MockStream : public Stream { MOCK_METHOD(uint32_t, bufferLimit, ()); MOCK_METHOD(const Network::Address::InstanceConstSharedPtr&, connectionLocalAddress, ()); MOCK_METHOD(void, setFlushTimeout, (std::chrono::milliseconds timeout)); + MOCK_METHOD(void, setAccount, (Buffer::BufferMemoryAccountSharedPtr)); std::list callbacks_{}; Network::Address::InstanceConstSharedPtr connection_local_address_; + Buffer::BufferMemoryAccountSharedPtr account_; void runHighWatermarkCallbacks() { for (auto* callback : callbacks_) { diff --git a/test/mocks/network/mocks.h b/test/mocks/network/mocks.h index c102194d40169..9b68638ea21a3 100644 --- a/test/mocks/network/mocks.h +++ b/test/mocks/network/mocks.h @@ -153,6 +153,7 @@ class MockUdpListenerCallbacks : public UdpListenerCallbacks { MOCK_METHOD(uint32_t, workerIndex, (), (const)); MOCK_METHOD(void, onDataWorker, (Network::UdpRecvData && data)); MOCK_METHOD(void, post, (Network::UdpRecvData && data)); + MOCK_METHOD(size_t, numPacketsExpectedPerEventLoop, (), (const)); }; class MockDrainDecision : public DrainDecision { diff --git a/test/mocks/router/mocks.h b/test/mocks/router/mocks.h index aa06b6e0a6620..dde676d080283 100644 --- a/test/mocks/router/mocks.h +++ b/test/mocks/router/mocks.h @@ -50,7 +50,7 @@ class MockDirectResponseEntry : public DirectResponseEntry { (Http::ResponseHeaderMap & headers, const StreamInfo::StreamInfo& stream_info), (const)); MOCK_METHOD(Http::HeaderTransforms, responseHeaderTransforms, - (const StreamInfo::StreamInfo& stream_info), (const)); + (const StreamInfo::StreamInfo& stream_info, bool do_formatting), (const)); MOCK_METHOD(std::string, newPath, (const Http::RequestHeaderMap& headers), (const)); MOCK_METHOD(void, rewritePathHeader, (Http::RequestHeaderMap & headers, bool insert_envoy_original_path), (const)); @@ -361,7 +361,7 @@ class MockRouteEntry : public RouteEntry { (Http::ResponseHeaderMap & headers, const StreamInfo::StreamInfo& stream_info), (const)); MOCK_METHOD(Http::HeaderTransforms, responseHeaderTransforms, - (const StreamInfo::StreamInfo& stream_info), (const)); + (const StreamInfo::StreamInfo& stream_info, bool do_formatting), (const)); MOCK_METHOD(const Http::HashPolicy*, hashPolicy, (), (const)); MOCK_METHOD(const HedgePolicy&, hedgePolicy, (), (const)); MOCK_METHOD(const Router::MetadataMatchCriteria*, metadataMatchCriteria, (), (const)); diff --git a/test/mocks/upstream/cluster_info.cc b/test/mocks/upstream/cluster_info.cc index 9c698a61ccd2a..5e755e1851df0 100644 --- a/test/mocks/upstream/cluster_info.cc +++ b/test/mocks/upstream/cluster_info.cc @@ -59,7 +59,8 @@ MockClusterInfo::MockClusterInfo() cluster_circuit_breakers_stat_names_)), resource_manager_(new Upstream::ResourceManagerImpl( runtime_, "fake_key", 1, 1024, 1024, 1, std::numeric_limits::max(), - circuit_breakers_stats_, absl::nullopt, absl::nullopt)) { + circuit_breakers_stats_, absl::nullopt, absl::nullopt)), + stats_scope_(stats_store_.createScope("test_scope")) { ON_CALL(*this, connectTimeout()).WillByDefault(Return(std::chrono::milliseconds(1))); ON_CALL(*this, idleTimeout()).WillByDefault(Return(absl::optional())); ON_CALL(*this, perUpstreamPreconnectRatio()).WillByDefault(Return(1.0)); @@ -126,15 +127,15 @@ MockClusterInfo::MockClusterInfo() MockClusterInfo::~MockClusterInfo() = default; Http::Http1::CodecStats& MockClusterInfo::http1CodecStats() const { - return Http::Http1::CodecStats::atomicGet(http1_codec_stats_, statsScope()); + return Http::Http1::CodecStats::atomicGet(http1_codec_stats_, *stats_scope_); } Http::Http2::CodecStats& MockClusterInfo::http2CodecStats() const { - return Http::Http2::CodecStats::atomicGet(http2_codec_stats_, statsScope()); + return Http::Http2::CodecStats::atomicGet(http2_codec_stats_, *stats_scope_); } Http::Http3::CodecStats& MockClusterInfo::http3CodecStats() const { - return Http::Http3::CodecStats::atomicGet(http3_codec_stats_, statsScope()); + return Http::Http3::CodecStats::atomicGet(http3_codec_stats_, *stats_scope_); } } // namespace Upstream diff --git a/test/mocks/upstream/cluster_info.h b/test/mocks/upstream/cluster_info.h index 5a3322536313c..ab597fade7fa4 100644 --- a/test/mocks/upstream/cluster_info.h +++ b/test/mocks/upstream/cluster_info.h @@ -200,6 +200,7 @@ class MockClusterInfo : public ClusterInfo { envoy::config::core::v3::Metadata metadata_; std::unique_ptr typed_metadata_; absl::optional max_stream_duration_; + Stats::ScopePtr stats_scope_; mutable Http::Http1::CodecStats::AtomicPtr http1_codec_stats_; mutable Http::Http2::CodecStats::AtomicPtr http2_codec_stats_; mutable Http::Http3::CodecStats::AtomicPtr http3_codec_stats_; diff --git a/test/server/config_validation/config_fuzz_test.cc b/test/server/config_validation/config_fuzz_test.cc index a94dbb01aeb3a..e6ce91e4440d5 100644 --- a/test/server/config_validation/config_fuzz_test.cc +++ b/test/server/config_validation/config_fuzz_test.cc @@ -41,8 +41,9 @@ DEFINE_PROTO_FUZZER(const envoy::config::bootstrap::v3::Bootstrap& input) { options.log_level_ = Fuzz::Runner::logLevel(); try { - validateConfig(options, Network::Address::InstanceConstSharedPtr(), component_factory, - Thread::threadFactoryForTest(), Filesystem::fileSystemForTest()); + validateConfig(options, std::make_shared("127.0.0.1"), + component_factory, Thread::threadFactoryForTest(), + Filesystem::fileSystemForTest()); } catch (const EnvoyException& ex) { ENVOY_LOG_MISC(debug, "Controlled EnvoyException exit: {}", ex.what()); } diff --git a/test/server/config_validation/xds_fuzz.cc b/test/server/config_validation/xds_fuzz.cc index d1543c238ad0c..3b4d23f4b1575 100644 --- a/test/server/config_validation/xds_fuzz.cc +++ b/test/server/config_validation/xds_fuzz.cc @@ -56,7 +56,7 @@ void XdsFuzzTest::updateRoute( XdsFuzzTest::XdsFuzzTest(const test::server::config_validation::XdsTestCase& input, envoy::config::core::v3::ApiVersion api_version) : HttpIntegrationTest( - Http::CodecClient::Type::HTTP2, TestEnvironment::getIpVersionsForTest()[0], + Http::CodecType::HTTP2, TestEnvironment::getIpVersionsForTest()[0], ConfigHelper::adsBootstrap(input.config().sotw_or_delta() == test::server::config_validation::Config::SOTW ? "GRPC" @@ -92,7 +92,7 @@ void XdsFuzzTest::initialize() { ads_cluster->MergeFrom(bootstrap.static_resources().clusters()[0]); ads_cluster->set_name("ads_cluster"); }); - setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); + setUpstreamProtocol(Http::CodecType::HTTP2); HttpIntegrationTest::initialize(); if (xds_stream_ == nullptr) { createXdsConnection(); diff --git a/test/server/server_corpus/start_async_client_when_validate_config b/test/server/server_corpus/start_async_client_when_validate_config new file mode 100644 index 0000000000000..68870032ba718 --- /dev/null +++ b/test/server/server_corpus/start_async_client_when_validate_config @@ -0,0 +1,220 @@ +static_resources { + clusters { + name: "z" + type: STRICT_DNS + connect_timeout { + seconds: 1000000 + } + lb_policy: MAGLEV + dns_lookup_family: V4_ONLY + metadata { + filter_metadata { + key: "V" + value { + } + } + } + ignore_health_on_host_removal: true + } + clusters { + name: "+" + connect_timeout { + nanos: 55040 + } + lb_policy: RING_HASH + dns_lookup_family: V4_ONLY + metadata { + } + load_balancing_policy { + } + } + clusters { + name: "V" + connect_timeout { + seconds: 256 + } + lb_policy: RING_HASH + http2_protocol_options { + allow_metadata: true + } + dns_lookup_family: V4_ONLY + outlier_detection { + } + common_http_protocol_options { + max_connection_duration { + nanos: 55040 + } + } + upstream_connection_options { + tcp_keepalive { + keepalive_probes { + value: 589951 + } + } + } + close_connections_on_host_health_failure: true + ignore_health_on_host_removal: true + use_tcp_for_dns_lookups: true + } + clusters { + name: "0" + connect_timeout { + seconds: 2304 + } + lb_policy: RING_HASH + http2_protocol_options { + initial_connection_window_size { + value: 67110913 + } + max_inbound_window_update_frames_per_data_frame_sent { + value: 589951 + } + } + cleanup_interval { + seconds: 2304 + } + alt_stat_name: "\r\002\000\000" + upstream_connection_options { + tcp_keepalive { + keepalive_interval { + value: 268435456 + } + } + } + maglev_lb_config { + } + } + clusters { + name: "a" + type: STRICT_DNS + connect_timeout { + seconds: 1000000 + nanos: 96 + } + lb_policy: RING_HASH + dns_lookup_family: V6_ONLY + metadata { + filter_metadata { + key: "V" + value { + } + } + } + upstream_connection_options { + } + least_request_lb_config { + } + connection_pool_per_downstream_connection: true + } + clusters { + name: ";" + connect_timeout { + seconds: 256 + } + lb_policy: RING_HASH + metadata { + } + alt_stat_name: "+" + upstream_connection_options { + tcp_keepalive { + } + } + respect_dns_ttl: true + } + clusters { + name: "\000\000\000\017~" + connect_timeout { + seconds: 2324 + } + lb_policy: RING_HASH + circuit_breakers { + } + http2_protocol_options { + initial_connection_window_size { + value: 67110913 + } + max_inbound_window_update_frames_per_data_frame_sent { + value: 589951 + } + } + dns_lookup_family: V6_ONLY + lb_subset_config { + } + metadata { + } + upstream_connection_options { + tcp_keepalive { + keepalive_time { + value: 2 + } + } + } + track_timeout_budgets: true + track_cluster_stats { + timeout_budgets: true + } + } +} +dynamic_resources { + ads_config { + api_type: DELTA_GRPC + grpc_services { + envoy_grpc { + cluster_name: "+" + } + } + set_node_on_first_message_only: true + transport_api_version: V3 + } +} +flags_path: "+" +stats_sinks { +} +stats_sinks { + name: "h" +} +stats_sinks { +} +stats_sinks { +} +stats_sinks { + name: "\026" + typed_config { + type_url: "type.googleapis.com/envoy.api.v2.route.Route" + value: "\022\000B\023\n\000\022\017\n\r\n\000\022\t*\007\n\005\n\001\026\022\000R\000R\000R\000R\000R\000R\000b\000b\000" + } +} +stats_sinks { +} +stats_sinks { +} +stats_sinks { +} +stats_sinks { + name: "?\026" +} +stats_sinks { +} +stats_sinks { +} +stats_sinks { + name: "\r\002\000\000" +} +stats_sinks { +} +stats_sinks { + name: "\026" +} +stats_sinks { + name: "\r\002\000\000" +} +stats_sinks { +} +stats_sinks { + name: "\026" +} +config_sources { + path: "+" + resource_api_version: V3 +} + diff --git a/test/server/server_fuzz_test.cc b/test/server/server_fuzz_test.cc index 30c399bc008a1..b18dc694832c6 100644 --- a/test/server/server_fuzz_test.cc +++ b/test/server/server_fuzz_test.cc @@ -93,8 +93,13 @@ bool validateLbSubsetConfig(const envoy::config::bootstrap::v3::Bootstrap& input subset_selectors++; if (subset_selector.single_host_per_subset()) { use_single_host_per_subset = true; + const auto& keys = subset_selector.keys(); // Only expect 1 key inside subset selector when use_single_host_per_subset is set to true. - if (subset_selector.keys().size() != 1) { + if (keys.size() != 1) { + return false; + } + // Expect key to be non-empty when use_single_host_per_subset is set to true. + if (keys[0].empty()) { return false; } } diff --git a/test/test_common/network_utility.cc b/test/test_common/network_utility.cc index 1e1eeba9f1c1a..7e413e3ac5ce5 100644 --- a/test/test_common/network_utility.cc +++ b/test/test_common/network_utility.cc @@ -211,6 +211,9 @@ struct SyncPacketProcessor : public Network::UdpPacketProcessor { } uint64_t maxDatagramSize() const override { return max_rx_datagram_size_; } void onDatagramsDropped(uint32_t) override {} + size_t numPacketsExpectedPerEventLoop() const override { + return Network::MAX_NUM_PACKETS_PER_EVENT_LOOP; + } std::list& data_; const uint64_t max_rx_datagram_size_; diff --git a/tools/code_format/requirements.txt b/tools/code_format/requirements.txt index 15ba3b2f5e01d..a2456ecee4229 100644 --- a/tools/code_format/requirements.txt +++ b/tools/code_format/requirements.txt @@ -1,9 +1,9 @@ flake8==3.9.2 \ --hash=sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907 \ --hash=sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b -importlib-metadata==4.0.1 \ - --hash=sha256:d7eb1dea6d6a6086f8be21784cc9e3bcfa55872b52309bc5fad53a8ea444465d \ - --hash=sha256:8c501196e49fb9df5df43833bdb1e4328f64847763ec8a50703148b73784d581 +importlib-metadata==4.2.0 \ + --hash=sha256:057e92c15bc8d9e8109738a48db0ccb31b4d9d5cfbee5a8670879a30be66304b \ + --hash=sha256:b7e52a1f8dec14a75ea73e0891f3060099ca1d8e6a462a4dff11c3e119ea1b31 mccabe==0.6.1 \ --hash=sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42 \ --hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f diff --git a/tools/config_validation/validate_fragment.py b/tools/config_validation/validate_fragment.py index e708a228a0cee..8059ab3f0c725 100644 --- a/tools/config_validation/validate_fragment.py +++ b/tools/config_validation/validate_fragment.py @@ -21,6 +21,37 @@ import argparse +class IgnoredKey(yaml.YAMLObject): + """Python support type for Envoy's config !ignore tag.""" + yaml_tag = '!ignore' + + def __init__(self, strval): + self.strval = strval + + def __repr__(self): + return f'IgnoredKey({str})' + + def __eq__(self, other): + return isinstance(other, IgnoredKey) and self.strval == other.strval + + def __hash__(self): + return hash((self.yaml_tag, self.strval)) + + @classmethod + def from_yaml(cls, loader, node): + return IgnoredKey(node.value) + + @classmethod + def to_yaml(cls, dumper, data): + return dumper.represent_scalar(cls.yaml_tag, data.strval) + + +def validate_yaml(type_name, content): + yaml.SafeLoader.add_constructor('!ignore', IgnoredKey.from_yaml) + yaml.SafeDumper.add_multi_representer(IgnoredKey, IgnoredKey.to_yaml) + validate_fragment(type_name, yaml.safe_load(content)) + + def validate_fragment(type_name, fragment): """Validate a dictionary representing a JSON/YAML fragment against an Envoy API proto3 type. @@ -33,7 +64,7 @@ def validate_fragment(type_name, fragment): fragment: a dictionary representing the parsed JSON/YAML configuration fragment. """ - json_fragment = json.dumps(fragment) + json_fragment = json.dumps(fragment, skipkeys=True) r = runfiles.Create() all_protos_pb_text_path = r.Rlocation( @@ -69,4 +100,4 @@ def parse_args(): message_type = parsed_args.message_type content = parsed_args.s if (parsed_args.fragment_path is None) else pathlib.Path( parsed_args.fragment_path).read_text() - validate_fragment(message_type, yaml.safe_load(content)) + validate_yaml(message_type, content) diff --git a/tools/dependency/requirements.txt b/tools/dependency/requirements.txt index 68995769e3133..374b4fa178912 100644 --- a/tools/dependency/requirements.txt +++ b/tools/dependency/requirements.txt @@ -140,9 +140,9 @@ six==1.16.0 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 # via pynacl -urllib3==1.26.4 \ - --hash=sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df \ - --hash=sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937 +urllib3==1.26.5 \ + --hash=sha256:753a0374df26658f99d826cfe40394a686d05985786d946fbe4165b5148f5a7c \ + --hash=sha256:a7acd0977125325f516bda9735fa7142b909a8d01e8b2e4c8108d0984e6e0098 # via # -r tools/dependency/requirements.txt # requests diff --git a/tools/deprecate_version/requirements.txt b/tools/deprecate_version/requirements.txt index e9024ac6f3bd0..d8b9929bf2756 100644 --- a/tools/deprecate_version/requirements.txt +++ b/tools/deprecate_version/requirements.txt @@ -125,9 +125,9 @@ smmap==4.0.0 \ # via # -r tools/deprecate_version/requirements.txt # gitdb -urllib3==1.26.4 \ - --hash=sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df \ - --hash=sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937 +urllib3==1.26.5 \ + --hash=sha256:753a0374df26658f99d826cfe40394a686d05985786d946fbe4165b5148f5a7c \ + --hash=sha256:a7acd0977125325f516bda9735fa7142b909a8d01e8b2e4c8108d0984e6e0098 # via # -r tools/deprecate_version/requirements.txt # requests diff --git a/tools/extensions/BUILD b/tools/extensions/BUILD index 43f21c21d2075..4054af1a4d2cb 100644 --- a/tools/extensions/BUILD +++ b/tools/extensions/BUILD @@ -7,19 +7,18 @@ licenses(["notice"]) # Apache 2 envoy_package() py_binary( - name = "generate_extension_db", - srcs = ["generate_extension_db.py"], + name = "validate_extensions", + srcs = ["validate_extensions.py"], data = [ "@com_github_bazelbuild_buildtools//buildozer:buildozer", + "//source/extensions:extensions_metadata.yaml", + "//source/extensions:extensions_build_config.bzl", + "//test/extensions/filters/network/common/fuzz:uber_per_readfilter.cc", ] + envoy_all_extensions(), - python_version = "PY3", - srcs_version = "PY3", - visibility = ["//visibility:public"], ) py_binary( name = "generate_extension_rst", srcs = ["generate_extension_rst.py"], - data = [":generate_extension_db"], - visibility = ["//visibility:public"], + data = ["//source/extensions:extensions_metadata.yaml"], ) diff --git a/tools/extensions/generate_extension_db.py b/tools/extensions/generate_extension_db.py deleted file mode 100644 index ec689e701e010..0000000000000 --- a/tools/extensions/generate_extension_db.py +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env python3 - -# Generate an extension database, a JSON file mapping from qualified well known -# extension name to metadata derived from the envoy_cc_extension target. - -# This script expects a copy of the envoy source to be located at /source -# Alternatively, you can specify a path to the source dir with `ENVOY_SRCDIR` - -# You must specify the target file to save the generated json db to. -# You can do this either as an arg to this script/target or with the env var -# `EXTENSION_DB_PATH` - -import ast -import json -import os -import pathlib -import re -import subprocess -import sys - -from importlib.util import spec_from_loader, module_from_spec -from importlib.machinery import SourceFileLoader - -BUILDOZER_PATH = os.path.abspath( - "external/com_github_bazelbuild_buildtools/buildozer/buildozer_/buildozer") - -ENVOY_SRCDIR = os.getenv('ENVOY_SRCDIR', '/source') - -if not os.path.exists(ENVOY_SRCDIR): - raise SystemExit( - "Envoy source must either be located at /source, or ENVOY_SRCDIR env var must be set") - -# source/extensions/extensions_build_config.bzl must have a .bzl suffix for Starlark -# import, so we are forced to do this workaround. -_extensions_build_config_spec = spec_from_loader( - 'extensions_build_config', - SourceFileLoader( - 'extensions_build_config', - os.path.join(ENVOY_SRCDIR, 'source/extensions/extensions_build_config.bzl'))) -extensions_build_config = module_from_spec(_extensions_build_config_spec) -_extensions_build_config_spec.loader.exec_module(extensions_build_config) - - -class ExtensionDbError(Exception): - pass - - -def is_missing(value): - return value == '(missing)' - - -def num_read_filters_fuzzed(): - data = pathlib.Path( - os.path.join( - ENVOY_SRCDIR, - 'test/extensions/filters/network/common/fuzz/uber_per_readfilter.cc')).read_text() - # Hack-ish! We only search the first 50 lines to capture the filters in filterNames(). - return len(re.findall('NetworkFilterNames::get()', ''.join(data.splitlines()[:50]))) - - -def num_robust_to_downstream_network_filters(db): - # Count number of network filters robust to untrusted downstreams. - return len([ - ext for ext, data in db.items() - if 'network' in ext and data['security_posture'] == 'robust_to_untrusted_downstream' - ]) - - -def get_extension_metadata(target): - if not BUILDOZER_PATH: - raise ExtensionDbError('Buildozer not found!') - r = subprocess.run( - [BUILDOZER_PATH, '-stdout', 'print security_posture status undocumented category', target], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - rout = r.stdout.decode('utf-8').strip().split(' ') - security_posture, status, undocumented = rout[:3] - categories = ' '.join(rout[3:]) - if is_missing(security_posture): - raise ExtensionDbError( - 'Missing security posture for %s. Please make sure the target is an envoy_cc_extension and security_posture is set' - % target) - if is_missing(categories): - raise ExtensionDbError( - 'Missing extension category for %s. Please make sure the target is an envoy_cc_extension and category is set' - % target) - # evaluate tuples/lists - # wrap strings in a list - categories = ( - ast.literal_eval(categories) if ('[' in categories or '(' in categories) else [categories]) - return { - 'security_posture': security_posture, - 'undocumented': False if is_missing(undocumented) else bool(undocumented), - 'status': 'stable' if is_missing(status) else status, - 'categories': categories, - } - - -if __name__ == '__main__': - try: - output_path = os.getenv("EXTENSION_DB_PATH") or sys.argv[1] - except IndexError: - raise SystemExit( - "Output path must be either specified as arg or with EXTENSION_DB_PATH env var") - - extension_db = {} - # Include all extensions from source/extensions/extensions_build_config.bzl - all_extensions = {} - all_extensions.update(extensions_build_config.EXTENSIONS) - for extension, target in all_extensions.items(): - extension_db[extension] = get_extension_metadata(target) - if num_robust_to_downstream_network_filters(extension_db) != num_read_filters_fuzzed(): - raise ExtensionDbError( - 'Check that all network filters robust against untrusted' - 'downstreams are fuzzed by adding them to filterNames() in' - 'test/extensions/filters/network/common/uber_per_readfilter.cc') - # The TLS and generic upstream extensions are hard-coded into the build, so - # not in source/extensions/extensions_build_config.bzl - # TODO(mattklein123): Read these special keys from all_extensions.bzl or a shared location to - # avoid duplicate logic. - extension_db['envoy.transport_sockets.tls'] = get_extension_metadata( - '//source/extensions/transport_sockets/tls:config') - extension_db['envoy.upstreams.http.generic'] = get_extension_metadata( - '//source/extensions/upstreams/http/generic:config') - extension_db['envoy.upstreams.tcp.generic'] = get_extension_metadata( - '//source/extensions/upstreams/tcp/generic:config') - extension_db['envoy.upstreams.http.http_protocol_options'] = get_extension_metadata( - '//source/extensions/upstreams/http:config') - extension_db['envoy.request_id.uuid'] = get_extension_metadata( - '//source/extensions/request_id/uuid:config') - - pathlib.Path(os.path.dirname(output_path)).mkdir(parents=True, exist_ok=True) - pathlib.Path(output_path).write_text(json.dumps(extension_db)) diff --git a/tools/extensions/generate_extension_rst.py b/tools/extensions/generate_extension_rst.py index 9199873f7079c..62e4b6250d90d 100644 --- a/tools/extensions/generate_extension_rst.py +++ b/tools/extensions/generate_extension_rst.py @@ -3,18 +3,18 @@ # Generate RST lists of extensions grouped by their security posture. from collections import defaultdict -import json import os import pathlib -import subprocess + +import yaml def format_item(extension, metadata): - if metadata['undocumented']: + if metadata.get('undocumented'): item = '* %s' % extension else: item = '* :ref:`%s `' % (extension, extension) - if metadata['status'] == 'alpha': + if metadata.get('status') == 'alpha': item += ' (alpha)' return item @@ -27,14 +27,8 @@ def format_item(extension, metadata): "Path to an output directory must be specified with GENERATED_RST_DIR env var") security_rst_root = os.path.join(generated_rst_dir, "intro/arch_overview/security") - try: - extension_db_path = os.environ["EXTENSION_DB_PATH"] - except KeyError: - raise SystemExit( - "Path to a json extension db must be specified with EXTENSION_DB_PATH env var") - if not os.path.exists(extension_db_path): - subprocess.run("tools/extensions/generate_extension_db".split(), check=True) - extension_db = json.loads(pathlib.Path(extension_db_path).read_text()) + with open("source/extensions/extensions_metadata.yaml") as f: + extension_db = yaml.safe_load(f.read()) pathlib.Path(security_rst_root).mkdir(parents=True, exist_ok=True) @@ -47,5 +41,5 @@ def format_item(extension, metadata): content = '\n'.join( format_item(extension, extension_db[extension]) for extension in sorted(extensions) - if extension_db[extension]['status'] != 'wip') + if extension_db[extension].get('status') != 'wip') output_path.write_text(content) diff --git a/tools/extensions/validate_extensions.py b/tools/extensions/validate_extensions.py new file mode 100644 index 0000000000000..8140c3e56f674 --- /dev/null +++ b/tools/extensions/validate_extensions.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 + +# Validate extension metadata + +# This script expects a copy of the envoy source to be located at /source +# Alternatively, you can specify a path to the source dir with `ENVOY_SRCDIR` + +import pathlib +import re +import sys +from importlib.util import spec_from_loader, module_from_spec +from importlib.machinery import SourceFileLoader + +import yaml + +BUILTIN_EXTENSIONS = ( + "envoy.request_id.uuid", "envoy.upstreams.tcp.generic", "envoy.transport_sockets.tls", + "envoy.upstreams.http.http_protocol_options", "envoy.upstreams.http.generic") + +# All Envoy extensions must be tagged with their security hardening stance with +# respect to downstream and upstream data plane threats. These are verbose +# labels intended to make clear the trust that operators may place in +# extensions. +EXTENSION_SECURITY_POSTURES = [ + # This extension is hardened against untrusted downstream traffic. It + # assumes that the upstream is trusted. + "robust_to_untrusted_downstream", + # This extension is hardened against both untrusted downstream and upstream + # traffic. + "robust_to_untrusted_downstream_and_upstream", + # This extension is not hardened and should only be used in deployments + # where both the downstream and upstream are trusted. + "requires_trusted_downstream_and_upstream", + # This is functionally equivalent to + # requires_trusted_downstream_and_upstream, but acts as a placeholder to + # allow us to identify extensions that need classifying. + "unknown", + # Not relevant to data plane threats, e.g. stats sinks. + "data_plane_agnostic", +] + +# Extension categories as defined by factories +EXTENSION_CATEGORIES = [ + "envoy.access_loggers", + "envoy.bootstrap", + "envoy.clusters", + "envoy.compression.compressor", + "envoy.compression.decompressor", + "envoy.filters.http", + "envoy.filters.http.cache", + "envoy.filters.listener", + "envoy.filters.network", + "envoy.filters.udp_listener", + "envoy.grpc_credentials", + "envoy.guarddog_actions", + "envoy.health_checkers", + "envoy.http.stateful_header_formatters", + "envoy.internal_redirect_predicates", + "envoy.io_socket", + "envoy.http.original_ip_detection", + "envoy.matching.common_inputs", + "envoy.matching.input_matchers", + "envoy.rate_limit_descriptors", + "envoy.request_id", + "envoy.resource_monitors", + "envoy.retry_host_predicates", + "envoy.retry_priorities", + "envoy.stats_sinks", + "envoy.thrift_proxy.filters", + "envoy.tracers", + "envoy.transport_sockets.downstream", + "envoy.transport_sockets.upstream", + "envoy.tls.cert_validator", + "envoy.upstreams", + "envoy.wasm.runtime", + "DELIBERATELY_OMITTED", +] + +EXTENSION_STATUS_VALUES = [ + # This extension is stable and is expected to be production usable. + "stable", + # This extension is functional but has not had substantial production burn + # time, use only with this caveat. + "alpha", + # This extension is work-in-progress. Functionality is incomplete and it is + # not intended for production use. + "wip", +] + +# source/extensions/extensions_build_config.bzl must have a .bzl suffix for Starlark +# import, so we are forced to do this workaround. +_extensions_build_config_spec = spec_from_loader( + 'extensions_build_config', + SourceFileLoader('extensions_build_config', 'source/extensions/extensions_build_config.bzl')) +extensions_build_config = module_from_spec(_extensions_build_config_spec) +_extensions_build_config_spec.loader.exec_module(extensions_build_config) + + +def num_read_filters_fuzzed(): + data = pathlib.Path( + 'test/extensions/filters/network/common/fuzz/uber_per_readfilter.cc').read_text() + # Hack-ish! We only search the first 50 lines to capture the filters in filterNames(). + return len(re.findall('NetworkFilterNames::get()', ''.join(data.splitlines()[:50]))) + + +def num_robust_to_downstream_network_filters(db): + # Count number of network filters robust to untrusted downstreams. + return len([ + ext for ext, data in db.items() + if 'network' in ext and data['security_posture'] == 'robust_to_untrusted_downstream' + ]) + + +# TODO(phlax): move this to a checker class, and add pytests +def validate_extensions(): + returns = 0 + + with open("source/extensions/extensions_metadata.yaml") as f: + metadata = yaml.safe_load(f.read()) + + all_extensions = set(extensions_build_config.EXTENSIONS.keys()) | set(BUILTIN_EXTENSIONS) + only_metadata = set(metadata.keys()) - all_extensions + missing_metadata = all_extensions - set(metadata.keys()) + + if only_metadata: + returns = 1 + print(f"Metadata for unused extensions found: {only_metadata}") + + if missing_metadata: + returns = 1 + print(f"Metadata missing for extensions: {missing_metadata}") + + if num_robust_to_downstream_network_filters(metadata) != num_read_filters_fuzzed(): + returns = 1 + print( + 'Check that all network filters robust against untrusted' + 'downstreams are fuzzed by adding them to filterNames() in' + 'test/extensions/filters/network/common/uber_per_readfilter.cc') + + for k, v in metadata.items(): + if not v["security_posture"]: + returns = 1 + print( + f"Missing security posture for {k}. " + "Please make sure the target is an envoy_cc_extension and security_posture is set") + elif v["security_posture"] not in EXTENSION_SECURITY_POSTURES: + print("Unknown extension security posture: {v['security_posture']}") + returns = 1 + if not v["categories"]: + returns = 1 + print( + f"Missing extension category for {k}. " + "Please make sure the target is an envoy_cc_extension and category is set") + else: + for cat in v["categories"]: + if cat not in EXTENSION_CATEGORIES: + returns = 1 + print(f"Unknown extension category for {k}: {cat}") + if v["status"] not in EXTENSION_STATUS_VALUES: + returns = 1 + print(f"Unknown extension status: {v['status']}") + + return returns + + +if __name__ == '__main__': + sys.exit(validate_extensions()) diff --git a/tools/protodoc/BUILD b/tools/protodoc/BUILD index d9eeb6ac203fb..d7df1f49ac10f 100644 --- a/tools/protodoc/BUILD +++ b/tools/protodoc/BUILD @@ -24,6 +24,7 @@ py_binary( data = [ "//docs:protodoc_manifest.yaml", "//docs:v2_mapping.json", + "//source/extensions:extensions_metadata.yaml", ], visibility = ["//visibility:public"], deps = [ diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index a66d6cf021047..2edcd91031491 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -6,8 +6,6 @@ from collections import defaultdict import json import functools -import os -import pathlib import sys from google.protobuf import json_format @@ -115,7 +113,10 @@ 'This extension is work-in-progress. Functionality is incomplete and it is not intended for production use.', } -EXTENSION_DB = json.loads(pathlib.Path(os.getenv('EXTENSION_DB_PATH')).read_text()) +r = runfiles.Create() + +with open(r.Rlocation("envoy/source/extensions/extensions_metadata.yaml")) as f: + EXTENSION_DB = yaml.safe_load(f.read()) # create an index of extension categories from extension db EXTENSION_CATEGORIES = {} @@ -241,7 +242,7 @@ def format_extension(extension): """ try: extension_metadata = EXTENSION_DB[extension] - status = EXTENSION_STATUS_VALUES.get(extension_metadata['status'], '') + status = EXTENSION_STATUS_VALUES.get(extension_metadata.get('status'), '') security_posture = EXTENSION_SECURITY_POSTURES[extension_metadata['security_posture']] categories = extension_metadata["categories"] except KeyError as e: @@ -662,8 +663,6 @@ class RstFormatVisitor(visitor.Visitor): """ def __init__(self): - r = runfiles.Create() - with open(r.Rlocation('envoy/docs/v2_mapping.json'), 'r') as f: self.v2_mapping = json.load(f) diff --git a/tools/spelling/spelling_dictionary.txt b/tools/spelling/spelling_dictionary.txt index 44481c60f7af5..93f6286e29c66 100644 --- a/tools/spelling/spelling_dictionary.txt +++ b/tools/spelling/spelling_dictionary.txt @@ -34,6 +34,7 @@ HEXDIG HEXDIGIT OWS Preconnecting +RCVBUF RTCP RTP STATNAME @@ -443,6 +444,8 @@ benchmarked bidi bignum bitfield +bitmask +bitmasks bitset bitwise blackhole