diff --git a/CODEOWNERS b/CODEOWNERS index 00a3e81fd1e53..14a4906ff863f 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -147,6 +147,7 @@ extensions/filters/common/original_src @snowp @klarose # Compression /*/extensions/compression/common @junr03 @rojkov /*/extensions/compression/gzip @junr03 @rojkov +/*/extensions/compression/brotli @junr03 @rojkov /*/extensions/filters/http/decompressor @rojkov @dio # Watchdog Extensions /*/extensions/watchdog/profile_action @kbaichoo @antoniovicente diff --git a/api/BUILD b/api/BUILD index 300420c002385..daee0fd944263 100644 --- a/api/BUILD +++ b/api/BUILD @@ -165,6 +165,8 @@ proto_library( "//envoy/extensions/common/matching/v3:pkg", "//envoy/extensions/common/ratelimit/v3:pkg", "//envoy/extensions/common/tap/v3:pkg", + "//envoy/extensions/compression/brotli/compressor/v3:pkg", + "//envoy/extensions/compression/brotli/decompressor/v3:pkg", "//envoy/extensions/compression/gzip/compressor/v3:pkg", "//envoy/extensions/compression/gzip/decompressor/v3:pkg", "//envoy/extensions/filters/common/dependency/v3:pkg", diff --git a/api/envoy/extensions/compression/brotli/compressor/v3/BUILD b/api/envoy/extensions/compression/brotli/compressor/v3/BUILD new file mode 100644 index 0000000000000..ee92fb652582e --- /dev/null +++ b/api/envoy/extensions/compression/brotli/compressor/v3/BUILD @@ -0,0 +1,9 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], +) diff --git a/api/envoy/extensions/compression/brotli/compressor/v3/brotli.proto b/api/envoy/extensions/compression/brotli/compressor/v3/brotli.proto new file mode 100644 index 0000000000000..cb2933dd5d385 --- /dev/null +++ b/api/envoy/extensions/compression/brotli/compressor/v3/brotli.proto @@ -0,0 +1,54 @@ +syntax = "proto3"; + +package envoy.extensions.compression.brotli.compressor.v3; + +import "google/protobuf/wrappers.proto"; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.compression.brotli.compressor.v3"; +option java_outer_classname = "BrotliProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Brotli Compressor] +// [#extension: envoy.compression.brotli.compressor] + +// [#next-free-field: 7] +message Brotli { + enum EncoderMode { + DEFAULT = 0; + GENERIC = 1; + TEXT = 2; + FONT = 3; + } + + // Value from 0 to 11 that controls the main compression speed-density lever. + // The higher quality, the slower compression. The default value is 3. + google.protobuf.UInt32Value quality = 1 [(validate.rules).uint32 = {lte: 11}]; + + // A value used to tune encoder for specific input. For more information about modes, + // please refer to brotli manual: https://brotli.org/encode.html#aa6f + // This field will be set to "DEFAULT" if not specified. + EncoderMode encoder_mode = 2 [(validate.rules).enum = {defined_only: true}]; + + // Value from 10 to 24 that represents the base two logarithmic of the compressor's window size. + // Larger window results in better compression at the expense of memory usage. The default is 18. + // For more details about this parameter, please refer to brotli manual: + // https://brotli.org/encode.html#a9a8 + google.protobuf.UInt32Value window_bits = 3 [(validate.rules).uint32 = {lte: 24 gte: 10}]; + + // Value from 16 to 24 that represents the base two logarithmic of the compressor's input block + // size. Larger input block results in better compression at the expense of memory usage. The + // default is 24. For more details about this parameter, please refer to brotli manual: + // https://brotli.org/encode.html#a9a8 + google.protobuf.UInt32Value input_block_bits = 4 [(validate.rules).uint32 = {lte: 24 gte: 16}]; + + // Value for compressor's next output buffer. If not set, defaults to 4096. + google.protobuf.UInt32Value chunk_size = 5 [(validate.rules).uint32 = {lte: 65536 gte: 4096}]; + + // If true, disables "literal context modeling" format feature. + // This flag is a "decoding-speed vs compression ratio" trade-off. + bool disable_literal_context_modeling = 6; +} diff --git a/api/envoy/extensions/compression/brotli/decompressor/v3/BUILD b/api/envoy/extensions/compression/brotli/decompressor/v3/BUILD new file mode 100644 index 0000000000000..ee92fb652582e --- /dev/null +++ b/api/envoy/extensions/compression/brotli/decompressor/v3/BUILD @@ -0,0 +1,9 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], +) diff --git a/api/envoy/extensions/compression/brotli/decompressor/v3/brotli.proto b/api/envoy/extensions/compression/brotli/decompressor/v3/brotli.proto new file mode 100644 index 0000000000000..24511861cf930 --- /dev/null +++ b/api/envoy/extensions/compression/brotli/decompressor/v3/brotli.proto @@ -0,0 +1,25 @@ +syntax = "proto3"; + +package envoy.extensions.compression.brotli.decompressor.v3; + +import "google/protobuf/wrappers.proto"; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.compression.brotli.decompressor.v3"; +option java_outer_classname = "BrotliProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Brotli Decompressor] +// [#extension: envoy.compression.brotli.decompressor] + +message Brotli { + // If true, disables "canny" ring buffer allocation strategy. + // Ring buffer is allocated according to window size, despite the real size of the content. + bool disable_ring_buffer_reallocation = 1; + + // Value for decompressor's next output buffer. If not set, defaults to 4096. + google.protobuf.UInt32Value chunk_size = 2 [(validate.rules).uint32 = {lte: 65536 gte: 4096}]; +} diff --git a/api/versioning/BUILD b/api/versioning/BUILD index d041dbc02fbcb..4668593ee22e0 100644 --- a/api/versioning/BUILD +++ b/api/versioning/BUILD @@ -48,6 +48,8 @@ proto_library( "//envoy/extensions/common/matching/v3:pkg", "//envoy/extensions/common/ratelimit/v3:pkg", "//envoy/extensions/common/tap/v3:pkg", + "//envoy/extensions/compression/brotli/compressor/v3:pkg", + "//envoy/extensions/compression/brotli/decompressor/v3:pkg", "//envoy/extensions/compression/gzip/compressor/v3:pkg", "//envoy/extensions/compression/gzip/decompressor/v3:pkg", "//envoy/extensions/filters/common/dependency/v3:pkg", diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 2b488819e041c..8b75e99a598be 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -158,6 +158,7 @@ def envoy_dependencies(skip_targets = []): _io_opentracing_cpp() _net_zlib() _com_github_zlib_ng_zlib_ng() + _org_brotli() _upb() _proxy_wasm_cpp_sdk() _proxy_wasm_cpp_host() @@ -352,6 +353,19 @@ def _com_github_zlib_ng_zlib_ng(): patches = ["@envoy//bazel/foreign_cc:zlib_ng.patch"], ) +def _org_brotli(): + external_http_archive( + name = "org_brotli", + ) + native.bind( + name = "brotlienc", + actual = "@org_brotli//:brotlienc", + ) + native.bind( + name = "brotlidec", + actual = "@org_brotli//:brotlidec", + ) + def _com_google_cel_cpp(): external_http_archive("com_google_cel_cpp") external_http_archive("rules_antlr") diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 3b0d623e3761f..532c4cce0edc3 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -406,6 +406,24 @@ REPOSITORY_LOCATIONS_SPEC = dict( release_date = "2019-04-14", cpe = "cpe:2.3:a:gnu:zlib:*", ), + org_brotli = dict( + project_name = "brotli", + project_desc = "brotli compression library", + project_url = "https://brotli.org", + # Use the dev branch of brotli to resolve compilation issues. + # TODO(rojkov): Remove when brotli > 1.0.9 is released. + version = "0cd2e3926e95e7e2930f57ae3f4885508d462a25", + sha256 = "93810780e60304b51f2c9645fe313a6e4640711063ed0b860cfa60999dd256c5", + strip_prefix = "brotli-{version}", + urls = ["https://github.com/google/brotli/archive/{version}.tar.gz"], + use_category = ["dataplane_ext"], + extensions = [ + "envoy.compression.brotli.compressor", + "envoy.compression.brotli.decompressor", + ], + release_date = "2020-09-08", + cpe = "cpe:2.3:a:google:brotli:*", + ), com_github_zlib_ng_zlib_ng = dict( project_name = "zlib-ng", project_desc = "zlib fork (higher performance)", diff --git a/docs/root/api-v3/config/compression/compression.rst b/docs/root/api-v3/config/compression/compression.rst index 80aa0ba927ccb..c4f81044a522b 100644 --- a/docs/root/api-v3/config/compression/compression.rst +++ b/docs/root/api-v3/config/compression/compression.rst @@ -6,3 +6,4 @@ Compression :maxdepth: 2 ../../extensions/compression/gzip/*/v3/* + ../../extensions/compression/brotli/*/v3/* diff --git a/docs/root/configuration/http/http_filters/compressor_filter.rst b/docs/root/configuration/http/http_filters/compressor_filter.rst index bb13424048ec8..8bd65154ef8f7 100644 --- a/docs/root/configuration/http/http_filters/compressor_filter.rst +++ b/docs/root/configuration/http/http_filters/compressor_filter.rst @@ -23,8 +23,9 @@ determine whether or not the content should be compressed. The content is compressed and then sent to the client with the appropriate headers, if response and request allow. -Currently the filter supports :ref:`gzip compression ` -only. Other compression libraries can be supported as extensions. +Currently the filter supports :ref:`gzip ` +and :ref:`brotli ` +compression only. Other compression libraries can be supported as extensions. An example configuration of the filter may look like the following: diff --git a/docs/root/configuration/http/http_filters/decompressor_filter.rst b/docs/root/configuration/http/http_filters/decompressor_filter.rst index e004fedfb0f3c..10919b5abb87e 100644 --- a/docs/root/configuration/http/http_filters/decompressor_filter.rst +++ b/docs/root/configuration/http/http_filters/decompressor_filter.rst @@ -16,8 +16,9 @@ determine whether or not the content should be decompressed. The content is decompressed and passed on to the rest of the filter chain. Note that decompression happens independently for request and responses based on the rules described below. -Currently the filter supports :ref:`gzip compression ` -only. Other compression libraries can be supported as extensions. +Currently the filter supports :ref:`gzip ` +and :ref:`brotli ` +compression only. Other compression libraries can be supported as extensions. An example configuration of the filter may look like the following: diff --git a/docs/root/intro/arch_overview/other_features/compression/libraries.rst b/docs/root/intro/arch_overview/other_features/compression/libraries.rst index 2bb4ea1bad4b5..c6dbe69991aa0 100644 --- a/docs/root/intro/arch_overview/other_features/compression/libraries.rst +++ b/docs/root/intro/arch_overview/other_features/compression/libraries.rst @@ -6,7 +6,8 @@ Compression Libraries Underlying implementation ------------------------- -Currently Envoy uses `zlib `_ as a compression library. +Currently Envoy uses `zlib `_ and `brotli `_ as compression +libraries. .. note:: diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index b9000660bcd42..223d9a7bba30a 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -71,6 +71,7 @@ New Features ------------ * access log: added the :ref:`formatters ` extension point for custom formatters (command operators). * access log: support command operator: %REQUEST_HEADERS_BYTES%, %RESPONSE_HEADERS_BYTES%, and %RESPONSE_TRAILERS_BYTES%. +* compression: add brotli :ref:`compressor ` and :ref:`decompressor `. * config: add `envoy.features.fail_on_any_deprecated_feature` runtime key, which matches the behaviour of compile-time flag `ENVOY_DISABLE_DEPRECATED_FEATURES`, i.e. use of deprecated fields will cause a crash. * dispatcher: supports a stack of `Envoy::ScopeTrackedObject` instead of a single tracked object. This will allow Envoy to dump more debug information on crash. * grpc_json_transcoder: added option :ref:`strict_http_request_validation ` to reject invalid requests early. diff --git a/generated_api_shadow/envoy/extensions/compression/brotli/compressor/v3/BUILD b/generated_api_shadow/envoy/extensions/compression/brotli/compressor/v3/BUILD new file mode 100644 index 0000000000000..ee92fb652582e --- /dev/null +++ b/generated_api_shadow/envoy/extensions/compression/brotli/compressor/v3/BUILD @@ -0,0 +1,9 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], +) diff --git a/generated_api_shadow/envoy/extensions/compression/brotli/compressor/v3/brotli.proto b/generated_api_shadow/envoy/extensions/compression/brotli/compressor/v3/brotli.proto new file mode 100644 index 0000000000000..cb2933dd5d385 --- /dev/null +++ b/generated_api_shadow/envoy/extensions/compression/brotli/compressor/v3/brotli.proto @@ -0,0 +1,54 @@ +syntax = "proto3"; + +package envoy.extensions.compression.brotli.compressor.v3; + +import "google/protobuf/wrappers.proto"; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.compression.brotli.compressor.v3"; +option java_outer_classname = "BrotliProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Brotli Compressor] +// [#extension: envoy.compression.brotli.compressor] + +// [#next-free-field: 7] +message Brotli { + enum EncoderMode { + DEFAULT = 0; + GENERIC = 1; + TEXT = 2; + FONT = 3; + } + + // Value from 0 to 11 that controls the main compression speed-density lever. + // The higher quality, the slower compression. The default value is 3. + google.protobuf.UInt32Value quality = 1 [(validate.rules).uint32 = {lte: 11}]; + + // A value used to tune encoder for specific input. For more information about modes, + // please refer to brotli manual: https://brotli.org/encode.html#aa6f + // This field will be set to "DEFAULT" if not specified. + EncoderMode encoder_mode = 2 [(validate.rules).enum = {defined_only: true}]; + + // Value from 10 to 24 that represents the base two logarithmic of the compressor's window size. + // Larger window results in better compression at the expense of memory usage. The default is 18. + // For more details about this parameter, please refer to brotli manual: + // https://brotli.org/encode.html#a9a8 + google.protobuf.UInt32Value window_bits = 3 [(validate.rules).uint32 = {lte: 24 gte: 10}]; + + // Value from 16 to 24 that represents the base two logarithmic of the compressor's input block + // size. Larger input block results in better compression at the expense of memory usage. The + // default is 24. For more details about this parameter, please refer to brotli manual: + // https://brotli.org/encode.html#a9a8 + google.protobuf.UInt32Value input_block_bits = 4 [(validate.rules).uint32 = {lte: 24 gte: 16}]; + + // Value for compressor's next output buffer. If not set, defaults to 4096. + google.protobuf.UInt32Value chunk_size = 5 [(validate.rules).uint32 = {lte: 65536 gte: 4096}]; + + // If true, disables "literal context modeling" format feature. + // This flag is a "decoding-speed vs compression ratio" trade-off. + bool disable_literal_context_modeling = 6; +} diff --git a/generated_api_shadow/envoy/extensions/compression/brotli/decompressor/v3/BUILD b/generated_api_shadow/envoy/extensions/compression/brotli/decompressor/v3/BUILD new file mode 100644 index 0000000000000..ee92fb652582e --- /dev/null +++ b/generated_api_shadow/envoy/extensions/compression/brotli/decompressor/v3/BUILD @@ -0,0 +1,9 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], +) diff --git a/generated_api_shadow/envoy/extensions/compression/brotli/decompressor/v3/brotli.proto b/generated_api_shadow/envoy/extensions/compression/brotli/decompressor/v3/brotli.proto new file mode 100644 index 0000000000000..24511861cf930 --- /dev/null +++ b/generated_api_shadow/envoy/extensions/compression/brotli/decompressor/v3/brotli.proto @@ -0,0 +1,25 @@ +syntax = "proto3"; + +package envoy.extensions.compression.brotli.decompressor.v3; + +import "google/protobuf/wrappers.proto"; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.compression.brotli.decompressor.v3"; +option java_outer_classname = "BrotliProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Brotli Decompressor] +// [#extension: envoy.compression.brotli.decompressor] + +message Brotli { + // If true, disables "canny" ring buffer allocation strategy. + // Ring buffer is allocated according to window size, despite the real size of the content. + bool disable_ring_buffer_reallocation = 1; + + // Value for decompressor's next output buffer. If not set, defaults to 4096. + google.protobuf.UInt32Value chunk_size = 2 [(validate.rules).uint32 = {lte: 65536 gte: 4096}]; +} diff --git a/source/common/http/headers.h b/source/common/http/headers.h index 9d1774fd17b5e..3d8f976a8eadb 100644 --- a/source/common/http/headers.h +++ b/source/common/http/headers.h @@ -96,6 +96,7 @@ class CustomHeaderValues { } CacheControlValues; struct { + const std::string Brotli{"br"}; const std::string Gzip{"gzip"}; } ContentEncodingValues; diff --git a/source/extensions/compression/brotli/common/BUILD b/source/extensions/compression/brotli/common/BUILD new file mode 100644 index 0000000000000..ddf53267cb59f --- /dev/null +++ b/source/extensions/compression/brotli/common/BUILD @@ -0,0 +1,18 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_library( + name = "brotli_base_lib", + srcs = ["base.cc"], + hdrs = ["base.h"], + deps = [ + "//source/common/buffer:buffer_lib", + ], +) diff --git a/source/extensions/compression/brotli/common/base.cc b/source/extensions/compression/brotli/common/base.cc new file mode 100644 index 0000000000000..12a9a944c8fdb --- /dev/null +++ b/source/extensions/compression/brotli/common/base.cc @@ -0,0 +1,36 @@ +#include "extensions/compression/brotli/common/base.h" + +namespace Envoy { +namespace Extensions { +namespace Compression { +namespace Brotli { +namespace Common { + +BrotliContext::BrotliContext(const uint32_t chunk_size) + : chunk_size_{chunk_size}, chunk_ptr_{std::make_unique(chunk_size)}, next_in_{}, + next_out_{chunk_ptr_.get()}, avail_in_{0}, avail_out_{chunk_size} {} + +void BrotliContext::updateOutput(Buffer::Instance& output_buffer) { + if (avail_out_ == 0) { + output_buffer.add(static_cast(chunk_ptr_.get()), chunk_size_); + resetOut(); + } +} + +void BrotliContext::finalizeOutput(Buffer::Instance& output_buffer) { + const size_t n_output = chunk_size_ - avail_out_; + if (n_output > 0) { + output_buffer.add(static_cast(chunk_ptr_.get()), n_output); + } +} + +void BrotliContext::resetOut() { + avail_out_ = chunk_size_; + next_out_ = chunk_ptr_.get(); +} + +} // namespace Common +} // namespace Brotli +} // namespace Compression +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/compression/brotli/common/base.h b/source/extensions/compression/brotli/common/base.h new file mode 100644 index 0000000000000..fed019c9a297d --- /dev/null +++ b/source/extensions/compression/brotli/common/base.h @@ -0,0 +1,35 @@ +#pragma once + +#include + +#include "envoy/buffer/buffer.h" + +namespace Envoy { +namespace Extensions { +namespace Compression { +namespace Brotli { +namespace Common { + +// Keeps a `Brotli` compression stream's state. +struct BrotliContext { + BrotliContext(const uint32_t chunk_size); + + void updateOutput(Buffer::Instance& output_buffer); + void finalizeOutput(Buffer::Instance& output_buffer); + + const uint32_t chunk_size_; + std::unique_ptr chunk_ptr_; + const uint8_t* next_in_; + uint8_t* next_out_; + size_t avail_in_; + size_t avail_out_; + +private: + void resetOut(); +}; + +} // namespace Common +} // namespace Brotli +} // namespace Compression +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/compression/brotli/compressor/BUILD b/source/extensions/compression/brotli/compressor/BUILD new file mode 100644 index 0000000000000..efd1739e00142 --- /dev/null +++ b/source/extensions/compression/brotli/compressor/BUILD @@ -0,0 +1,35 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_cc_library", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_library( + name = "compressor_lib", + srcs = ["brotli_compressor_impl.cc"], + hdrs = ["brotli_compressor_impl.h"], + external_deps = ["brotlienc"], + deps = [ + "//include/envoy/compression/compressor:compressor_interface", + "//source/common/buffer:buffer_lib", + "//source/extensions/compression/brotli/common:brotli_base_lib", + ], +) + +envoy_cc_extension( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + security_posture = "robust_to_untrusted_downstream", + deps = [ + ":compressor_lib", + "//source/common/http:headers_lib", + "//source/extensions/compression/common/compressor:compressor_factory_base_lib", + "@envoy_api//envoy/extensions/compression/brotli/compressor/v3:pkg_cc_proto", + ], +) diff --git a/source/extensions/compression/brotli/compressor/brotli_compressor_impl.cc b/source/extensions/compression/brotli/compressor/brotli_compressor_impl.cc new file mode 100644 index 0000000000000..4b4366fbe3aff --- /dev/null +++ b/source/extensions/compression/brotli/compressor/brotli_compressor_impl.cc @@ -0,0 +1,84 @@ +#include "extensions/compression/brotli/compressor/brotli_compressor_impl.h" + +#include "common/buffer/buffer_impl.h" + +namespace Envoy { +namespace Extensions { +namespace Compression { +namespace Brotli { +namespace Compressor { + +BrotliCompressorImpl::BrotliCompressorImpl(const uint32_t quality, const uint32_t window_bits, + const uint32_t input_block_bits, + const bool disable_literal_context_modeling, + const EncoderMode mode, const uint32_t chunk_size) + : chunk_size_{chunk_size}, state_(BrotliEncoderCreateInstance(nullptr, nullptr, nullptr), + &BrotliEncoderDestroyInstance) { + RELEASE_ASSERT(quality <= BROTLI_MAX_QUALITY, ""); + BROTLI_BOOL result = BrotliEncoderSetParameter(state_.get(), BROTLI_PARAM_QUALITY, quality); + RELEASE_ASSERT(result == BROTLI_TRUE, ""); + + RELEASE_ASSERT(window_bits >= BROTLI_MIN_WINDOW_BITS && window_bits <= BROTLI_MAX_WINDOW_BITS, + ""); + result = BrotliEncoderSetParameter(state_.get(), BROTLI_PARAM_LGWIN, window_bits); + RELEASE_ASSERT(result == BROTLI_TRUE, ""); + + RELEASE_ASSERT(input_block_bits >= BROTLI_MIN_INPUT_BLOCK_BITS && + input_block_bits <= BROTLI_MAX_INPUT_BLOCK_BITS, + ""); + result = BrotliEncoderSetParameter(state_.get(), BROTLI_PARAM_LGBLOCK, input_block_bits); + RELEASE_ASSERT(result == BROTLI_TRUE, ""); + + result = BrotliEncoderSetParameter(state_.get(), BROTLI_PARAM_DISABLE_LITERAL_CONTEXT_MODELING, + disable_literal_context_modeling); + RELEASE_ASSERT(result == BROTLI_TRUE, ""); + + result = BrotliEncoderSetParameter(state_.get(), BROTLI_PARAM_MODE, static_cast(mode)); + RELEASE_ASSERT(result == BROTLI_TRUE, ""); +} + +void BrotliCompressorImpl::compress(Buffer::Instance& buffer, + Envoy::Compression::Compressor::State state) { + Common::BrotliContext ctx(chunk_size_); + + Buffer::OwnedImpl accumulation_buffer; + for (const Buffer::RawSlice& input_slice : buffer.getRawSlices()) { + ctx.avail_in_ = input_slice.len_; + ctx.next_in_ = static_cast(input_slice.mem_); + + while (ctx.avail_in_ > 0) { + process(ctx, accumulation_buffer, BROTLI_OPERATION_PROCESS); + } + + buffer.drain(input_slice.len_); + } + + ASSERT(buffer.length() == 0); + buffer.move(accumulation_buffer); + + // The encoder's internal buffer can still hold data not flushed to the + // output chunk which in turn can be almost full and not fit to accommodate + // the flushed data. And in case of the `Finish` operation the encoder may add + // `ISLAST` and `ISLASTEMPTY` headers to the output. Thus keep processing + // until the encoder's output is fully depleted. + do { + process(ctx, buffer, + state == Envoy::Compression::Compressor::State::Finish ? BROTLI_OPERATION_FINISH + : BROTLI_OPERATION_FLUSH); + } while (BrotliEncoderHasMoreOutput(state_.get())); + + ctx.finalizeOutput(buffer); +} + +void BrotliCompressorImpl::process(Common::BrotliContext& ctx, Buffer::Instance& output_buffer, + const BrotliEncoderOperation op) { + BROTLI_BOOL result = BrotliEncoderCompressStream(state_.get(), op, &ctx.avail_in_, &ctx.next_in_, + &ctx.avail_out_, &ctx.next_out_, nullptr); + RELEASE_ASSERT(result == BROTLI_TRUE, "unable to compress"); + ctx.updateOutput(output_buffer); +} +} // namespace Compressor +} // namespace Brotli +} // namespace Compression +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/compression/brotli/compressor/brotli_compressor_impl.h b/source/extensions/compression/brotli/compressor/brotli_compressor_impl.h new file mode 100644 index 0000000000000..ff153e81b6ad7 --- /dev/null +++ b/source/extensions/compression/brotli/compressor/brotli_compressor_impl.h @@ -0,0 +1,67 @@ +#pragma once + +#include "envoy/compression/compressor/compressor.h" + +#include "extensions/compression/brotli/common/base.h" + +#include "brotli/encode.h" + +namespace Envoy { +namespace Extensions { +namespace Compression { +namespace Brotli { +namespace Compressor { + +/** + * Implementation of compressor's interface. + */ +class BrotliCompressorImpl : public Envoy::Compression::Compressor::Compressor, NonCopyable { +public: + /** + * Enum values are used for setting the encoder mode. + * Generic: in this mode the compressor does not know anything in advance about the properties of + * the input; + * Text: compression mode for UTF-8 formatted text input; + * Font: compression mode used in `WOFF` 2.0; + * Default: compression mode used by brotli encoder by default which is Generic currently. + * @see BROTLI_DEFAULT_MODE in brotli manual. + */ + enum class EncoderMode : uint32_t { + Generic = BROTLI_MODE_GENERIC, + Text = BROTLI_MODE_TEXT, + Font = BROTLI_MODE_FONT, + Default = BROTLI_DEFAULT_MODE, + }; + + /** + * Constructor. + * @param quality sets compression level. The higher the quality, the slower the + * compression. @see BROTLI_PARAM_QUALITY (brotli manual). + * @param window_bits sets recommended sliding `LZ77` window size. + * @param input_block_bits sets recommended input block size. Bigger input block size allows + * better compression, but consumes more memory. + * @param disable_literal_context_modeling affects usage of "literal context modeling" format + * feature. This flag is a "decoding-speed vs compression ratio" trade-off. + * @param mode tunes encoder for specific input. @see EncoderMode enum. + * @param chunk_size amount of memory reserved for the compressor output. + */ + BrotliCompressorImpl(const uint32_t quality, const uint32_t window_bits, + const uint32_t input_block_bits, const bool disable_literal_context_modeling, + const EncoderMode mode, const uint32_t chunk_size); + + // Compression::Compressor::Compressor + void compress(Buffer::Instance& buffer, Envoy::Compression::Compressor::State state) override; + +private: + void process(Common::BrotliContext& ctx, Buffer::Instance& output_buffer, + const BrotliEncoderOperation op); + + const uint32_t chunk_size_; + std::unique_ptr state_; +}; + +} // namespace Compressor +} // namespace Brotli +} // namespace Compression +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/compression/brotli/compressor/config.cc b/source/extensions/compression/brotli/compressor/config.cc new file mode 100644 index 0000000000000..4ca5f639f1a0d --- /dev/null +++ b/source/extensions/compression/brotli/compressor/config.cc @@ -0,0 +1,69 @@ +#include "extensions/compression/brotli/compressor/config.h" + +namespace Envoy { +namespace Extensions { +namespace Compression { +namespace Brotli { +namespace Compressor { + +namespace { +// Default input block size. +const uint32_t DefaultInputBlockBits = 24; + +// Default compression window size. +const uint32_t DefaultWindowBits = 18; + +// Default quality. +const uint32_t DefaultQuality = 3; + +// Default zlib chunk size. +const uint32_t DefaultChunkSize = 4096; +} // namespace + +BrotliCompressorFactory::BrotliCompressorFactory( + const envoy::extensions::compression::brotli::compressor::v3::Brotli& brotli) + : chunk_size_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(brotli, chunk_size, DefaultChunkSize)), + disable_literal_context_modeling_(brotli.disable_literal_context_modeling()), + encoder_mode_(encoderModeEnum(brotli.encoder_mode())), + input_block_bits_( + PROTOBUF_GET_WRAPPED_OR_DEFAULT(brotli, input_block_bits, DefaultInputBlockBits)), + quality_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(brotli, quality, DefaultQuality)), + window_bits_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(brotli, window_bits, DefaultWindowBits)) {} + +Envoy::Compression::Compressor::CompressorPtr BrotliCompressorFactory::createCompressor() { + return std::make_unique(quality_, window_bits_, input_block_bits_, + disable_literal_context_modeling_, encoder_mode_, + chunk_size_); +} + +BrotliCompressorImpl::EncoderMode BrotliCompressorFactory::encoderModeEnum( + envoy::extensions::compression::brotli::compressor::v3::Brotli::EncoderMode encoder_mode) { + switch (encoder_mode) { + case envoy::extensions::compression::brotli::compressor::v3::Brotli::GENERIC: + return BrotliCompressorImpl::EncoderMode::Generic; + case envoy::extensions::compression::brotli::compressor::v3::Brotli::TEXT: + return BrotliCompressorImpl::EncoderMode::Text; + case envoy::extensions::compression::brotli::compressor::v3::Brotli::FONT: + return BrotliCompressorImpl::EncoderMode::Font; + default: + return BrotliCompressorImpl::EncoderMode::Default; + } +} + +Envoy::Compression::Compressor::CompressorFactoryPtr +BrotliCompressorLibraryFactory::createCompressorFactoryFromProtoTyped( + const envoy::extensions::compression::brotli::compressor::v3::Brotli& proto_config) { + return std::make_unique(proto_config); +} + +/** + * Static registration for the brotli compressor library. @see NamedCompressorLibraryConfigFactory. + */ +REGISTER_FACTORY(BrotliCompressorLibraryFactory, + Envoy::Compression::Compressor::NamedCompressorLibraryConfigFactory); + +} // namespace Compressor +} // namespace Brotli +} // namespace Compression +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/compression/brotli/compressor/config.h b/source/extensions/compression/brotli/compressor/config.h new file mode 100644 index 0000000000000..7046676421337 --- /dev/null +++ b/source/extensions/compression/brotli/compressor/config.h @@ -0,0 +1,67 @@ +#pragma once + +#include "envoy/compression/compressor/factory.h" +#include "envoy/extensions/compression/brotli/compressor/v3/brotli.pb.h" +#include "envoy/extensions/compression/brotli/compressor/v3/brotli.pb.validate.h" + +#include "common/http/headers.h" + +#include "extensions/compression/brotli/compressor/brotli_compressor_impl.h" +#include "extensions/compression/common/compressor/factory_base.h" + +namespace Envoy { +namespace Extensions { +namespace Compression { +namespace Brotli { +namespace Compressor { + +namespace { + +const std::string& brotliStatsPrefix() { CONSTRUCT_ON_FIRST_USE(std::string, "brotli."); } +const std::string& brotliExtensionName() { + CONSTRUCT_ON_FIRST_USE(std::string, "envoy.compression.brotli.compressor"); +} + +} // namespace + +class BrotliCompressorFactory : public Envoy::Compression::Compressor::CompressorFactory { +public: + BrotliCompressorFactory( + const envoy::extensions::compression::brotli::compressor::v3::Brotli& brotli); + + // Envoy::Compression::Compressor::CompressorFactory + Envoy::Compression::Compressor::CompressorPtr createCompressor() override; + const std::string& statsPrefix() const override { return brotliStatsPrefix(); } + const std::string& contentEncoding() const override { + return Http::CustomHeaders::get().ContentEncodingValues.Brotli; + } + +private: + static BrotliCompressorImpl::EncoderMode encoderModeEnum( + envoy::extensions::compression::brotli::compressor::v3::Brotli::EncoderMode encoder_mode); + const uint32_t chunk_size_; + const bool disable_literal_context_modeling_; + const BrotliCompressorImpl::EncoderMode encoder_mode_; + const uint32_t input_block_bits_; + const uint32_t quality_; + const uint32_t window_bits_; +}; + +class BrotliCompressorLibraryFactory + : public Compression::Common::Compressor::CompressorLibraryFactoryBase< + envoy::extensions::compression::brotli::compressor::v3::Brotli> { +public: + BrotliCompressorLibraryFactory() : CompressorLibraryFactoryBase(brotliExtensionName()) {} + +private: + Envoy::Compression::Compressor::CompressorFactoryPtr createCompressorFactoryFromProtoTyped( + const envoy::extensions::compression::brotli::compressor::v3::Brotli& config) override; +}; + +DECLARE_FACTORY(BrotliCompressorLibraryFactory); + +} // namespace Compressor +} // namespace Brotli +} // namespace Compression +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/compression/brotli/decompressor/BUILD b/source/extensions/compression/brotli/decompressor/BUILD new file mode 100644 index 0000000000000..3cc016e2f215f --- /dev/null +++ b/source/extensions/compression/brotli/decompressor/BUILD @@ -0,0 +1,37 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_cc_library", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_library( + name = "decompressor_lib", + srcs = ["brotli_decompressor_impl.cc"], + hdrs = ["brotli_decompressor_impl.h"], + external_deps = ["brotlidec"], + deps = [ + "//include/envoy/compression/decompressor:decompressor_interface", + "//include/envoy/stats:stats_interface", + "//include/envoy/stats:stats_macros", + "//source/common/buffer:buffer_lib", + "//source/extensions/compression/brotli/common:brotli_base_lib", + ], +) + +envoy_cc_extension( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + security_posture = "robust_to_untrusted_downstream", + deps = [ + ":decompressor_lib", + "//source/common/http:headers_lib", + "//source/extensions/compression/common/decompressor:decompressor_factory_base_lib", + "@envoy_api//envoy/extensions/compression/brotli/decompressor/v3:pkg_cc_proto", + ], +) diff --git a/source/extensions/compression/brotli/decompressor/brotli_decompressor_impl.cc b/source/extensions/compression/brotli/decompressor/brotli_decompressor_impl.cc new file mode 100644 index 0000000000000..634c832b80332 --- /dev/null +++ b/source/extensions/compression/brotli/decompressor/brotli_decompressor_impl.cc @@ -0,0 +1,70 @@ +#include "extensions/compression/brotli/decompressor/brotli_decompressor_impl.h" + +#include + +namespace Envoy { +namespace Extensions { +namespace Compression { +namespace Brotli { +namespace Decompressor { + +BrotliDecompressorImpl::BrotliDecompressorImpl(Stats::Scope& scope, const std::string& stats_prefix, + const uint32_t chunk_size, + const bool disable_ring_buffer_reallocation) + : chunk_size_{chunk_size}, + state_(BrotliDecoderCreateInstance(nullptr, nullptr, nullptr), &BrotliDecoderDestroyInstance), + stats_(generateStats(stats_prefix, scope)) { + BROTLI_BOOL result = + BrotliDecoderSetParameter(state_.get(), BROTLI_DECODER_PARAM_DISABLE_RING_BUFFER_REALLOCATION, + disable_ring_buffer_reallocation ? BROTLI_TRUE : BROTLI_FALSE); + RELEASE_ASSERT(result == BROTLI_TRUE, ""); +} + +void BrotliDecompressorImpl::decompress(const Buffer::Instance& input_buffer, + Buffer::Instance& output_buffer) { + Common::BrotliContext ctx(chunk_size_); + + for (const Buffer::RawSlice& input_slice : input_buffer.getRawSlices()) { + ctx.avail_in_ = input_slice.len_; + ctx.next_in_ = static_cast(input_slice.mem_); + + while (ctx.avail_in_ > 0) { + if (!process(ctx, output_buffer)) { + ctx.finalizeOutput(output_buffer); + return; + } + } + } + + // Even though the input has been fully consumed by the decoder it still can + // be unfolded into output not fitting the output chunk. Thus keep processing + // until the decoder's output is fully depleted. + bool success; + do { + success = process(ctx, output_buffer); + } while (success && BrotliDecoderHasMoreOutput(state_.get())); + + ctx.finalizeOutput(output_buffer); +} + +bool BrotliDecompressorImpl::process(Common::BrotliContext& ctx, Buffer::Instance& output_buffer) { + BrotliDecoderResult result; + result = BrotliDecoderDecompressStream(state_.get(), &ctx.avail_in_, &ctx.next_in_, + &ctx.avail_out_, &ctx.next_out_, nullptr); + if (result == BROTLI_DECODER_RESULT_ERROR) { + // TODO(rojkov): currently the Brotli library doesn't specify possible errors in its API. Add + // more detailed stats when they are documented. + stats_.brotli_error_.inc(); + return false; + } + + ctx.updateOutput(output_buffer); + + return true; +} + +} // namespace Decompressor +} // namespace Brotli +} // namespace Compression +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/compression/brotli/decompressor/brotli_decompressor_impl.h b/source/extensions/compression/brotli/decompressor/brotli_decompressor_impl.h new file mode 100644 index 0000000000000..26472903410f2 --- /dev/null +++ b/source/extensions/compression/brotli/decompressor/brotli_decompressor_impl.h @@ -0,0 +1,64 @@ +#pragma once + +#include "envoy/compression/decompressor/decompressor.h" +#include "envoy/stats/scope.h" +#include "envoy/stats/stats_macros.h" + +#include "extensions/compression/brotli/common/base.h" + +#include "brotli/decode.h" + +namespace Envoy { +namespace Extensions { +namespace Compression { +namespace Brotli { +namespace Decompressor { + +/** + * All brotli decompressor stats. @see stats_macros.h + */ +#define ALL_BROTLI_DECOMPRESSOR_STATS(COUNTER) COUNTER(brotli_error) + +/** + * Struct definition for brotli decompressor stats. @see stats_macros.h + */ +struct BrotliDecompressorStats { + ALL_BROTLI_DECOMPRESSOR_STATS(GENERATE_COUNTER_STRUCT) +}; + +/** + * Implementation of decompressor's interface. + */ +class BrotliDecompressorImpl : public Envoy::Compression::Decompressor::Decompressor, NonCopyable { +public: + /** + * Constructor. + * @param chunk_size amount of memory reserved for the decompressor output. + * @param disable_ring_buffer_reallocation if true disables "canny" ring buffer allocation + * strategy. Ring buffer is allocated according to window size, despite the real size of the + * content. + */ + BrotliDecompressorImpl(Stats::Scope& scope, const std::string& stats_prefix, + const uint32_t chunk_size, bool disable_ring_buffer_reallocation); + + // Envoy::Compression::Decompressor::Decompressor + void decompress(const Buffer::Instance& input_buffer, Buffer::Instance& output_buffer) override; + +private: + static BrotliDecompressorStats generateStats(const std::string& prefix, Stats::Scope& scope) { + return BrotliDecompressorStats{ + ALL_BROTLI_DECOMPRESSOR_STATS(POOL_COUNTER_PREFIX(scope, prefix))}; + } + + bool process(Common::BrotliContext& ctx, Buffer::Instance& output_buffer); + + const uint32_t chunk_size_; + std::unique_ptr state_; + const BrotliDecompressorStats stats_; +}; + +} // namespace Decompressor +} // namespace Brotli +} // namespace Compression +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/compression/brotli/decompressor/config.cc b/source/extensions/compression/brotli/decompressor/config.cc new file mode 100644 index 0000000000000..63ae34cd2f994 --- /dev/null +++ b/source/extensions/compression/brotli/decompressor/config.cc @@ -0,0 +1,44 @@ +#include "extensions/compression/brotli/decompressor/config.h" + +namespace Envoy { +namespace Extensions { +namespace Compression { +namespace Brotli { +namespace Decompressor { + +namespace { + +const uint32_t DefaultChunkSize = 4096; + +} // namespace + +BrotliDecompressorFactory::BrotliDecompressorFactory( + const envoy::extensions::compression::brotli::decompressor::v3::Brotli& brotli, + Stats::Scope& scope) + : scope_(scope), + chunk_size_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(brotli, chunk_size, DefaultChunkSize)), + disable_ring_buffer_reallocation_{brotli.disable_ring_buffer_reallocation()} {} + +Envoy::Compression::Decompressor::DecompressorPtr +BrotliDecompressorFactory::createDecompressor(const std::string& stats_prefix) { + return std::make_unique(scope_, stats_prefix, chunk_size_, + disable_ring_buffer_reallocation_); +} + +Envoy::Compression::Decompressor::DecompressorFactoryPtr +BrotliDecompressorLibraryFactory::createDecompressorFactoryFromProtoTyped( + const envoy::extensions::compression::brotli::decompressor::v3::Brotli& proto_config, + Server::Configuration::FactoryContext& context) { + return std::make_unique(proto_config, context.scope()); +} + +/** + * Static registration for the brotli decompressor. @see NamedDecompressorLibraryConfigFactory. + */ +REGISTER_FACTORY(BrotliDecompressorLibraryFactory, + Envoy::Compression::Decompressor::NamedDecompressorLibraryConfigFactory); +} // namespace Decompressor +} // namespace Brotli +} // namespace Compression +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/compression/brotli/decompressor/config.h b/source/extensions/compression/brotli/decompressor/config.h new file mode 100644 index 0000000000000..111e9b5ebe77e --- /dev/null +++ b/source/extensions/compression/brotli/decompressor/config.h @@ -0,0 +1,64 @@ +#pragma once + +#include "envoy/compression/decompressor/config.h" +#include "envoy/extensions/compression/brotli/decompressor/v3/brotli.pb.h" +#include "envoy/extensions/compression/brotli/decompressor/v3/brotli.pb.validate.h" + +#include "common/http/headers.h" + +#include "extensions/compression/brotli/decompressor/brotli_decompressor_impl.h" +#include "extensions/compression/common/decompressor/factory_base.h" + +namespace Envoy { +namespace Extensions { +namespace Compression { +namespace Brotli { +namespace Decompressor { + +namespace { +const std::string& brotliStatsPrefix() { CONSTRUCT_ON_FIRST_USE(std::string, "brotli."); } +const std::string& brotliExtensionName() { + CONSTRUCT_ON_FIRST_USE(std::string, "envoy.compression.brotli.decompressor"); +} + +} // namespace + +class BrotliDecompressorFactory : public Envoy::Compression::Decompressor::DecompressorFactory { +public: + BrotliDecompressorFactory( + const envoy::extensions::compression::brotli::decompressor::v3::Brotli& brotli, + Stats::Scope& scope); + + // Envoy::Compression::Decompressor::DecompressorFactory + Envoy::Compression::Decompressor::DecompressorPtr + createDecompressor(const std::string& stats_prefix) override; + const std::string& statsPrefix() const override { return brotliStatsPrefix(); } + const std::string& contentEncoding() const override { + return Http::CustomHeaders::get().ContentEncodingValues.Brotli; + } + +private: + Stats::Scope& scope_; + const uint32_t chunk_size_; + const bool disable_ring_buffer_reallocation_; +}; + +class BrotliDecompressorLibraryFactory + : public Compression::Common::Decompressor::DecompressorLibraryFactoryBase< + envoy::extensions::compression::brotli::decompressor::v3::Brotli> { +public: + BrotliDecompressorLibraryFactory() : DecompressorLibraryFactoryBase(brotliExtensionName()) {} + +private: + Envoy::Compression::Decompressor::DecompressorFactoryPtr createDecompressorFactoryFromProtoTyped( + const envoy::extensions::compression::brotli::decompressor::v3::Brotli& proto_config, + Server::Configuration::FactoryContext& context) override; +}; + +DECLARE_FACTORY(BrotliDecompressorLibraryFactory); + +} // namespace Decompressor +} // namespace Brotli +} // namespace Compression +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index 1f861f0a22998..8b2565a0fa848 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -23,6 +23,8 @@ EXTENSIONS = { "envoy.compression.gzip.compressor": "//source/extensions/compression/gzip/compressor:config", "envoy.compression.gzip.decompressor": "//source/extensions/compression/gzip/decompressor:config", + "envoy.compression.brotli.compressor": "//source/extensions/compression/brotli/compressor:config", + "envoy.compression.brotli.decompressor": "//source/extensions/compression/brotli/decompressor:config", # # gRPC Credentials Plugins diff --git a/test/extensions/compression/brotli/compressor/BUILD b/test/extensions/compression/brotli/compressor/BUILD new file mode 100644 index 0000000000000..1427caffdc444 --- /dev/null +++ b/test/extensions/compression/brotli/compressor/BUILD @@ -0,0 +1,24 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_extension_cc_test( + name = "compressor_test", + srcs = ["brotli_compressor_impl_test.cc"], + extension_name = "envoy.compression.brotli.compressor", + deps = [ + "//source/extensions/compression/brotli/compressor:config", + "//source/extensions/compression/brotli/decompressor:decompressor_lib", + "//test/mocks/server:factory_context_mocks", + "//test/test_common:utility_lib", + ], +) diff --git a/test/extensions/compression/brotli/compressor/brotli_compressor_impl_test.cc b/test/extensions/compression/brotli/compressor/brotli_compressor_impl_test.cc new file mode 100644 index 0000000000000..8a26cd53e1251 --- /dev/null +++ b/test/extensions/compression/brotli/compressor/brotli_compressor_impl_test.cc @@ -0,0 +1,142 @@ +#include "common/buffer/buffer_impl.h" +#include "common/stats/isolated_store_impl.h" + +#include "extensions/compression/brotli/compressor/config.h" +#include "extensions/compression/brotli/decompressor/brotli_decompressor_impl.h" + +#include "test/mocks/server/factory_context.h" +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace Compression { +namespace Brotli { +namespace Compressor { +namespace { + +class BrotliCompressorImplTest : public testing::Test { +protected: + void drainBuffer(Buffer::OwnedImpl& buffer) { buffer.drain(buffer.length()); } + + void verifyWithDecompressor(Envoy::Compression::Compressor::CompressorPtr compressor) { + Buffer::OwnedImpl buffer; + Buffer::OwnedImpl accumulation_buffer; + std::string original_text{}; + for (uint64_t i = 0; i < 10; i++) { + TestUtility::feedBufferWithRandomCharacters(buffer, default_input_size * i, i); + original_text.append(buffer.toString()); + ASSERT_EQ(default_input_size * i, buffer.length()); + compressor->compress(buffer, Envoy::Compression::Compressor::State::Flush); + accumulation_buffer.add(buffer); + drainBuffer(buffer); + ASSERT_EQ(0, buffer.length()); + } + + compressor->compress(buffer, Envoy::Compression::Compressor::State::Finish); + accumulation_buffer.add(buffer); + drainBuffer(buffer); + + Stats::IsolatedStoreImpl stats_store{}; + Compression::Brotli::Decompressor::BrotliDecompressorImpl decompressor{stats_store, "test.", + 4096, false}; + + decompressor.decompress(accumulation_buffer, buffer); + std::string decompressed_text{buffer.toString()}; + + ASSERT_EQ(original_text.length(), decompressed_text.length()); + EXPECT_EQ(original_text, decompressed_text); + } + + static constexpr uint32_t default_quality{11}; + static constexpr uint32_t default_window_bits{22}; + static constexpr uint32_t default_input_block_bits{22}; + static constexpr uint32_t default_input_size{796}; +}; + +TEST_F(BrotliCompressorImplTest, CompressorDeathTest) { + EXPECT_DEATH( + { + BrotliCompressorImpl compressor(1000, default_window_bits, default_input_block_bits, false, + BrotliCompressorImpl::EncoderMode::Generic, 4096); + }, + "assert failure: quality <= BROTLI_MAX_QUALITY"); + EXPECT_DEATH( + { + BrotliCompressorImpl compressor(default_quality, 1, default_input_block_bits, false, + BrotliCompressorImpl::EncoderMode::Generic, 4096); + }, + "assert failure: window_bits >= BROTLI_MIN_WINDOW_BITS && window_bits <= " + "BROTLI_MAX_WINDOW_BITS"); + EXPECT_DEATH( + { + BrotliCompressorImpl compressor(default_quality, default_window_bits, 30, false, + BrotliCompressorImpl::EncoderMode::Generic, 4096); + }, + "assert failure: input_block_bits >= BROTLI_MIN_INPUT_BLOCK_BITS && input_block_bits <= " + "BROTLI_MAX_INPUT_BLOCK_BITS"); +} + +TEST_F(BrotliCompressorImplTest, CallingFinishOnly) { + Buffer::OwnedImpl buffer; + BrotliCompressorImpl compressor(default_quality, default_window_bits, default_input_block_bits, + false, BrotliCompressorImpl::EncoderMode::Default, 4096); + + TestUtility::feedBufferWithRandomCharacters(buffer, 4096); + compressor.compress(buffer, Envoy::Compression::Compressor::State::Finish); +} + +TEST_F(BrotliCompressorImplTest, CallingFlushOnly) { + Buffer::OwnedImpl buffer; + BrotliCompressorImpl compressor(default_quality, default_window_bits, default_input_block_bits, + false, BrotliCompressorImpl::EncoderMode::Default, 4096); + + TestUtility::feedBufferWithRandomCharacters(buffer, 4096); + compressor.compress(buffer, Envoy::Compression::Compressor::State::Flush); +} + +TEST_F(BrotliCompressorImplTest, CompressWithSmallChunkSize) { + auto compressor = std::make_unique( + default_quality, default_window_bits, default_input_block_bits, false, + BrotliCompressorImpl::EncoderMode::Default, 8); + verifyWithDecompressor(std::move(compressor)); +} + +class ConfigTest : public BrotliCompressorImplTest, + public testing::WithParamInterface {}; + +INSTANTIATE_TEST_SUITE_P(ConfigTestSuite, ConfigTest, + testing::Values("generic", "default", "font", "text")); + +TEST_P(ConfigTest, LoadConfig) { + absl::string_view encoder_mode = GetParam(); + + std::string json{fmt::format(R"EOF({{ + "disable_literal_context_modeling": true, + "quality": 11, + "encoder_mode": "{}", + "window_bits": 22, + "input_block_bits": 24, + "chunk_size": 4096 +}})EOF", + encoder_mode)}; + envoy::extensions::compression::brotli::compressor::v3::Brotli brotli; + TestUtility::loadFromJson(json, brotli); + + BrotliCompressorLibraryFactory lib_factory; + NiceMock context; + Envoy::Compression::Compressor::CompressorFactoryPtr factory = + lib_factory.createCompressorFactoryFromProto(brotli, context); + EXPECT_EQ("brotli.", factory->statsPrefix()); + EXPECT_EQ("br", factory->contentEncoding()); + + verifyWithDecompressor(factory->createCompressor()); +} + +} // namespace +} // namespace Compressor +} // namespace Brotli +} // namespace Compression +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/compression/brotli/decompressor/BUILD b/test/extensions/compression/brotli/decompressor/BUILD new file mode 100644 index 0000000000000..4bd8afe792d75 --- /dev/null +++ b/test/extensions/compression/brotli/decompressor/BUILD @@ -0,0 +1,25 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_extension_cc_test( + name = "brotli_decompressor_impl_test", + srcs = ["brotli_decompressor_impl_test.cc"], + extension_name = "envoy.compression.brotli.decompressor", + deps = [ + "//source/common/stats:isolated_store_lib", + "//source/extensions/compression/brotli/compressor:compressor_lib", + "//source/extensions/compression/brotli/decompressor:config", + "//test/mocks/server:factory_context_mocks", + "//test/test_common:utility_lib", + ], +) diff --git a/test/extensions/compression/brotli/decompressor/brotli_decompressor_impl_test.cc b/test/extensions/compression/brotli/decompressor/brotli_decompressor_impl_test.cc new file mode 100644 index 0000000000000..cd5d3ee1fd2a3 --- /dev/null +++ b/test/extensions/compression/brotli/decompressor/brotli_decompressor_impl_test.cc @@ -0,0 +1,271 @@ +#include "common/buffer/buffer_impl.h" +#include "common/stats/isolated_store_impl.h" + +#include "extensions/compression/brotli/compressor/brotli_compressor_impl.h" +#include "extensions/compression/brotli/decompressor/config.h" + +#include "test/mocks/server/factory_context.h" +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace Compression { +namespace Brotli { +namespace Decompressor { +namespace { + +class BrotliDecompressorImplTest : public testing::Test { +protected: + void drainBuffer(Buffer::OwnedImpl& buffer) { buffer.drain(buffer.length()); } + + static constexpr uint32_t default_quality{2}; + static constexpr uint32_t default_window_bits{22}; + static constexpr uint32_t default_input_block_bits{22}; + static constexpr uint32_t default_input_size{796}; +}; + +// Exercises compression and decompression by compressing some data, decompressing it and then +// comparing compressor's input/checksum with decompressor's output/checksum. +TEST_F(BrotliDecompressorImplTest, CompressAndDecompress) { + Buffer::OwnedImpl buffer; + Buffer::OwnedImpl accumulation_buffer; + + Brotli::Compressor::BrotliCompressorImpl compressor{ + default_quality, + default_window_bits, + default_input_block_bits, + false, + Brotli::Compressor::BrotliCompressorImpl::EncoderMode::Default, + 4096}; + + std::string original_text{}; + for (uint64_t i = 0; i < 20; ++i) { + TestUtility::feedBufferWithRandomCharacters(buffer, default_input_size * i, i); + original_text.append(buffer.toString()); + compressor.compress(buffer, Envoy::Compression::Compressor::State::Flush); + accumulation_buffer.add(buffer); + drainBuffer(buffer); + } + + ASSERT_EQ(0, buffer.length()); + + compressor.compress(buffer, Envoy::Compression::Compressor::State::Finish); + ASSERT_GE(10, buffer.length()); + + accumulation_buffer.add(buffer); + + drainBuffer(buffer); + ASSERT_EQ(0, buffer.length()); + + std::string json{R"EOF({ + "disable_ring_buffer_reallocation": false, + "chunk_size": 4096 +})EOF"}; + envoy::extensions::compression::brotli::decompressor::v3::Brotli brotli; + TestUtility::loadFromJson(json, brotli); + + BrotliDecompressorLibraryFactory lib_factory; + NiceMock context; + Envoy::Compression::Decompressor::DecompressorFactoryPtr factory = + lib_factory.createDecompressorFactoryFromProto(brotli, context); + EXPECT_EQ("brotli.", factory->statsPrefix()); + EXPECT_EQ("br", factory->contentEncoding()); + + Envoy::Compression::Decompressor::DecompressorPtr decompressor = + factory->createDecompressor("test."); + decompressor->decompress(accumulation_buffer, buffer); + std::string decompressed_text{buffer.toString()}; + ASSERT_EQ(original_text.length(), decompressed_text.length()); + EXPECT_EQ(original_text, decompressed_text); +} + +// Exercises decompression with a very small output buffer. +TEST_F(BrotliDecompressorImplTest, DecompressWithSmallOutputBuffer) { + Buffer::OwnedImpl buffer; + Buffer::OwnedImpl accumulation_buffer; + + Brotli::Compressor::BrotliCompressorImpl compressor{ + default_quality, + default_window_bits, + default_input_block_bits, + false, + Brotli::Compressor::BrotliCompressorImpl::EncoderMode::Default, + 4096}; + + std::string original_text{}; + for (uint64_t i = 0; i < 20; ++i) { + TestUtility::feedBufferWithRandomCharacters(buffer, default_input_size * i, i); + original_text.append(buffer.toString()); + compressor.compress(buffer, Envoy::Compression::Compressor::State::Flush); + accumulation_buffer.add(buffer); + drainBuffer(buffer); + } + + ASSERT_EQ(0, buffer.length()); + + compressor.compress(buffer, Envoy::Compression::Compressor::State::Finish); + ASSERT_GE(10, buffer.length()); + + accumulation_buffer.add(buffer); + + drainBuffer(buffer); + ASSERT_EQ(0, buffer.length()); + + Stats::IsolatedStoreImpl stats_store{}; + BrotliDecompressorImpl decompressor{stats_store, "test.", 16, false}; + + decompressor.decompress(accumulation_buffer, buffer); + std::string decompressed_text{buffer.toString()}; + + ASSERT_EQ(original_text.length(), decompressed_text.length()); + EXPECT_EQ(original_text, decompressed_text); + EXPECT_EQ(0, stats_store.counterFromString("test.brotli_error").value()); +} + +TEST_F(BrotliDecompressorImplTest, WrongInput) { + Buffer::OwnedImpl buffer; + Buffer::OwnedImpl output_buffer; + const char zeros[20]{}; + + Buffer::BufferFragmentImpl* frag = new Buffer::BufferFragmentImpl( + zeros, 20, [](const void*, size_t, const Buffer::BufferFragmentImpl* frag) { delete frag; }); + buffer.addBufferFragment(*frag); + Stats::IsolatedStoreImpl stats_store{}; + BrotliDecompressorImpl decompressor{stats_store, "test.", 16, false}; + decompressor.decompress(buffer, output_buffer); + EXPECT_EQ(1, stats_store.counterFromString("test.brotli_error").value()); +} + +TEST_F(BrotliDecompressorImplTest, CompressDecompressOfMultipleSlices) { + Buffer::OwnedImpl buffer; + Buffer::OwnedImpl accumulation_buffer; + + const std::string sample{"slice, slice, slice, slice, slice, "}; + std::string original_text; + for (uint64_t i = 0; i < 20; ++i) { + Buffer::BufferFragmentImpl* frag = new Buffer::BufferFragmentImpl( + sample.c_str(), sample.size(), + [](const void*, size_t, const Buffer::BufferFragmentImpl* frag) { delete frag; }); + + buffer.addBufferFragment(*frag); + original_text.append(sample); + } + + const uint64_t num_slices = buffer.getRawSlices().size(); + EXPECT_EQ(num_slices, 20); + + Brotli::Compressor::BrotliCompressorImpl compressor{ + default_quality, + default_window_bits, + default_input_block_bits, + false, + Brotli::Compressor::BrotliCompressorImpl::EncoderMode::Default, + 4096}; + + compressor.compress(buffer, Envoy::Compression::Compressor::State::Flush); + accumulation_buffer.add(buffer); + + Stats::IsolatedStoreImpl stats_store{}; + BrotliDecompressorImpl decompressor{stats_store, "test.", 16, false}; + + drainBuffer(buffer); + ASSERT_EQ(0, buffer.length()); + + decompressor.decompress(accumulation_buffer, buffer); + std::string decompressed_text{buffer.toString()}; + + ASSERT_EQ(original_text.length(), decompressed_text.length()); + EXPECT_EQ(original_text, decompressed_text); + EXPECT_EQ(0, stats_store.counterFromString("test.brotli_error").value()); +} + +class UncommonParamsTest : public BrotliDecompressorImplTest, + public testing::WithParamInterface> { +protected: + void testcompressDecompressWithUncommonParams( + const uint32_t quality, const uint32_t window_bits, const uint32_t input_block_bits, + const bool disable_literal_context_modeling, + const Compression::Brotli::Compressor::BrotliCompressorImpl::EncoderMode encoder_mode, + const bool disable_ring_buffer_reallocation) { + Buffer::OwnedImpl buffer; + Buffer::OwnedImpl accumulation_buffer; + + Compression::Brotli::Compressor::BrotliCompressorImpl compressor{ + quality, window_bits, input_block_bits, disable_literal_context_modeling, + encoder_mode, 4096}; + + std::string original_text{}; + for (uint64_t i = 0; i < 30; ++i) { + TestUtility::feedBufferWithRandomCharacters(buffer, default_input_size * i, i); + original_text.append(buffer.toString()); + compressor.compress(buffer, Envoy::Compression::Compressor::State::Flush); + accumulation_buffer.add(buffer); + drainBuffer(buffer); + } + ASSERT_EQ(0, buffer.length()); + + compressor.compress(buffer, Envoy::Compression::Compressor::State::Finish); + accumulation_buffer.add(buffer); + + drainBuffer(buffer); + ASSERT_EQ(0, buffer.length()); + + Stats::IsolatedStoreImpl stats_store{}; + BrotliDecompressorImpl decompressor{stats_store, "test.", 4096, + disable_ring_buffer_reallocation}; + + decompressor.decompress(accumulation_buffer, buffer); + std::string decompressed_text{buffer.toString()}; + + ASSERT_EQ(original_text.length(), decompressed_text.length()); + EXPECT_EQ(original_text, decompressed_text); + EXPECT_EQ(0, stats_store.counterFromString("test.brotli_error").value()); + } +}; + +INSTANTIATE_TEST_SUITE_P(UncommonParamsTestSuite, UncommonParamsTest, + testing::Values(std::make_tuple(false, false), + std::make_tuple(false, true), std::make_tuple(true, false), + std::make_tuple(true, true))); + +// Exercises decompression with other supported brotli initialization params. +TEST_P(UncommonParamsTest, Validate) { + const bool disable_literal_context_modeling = std::get<0>(GetParam()); + const bool disable_ring_buffer_reallocation = std::get<1>(GetParam()); + + // Test with different memory levels. + for (uint32_t i = 1; i < 8; ++i) { + testcompressDecompressWithUncommonParams( + i - 1, // quality + default_window_bits, // window_bits + default_input_block_bits, // input_block_bits + disable_literal_context_modeling, + Brotli::Compressor::BrotliCompressorImpl::EncoderMode::Font, + disable_ring_buffer_reallocation); + + testcompressDecompressWithUncommonParams( + default_quality, // quality + default_window_bits, // window_bits + i + 15, // input_block_bits + disable_literal_context_modeling, + Brotli::Compressor::BrotliCompressorImpl::EncoderMode::Text, + disable_ring_buffer_reallocation); + + testcompressDecompressWithUncommonParams( + default_quality, // quality + i + 10, // window_bits + default_input_block_bits, // input_block_bits + disable_literal_context_modeling, + Brotli::Compressor::BrotliCompressorImpl::EncoderMode::Generic, + disable_ring_buffer_reallocation); + } +} + +} // namespace +} // namespace Decompressor +} // namespace Brotli +} // namespace Compression +} // namespace Extensions +} // namespace Envoy diff --git a/tools/spelling/spelling_dictionary.txt b/tools/spelling/spelling_dictionary.txt index 48f4c1d953de0..7bdd63cce4f75 100644 --- a/tools/spelling/spelling_dictionary.txt +++ b/tools/spelling/spelling_dictionary.txt @@ -445,6 +445,7 @@ bools boringssl borks broadcasted +brotli buf buflen bugprone