compression: add brotli compressor and decompressor #12998
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| # DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. | ||
|
|
||
| load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") | ||
|
|
||
| licenses(["notice"]) # Apache 2 | ||
|
|
||
| api_proto_package( | ||
| deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], | ||
| ) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,54 @@ | ||
| syntax = "proto3"; | ||
|
|
||
| package envoy.extensions.compression.brotli.compressor.v3; | ||
|
|
||
| import "google/protobuf/wrappers.proto"; | ||
|
|
||
| import "udpa/annotations/status.proto"; | ||
| import "validate/validate.proto"; | ||
|
|
||
| option java_package = "io.envoyproxy.envoy.extensions.compression.brotli.compressor.v3"; | ||
| option java_outer_classname = "BrotliProto"; | ||
| option java_multiple_files = true; | ||
| option (udpa.annotations.file_status).package_version_status = ACTIVE; | ||
|
|
||
| // [#protodoc-title: Brotli Compressor] | ||
| // [#extension: envoy.compression.brotli.compressor] | ||
|
|
||
| // [#next-free-field: 7] | ||
| message Brotli { | ||
| enum EncoderMode { | ||
| DEFAULT = 0; | ||
| GENERIC = 1; | ||
| TEXT = 2; | ||
| FONT = 3; | ||
| } | ||
|
|
||
| // Value from 0 to 11 that controls the main compression speed-density lever. | ||
| // The higher quality, the slower compression. The default value is 3. | ||
| google.protobuf.UInt32Value quality = 1 [(validate.rules).uint32 = {lte: 11}]; | ||
|
|
||
| // A value used to tune encoder for specific input. For more information about modes, | ||
| // please refer to brotli manual: https://brotli.org/encode.html#aa6f | ||
| // This field will be set to "DEFAULT" if not specified. | ||
| EncoderMode encoder_mode = 2 [(validate.rules).enum = {defined_only: true}]; | ||
|
|
||
| // Value from 10 to 24 that represents the base two logarithmic of the compressor's window size. | ||
|
Contributor
There was a problem hiding this comment. These defaults may be a bit too compression-ratio optimized:
Member
Author
There was a problem hiding this comment. I agree. Saner defaults wouldn't harm definitely. What would be the best choices for your use cases?
Contributor
There was a problem hiding this comment. Looking at cloudflare they use
Member
Author
There was a problem hiding this comment. Thanks! Updated the defaults. |
||
| // Larger window results in better compression at the expense of memory usage. The default is 18. | ||
| // For more details about this parameter, please refer to brotli manual: | ||
| // https://brotli.org/encode.html#a9a8 | ||
| google.protobuf.UInt32Value window_bits = 3 [(validate.rules).uint32 = {lte: 24 gte: 10}]; | ||
|
|
||
| // Value from 16 to 24 that represents the base two logarithmic of the compressor's input block | ||
| // size. Larger input block results in better compression at the expense of memory usage. The | ||
| // default is 24. For more details about this parameter, please refer to brotli manual: | ||
| // https://brotli.org/encode.html#a9a8 | ||
| google.protobuf.UInt32Value input_block_bits = 4 [(validate.rules).uint32 = {lte: 24 gte: 16}]; | ||
|
|
||
| // Value for compressor's next output buffer. If not set, defaults to 4096. | ||
| google.protobuf.UInt32Value chunk_size = 5 [(validate.rules).uint32 = {lte: 65536 gte: 4096}]; | ||
|
|
||
| // If true, disables "literal context modeling" format feature. | ||
| // This flag is a "decoding-speed vs compression ratio" trade-off. | ||
| bool disable_literal_context_modeling = 6; | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| # DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. | ||
|
|
||
| load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") | ||
|
|
||
| licenses(["notice"]) # Apache 2 | ||
|
|
||
| api_proto_package( | ||
| deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], | ||
| ) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| syntax = "proto3"; | ||
|
|
||
| package envoy.extensions.compression.brotli.decompressor.v3; | ||
|
|
||
| import "google/protobuf/wrappers.proto"; | ||
|
|
||
| import "udpa/annotations/status.proto"; | ||
| import "validate/validate.proto"; | ||
|
|
||
| option java_package = "io.envoyproxy.envoy.extensions.compression.brotli.decompressor.v3"; | ||
| option java_outer_classname = "BrotliProto"; | ||
| option java_multiple_files = true; | ||
| option (udpa.annotations.file_status).package_version_status = ACTIVE; | ||
|
|
||
| // [#protodoc-title: Brotli Decompressor] | ||
| // [#extension: envoy.compression.brotli.decompressor] | ||
|
|
||
| message Brotli { | ||
| // If true, disables "canny" ring buffer allocation strategy. | ||
| // Ring buffer is allocated according to window size, despite the real size of the content. | ||
| bool disable_ring_buffer_reallocation = 1; | ||
|
|
||
| // Value for decompressor's next output buffer. If not set, defaults to 4096. | ||
| google.protobuf.UInt32Value chunk_size = 2 [(validate.rules).uint32 = {lte: 65536 gte: 4096}]; | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| load( | ||
| "//bazel:envoy_build_system.bzl", | ||
| "envoy_cc_library", | ||
| "envoy_extension_package", | ||
| ) | ||
|
|
||
| licenses(["notice"]) # Apache 2 | ||
|
|
||
| envoy_extension_package() | ||
|
|
||
| envoy_cc_library( | ||
| name = "brotli_base_lib", | ||
| srcs = ["base.cc"], | ||
| hdrs = ["base.h"], | ||
| deps = [ | ||
| "//source/common/buffer:buffer_lib", | ||
| ], | ||
| ) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| #include "extensions/compression/brotli/common/base.h" | ||
|
|
||
| namespace Envoy { | ||
| namespace Extensions { | ||
| namespace Compression { | ||
| namespace Brotli { | ||
| namespace Common { | ||
|
|
||
| BrotliContext::BrotliContext(const uint32_t chunk_size) | ||
| : chunk_size_{chunk_size}, chunk_ptr_{std::make_unique<uint8_t[]>(chunk_size)}, next_in_{}, | ||
| next_out_{chunk_ptr_.get()}, avail_in_{0}, avail_out_{chunk_size} {} | ||
|
|
||
| void BrotliContext::updateOutput(Buffer::Instance& output_buffer) { | ||
| if (avail_out_ == 0) { | ||
| output_buffer.add(static_cast<void*>(chunk_ptr_.get()), chunk_size_); | ||
| resetOut(); | ||
| } | ||
| } | ||
|
|
||
| void BrotliContext::finalizeOutput(Buffer::Instance& output_buffer) { | ||
| const size_t n_output = chunk_size_ - avail_out_; | ||
| if (n_output > 0) { | ||
| output_buffer.add(static_cast<void*>(chunk_ptr_.get()), n_output); | ||
| } | ||
| } | ||
|
|
||
| void BrotliContext::resetOut() { | ||
| avail_out_ = chunk_size_; | ||
| next_out_ = chunk_ptr_.get(); | ||
| } | ||
|
|
||
| } // namespace Common | ||
| } // namespace Brotli | ||
| } // namespace Compression | ||
| } // namespace Extensions | ||
| } // namespace Envoy |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Where do these come from? How will users decide which to use?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I see, you comment below on this.