compression: add zstd compressor and decompressor

CODEOWNERS

@@ -201,6 +201,7 @@ extensions/filters/http/oauth2 @rgs1 @derekargueta @snowp
 /*/extensions/compression/gzip/decompressor @rojkov @kbaichoo
 /*/extensions/compression/gzip/compressor @rojkov @kbaichoo
 /*/extensions/compression/gzip/common @rojkov @kbaichoo
+/*/extensions/compression/zstd @rojkov @rainingmaster
 # cel
 /*/extensions/access_loggers/filters/cel @kyessenov @douglas-reid @adisuissa
 # health cehck

api/BUILD

@@ -133,6 +133,8 @@ proto_library(
         "//envoy/extensions/compression/brotli/decompressor/v3:pkg",
         "//envoy/extensions/compression/gzip/compressor/v3:pkg",
         "//envoy/extensions/compression/gzip/decompressor/v3:pkg",
+        "//envoy/extensions/compression/zstd/compressor/v3:pkg",
+        "//envoy/extensions/compression/zstd/decompressor/v3:pkg",
         "//envoy/extensions/config/validators/minimum_clusters/v3:pkg",
         "//envoy/extensions/filters/common/dependency/v3:pkg",
         "//envoy/extensions/filters/common/fault/v3:pkg",

api/envoy/extensions/compression/zstd/compressor/v3/BUILD

@@ -0,0 +1,12 @@
+# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py.
+
+load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package")
+
+licenses(["notice"])  # Apache 2
+
+api_proto_package(
+    deps = [
+        "//envoy/config/core/v3:pkg",
+        "@com_github_cncf_udpa//udpa/annotations:pkg",
+    ],
+)

api/envoy/extensions/compression/zstd/compressor/v3/zstd.proto

@@ -0,0 +1,64 @@
+syntax = "proto3";
+
+package envoy.extensions.compression.zstd.compressor.v3;
+
+import "envoy/config/core/v3/base.proto";
+
+import "google/protobuf/wrappers.proto";
+
+import "udpa/annotations/status.proto";
+import "validate/validate.proto";
+
+option java_package = "io.envoyproxy.envoy.extensions.compression.zstd.compressor.v3";
+option java_outer_classname = "ZstdProto";
+option java_multiple_files = true;
+option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/compression/zstd/compressor/v3;compressorv3";
+option (udpa.annotations.file_status).package_version_status = ACTIVE;
+
+// [#protodoc-title: Zstd Compressor]
+// [#extension: envoy.compression.zstd.compressor]
+
+// [#next-free-field: 6]
+message Zstd {
+  // Reference to http://facebook.github.io/zstd/zstd_manual.html
+  enum Strategy {
+    DEFAULT = 0;
+    FAST = 1;
+    DFAST = 2;
+    GREEDY = 3;
+    LAZY = 4;
+    LAZY2 = 5;
+    BTLAZY2 = 6;
+    BTOPT = 7;
+    BTULTRA = 8;
+    BTULTRA2 = 9;
+  }
+
+  // Set compression parameters according to pre-defined compression level table.
+  // Note that exact compression parameters are dynamically determined,
+  // depending on both compression level and source content size (when known).
+  // Value 0 means default, and default level is 3.
+  // Setting a level does not automatically set all other compression parameters
+  // to default. Setting this will however eventually dynamically impact the compression
+  // parameters which have not been manually set. The manually set
+  // ones will 'stick'.
+  google.protobuf.UInt32Value compression_level = 1;
+
+  // A 32-bits checksum of content is written at end of frame. If not set, defaults to false.
+  bool enable_checksum = 2;
+
+  // The higher the value of selected strategy, the more complex it is,
+  // resulting in stronger and slower compression.
+  // Special: value 0 means "use default strategy".
+  Strategy strategy = 3 [(validate.rules).enum = {defined_only: true}];
+
+  // A dictionary for compression. Zstd offers dictionary compression, which greatly improves
+  // efficiency on small files and messages. Each dictionary will be generated with a dictionary ID
+  // that can be used to search the same dictionary during decompression.
+  // Please refer to `zstd manual <https://github.com/facebook/zstd/blob/dev/programs/zstd.1.md#dictionary-builder>`_
+  // to train a specific dictionary for compression.
+  config.core.v3.DataSource dictionary = 4;
+
+  // Value for compressor's next output buffer. If not set, defaults to 4096.
+  google.protobuf.UInt32Value chunk_size = 5 [(validate.rules).uint32 = {lte: 65536 gte: 4096}];
+}

api/envoy/extensions/compression/zstd/decompressor/v3/BUILD

@@ -0,0 +1,12 @@
+# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py.
+
+load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package")
+
+licenses(["notice"])  # Apache 2
+
+api_proto_package(
+    deps = [
+        "//envoy/config/core/v3:pkg",
+        "@com_github_cncf_udpa//udpa/annotations:pkg",
+    ],
+)

api/envoy/extensions/compression/zstd/decompressor/v3/zstd.proto

@@ -0,0 +1,33 @@
+syntax = "proto3";
+
+package envoy.extensions.compression.zstd.decompressor.v3;
+
+import "envoy/config/core/v3/base.proto";
+
+import "google/protobuf/wrappers.proto";
+
+import "udpa/annotations/status.proto";
+import "validate/validate.proto";
+
+option java_package = "io.envoyproxy.envoy.extensions.compression.zstd.decompressor.v3";
+option java_outer_classname = "ZstdProto";
+option java_multiple_files = true;
+option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/compression/zstd/decompressor/v3;decompressorv3";
+option (udpa.annotations.file_status).package_version_status = ACTIVE;
+
+// [#protodoc-title: Zstd Decompressor]
+// [#extension: envoy.compression.zstd.decompressor]
+
+message Zstd {
+  // Dictionaries for decompression. Zstd offers dictionary compression, which greatly improves
+  // efficiency on small files and messages. It is necessary to ensure that the dictionary used for
+  // decompression is the same as the compression dictionary. Multiple dictionaries can be set, and the
+  // dictionary will be automatically selected for decompression according to the dictionary ID in the
+  // source content.
+  // Please refer to `zstd manual <https://github.com/facebook/zstd/blob/dev/programs/zstd.1.md#dictionary-builder>`_
+  // to train specific dictionaries for decompression.
+  repeated config.core.v3.DataSource dictionaries = 1;
+
+  // Value for decompressor's next output buffer. If not set, defaults to 4096.
+  google.protobuf.UInt32Value chunk_size = 2 [(validate.rules).uint32 = {lte: 65536 gte: 4096}];
+}

api/versioning/BUILD

@@ -70,6 +70,8 @@ proto_library(
         "//envoy/extensions/compression/brotli/decompressor/v3:pkg",
         "//envoy/extensions/compression/gzip/compressor/v3:pkg",
         "//envoy/extensions/compression/gzip/decompressor/v3:pkg",
+        "//envoy/extensions/compression/zstd/compressor/v3:pkg",
+        "//envoy/extensions/compression/zstd/decompressor/v3:pkg",
         "//envoy/extensions/config/validators/minimum_clusters/v3:pkg",
         "//envoy/extensions/filters/common/dependency/v3:pkg",
         "//envoy/extensions/filters/common/fault/v3:pkg",

bazel/foreign_cc/BUILD

@@ -480,3 +480,19 @@ envoy_cmake(
         "//conditions:default": ["libz.a"],
     }),
 )
+
+envoy_cmake(
+    name = "zstd",
+    build_data = ["@com_github_facebook_zstd//:all"],
+    cache_entries = {
+        "CMAKE_BUILD_TYPE": "Release",
+        "ZSTD_BUILD_SHARED": "off",
+        "ZSTD_BUILD_STATIC": "on",
+    },
+    lib_source = "@com_github_facebook_zstd//:all",
+    out_static_libs = select({
+        "//bazel:windows_x86_64": ["zstd_static.lib"],
+        "//conditions:default": ["libzstd.a"],
+    }),
+    working_directory = "build/cmake",
+)

bazel/repositories.bzl

@@ -202,6 +202,7 @@ def envoy_dependencies(skip_targets = []):
     _com_github_zlib_ng_zlib_ng()
     _org_boost()
     _org_brotli()
+    _com_github_facebook_zstd()
     _re2()
     _upb()
     _proxy_wasm_cpp_sdk()
@@ -455,6 +456,17 @@ def _org_brotli():
         actual = "@org_brotli//:brotlidec",
     )
 
+def _com_github_facebook_zstd():
+    external_http_archive(
+        name = "com_github_facebook_zstd",
+        build_file_content = BUILD_ALL_CONTENT,
+    )
+
+    native.bind(
+        name = "zstd",
+        actual = "@envoy//bazel/foreign_cc:zstd",
+    )
+
 def _com_google_cel_cpp():
     external_http_archive("com_google_cel_cpp")
     external_http_archive("rules_antlr")

bazel/repository_locations.bzl

@@ -500,6 +500,22 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         release_date = "2020-09-08",
         cpe = "cpe:2.3:a:google:brotli:*",
     ),
+    com_github_facebook_zstd = dict(
+        project_name = "zstd",
+        project_desc = "zstd compression library",
+        project_url = "https://facebook.github.io/zstd",
+        version = "1.5.2",
+        sha256 = "f7de13462f7a82c29ab865820149e778cbfe01087b3a55b5332707abf9db4a6e",
+        strip_prefix = "zstd-{version}",
+        urls = ["https://github.com/facebook/zstd/archive/v{version}.tar.gz"],
+        use_category = ["dataplane_ext"],
+        extensions = [
+            "envoy.compression.zstd.compressor",
+            "envoy.compression.zstd.decompressor",
+        ],
+        release_date = "2022-01-20",
+        cpe = "cpe:2.3:a:facebook:zstandard:*",
+    ),
     com_github_zlib_ng_zlib_ng = dict(
         project_name = "zlib-ng",
         project_desc = "zlib fork (higher performance)",

docs/root/api-v3/config/compression/compression.rst

@@ -7,3 +7,4 @@ Compression
 
   ../../extensions/compression/gzip/*/v3/*
   ../../extensions/compression/brotli/*/v3/*
+  ../../extensions/compression/zstd/*/v3/*

docs/root/configuration/http/http_filters/compressor_filter.rst

@@ -19,8 +19,9 @@ determine whether or not the content should be compressed. The content is
 compressed and then sent to the client with the appropriate headers, if
 response and request allow.
 
-Currently the filter supports :ref:`gzip <envoy_v3_api_msg_extensions.compression.gzip.compressor.v3.Gzip>`
-and :ref:`brotli <envoy_v3_api_msg_extensions.compression.brotli.compressor.v3.Brotli>`
+Currently the filter supports :ref:`gzip <envoy_v3_api_msg_extensions.compression.gzip.compressor.v3.Gzip>`,
+:ref:`brotli <envoy_v3_api_msg_extensions.compression.brotli.compressor.v3.Brotli>`
+and :ref:`zstd <envoy_v3_api_msg_extensions.compression.zstd.compressor.v3.Zstd>`
 compression only. Other compression libraries can be supported as extensions.
 
 An example configuration of the filter may look like the following:

docs/root/configuration/http/http_filters/decompressor_filter.rst

@@ -16,8 +16,9 @@ determine whether or not the content should be decompressed. The content is
 decompressed and passed on to the rest of the filter chain. Note that decompression happens
 independently for request and responses based on the rules described below.
 
-Currently the filter supports :ref:`gzip <envoy_v3_api_msg_extensions.compression.gzip.decompressor.v3.Gzip>`
-and :ref:`brotli <envoy_v3_api_msg_extensions.compression.brotli.decompressor.v3.Brotli>`
+Currently the filter supports :ref:`gzip <envoy_v3_api_msg_extensions.compression.gzip.decompressor.v3.Gzip>`,
+:ref:`brotli <envoy_v3_api_msg_extensions.compression.brotli.decompressor.v3.Brotli>`
+and :ref:`zstd <envoy_v3_api_msg_extensions.compression.zstd.decompressor.v3.Zstd>`
 compression only. Other compression libraries can be supported as extensions.
 
 An example configuration of the filter may look like the following:

docs/root/intro/arch_overview/other_features/compression/libraries.rst

@@ -6,8 +6,8 @@ Compression Libraries
 Underlying implementation
 -------------------------
 
-Currently Envoy uses `zlib <http://zlib.net>`_ and `brotli <https://brotli.org>`_ as compression
-libraries.
+Currently Envoy uses `zlib <http://zlib.net>`_, `brotli <https://brotli.org>`_ and
+`zstd <https://facebook.github.io/zstd>`_ as compression libraries.
 
 .. note::
 

docs/root/start/sandboxes/index.rst

@@ -75,3 +75,4 @@ The following sandboxes are available:
     websocket
     win32_front_proxy
     zipkin_tracing
+    zstd

docs/root/start/sandboxes/zstd.rst

@@ -0,0 +1,95 @@
+.. _install_sandboxes_zstd:
+
+Zstd
+======
+
+.. sidebar:: Requirements
+
+   .. include:: _include/docker-env-setup-link.rst
+
+   :ref:`curl <start_sandboxes_setup_curl>`
+        Used to make ``HTTP`` requests.
+
+By enabling compression in Envoy you can save some network bandwidth, at the expense of increased processor usage.
+
+Envoy supports compression and decompression for both requests and responses.
+
+This sandbox provides an example of response compression served over ``HTTPS``.
+
+The sandbox covers two scenarios:
+
+- compression of files from an upstream server
+- compression of Envoy's own statistics
+
+Step 1: Start all of our containers
+***********************************
+
+Change to the ``examples/zstd`` directory and bring up the docker composition.
+
+.. code-block:: console
+
+    $ pwd
+    envoy/examples/zstd
+    $ docker-compose build --pull
+    $ docker-compose up -d
+    $ docker-compose ps
+    Name                 Command                        State   Ports
+    --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+    zstd_envoy-stats_1   /docker-entrypoint.sh /usr ... Up      0.0.0.0:10000->10000/tcp,:::10000->10000/tcp, 0.0.0.0:9901->9901/tcp,:::9901->9901/tcp, 0.0.0.0:9902->9902/tcp,:::9902->9902/tcp
+    zstd_service_1       python3 /code/service.py       Up
+
+Step 2: Test Envoy’s compression of upstream files
+**************************************************
+
+The sandbox is configured with two endpoints on port ``10000`` for serving upstream files:
+
+- ``/file.txt``
+- ``/file.json``
+
+Only ``/file.json`` is configured to be compressed.
+
+Use ``curl`` to check that the response from requesting ``file.json`` contains the ``content-encoding: zstd`` header.
+
+You will need to add an ``accept-encoding: zstd`` request header.
+
+.. code-block:: console
+
+    $ curl -ski -H "Accept-Encoding: zstd" https://localhost:10000/file.json | grep "content-encoding"
+    content-encoding: zstd
+
+As only files with a content-type of ``application/json`` are configured to be compressed, the response from requesting ``file.txt`` should not contain the ``content-encoding: zstd`` header, and the file will not be compressed:
+
+.. code-block:: console
+
+    $ curl -ski -H "Accept-Encoding: zstd" https://localhost:10000/file.txt | grep "content-encoding"
+
+Step 3: Test compression of Envoy’s statistics
+**********************************************
+
+The sandbox is configured with two ports serving Envoy’s admin and statistics interface:
+
+- ``9901`` exposes the standard admin interface without tls
+- ``9902`` exposes a compressed version of the admin interface with tls
+
+Use ``curl`` to make a request for uncompressed statistics on port ``9901``, it should not contain the ``content-encoding`` header in the response:
+
+.. code-block:: console
+
+    $ curl -ski -H "Accept-Encoding: zstd" http://localhost:9901/stats/prometheus | grep "content-encoding"
+
+Now, use ``curl`` to make a request for the compressed statistics:
+
+.. code-block:: console
+
+    $ curl -ski -H "Accept-Encoding: zstd" https://localhost:9902/stats/prometheus | grep "content-encoding"
+    content-encoding: zstd
+
+.. seealso::
+   :ref:`Zstd API <envoy_v3_api_msg_extensions.compression.zstd.compressor.v3.Zstd>`
+      API and configuration reference for Envoy's zstd compression.
+
+   :ref:`Compression configuration <config_http_filters_compressor>`
+      Reference documentation for Envoy's compressor filter.
+
+   :ref:`Envoy admin quick start guide <start_quick_start_admin>`
+      Quick start guide to the Envoy admin interface.

docs/root/version_history/current.rst

@@ -106,6 +106,7 @@ New Features
 * cluster: added support for per host limits in :ref:`circuit breakers settings <envoy_v3_api_msg_config.cluster.v3.CircuitBreakers>`. Currently only  :ref:`max_connections <envoy_v3_api_field_config.cluster.v3.CircuitBreakers.Thresholds.max_connections>` is supported.
 * cluster: added support to restore original destination address from any desired header via setting :ref:`http_header_name <envoy_v3_api_field_config.cluster.v3.Cluster.OriginalDstLbConfig.http_header_name>`.
 * cluster: support :ref:`override host status restriction <envoy_v3_api_field_config.cluster.v3.Cluster.CommonLbConfig.override_host_status>`.
+* compression: add zstd :ref:`compressor <envoy_v3_api_msg_extensions.compression.zstd.compressor.v3.Zstd>` and :ref:`decompressor <envoy_v3_api_msg_extensions.compression.zstd.decompressor.v3.Zstd>`.
 * config: added new file based xDS configuration via :ref:`path_config_source <envoy_v3_api_field_config.core.v3.ConfigSource.path_config_source>`.
   :ref:`watched_directory <envoy_v3_api_field_config.core.v3.PathConfigSource.watched_directory>` can
   be used to setup an independent watch for when to reload the file path, for example when using

examples/zstd/Dockerfile-service

@@ -0,0 +1,13 @@
+FROM debian:buster-slim
+
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y python3 python3-pip python3-setuptools \
+    && apt-get autoremove -y && apt-get clean \
+    && rm -rf /tmp/* /var/tmp/* \
+    && rm -rf /var/lib/apt/lists/*
+RUN pip3 install -q Flask==2.0.3
+RUN mkdir -p /code/data
+RUN dd if=/dev/zero of="/code/data/file.txt" bs=1024 count=10240 \
+    && dd if=/dev/zero of="/code/data/file.json" bs=1024 count=10240
+ADD ./service.py /code
+ENTRYPOINT ["python3", "/code/service.py"]

examples/zstd/Dockerfile-zstd

@@ -0,0 +1,5 @@
+FROM envoyproxy/envoy-dev:latest
+
+COPY ./zstd-envoy.yaml /etc/zstd-envoy.yaml
+RUN chmod go+r /etc/zstd-envoy.yaml
+CMD ["/usr/local/bin/envoy", "-c", "/etc/zstd-envoy.yaml", "--service-cluster", "zstd"]

examples/zstd/README.md

@@ -0,0 +1,2 @@
+To learn about this sandbox and for instructions on how to run it please head over
+to the [envoy docs](https://www.envoyproxy.io/docs/envoy/latest/start/sandboxes/zstd.html)