zlib: add zstd support

Fixes: #48412
PR-URL: #52100
Reviewed-By: Yagiz Nizipli <[email protected]>

Author: hybrist

benchmark/zlib/creation.js

@@ -5,7 +5,7 @@ const zlib = require('zlib');
 const bench = common.createBenchmark(main, {
   type: [
     'Deflate', 'DeflateRaw', 'Inflate', 'InflateRaw', 'Gzip', 'Gunzip', 'Unzip',
-    'BrotliCompress', 'BrotliDecompress',
+    'BrotliCompress', 'BrotliDecompress', 'ZstdCompress', 'ZstdDecompress',
   ],
   options: ['true', 'false'],
   n: [5e5],

benchmark/zlib/pipe.js

@@ -7,22 +7,27 @@ const bench = common.createBenchmark(main, {
   inputLen: [1024],
   duration: [5],
   type: ['string', 'buffer'],
-  algorithm: ['gzip', 'brotli'],
+  algorithm: ['gzip', 'brotli', 'zstd'],
 }, {
   test: {
     inputLen: 1024,
     duration: 0.2,
   },
 });
 
+const algorithms = {
+  'gzip': [zlib.createGzip, zlib.createGunzip],
+  'brotli': [zlib.createBrotliCompress, zlib.createBrotliDecompress],
+  'zstd': [zlib.createZstdCompress, zlib.createZstdDecompress],
+};
+
 function main({ inputLen, duration, type, algorithm }) {
   const buffer = Buffer.alloc(inputLen, fs.readFileSync(__filename));
   const chunk = type === 'buffer' ? buffer : buffer.toString('utf8');
 
-  const input = algorithm === 'gzip' ?
-    zlib.createGzip() : zlib.createBrotliCompress();
-  const output = algorithm === 'gzip' ?
-    zlib.createGunzip() : zlib.createBrotliDecompress();
+  const [createCompress, createUncompress] = algorithms[algorithm];
+  const input = createCompress();
+  const output = createUncompress();
 
   let readFromOutput = 0;
   input.pipe(output);

doc/api/errors.md

@@ -3314,6 +3314,12 @@ The requested functionality is not supported in worker threads.
 
 Creation of a [`zlib`][] object failed due to incorrect configuration.
 
+<a id="ERR_ZSTD_INVALID_PARAM"></a>
+
+### `ERR_ZSTD_INVALID_PARAM`
+
+An invalid parameter key was passed during construction of a Zstd stream.
+
 <a id="HPE_CHUNK_EXTENSIONS_OVERFLOW"></a>
 
 ### `HPE_CHUNK_EXTENSIONS_OVERFLOW`

doc/api/zlib.md

@@ -7,7 +7,7 @@
 <!-- source_link=lib/zlib.js -->
 
 The `node:zlib` module provides compression functionality implemented using
-Gzip, Deflate/Inflate, and Brotli.
+Gzip, Deflate/Inflate, Brotli, and Zstd.
 
 To access it:
 
@@ -220,8 +220,8 @@ operations be cached to avoid duplication of effort.
 
 ## Compressing HTTP requests and responses
 
-The `node:zlib` module can be used to implement support for the `gzip`, `deflate`
-and `br` content-encoding mechanisms defined by
+The `node:zlib` module can be used to implement support for the `gzip`, `deflate`,
+`br`, and `zstd` content-encoding mechanisms defined by
 [HTTP](https://tools.ietf.org/html/rfc7230#section-4.2).
 
 The HTTP [`Accept-Encoding`][] header is used within an HTTP request to identify
@@ -284,7 +284,7 @@ const { pipeline } = require('node:stream');
 const request = http.get({ host: 'example.com',
                            path: '/',
                            port: 80,
-                           headers: { 'Accept-Encoding': 'br,gzip,deflate' } });
+                           headers: { 'Accept-Encoding': 'br,gzip,deflate,zstd' } });
 request.on('response', (response) => {
   const output = fs.createWriteStream('example.com_index.html');
 
@@ -306,6 +306,9 @@ request.on('response', (response) => {
     case 'deflate':
       pipeline(response, zlib.createInflate(), output, onError);
       break;
+    case 'zstd':
+      pipeline(response, zlib.createZstdDecompress(), output, onError);
+      break;
     default:
       pipeline(response, output, onError);
       break;
@@ -396,6 +399,9 @@ http.createServer((request, response) => {
   } else if (/\bbr\b/.test(acceptEncoding)) {
     response.writeHead(200, { 'Content-Encoding': 'br' });
     pipeline(raw, zlib.createBrotliCompress(), response, onError);
+  } else if (/\bzstd\b/.test(acceptEncoding)) {
+    response.writeHead(200, { 'Content-Encoding': 'zstd' });
+    pipeline(raw, zlib.createZstdCompress(), response, onError);
   } else {
     response.writeHead(200, {});
     pipeline(raw, response, onError);
@@ -416,6 +422,7 @@ const buffer = Buffer.from('eJzT0yMA', 'base64');
 zlib.unzip(
   buffer,
   // For Brotli, the equivalent is zlib.constants.BROTLI_OPERATION_FLUSH.
+  // For Zstd, the equivalent is zlib.constants.ZSTD_e_flush.
   { finishFlush: zlib.constants.Z_SYNC_FLUSH },
   (err, buffer) => {
     if (err) {
@@ -487,6 +494,16 @@ these options have different ranges than the zlib ones:
 
 See [below][Brotli parameters] for more details on Brotli-specific options.
 
+### For Zstd-based streams
+
+There are equivalents to the zlib options for Zstd-based streams, although
+these options have different ranges than the zlib ones:
+
+* zlib's `level` option matches Zstd's `ZSTD_c_compressionLevel` option.
+* zlib's `windowBits` option matches Zstd's `ZSTD_c_windowLog` option.
+
+See [below][Zstd parameters] for more details on Zstd-specific options.
+
 ## Flushing
 
 Calling [`.flush()`][] on a compression stream will make `zlib` return as much
@@ -701,6 +718,50 @@ These advanced options are available for controlling decompression:
   * Boolean flag enabling “Large Window Brotli” mode (not compatible with the
     Brotli format as standardized in [RFC 7932][]).
 
+### Zstd constants
+
+<!-- YAML
+added: REPLACEME
+-->
+
+There are several options and other constants available for Zstd-based
+streams:
+
+#### Flush operations
+
+The following values are valid flush operations for Zstd-based streams:
+
+* `zlib.constants.ZSTD_e_continue` (default for all operations)
+* `zlib.constants.ZSTD_e_flush` (default when calling `.flush()`)
+* `zlib.constants.ZSTD_e_end` (default for the last chunk)
+
+#### Compressor options
+
+There are several options that can be set on Zstd encoders, affecting
+compression efficiency and speed. Both the keys and the values can be accessed
+as properties of the `zlib.constants` object.
+
+The most important options are:
+
+* `ZSTD_c_compressionLevel`
+  * Set compression parameters according to pre-defined cLevel table. Default
+    level is ZSTD\_CLEVEL\_DEFAULT==3.
+
+#### Pledged Source Size
+
+It's possible to specify the expected total size of the uncompressed input via
+`opts.pledgedSrcSize`. If the size doesn't match at the end of the input,
+compression will fail with the code `ZSTD_error_srcSize_wrong`.
+
+#### Decompressor options
+
+These advanced options are available for controlling decompression:
+
+* `ZSTD_d_windowLogMax`
+  * Select a size limit (in power of 2) beyond which the streaming API will
+    refuse to allocate memory buffer in order to protect the host from
+    unreasonable memory requirements.
+
 ## Class: `Options`
 
 <!-- YAML
@@ -962,6 +1023,51 @@ added: v0.7.0
 Reset the compressor/decompressor to factory defaults. Only applicable to
 the inflate and deflate algorithms.
 
+## Class: `ZstdOptions`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+<!--type=misc-->
+
+Each Zstd-based class takes an `options` object. All options are optional.
+
+* `flush` {integer} **Default:** `zlib.constants.ZSTD_e_continue`
+* `finishFlush` {integer} **Default:** `zlib.constants.ZSTD_e_end`
+* `chunkSize` {integer} **Default:** `16 * 1024`
+* `params` {Object} Key-value object containing indexed [Zstd parameters][].
+* `maxOutputLength` {integer} Limits output size when using
+  [convenience methods][]. **Default:** [`buffer.kMaxLength`][]
+
+For example:
+
+```js
+const stream = zlib.createZstdCompress({
+  chunkSize: 32 * 1024,
+  params: {
+    [zlib.constants.ZSTD_c_compressionLevel]: 10,
+    [zlib.constants.ZSTD_c_checksumFlag]: 1,
+  },
+});
+```
+
+## Class: `zlib.ZstdCompress`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+Compress data using the Zstd algorithm.
+
+## Class: `zlib.ZstdDecompress`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+Decompress data using the Zstd algorithm.
+
 ## `zlib.constants`
 
 <!-- YAML
@@ -1135,6 +1241,26 @@ added: v0.5.8
 
 Creates and returns a new [`Unzip`][] object.
 
+## `zlib.createZstdCompress([options])`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* `options` {zstd options}
+
+Creates and returns a new [`ZstdCompress`][] object.
+
+## `zlib.createZstdDecompress([options])`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* `options` {zstd options}
+
+Creates and returns a new [`ZstdDecompress`][] object.
+
 ## Convenience methods
 
 <!--type=misc-->
@@ -1481,11 +1607,54 @@ changes:
 
 Decompress a chunk of data with [`Unzip`][].
 
+### `zlib.zstdCompress(buffer[, options], callback)`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* `buffer` {Buffer|TypedArray|DataView|ArrayBuffer|string}
+* `options` {zstd options}
+* `callback` {Function}
+
+### `zlib.zstdCompressSync(buffer[, options])`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* `buffer` {Buffer|TypedArray|DataView|ArrayBuffer|string}
+* `options` {zstd options}
+
+Compress a chunk of data with [`ZstdCompress`][].
+
+### `zlib.zstdDecompress(buffer[, options], callback)`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* `buffer` {Buffer|TypedArray|DataView|ArrayBuffer|string}
+* `options` {zstd options}
+* `callback` {Function}
+
+### `zlib.zstdDecompressSync(buffer[, options])`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* `buffer` {Buffer|TypedArray|DataView|ArrayBuffer|string}
+* `options` {zstd options}
+
+Decompress a chunk of data with [`ZstdDecompress`][].
+
 [Brotli parameters]: #brotli-constants
 [Cyclic redundancy check]: https://en.wikipedia.org/wiki/Cyclic_redundancy_check
 [Memory usage tuning]: #memory-usage-tuning
 [RFC 7932]: https://www.rfc-editor.org/rfc/rfc7932.txt
 [Streams API]: stream.md
+[Zstd parameters]: #zstd-constants
 [`.flush()`]: #zlibflushkind-callback
 [`Accept-Encoding`]: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.3
 [`BrotliCompress`]: #class-zlibbrotlicompress
@@ -1498,6 +1667,8 @@ Decompress a chunk of data with [`Unzip`][].
 [`InflateRaw`]: #class-zlibinflateraw
 [`Inflate`]: #class-zlibinflate
 [`Unzip`]: #class-zlibunzip
+[`ZstdCompress`]: #class-zlibzstdcompress
+[`ZstdDecompress`]: #class-zlibzstddecompress
 [`buffer.kMaxLength`]: buffer.md#bufferkmaxlength
 [`deflateInit2` and `inflateInit2`]: https://zlib.net/manual.html#Advanced
 [`stream.Transform`]: stream.md#class-streamtransform

lib/internal/errors.js

@@ -1889,3 +1889,4 @@ E('ERR_WORKER_UNSERIALIZABLE_ERROR',
   'Serializing an uncaught exception failed', Error);
 E('ERR_WORKER_UNSUPPORTED_OPERATION',
   '%s is not supported in workers', TypeError);
+E('ERR_ZSTD_INVALID_PARAM', '%s is not a valid zstd parameter', RangeError);