From 275264c7e0dd46629d78511158a34e980b932cfb Mon Sep 17 00:00:00 2001 From: Naman Jain Date: Tue, 8 Jun 2021 11:38:48 +0530 Subject: [PATCH] feat(zstd): replace datadog's zstd with Klauspost's zstd (#1709) Remove Datadog's ZSTD that requires CGO Make Klauspost's ZSTD as default --- README.md | 2 -- go.mod | 1 - go.sum | 2 -- options.go | 4 ++-- y/{zstd_nocgo.go => zstd.go} | 2 -- y/zstd_cgo.go | 38 ------------------------------------ 6 files changed, 2 insertions(+), 47 deletions(-) rename y/{zstd_nocgo.go => zstd.go} (99%) delete mode 100644 y/zstd_cgo.go diff --git a/README.md b/README.md index efa8c4399..c19dae4c3 100644 --- a/README.md +++ b/README.md @@ -58,8 +58,6 @@ $ go get github.com/dgraph-io/badger/v3 ``` This will retrieve the library. -##### Note: Badger does not directly use CGO but it relies on https://github.com/DataDog/zstd for compression and it requires gcc/cgo. If you wish to use badger without gcc/cgo, you can run `CGO_ENABLED=0 go get github.com/dgraph-io/badger/v3` which will download badger with https://github.com/klauspost/compress ZSTD compression which does not require CGO. - #### Installing Badger Command Line Tool Download and extract the latest Badger DB release from https://github.com/dgraph-io/badger/releases and then run the following commands. diff --git a/go.mod b/go.mod index 36b0f0327..b80c145c4 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.12 // replace github.com/dgraph-io/ristretto => /home/amanbansal/go/src/github.com/dgraph-io/ristretto require ( - github.com/DataDog/zstd v1.4.6-0.20210216161059-8cb8bacba7ba github.com/cespare/xxhash v1.1.0 github.com/dgraph-io/ristretto v0.0.4-0.20210504190834-0bf2acd73aa3 github.com/dustin/go-humanize v1.0.0 diff --git a/go.sum b/go.sum index 45d83b606..6ecb7fe8c 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,5 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/DataDog/zstd v1.4.6-0.20210216161059-8cb8bacba7ba h1:3qB2yylqW3kVPr9QoPZtTJOXsJOUdNWT2CrZcifhs5g= -github.com/DataDog/zstd v1.4.6-0.20210216161059-8cb8bacba7ba/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= diff --git a/options.go b/options.go index 9bf977e22..79af6fc2d 100644 --- a/options.go +++ b/options.go @@ -163,7 +163,7 @@ func DefaultOptions(path string) Options { // compression is ratio supposed to increase with increasing compression level but since the // input for compression algorithm is small (4 KB), we don't get significant benefit at // level 3. - // NOTE: The benchmarks are with DataDog ZSTD that requires CGO. + // NOTE: The benchmarks are with DataDog ZSTD that requires CGO. Hence, no longer valid. // no_compression-16 10 502848865 ns/op 165.46 MB/s - // zstd_compression/level_1-16 7 739037966 ns/op 112.58 MB/s 2.93 // zstd_compression/level_3-16 7 756950250 ns/op 109.91 MB/s 2.72 @@ -730,7 +730,7 @@ func (opt Options) WithInMemory(b bool) Options { // algorithm is small (4 KB), we don't get significant benefit at level 3. It is advised to write // your own benchmarks before choosing a compression algorithm or level. // -// NOTE: The benchmarks are with DataDog ZSTD that requires CGO. +// NOTE: The benchmarks are with DataDog ZSTD that requires CGO. Hence, no longer valid. // no_compression-16 10 502848865 ns/op 165.46 MB/s - // zstd_compression/level_1-16 7 739037966 ns/op 112.58 MB/s 2.93 // zstd_compression/level_3-16 7 756950250 ns/op 109.91 MB/s 2.72 diff --git a/y/zstd_nocgo.go b/y/zstd.go similarity index 99% rename from y/zstd_nocgo.go rename to y/zstd.go index 929b94aab..57018680a 100644 --- a/y/zstd_nocgo.go +++ b/y/zstd.go @@ -1,5 +1,3 @@ -// +build !cgo - /* * Copyright 2019 Dgraph Labs, Inc. and Contributors * diff --git a/y/zstd_cgo.go b/y/zstd_cgo.go deleted file mode 100644 index ddb4850b0..000000000 --- a/y/zstd_cgo.go +++ /dev/null @@ -1,38 +0,0 @@ -// +build cgo - -/* - * Copyright 2019 Dgraph Labs, Inc. and Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package y - -import ( - "github.com/DataDog/zstd" -) - -// ZSTDDecompress decompresses a block using ZSTD algorithm. -func ZSTDDecompress(dst, src []byte) ([]byte, error) { - return zstd.Decompress(dst, src) -} - -// ZSTDCompress compresses a block using ZSTD algorithm. -func ZSTDCompress(dst, src []byte, compressionLevel int) ([]byte, error) { - return zstd.CompressLevel(dst, src, compressionLevel) -} - -// ZSTDCompressBound returns the worst case size needed for a destination buffer. -func ZSTDCompressBound(srcSize int) int { - return zstd.CompressBound(srcSize) -}