meta: suport foundationdb as meta engine (#2581)

.github/workflows/unittests.yml

@@ -73,6 +73,8 @@ jobs:
           wget -O /home/travis/.m2/rclone-v1.57.0-linux-amd64.zip --no-check-certificate https://downloads.rclone.org/v1.57.0/rclone-v1.57.0-linux-amd64.zip
           wget -O /home/travis/.m2/litmus-0.13.tar.gz http://www.webdav.org/neon/litmus/litmus-0.13.tar.gz
           wget -O /home/travis/.m2/etcd-v3.5.2-linux-amd64.tar.gz  https://github.com/etcd-io/etcd/releases/download/v3.5.2/etcd-v3.5.2-linux-amd64.tar.gz
+          wget -O /home/travis/.m2/foundationdb-clients_6.3.23-1_amd64.deb https://github.com/apple/foundationdb/releases/download/6.3.23/foundationdb-clients_6.3.23-1_amd64.deb
+          wget -O /home/travis/.m2/foundationdb-server_6.3.23-1_amd64.deb https://github.com/apple/foundationdb/releases/download/6.3.23/foundationdb-server_6.3.23-1_amd64.deb
           unzip /home/travis/.m2/rclone-v1.57.0-linux-amd64.zip -d /home/travis/.m2/
           tar -zxvf /home/travis/.m2/litmus-0.13.tar.gz -C /home/travis/.m2/
           cd  /home/travis/.m2/litmus-0.13/ && ./configure && make && cd -
@@ -98,6 +100,7 @@ jobs:
           source /home/runner/.bash_profile
           tiup -v
           nohup tiup playground --mode tikv-slim >> output.log 2>&1 &
+          sudo dpkg -i /home/travis/.m2/foundationdb-clients_6.3.23-1_amd64.deb /home/travis/.m2/foundationdb-server_6.3.23-1_amd64.deb
           docker run -d -p 9000:9000 -p 9001:9001 -e "MINIO_ROOT_USER=testUser" -e "MINIO_ROOT_PASSWORD=testUserPassword" quay.io/minio/minio:RELEASE.2022-01-25T19-56-04Z server /data --console-address ":9001"
           go install github.com/minio/[email protected] && mc config host add local http://127.0.0.1:9000 testUser testUserPassword && mc mb local/testbucket
           nohup /home/travis/.m2/rclone-v1.57.0-linux-amd64/rclone serve webdav local --addr 127.0.0.1:9007 >> rclone.log 2>&1 &
@@ -109,7 +112,7 @@ jobs:
           chmod +x travis-setup-hdfs.sh
           sudo lsof -i :8020 || true
           ./travis-setup-hdfs.sh
-          for i in {2222,3306,5432,8020,9000,9007} ; do echo "lsof port:"$i;sudo lsof -i :$i;if [ $? != 0 ];then sleep 5; else continue; fi;sudo lsof -i :$i; if [ $? != 0 ];then echo "service not ready, port:"$i; exit 1;fi; done
+          for i in {2222,3306,5432,8020,9000,9007,4500} ; do echo "lsof port:"$i;sudo lsof -i :$i;if [ $? != 0 ];then sleep 5; else continue; fi;sudo lsof -i :$i; if [ $? != 0 ];then echo "service not ready, port:"$i; exit 1;fi; done
           sudo lsof -i :2379;if [ $? != 0 ];then echo "tikv is not ready";cat output.log;exit 1; fi
 
 
@@ -120,6 +123,8 @@ jobs:
           make test
           cat cov1.out >> coverage.txt
           cat cov2.out >> coverage.txt
+          make test.fdb
+          cat cov3.out >> coverage.txt
           sudo ./juicefs umount /jfs || sudo lsof /jfs && sudo ./juicefs umount --force /jfs || true
           sudo ./juicefs gc localhost
           sudo ./juicefs fsck localhost

.github/workflows/verify.yml

@@ -73,4 +73,10 @@ jobs:
       run: |
         sudo apt-get update
         sudo apt-get install librados-dev
-        GOPATH=$HOME/go make juicefs.ceph
+        GOPATH=$HOME/go make juicefs.ceph
+
+    - name: build fdb
+      run: |
+        wget https://github.com/apple/foundationdb/releases/download/6.3.23/foundationdb-clients_6.3.23-1_amd64.deb
+        sudo dpkg -i foundationdb-clients_6.3.23-1_amd64.deb
+        GOPATH=$HOME/go make juicefs.fdb

Makefile

@@ -30,6 +30,8 @@ juicefs.lite: Makefile cmd/*.go pkg/*/*.go
 juicefs.ceph: Makefile cmd/*.go pkg/*/*.go
 	go build -tags ceph -ldflags="$(LDFLAGS)"  -o juicefs.ceph .
 
+juicefs.fdb: Makefile cmd/*.go pkg/*/*.go
+	go build -tags fdb -ldflags="$(LDFLAGS)"  -o juicefs.fdb .
 
 # This is the script for compiling the Linux version on the MacOS platform.
 # Please execute the `brew install FiloSottile/musl-cross/musl-cross` command before using it.
@@ -71,3 +73,6 @@ release:
 test:
 	go test -v -cover -count=1 -timeout=8m ./pkg/... -coverprofile=cov1.out
 	sudo JFS_GC_SKIPPEDTIME=1 MINIO_ACCESS_KEY=testUser MINIO_SECRET_KEY=testUserPassword go test -v -count=1 -cover -timeout=8m ./cmd/... -coverprofile=cov2.out -coverpkg=./pkg/...,./cmd/...
+
+test.fdb:
+	go test -v -cover -count=1 -timeout=8m ./pkg/meta/ -tags fdb -run=TestFdb -coverprofile=cov3.out

go.mod

@@ -85,6 +85,7 @@ require (
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
 	github.com/alecthomas/participle v0.2.1 // indirect
+	github.com/apple/foundationdb/bindings/go v0.0.0-20211207225159-47b9a81d1c10
 	github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da // indirect
 	github.com/bcicen/jstream v1.0.1 // indirect
 	github.com/beevik/ntp v0.3.0 // indirect

go.sum

@@ -148,6 +148,8 @@ github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQY
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
+github.com/apple/foundationdb/bindings/go v0.0.0-20211207225159-47b9a81d1c10 h1:xU6bzJilZ630rLUhRsqWgJjSl2PCn5uLrehoG6ntwls=
+github.com/apple/foundationdb/bindings/go v0.0.0-20211207225159-47b9a81d1c10/go.mod h1:w63jdZTFCtvdjsUj5yrdKgjxaAD5uXQX6hJ7EaiLFRs=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I=

pkg/meta/tkv_fdb.go

@@ -0,0 +1,185 @@
+//go:build fdb
+// +build fdb
+
+/*
+ * JuiceFS, Copyright 2022 Juicedata, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package meta
+
+import (
+	"fmt"
+	"net/url"
+
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
+)
+
+func init() {
+	Register("fdb", newKVMeta)
+	drivers["fdb"] = newFdbClient
+}
+
+type fdbTxn struct {
+	fdb.Transaction
+}
+
+type fdbClient struct {
+	client fdb.Database
+}
+
+func newFdbClient(addr string) (tkvClient, error) {
+	err := fdb.APIVersion(630)
+	if err != nil {
+		return nil, fmt.Errorf("set API version: %s", err)
+	}
+	u, err := url.Parse("fdb://" + addr)
+	if err != nil {
+		return nil, err
+	}
+	db, err := fdb.OpenDatabase(u.Path)
+	if err != nil {
+		return nil, fmt.Errorf("open database: %s", err)
+	}
+	return withPrefix(&fdbClient{db}, append([]byte(u.Query().Get("prefix")), 0xFD)), nil
+}
+
+func (c *fdbClient) name() string {
+	return "fdb"
+}
+
+func (c *fdbClient) txn(f func(kvTxn) error) error {
+	_, err := c.client.Transact(func(t fdb.Transaction) (interface{}, error) {
+		e := f(&fdbTxn{t})
+		return nil, e
+	})
+	return err
+}
+
+func (c *fdbClient) scan(prefix []byte, handler func(key, value []byte)) error {
+	_, err := c.client.ReadTransact(func(t fdb.ReadTransaction) (interface{}, error) {
+		snapshot := t.Snapshot()
+		iter := snapshot.GetRange(
+			fdb.KeyRange{Begin: fdb.Key(prefix), End: fdb.Key(nextKey(prefix))},
+			fdb.RangeOptions{Mode: fdb.StreamingModeWantAll},
+		).Iterator()
+		for iter.Advance() {
+			r := iter.MustGet()
+			handler(r.Key, r.Value)
+		}
+		return nil, nil
+	})
+	return err
+}
+
+func (c *fdbClient) reset(prefix []byte) error {
+	_, err := c.client.Transact(func(t fdb.Transaction) (interface{}, error) {
+		t.ClearRange(fdb.KeyRange{
+			Begin: fdb.Key(prefix),
+			End:   fdb.Key(nextKey(prefix)),
+		})
+		return nil, nil
+	})
+	return err
+}
+
+func (c *fdbClient) close() error {
+	// c = &fdbClient{}
+	return nil
+}
+
+func (c *fdbClient) shouldRetry(err error) bool {
+	return false
+}
+
+func (tx *fdbTxn) get(key []byte) []byte {
+	return tx.Get(fdb.Key(key)).MustGet()
+}
+
+func (tx *fdbTxn) gets(keys ...[]byte) [][]byte {
+	ret := make([][]byte, len(keys))
+	for i, key := range keys {
+		val := tx.Get(fdb.Key(key)).MustGet()
+		ret[i] = val
+	}
+	return ret
+}
+
+func (tx *fdbTxn) range0(begin, end []byte) *fdb.RangeIterator {
+	return tx.GetRange(
+		fdb.KeyRange{Begin: fdb.Key(begin), End: fdb.Key(end)},
+		fdb.RangeOptions{Mode: fdb.StreamingModeWantAll},
+	).Iterator()
+}
+
+func (tx *fdbTxn) scanRange(begin, end []byte) map[string][]byte {
+	ret := make(map[string][]byte)
+	iter := tx.range0(begin, end)
+	for iter.Advance() {
+		r := iter.MustGet()
+		ret[string(r.Key)] = r.Value
+	}
+	return ret
+}
+
+func (tx *fdbTxn) scanKeys(prefix []byte) [][]byte {
+	ret := make([][]byte, 0)
+	iter := tx.range0(prefix, nextKey(prefix))
+	for iter.Advance() {
+		ret = append(ret, iter.MustGet().Key)
+	}
+	return ret
+}
+
+func (tx *fdbTxn) scanValues(prefix []byte, limit int, filter func(k, v []byte) bool) map[string][]byte {
+	ret := make(map[string][]byte)
+	iter := tx.range0(prefix, nextKey(prefix))
+	for iter.Advance() {
+		r := iter.MustGet()
+		if filter == nil || filter(r.Key, r.Value) {
+			ret[string(r.Key)] = r.Value
+			if limit > 0 {
+				if limit--; limit == 0 {
+					break
+				}
+			}
+		}
+	}
+	return ret
+}
+
+func (tx *fdbTxn) exist(prefix []byte) bool {
+	iter := tx.range0(prefix, nextKey(prefix))
+	return iter.Advance()
+}
+
+func (tx *fdbTxn) set(key, value []byte) {
+	tx.Set(fdb.Key(key), value)
+}
+
+func (tx *fdbTxn) append(key []byte, value []byte) []byte {
+	tx.AppendIfFits(fdb.Key(key), fdb.Key(value))
+	return tx.Get(fdb.Key(key)).MustGet()
+}
+
+func (tx *fdbTxn) incrBy(key []byte, value int64) int64 {
+	tx.Add(fdb.Key(key), packCounter(value))
+	return parseCounter(tx.Get(fdb.Key(key)).MustGet())
+}
+
+func (tx *fdbTxn) dels(keys ...[]byte) {
+	for _, key := range keys {
+		tx.Clear(fdb.Key(key))
+	}
+}

pkg/meta/tkv_fdb_test.go

@@ -0,0 +1,41 @@
+//go:build fdb
+// +build fdb
+
+/*
+ * JuiceFS, Copyright 2022 Juicedata, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+//nolint:errcheck
+package meta
+
+import (
+	"testing"
+)
+
+func TestFdbClient(t *testing.T) {
+	m, err := newKVMeta("fdb", "/etc/foundationdb/fdb.cluster?prefix=test2", &Config{})
+	if err != nil {
+		t.Fatalf("create meta: %s", err)
+	}
+	testMeta(t, m)
+}
+
+func TestFdb(t *testing.T) {
+	c, err := newFdbClient("/etc/foundationdb/fdb.cluster?prefix=test1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	testTKV(t, c)
+}

pkg/meta/tkv_mem.go

@@ -1,6 +1,3 @@
-//go:build !fdb
-// +build !fdb
-
 /*
  * JuiceFS, Copyright 2021 Juicedata, Inc.
  *