From bb1beb67cbbcf9f1577f83d2102481e10718d28d Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Wed, 23 Mar 2022 23:11:42 +0800 Subject: [PATCH 01/19] support etcd as meta engine --- pkg/meta/tkv_etcd.go | 245 +++++++++++++++++++++++++++++++++++++++++++ pkg/meta/tkv_test.go | 30 ++++++ 2 files changed, 275 insertions(+) create mode 100644 pkg/meta/tkv_etcd.go diff --git a/pkg/meta/tkv_etcd.go b/pkg/meta/tkv_etcd.go new file mode 100644 index 000000000000..d3aba309deca --- /dev/null +++ b/pkg/meta/tkv_etcd.go @@ -0,0 +1,245 @@ +//go:build !noetcd +// +build !noetcd + +/* + * JuiceFS, Copyright 2022 Juicedata, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package meta + +import ( + "bytes" + "context" + "time" + + etcd "go.etcd.io/etcd/clientv3" +) + +type etcdTxn struct { + kv etcd.KV + observed map[string]int + buffer map[string][]byte +} + +func (tx *etcdTxn) get(key []byte) []byte { + k := string(key) + if v, ok := tx.buffer[k]; ok { + return v + } + resp, err := tx.kv.Get(context.Background(), string(key), etcd.WithLimit(1), etcd.WithSerializable()) + if err != nil { + panic(err) + } + if resp.Count == 0 { + tx.observed[k] = 0 + return nil + } + for _, pair := range resp.Kvs { + if bytes.Equal(pair.Key, key) { + tx.observed[k] = int(pair.ModRevision) + return pair.Value + } + } + panic("not found") +} + +func (tx *etcdTxn) gets(keys ...[]byte) [][]byte { + // TODO: batch + values := make([][]byte, len(keys)) + for i, key := range keys { + values[i] = tx.get(key) + } + return values +} + +func (tx *etcdTxn) scanRange(begin_, end_ []byte) map[string][]byte { + resp, err := tx.kv.Get(context.Background(), string(begin_), etcd.WithRange(string(end_))) + if err != nil { + panic(err) + } + ret := make(map[string][]byte) + for _, kv := range resp.Kvs { + tx.observed[string(kv.Key)] = int(kv.ModRevision) + ret[string(kv.Key)] = kv.Value + } + return ret +} + +func (tx *etcdTxn) scan(prefix []byte, handler func(key []byte, value []byte)) { + resp, err := tx.kv.Get(context.Background(), string(prefix), etcd.WithPrefix(), + etcd.WithSort(etcd.SortByKey, etcd.SortAscend)) + if err != nil { + panic(err) + } + for _, kv := range resp.Kvs { + tx.observed[string(kv.Key)] = int(kv.ModRevision) + handler(kv.Key, kv.Value) + } +} + +func (tx *etcdTxn) scanKeys(prefix_ []byte) [][]byte { + resp, err := tx.kv.Get(context.Background(), string(prefix_), etcd.WithPrefix(), etcd.WithKeysOnly(), + etcd.WithSort(etcd.SortByKey, etcd.SortAscend)) + if err != nil { + panic(err) + } + var keys [][]byte + for _, kv := range resp.Kvs { + tx.observed[string(kv.Key)] = int(kv.ModRevision) + keys = append(keys, kv.Key) + } + return keys +} + +func (tx *etcdTxn) scanValues(prefix []byte, limit int, filter func(k, v []byte) bool) map[string][]byte { + if limit == 0 { + return nil + } + + res := tx.scanRange(prefix, nextKey(prefix)) + for k, v := range res { + if filter != nil && !filter([]byte(k), v) { + delete(res, k) + } + } + if n := len(res) - limit; limit > 0 && n > 0 { + for k := range res { + delete(res, k) + if n--; n == 0 { + break + } + } + } + return res +} + +func (tx *etcdTxn) exist(prefix []byte) bool { + resp, err := tx.kv.Get(context.Background(), string(prefix), etcd.WithPrefix(), etcd.WithCountOnly()) + if err != nil { + panic(err) + } + return resp.Count > 0 +} + +func (tx *etcdTxn) set(key, value []byte) { + tx.buffer[string(key)] = value +} + +func (tx *etcdTxn) append(key []byte, value []byte) []byte { + new := append(tx.get(key), value...) + tx.set(key, new) + return new +} + +func (tx *etcdTxn) incrBy(key []byte, value int64) int64 { + buf := tx.get(key) + new := parseCounter(buf) + if value != 0 { + new += value + tx.set(key, packCounter(new)) + } + return new +} + +func (tx *etcdTxn) dels(keys ...[]byte) { + for _, key := range keys { + tx.buffer[string(key)] = nil + } +} + +type etcdClient struct { + client *etcd.Client + kv etcd.KV +} + +func (c *etcdClient) name() string { + return "etcd" +} + +func (c *etcdClient) shouldRetry(err error) bool { + return false +} + +func (c *etcdClient) txn(f func(kvTxn) error) (err error) { + tx := &etcdTxn{ + c.kv, + make(map[string]int), + make(map[string][]byte), + } + start := time.Now() + defer func() { + if r := recover(); r != nil { + fe, ok := r.(error) + if ok { + err = fe + } else { + panic(r) + } + } + }() + err = f(tx) + if err != nil { + return err + } + if len(tx.buffer) == 0 { + return nil // read only + } + var conds []etcd.Cmp + var ops []etcd.Op + for k, v := range tx.observed { + conds = append(conds, etcd.Compare(etcd.ModRevision(k), "=", v)) + } + for k, v := range tx.buffer { + var op etcd.Op + if v == nil { + op = etcd.OpDelete(string(k)) + } else { + op = etcd.OpPut(string(k), string(v)) + } + ops = append(ops, op) + } + resp, err := c.kv.Txn(context.Background()).If(conds...).Then(ops...).Commit() + logger.Infof("txt with %d cond %d op took %s", len(conds), len(ops), time.Since(start)) + if err != nil { + return err + } + if resp.Succeeded { + return nil + } + // Try again + return +} + +func (c *etcdClient) reset(prefix []byte) error { + _, err := c.kv.Delete(context.Background(), string(prefix), etcd.WithPrefix()) + return err +} + +func (c *etcdClient) close() error { + return c.client.Close() +} + +func newEtcdClient(addr string) (tkvClient, error) { + c, err := etcd.NewFromURL(addr) + if err != nil { + return nil, err + } + return &etcdClient{c, etcd.NewKV(c)}, err +} + +func init() { + Register("etcd", newKVMeta) + drivers["etcd"] = newEtcdClient +} diff --git a/pkg/meta/tkv_test.go b/pkg/meta/tkv_test.go index 6e9da80705ec..51c49f96a343 100644 --- a/pkg/meta/tkv_test.go +++ b/pkg/meta/tkv_test.go @@ -48,6 +48,14 @@ func TestBadgerClient(t *testing.T) { testMeta(t, m) } +func TestEtcdClient(t *testing.T) { + m, err := newKVMeta("etcd", "http://localhost:2379", &Config{}) + if err != nil || m.Name() != "etcd" { + t.Fatalf("create meta: %s", err) + } + testMeta(t, m) +} + func testTKV(t *testing.T, c tkvClient) { txn := func(f func(kt kvTxn)) { if err := c.txn(func(kt kvTxn) error { @@ -154,6 +162,19 @@ func testTKV(t *testing.T, c tkvClient) { if count != 1 { t.Fatalf("counter should be 1, but got %d", count) } + + // key with zeros + k = []byte("k\x001") + txn(func(kt kvTxn) { + kt.set(k, v) + }) + var v2 []byte + txn(func(kt kvTxn) { + v2 = kt.get(k) + }) + if !bytes.Equal(v2, v) { + t.Fatalf("expect %v but got %v", v, v2) + } } func TestBadgerKV(t *testing.T) { @@ -164,6 +185,15 @@ func TestBadgerKV(t *testing.T) { testTKV(t, c) } +func TestEtcd(t *testing.T) { + c, err := newEtcdClient("http://localhost:2380") + if err != nil { + t.Fatal(err) + } + c = withPrefix(c, []byte("jfs")) + testTKV(t, c) +} + func TestMemKV(t *testing.T) { c, _ := newTkvClient("memkv", "") c = withPrefix(c, []byte("jfs")) From 416bbbe8bc3ebf1bcbbe167880822efcb119234b Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Thu, 24 Mar 2022 00:36:14 +0800 Subject: [PATCH 02/19] retry conflicted transaction --- pkg/meta/tkv_etcd.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/meta/tkv_etcd.go b/pkg/meta/tkv_etcd.go index d3aba309deca..f43915db580b 100644 --- a/pkg/meta/tkv_etcd.go +++ b/pkg/meta/tkv_etcd.go @@ -24,7 +24,8 @@ import ( "context" "time" - etcd "go.etcd.io/etcd/clientv3" + "github.com/pkg/errors" + etcd "go.etcd.io/etcd/client/v3" ) type etcdTxn struct { @@ -169,7 +170,7 @@ func (c *etcdClient) name() string { } func (c *etcdClient) shouldRetry(err error) bool { - return false + return errors.Is(err, conflicted) } func (c *etcdClient) txn(f func(kvTxn) error) (err error) { @@ -218,10 +219,11 @@ func (c *etcdClient) txn(f func(kvTxn) error) (err error) { if resp.Succeeded { return nil } - // Try again - return + return conflicted } +var conflicted = errors.New("conflicted transaction") + func (c *etcdClient) reset(prefix []byte) error { _, err := c.kv.Delete(context.Background(), string(prefix), etcd.WithPrefix()) return err From 29fa6fd70fd53b37267196977a89d65d96fb6ba0 Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Thu, 24 Mar 2022 00:37:40 +0800 Subject: [PATCH 03/19] fix build --- go.mod | 24 ++++++++++------ go.sum | 90 ++++++++++++++++++++++++++++++---------------------------- 2 files changed, 61 insertions(+), 53 deletions(-) diff --git a/go.mod b/go.mod index 3f5dd34177de..ea953bd8212d 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/go-redis/redis/v8 v8.4.0 github.com/go-sql-driver/mysql v1.6.0 github.com/gofrs/flock v0.8.1 - github.com/golang/protobuf v1.4.3 + github.com/golang/protobuf v1.5.2 github.com/google/btree v1.0.1 github.com/google/gops v0.3.13 github.com/google/uuid v1.1.2 @@ -46,9 +46,9 @@ require ( github.com/pkg/errors v0.9.1 github.com/pkg/sftp v1.10.0 github.com/pkg/xattr v0.4.4 - github.com/prometheus/client_golang v1.9.0 + github.com/prometheus/client_golang v1.11.0 github.com/prometheus/client_model v0.2.0 - github.com/prometheus/common v0.15.0 + github.com/prometheus/common v0.26.0 github.com/qingstor/qingstor-sdk-go/v4 v4.4.0 github.com/qiniu/api.v7/v7 v7.8.0 github.com/satori/go.uuid v1.2.0 @@ -60,9 +60,10 @@ require ( github.com/urfave/cli/v2 v2.3.0 github.com/vbauerster/mpb/v7 v7.0.3 github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8 + go.etcd.io/etcd/client/v3 v3.5.2 golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 - golang.org/x/oauth2 v0.0.0-20190517181255-950ef44c6e07 + golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 google.golang.org/api v0.5.0 @@ -96,6 +97,7 @@ require ( github.com/coredns/coredns v1.4.0 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e // indirect + github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect @@ -152,7 +154,7 @@ require ( github.com/jcmturner/goidentity/v6 v6.0.1 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/json-iterator/go v1.1.10 // indirect + github.com/json-iterator/go v1.1.11 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/klauspost/compress v1.13.4 // indirect github.com/klauspost/cpuid v1.3.1 // indirect @@ -194,7 +196,7 @@ require ( github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd // indirect github.com/pingcap/kvproto v0.0.0-20211122024046-03abd340988f // indirect github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7 // indirect - github.com/prometheus/procfs v0.2.0 // indirect + github.com/prometheus/procfs v0.6.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/rjeczalik/notify v0.9.2 // indirect @@ -225,6 +227,8 @@ require ( github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c // indirect github.com/xdg/stringprep v1.0.0 // indirect go.etcd.io/etcd v0.5.0-alpha.5.0.20200824191128-ae9734ed278b // indirect + go.etcd.io/etcd/api/v3 v3.5.2 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.2 // indirect go.opencensus.io v0.22.5 // indirect go.opentelemetry.io/otel v0.14.0 // indirect go.uber.org/atomic v1.7.0 // indirect @@ -234,9 +238,9 @@ require ( golang.org/x/text v0.3.6 // indirect golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 // indirect google.golang.org/appengine v1.6.5 // indirect - google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63 // indirect - google.golang.org/grpc v1.27.1 // indirect - google.golang.org/protobuf v1.23.0 // indirect + google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect + google.golang.org/grpc v1.38.0 // indirect + google.golang.org/protobuf v1.26.0 // indirect gopkg.in/ini.v1 v1.57.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/square/go-jose.v2 v2.3.1 // indirect @@ -251,3 +255,5 @@ replace github.com/hanwen/go-fuse/v2 v2.1.1-0.20210611132105-24a1dfe6b4f8 => git replace github.com/dgrijalva/jwt-go v3.2.0+incompatible => github.com/golang-jwt/jwt v3.2.1+incompatible replace github.com/vbauerster/mpb/v7 v7.0.3 => github.com/juicedata/mpb/v7 v7.0.4-0.20220216145631-6e0757f14703 + +replace google.golang.org/grpc => google.golang.org/grpc v1.29.0 diff --git a/go.sum b/go.sum index c3a06cd1032a..1a94c4f07caf 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,3 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.39.0 h1:UgQP9na6OTfp4dsAiz/eFpFA1C6tPdH5wiRdi19tuMw= cloud.google.com/go v0.39.0/go.mod h1:rVLT6fkc8chs9sfPtFc1SBH6em7n+ZoXaG+87tDISts= @@ -88,6 +87,7 @@ github.com/aliyun/aliyun-oss-go-sdk v2.1.0+incompatible/go.mod h1:T/Aws4fEfogEE9 github.com/alvaroloes/enumer v1.1.2/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT0m65DJ+Wo= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/appleboy/gin-jwt/v2 v2.6.3/go.mod h1:MfPYA4ogzvOcVkRwAxT7quHOtQmVKDpTwxyUrC2DNw0= @@ -144,7 +144,7 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= @@ -161,6 +161,8 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= @@ -216,9 +218,7 @@ github.com/emersion/go-ical v0.0.0-20200224201310-cd514449c39e/go.mod h1:4xVTBPc github.com/emersion/go-vcard v0.0.0-20191221110513-5f81fa0d3cc7/go.mod h1:HMJKR5wlh/ziNp+sHEDV2ltblO4JD2+IdDOWtGcQBTM= github.com/emersion/go-webdav v0.3.0 h1:I1J9xf7fa1NxXFWCKyN5Ju3Sa3jJeNMI3uJQKHYg0SY= github.com/emersion/go-webdav v0.3.0/go.mod h1:uSM1VveeKtogBVWaYccTksToczooJ0rrVGNsgnDsr4Q= -github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erikdubbelboer/gspt v0.0.0-20210805194459-ce36a5128377 h1:gT+RM6gdTIAzMT7HUvmT5mL8SyG8Wx7iS3+L0V34Km4= github.com/erikdubbelboer/gspt v0.0.0-20210805194459-ce36a5128377/go.mod h1:v6o7m/E9bfvm79dE1iFiF+3T7zLBnrjYjkWMa1J+Hv0= @@ -257,6 +257,7 @@ github.com/go-echarts/go-echarts v1.0.0/go.mod h1:qbmyAb/Rl1f2w7wKba1D4LoNq4U164 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-ldap/ldap v3.0.2+incompatible h1:kD5HQcAzlQ7yrhfn+h+MSABeAy/jAJhvIJ/QDllP44g= github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc= github.com/go-ldap/ldap/v3 v3.2.4 h1:PFavAq2xTgzo/loE8qNXcQaofAaqIpI4WgaLdv+1l3E= @@ -292,11 +293,11 @@ github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/goccy/go-graphviz v0.0.5/go.mod h1:wXVsXxmyMQU6TN3zGRttjNn3h+iCAS7xQFC6TlNvLhk= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -332,8 +333,10 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -353,8 +356,9 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -401,8 +405,9 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92Bcuy github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.12.1 h1:zCy2xE9ablevUOrUZc3Dl72Dt+ya2FNAvC2yLYMHzi4= github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69/go.mod h1:YLEMZOtU+AZ7dhN9T/IpGhXVGly2bvkJQ+zxj3WeVQo= github.com/hanwen/go-fuse v1.0.0 h1:GxS9Zrn6c35/BnfiVsZVWmsG803xwE7eVRDvcf/BEVc= github.com/hanwen/go-fuse v1.0.0/go.mod h1:unqXarDXqzAk0rt98O2tVndEPIpUgLD9+rwFisZH3Ok= @@ -519,8 +524,9 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -581,7 +587,6 @@ github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= -github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -815,8 +820,8 @@ github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeD github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.8.0/go.mod h1:O9VU6huf47PktckDQfMTX0Y8tY0/7TSWwj+ITvv0TnM= -github.com/prometheus/client_golang v1.9.0 h1:Rrch9mh17XcxvEu9D9DEpb4isxjGBtcevQjKvxPRQIU= -github.com/prometheus/client_golang v1.9.0/go.mod h1:FqZLKOZnGdFAhOK4nqGHa7D66IdsO+O441Eve7ptJDU= +github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -834,8 +839,8 @@ github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt2 github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.14.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= -github.com/prometheus/common v0.15.0 h1:4fgOnadei3EZvgRwxJ7RMpG1k1pOZth5Pc13tyspaKM= -github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= +github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -845,8 +850,9 @@ github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/qingstor/qingstor-sdk-go/v4 v4.4.0 h1:tbItWtGB1TDfYzqK8dtm6tV+xWU5iYMwL37C6AL5dDs= github.com/qingstor/qingstor-sdk-go/v4 v4.4.0/go.mod h1:mDVFtA7+bXQ5xoELTWkoFy1Ad13wtp8jtlnl/RU+zzM= @@ -1039,6 +1045,12 @@ go.etcd.io/etcd v0.0.0-20201125193152-8a03d2e9614b/go.mod h1:yVHk9ub3CSBatqGNg7G go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd v0.5.0-alpha.5.0.20200824191128-ae9734ed278b h1:3kC4J3eQF6p1UEfQTkC67eEeb3rTk+shQqdX6tFyq9Q= go.etcd.io/etcd v0.5.0-alpha.5.0.20200824191128-ae9734ed278b/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= +go.etcd.io/etcd/api/v3 v3.5.2 h1:tXok5yLlKyuQ/SXSjtqHc4uzNaMqZi2XsoSPr/LlJXI= +go.etcd.io/etcd/api/v3 v3.5.2/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= +go.etcd.io/etcd/client/pkg/v3 v3.5.2 h1:4hzqQ6hIb3blLyQ8usCU4h3NghkqcsohEQ3o3VetYxE= +go.etcd.io/etcd/client/pkg/v3 v3.5.2/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v3 v3.5.2 h1:WdnejrUtQC4nCxK0/dLTMqKOB+U5TP/2Ya0BJL+1otA= +go.etcd.io/etcd/client/v3 v3.5.2/go.mod h1:kOOaWFFgHygyT0WlSmL8TJiXmMysO/nNUlEsSsN6W4o= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -1072,6 +1084,7 @@ go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/arch v0.0.0-20201008161808-52c3e6f60cff/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4= @@ -1105,7 +1118,6 @@ golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -1152,6 +1164,7 @@ golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -1161,8 +1174,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190517181255-950ef44c6e07 h1:XC1K3wNjuz44KaI+cj85C9TW85w/46RH7J+DTXNH5Wk= -golang.org/x/oauth2 v0.0.0-20190517181255-950ef44c6e07/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1226,13 +1239,14 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e h1:XMgFehsDnnLGtjvjOfqWSUzt0alpTR1RSEuznObga2c= @@ -1245,6 +1259,7 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1255,10 +1270,8 @@ golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 h1:NusfzzA6yGQ+ua51ck7E3omN golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -1287,6 +1300,7 @@ golang.org/x/tools v0.0.0-20201105001634-bc3cf281b174/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210112230658-8b4aab62c064/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4 h1:cVngSRcfgyZCzys3KYOpCFa+4dqX/Oub9tAq00ttGVs= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1299,7 +1313,6 @@ gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6d google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.5.0 h1:lj9SyhMzyoa38fgFF0oO2T6pjs5IzkLPKfVtxpyCRMM= google.golang.org/api v0.5.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= @@ -1313,33 +1326,22 @@ google.golang.org/genproto v0.0.0-20190508193815-b515fa19cec8/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63 h1:YzfoEYWbODU5Fbt37+h7X16BWQbad7Q4S6gclTKFXM8= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= -google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= -google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c h1:wtujag7C+4D6KMoulW9YauvK2lgdvCMS260jsqqBXr0= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/grpc v1.29.0 h1:2pJjwYOdkZ9HlN4sWRYBg9ttH5bCOlsueaM+b/oYjwo= +google.golang.org/grpc v1.29.0/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/gometalinter.v2 v2.0.12/go.mod h1:NDRytsqEZyolNuAgTzJkZMkSQM7FIKyzVzGhjB/qfYo= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20180810215634-df19058c872c/go.mod h1:3HH7i1SgMqlzxCcBmUHW657sD4Kvv9sC3HpL3YukzwA= @@ -1393,7 +1395,6 @@ gorm.io/driver/mysql v1.0.6/go.mod h1:KdrTanmfLPPyAOeYGyG+UpDys7/7eeWT1zCq+oekYn gorm.io/driver/sqlite v1.1.4/go.mod h1:mJCeTFr7+crvS+TRnWc5Z3UvwxUN1BGBLMrf5LA9DYw= gorm.io/gorm v1.20.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v1.21.9/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= -honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1403,8 +1404,9 @@ k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= moul.io/zapgorm2 v1.1.0/go.mod h1:emRfKjNqSzVj5lcgasBdovIXY1jSOwFz2GQZn1Rddks= rsc.io/goversion v1.2.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= -sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= xorm.io/builder v0.3.7 h1:2pETdKRK+2QG4mLX4oODHEhn5Z8j1m8sXa7jfu+/SZI= xorm.io/builder v0.3.7/go.mod h1:aUW0S9eb9VCaPohFCH3j7czOx1PMW3i1HrSzbLYGBSE= From 4eb34d2756c0f5f5f6d5c4330c567acdf16afd15 Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Thu, 24 Mar 2022 11:55:52 +0800 Subject: [PATCH 04/19] cleanup --- pkg/meta/tkv_etcd.go | 87 +++++++++++++++++++++++++++++--------------- 1 file changed, 57 insertions(+), 30 deletions(-) diff --git a/pkg/meta/tkv_etcd.go b/pkg/meta/tkv_etcd.go index f43915db580b..2a2e9574b266 100644 --- a/pkg/meta/tkv_etcd.go +++ b/pkg/meta/tkv_etcd.go @@ -22,6 +22,9 @@ package meta import ( "bytes" "context" + "fmt" + "net/url" + "strings" "time" "github.com/pkg/errors" @@ -29,8 +32,9 @@ import ( ) type etcdTxn struct { + ctx context.Context kv etcd.KV - observed map[string]int + observed map[string]int64 buffer map[string][]byte } @@ -39,21 +43,26 @@ func (tx *etcdTxn) get(key []byte) []byte { if v, ok := tx.buffer[k]; ok { return v } - resp, err := tx.kv.Get(context.Background(), string(key), etcd.WithLimit(1), etcd.WithSerializable()) + resp, err := tx.kv.Get(tx.ctx, k, etcd.WithLimit(1), etcd.WithSerializable()) if err != nil { - panic(err) + panic(fmt.Errorf("get %v: %s", k, err)) } if resp.Count == 0 { tx.observed[k] = 0 return nil } + if resp.Count > 1 { + panic(fmt.Errorf("expect 1 keys but got %d", resp.Count)) + } for _, pair := range resp.Kvs { if bytes.Equal(pair.Key, key) { - tx.observed[k] = int(pair.ModRevision) + tx.observed[k] = pair.ModRevision return pair.Value + } else { + panic(fmt.Errorf("expect key %v, but got %v", k, string(pair.Key))) } } - panic("not found") + panic("unreachable") } func (tx *etcdTxn) gets(keys ...[]byte) [][]byte { @@ -66,39 +75,46 @@ func (tx *etcdTxn) gets(keys ...[]byte) [][]byte { } func (tx *etcdTxn) scanRange(begin_, end_ []byte) map[string][]byte { - resp, err := tx.kv.Get(context.Background(), string(begin_), etcd.WithRange(string(end_))) + return tx.scanRange0(begin_, end_, nil) +} + +func (tx *etcdTxn) scanRange0(begin_, end_ []byte, filter func(k, v []byte) bool) map[string][]byte { + resp, err := tx.kv.Get(tx.ctx, string(begin_), etcd.WithRange(string(end_))) if err != nil { - panic(err) + panic(fmt.Errorf("get range [%v-%v): %s", string(begin_), string(end_), err)) } ret := make(map[string][]byte) for _, kv := range resp.Kvs { - tx.observed[string(kv.Key)] = int(kv.ModRevision) - ret[string(kv.Key)] = kv.Value + if filter == nil || filter(kv.Key, kv.Value) { + k := string(kv.Key) + tx.observed[k] = kv.ModRevision + ret[k] = kv.Value + } } return ret } func (tx *etcdTxn) scan(prefix []byte, handler func(key []byte, value []byte)) { - resp, err := tx.kv.Get(context.Background(), string(prefix), etcd.WithPrefix(), + resp, err := tx.kv.Get(tx.ctx, string(prefix), etcd.WithPrefix(), etcd.WithSort(etcd.SortByKey, etcd.SortAscend)) if err != nil { - panic(err) + panic(fmt.Errorf("get prefix %v: %s", string(prefix), err)) } for _, kv := range resp.Kvs { - tx.observed[string(kv.Key)] = int(kv.ModRevision) + tx.observed[string(kv.Key)] = kv.ModRevision handler(kv.Key, kv.Value) } } -func (tx *etcdTxn) scanKeys(prefix_ []byte) [][]byte { - resp, err := tx.kv.Get(context.Background(), string(prefix_), etcd.WithPrefix(), etcd.WithKeysOnly(), +func (tx *etcdTxn) scanKeys(prefix []byte) [][]byte { + resp, err := tx.kv.Get(tx.ctx, string(prefix), etcd.WithPrefix(), etcd.WithKeysOnly(), etcd.WithSort(etcd.SortByKey, etcd.SortAscend)) if err != nil { - panic(err) + panic(fmt.Errorf("get prefix %v with keys only: %s", string(prefix), err)) } var keys [][]byte for _, kv := range resp.Kvs { - tx.observed[string(kv.Key)] = int(kv.ModRevision) + tx.observed[string(kv.Key)] = kv.ModRevision keys = append(keys, kv.Key) } return keys @@ -108,13 +124,8 @@ func (tx *etcdTxn) scanValues(prefix []byte, limit int, filter func(k, v []byte) if limit == 0 { return nil } - - res := tx.scanRange(prefix, nextKey(prefix)) - for k, v := range res { - if filter != nil && !filter([]byte(k), v) { - delete(res, k) - } - } + // TODO: use Limit option if filter is nil + res := tx.scanRange0(prefix, nextKey(prefix), filter) if n := len(res) - limit; limit > 0 && n > 0 { for k := range res { delete(res, k) @@ -127,9 +138,9 @@ func (tx *etcdTxn) scanValues(prefix []byte, limit int, filter func(k, v []byte) } func (tx *etcdTxn) exist(prefix []byte) bool { - resp, err := tx.kv.Get(context.Background(), string(prefix), etcd.WithPrefix(), etcd.WithCountOnly()) + resp, err := tx.kv.Get(tx.ctx, string(prefix), etcd.WithPrefix(), etcd.WithCountOnly()) if err != nil { - panic(err) + panic(fmt.Errorf("get prefix %v with count only: %s", string(prefix), err)) } return resp.Count > 0 } @@ -174,9 +185,11 @@ func (c *etcdClient) shouldRetry(err error) bool { } func (c *etcdClient) txn(f func(kvTxn) error) (err error) { + ctx := context.Background() tx := &etcdTxn{ + ctx, c.kv, - make(map[string]int), + make(map[string]int64), make(map[string][]byte), } start := time.Now() @@ -211,8 +224,10 @@ func (c *etcdClient) txn(f func(kvTxn) error) (err error) { } ops = append(ops, op) } - resp, err := c.kv.Txn(context.Background()).If(conds...).Then(ops...).Commit() - logger.Infof("txt with %d cond %d op took %s", len(conds), len(ops), time.Since(start)) + resp, err := c.kv.Txn(ctx).If(conds...).Then(ops...).Commit() + if time.Since(start) > time.Millisecond*10 { + logger.Debugf("txn with %d conds and %d ops took %s", len(conds), len(ops), time.Since(start)) + } if err != nil { return err } @@ -234,11 +249,23 @@ func (c *etcdClient) close() error { } func newEtcdClient(addr string) (tkvClient, error) { - c, err := etcd.NewFromURL(addr) + u, err := url.Parse(addr) + if err != nil { + return nil, fmt.Errorf("parse %s: %s", addr, err) + } + passwd, _ := u.User.Password() + logger.Infof("user: %s, pass: %s", u.User.Username(), passwd) + conf := etcd.Config{ + Endpoints: strings.Split(u.Host, ","), + // Username: u.User.Username(), + // Password: passwd, + } + c, err := etcd.New(conf) if err != nil { return nil, err } - return &etcdClient{c, etcd.NewKV(c)}, err + var prefix string = u.Path + "\xFD" + return withPrefix(&etcdClient{c, etcd.NewKV(c)}, []byte(prefix)), nil } func init() { From 4b09bc4ea10c27883692db87da53bfbcfd36076b Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Thu, 24 Mar 2022 17:59:54 +0800 Subject: [PATCH 05/19] test in CI --- .travis.yml | 4 +++- pkg/meta/tkv_test.go | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index adc75da7a8d6..8f0eddf2b1ac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,6 +40,7 @@ before_script: - if [ ! -f "/home/travis/.m2/install.sh" ];then wget -O /home/travis/.m2/install.sh https://tiup-mirrors.pingcap.com/install.sh; fi - if [ ! -f "/home/travis/.m2/rclone-v1.57.0-linux-amd64.zip" ];then wget -O /home/travis/.m2/rclone-v1.57.0-linux-amd64.zip --no-check-certificate https://downloads.rclone.org/v1.57.0/rclone-v1.57.0-linux-amd64.zip && unzip /home/travis/.m2/rclone-v1.57.0-linux-amd64.zip -d /home/travis/.m2; fi - if [ ! -f "/home/travis/.m2/litmus-0.13.tar.g" ];then wget -O /home/travis/.m2/litmus-0.13.tar.gz http://www.webdav.org/neon/litmus/litmus-0.13.tar.gz; tar -zxvf /home/travis/.m2/litmus-0.13.tar.gz -C /home/travis/.m2;cd /home/travis/.m2/litmus-0.13;./configure;make;cd -; fi + - if [ ! -f "/home/travis/.m2/etcd-v3.5.2-linux-amd64.tar.gz"];then wget -O /home/travis/.m2/etcd-v3.5.2-linux-amd64.tar.gz https://github.com/etcd-io/etcd/releases/download/v3.5.2/etcd-v3.5.2-linux-amd64.tar.gz; tar -zxf /home/travis/.m2/etcd-v3.5.2-linux-amd64.tar.gz -C /home/travis/.m2; fi - docker images - sh /home/travis/.m2/install.sh && source ~/.bash_profile && nohup tiup playground --mode tikv-slim >> output.log 2>&1 & - docker run -d -p 9000:9000 -p 9001:9001 -e "MINIO_ROOT_USER=testUser" -e "MINIO_ROOT_PASSWORD=testUserPassword" quay.io/minio/minio:RELEASE.2022-01-25T19-56-04Z server /data --console-address ":9001" @@ -48,11 +49,12 @@ before_script: - sudo chmod 777 /usr/local/maven-3.6.3/conf/settings.xml - sudo sed -i "s??/home/travis/.m2/repository?" /usr/local/maven-3.6.3/conf/settings.xml - docker run -d --name sftp -p 2222:22 juicedata/ci-sftp + - /home/travis/.m2/etcd-v3.5.2-linux-amd64/etcd --unsafe-no-fsync & - make - sudo make -C fstests setup - chmod +x travis-setup-hdfs.sh - ./travis-setup-hdfs.sh - - for i in {2222,3306,5432,8020,9000,9007} ; do nc -z localhost $i;if [ $? != 0 ];then sleep 5; else continue; fi; nc -z localhost $i; if [ $? != 0 ];then echo "service not ready, port:"$i; exit 1;fi; done + - for i in {2222,3306,5432,8020,9000,9007,2379} ; do nc -z localhost $i;if [ $? != 0 ];then sleep 5; else continue; fi; nc -z localhost $i; if [ $? != 0 ];then echo "service not ready, port:"$i; exit 1;fi; done - nc -z localhost 2379;if [ $? != 0 ];then echo "tikv is not ready";cat output.log;exit 1; fi script: - sudo chmod 777 /var/jfsCache diff --git a/pkg/meta/tkv_test.go b/pkg/meta/tkv_test.go index 867fc703b83c..2dae40ae6aae 100644 --- a/pkg/meta/tkv_test.go +++ b/pkg/meta/tkv_test.go @@ -51,7 +51,7 @@ func TestBadgerClient(t *testing.T) { func TestEtcdClient(t *testing.T) { m, err := newKVMeta("etcd", "http://localhost:2379", &Config{}) if err != nil { - t.Skipf("create meta: %s", err) + t.Fatalf("create meta: %s", err) } testMeta(t, m) } @@ -188,7 +188,7 @@ func TestBadgerKV(t *testing.T) { func TestEtcd(t *testing.T) { c, err := newEtcdClient("http://localhost:2379/jfs") if err != nil { - t.Skip(err) + t.Fatal(err) } testTKV(t, c) } From df742f45993c253a10f848f5868718e71461dd51 Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Fri, 25 Mar 2022 13:22:36 +0800 Subject: [PATCH 06/19] get multiple keys in batch --- pkg/meta/tkv_etcd.go | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/pkg/meta/tkv_etcd.go b/pkg/meta/tkv_etcd.go index 61bfcf896db2..c6d8e4525e67 100644 --- a/pkg/meta/tkv_etcd.go +++ b/pkg/meta/tkv_etcd.go @@ -67,10 +67,29 @@ func (tx *etcdTxn) get(key []byte) []byte { } func (tx *etcdTxn) gets(keys ...[]byte) [][]byte { - // TODO: batch + ops := make([]etcd.Op, len(keys)) + for i, key := range keys { + ops[i] = etcd.OpGet(string(key), etcd.WithSerializable()) + } + r, err := tx.kv.Do(tx.ctx, etcd.OpTxn(nil, ops, nil)) + if err != nil { + panic(fmt.Errorf("batch get with %d keys: %s", len(keys), err)) + } + rs := make(map[string][]byte) + for _, res := range r.Txn().Responses { + for _, p := range res.GetResponseRange().Kvs { + k := string(p.Key) + tx.observed[k] = p.ModRevision + rs[k] = p.Value + } + } values := make([][]byte, len(keys)) for i, key := range keys { - values[i] = tx.get(key) + k := string(key) + values[i] = rs[k] + if len(values[i]) == 0 { + tx.observed[k] = 0 + } } return values } @@ -270,9 +289,10 @@ func newEtcdClient(addr string) (tkvClient, error) { } passwd, _ := u.User.Password() conf := etcd.Config{ - Endpoints: strings.Split(u.Host, ","), - Username: u.User.Username(), - Password: passwd, + Endpoints: strings.Split(u.Host, ","), + Username: u.User.Username(), + Password: passwd, + AutoSyncInterval: time.Minute, } c, err := etcd.New(conf) if err != nil { From 5ca494e8ca58c6fd2cff8e600dedf21d6e05c3a6 Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Fri, 25 Mar 2022 13:49:49 +0800 Subject: [PATCH 07/19] cleanup --- pkg/meta/tkv_etcd.go | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/pkg/meta/tkv_etcd.go b/pkg/meta/tkv_etcd.go index c6d8e4525e67..f30105a1c949 100644 --- a/pkg/meta/tkv_etcd.go +++ b/pkg/meta/tkv_etcd.go @@ -95,10 +95,6 @@ func (tx *etcdTxn) gets(keys ...[]byte) [][]byte { } func (tx *etcdTxn) scanRange(begin_, end_ []byte) map[string][]byte { - return tx.scanRange0(begin_, end_, nil) -} - -func (tx *etcdTxn) scanRange0(begin_, end_ []byte, filter func(k, v []byte) bool) map[string][]byte { resp, err := tx.kv.Get(tx.ctx, string(begin_), etcd.WithRange(string(end_)), @@ -108,11 +104,9 @@ func (tx *etcdTxn) scanRange0(begin_, end_ []byte, filter func(k, v []byte) bool } ret := make(map[string][]byte) for _, kv := range resp.Kvs { - if filter == nil || filter(kv.Key, kv.Value) { - k := string(kv.Key) - tx.observed[k] = kv.ModRevision - ret[k] = kv.Value - } + k := string(kv.Key) + tx.observed[k] = kv.ModRevision + ret[k] = kv.Value } return ret } @@ -121,7 +115,6 @@ func (tx *etcdTxn) scan(prefix []byte, handler func(key []byte, value []byte)) { resp, err := tx.kv.Get(tx.ctx, string(prefix), etcd.WithPrefix(), - etcd.WithSort(etcd.SortByKey, etcd.SortAscend), etcd.WithSerializable()) if err != nil { panic(fmt.Errorf("get prefix %v: %s", string(prefix), err)) @@ -137,7 +130,6 @@ func (tx *etcdTxn) scanKeys(prefix []byte) [][]byte { string(prefix), etcd.WithPrefix(), etcd.WithKeysOnly(), - etcd.WithSort(etcd.SortByKey, etcd.SortAscend), etcd.WithSerializable()) if err != nil { panic(fmt.Errorf("get prefix %v with keys only: %s", string(prefix), err)) @@ -154,17 +146,22 @@ func (tx *etcdTxn) scanValues(prefix []byte, limit int, filter func(k, v []byte) if limit == 0 { return nil } - // TODO: use Limit option if filter is nil - res := tx.scanRange0(prefix, nextKey(prefix), filter) - if n := len(res) - limit; limit > 0 && n > 0 { - for k := range res { - delete(res, k) - if n--; n == 0 { + resp, err := tx.kv.Get(tx.ctx, string(prefix), etcd.WithPrefix(), etcd.WithSerializable()) + if err != nil { + panic(fmt.Errorf("get prefix %s: %s", string(prefix), err)) + } + ret := make(map[string][]byte) + for _, kv := range resp.Kvs { + if filter == nil || filter(kv.Key, kv.Value) { + k := string(kv.Key) + tx.observed[k] = kv.ModRevision + ret[k] = kv.Value + if limit > 0 && len(ret) >= limit { break } } } - return res + return ret } func (tx *etcdTxn) exist(prefix []byte) bool { From b3f7a1b705d25f59db413108d86466093cd2bee9 Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Fri, 25 Mar 2022 14:02:56 +0800 Subject: [PATCH 08/19] limit batch to 128 --- pkg/meta/tkv_etcd.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pkg/meta/tkv_etcd.go b/pkg/meta/tkv_etcd.go index f30105a1c949..af378e86a34c 100644 --- a/pkg/meta/tkv_etcd.go +++ b/pkg/meta/tkv_etcd.go @@ -66,7 +66,21 @@ func (tx *etcdTxn) get(key []byte) []byte { panic("unreachable") } +func min(a, b int) int { + if a < b { + return a + } + return b +} + func (tx *etcdTxn) gets(keys ...[]byte) [][]byte { + if len(keys) > 128 { + var rs = make([][]byte, 0, len(keys)) + for i := 0; i < len(keys); i += 128 { + rs = append(rs, tx.gets(keys[i:min(i+128, len(keys))]...)...) + } + return rs + } ops := make([]etcd.Op, len(keys)) for i, key := range keys { ops[i] = etcd.OpGet(string(key), etcd.WithSerializable()) From ad9e5e7e5890cb8febf48599ad2f2674f5f8fba2 Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Fri, 25 Mar 2022 15:19:57 +0800 Subject: [PATCH 09/19] use tls connections --- go.mod | 4 ++++ go.sum | 7 +++++++ pkg/meta/tkv_etcd.go | 20 ++++++++++++++++++++ pkg/meta/tkv_test.go | 4 ++-- 4 files changed, 33 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 3bd008c788df..ed839a7ae0c5 100644 --- a/go.mod +++ b/go.mod @@ -95,8 +95,11 @@ require ( github.com/cespare/xxhash/v2 v2.1.1 // indirect github.com/cheggaaa/pb v1.0.29 // indirect github.com/coredns/coredns v1.4.0 // indirect + github.com/coreos/etcd v3.3.10+incompatible // indirect github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect + github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d // indirect github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect github.com/dchest/siphash v1.2.1 // indirect @@ -210,6 +213,7 @@ require ( github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a // indirect github.com/willf/bitset v1.1.11 // indirect github.com/willf/bloom v2.0.3+incompatible // indirect + go.etcd.io/etcd v3.3.27+incompatible // indirect go.etcd.io/etcd/api/v3 v3.5.2 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.2 // indirect go.opencensus.io v0.22.5 // indirect diff --git a/go.sum b/go.sum index 3501d62e18fe..c652a559ed56 100644 --- a/go.sum +++ b/go.sum @@ -134,13 +134,18 @@ github.com/colinmarc/hdfs/v2 v2.2.0 h1:4AaIlTq+/sWmeqYhI0dX8bD4YrMQM990tRjm636Fk github.com/colinmarc/hdfs/v2 v2.2.0/go.mod h1:Wss6n3mtaZyRwWaqtSH+6ge01qT0rw9dJJmvoUnIQ/E= github.com/coredns/coredns v1.4.0 h1:RubBkYmkByUqZWWkjRHvNLnUHgkRVqAWgSMmRFvpE1A= github.com/coredns/coredns v1.4.0/go.mod h1:zASH/MVDgR6XZTbxvOnsZfffS+31vg6Ackf/wo1+AM0= +github.com/coreos/etcd v3.3.10+incompatible h1:jFneRYjIvLMLhDLCzuTuU4rSJUjRplcJQ7pD7MnhC04= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= +github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= @@ -751,6 +756,8 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= +go.etcd.io/etcd v3.3.27+incompatible h1:5hMrpf6REqTHV2LW2OclNpRtxI0k9ZplMemJsMSWju0= +go.etcd.io/etcd v3.3.27+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI= go.etcd.io/etcd/api/v3 v3.5.2 h1:tXok5yLlKyuQ/SXSjtqHc4uzNaMqZi2XsoSPr/LlJXI= go.etcd.io/etcd/api/v3 v3.5.2/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= go.etcd.io/etcd/client/pkg/v3 v3.5.2 h1:4hzqQ6hIb3blLyQ8usCU4h3NghkqcsohEQ3o3VetYxE= diff --git a/pkg/meta/tkv_etcd.go b/pkg/meta/tkv_etcd.go index af378e86a34c..1f34ce8cab71 100644 --- a/pkg/meta/tkv_etcd.go +++ b/pkg/meta/tkv_etcd.go @@ -22,6 +22,7 @@ package meta import ( "bytes" "context" + "crypto/tls" "fmt" "net/url" "strings" @@ -29,6 +30,7 @@ import ( "github.com/pkg/errors" etcd "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/pkg/transport" ) type etcdTxn struct { @@ -290,6 +292,20 @@ func (c *etcdClient) close() error { return c.client.Close() } +func buildTlsConfig(u *url.URL) (*tls.Config, error) { + var tsinfo transport.TLSInfo + q := u.Query() + tsinfo.CAFile = q.Get("cacert") + tsinfo.CertFile = q.Get("cert") + tsinfo.KeyFile = q.Get("key") + tsinfo.ServerName = q.Get("server-name") + tsinfo.InsecureSkipVerify = q.Get("insecure-skip-verify") != "" + if tsinfo.CAFile != "" || tsinfo.CertFile != "" || tsinfo.KeyFile != "" || tsinfo.ServerName != "" { + return tsinfo.ClientConfig() + } + return nil, nil +} + func newEtcdClient(addr string) (tkvClient, error) { if !strings.Contains(addr, "://") { addr = "http://" + addr @@ -305,6 +321,10 @@ func newEtcdClient(addr string) (tkvClient, error) { Password: passwd, AutoSyncInterval: time.Minute, } + conf.TLS, err = buildTlsConfig(u) + if err != nil { + return nil, fmt.Errorf("build tls config from %s: %s", u.RawQuery, err) + } c, err := etcd.New(conf) if err != nil { return nil, err diff --git a/pkg/meta/tkv_test.go b/pkg/meta/tkv_test.go index 2dae40ae6aae..ecbbbc6342e6 100644 --- a/pkg/meta/tkv_test.go +++ b/pkg/meta/tkv_test.go @@ -49,7 +49,7 @@ func TestBadgerClient(t *testing.T) { } func TestEtcdClient(t *testing.T) { - m, err := newKVMeta("etcd", "http://localhost:2379", &Config{}) + m, err := newKVMeta("etcd", "localhost:2379", &Config{}) if err != nil { t.Fatalf("create meta: %s", err) } @@ -186,7 +186,7 @@ func TestBadgerKV(t *testing.T) { } func TestEtcd(t *testing.T) { - c, err := newEtcdClient("http://localhost:2379/jfs") + c, err := newEtcdClient("localhost:2379/jfs") if err != nil { t.Fatal(err) } From 3bf8b5efe8123ad2f77870a37b61c0cdd8e6f7ac Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Wed, 30 Mar 2022 15:43:45 +0800 Subject: [PATCH 10/19] limit number of slices in a compaction --- pkg/meta/tkv.go | 3 +++ pkg/meta/tkv_etcd.go | 5 ++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/meta/tkv.go b/pkg/meta/tkv.go index b9ee1af21d6c..73622f922842 100644 --- a/pkg/meta/tkv.go +++ b/pkg/meta/tkv.go @@ -1806,6 +1806,9 @@ func (m *kvMeta) compactChunk(inode Ino, indx uint32, force bool) { return } + if len(buf) > sliceBytes*100 { + buf = buf[:sliceBytes*100] + } ss := readSliceBuf(buf) skipped := skipSome(ss) ss = ss[skipped:] diff --git a/pkg/meta/tkv_etcd.go b/pkg/meta/tkv_etcd.go index 1f34ce8cab71..780a359e5ba6 100644 --- a/pkg/meta/tkv_etcd.go +++ b/pkg/meta/tkv_etcd.go @@ -45,8 +45,7 @@ func (tx *etcdTxn) get(key []byte) []byte { if v, ok := tx.buffer[k]; ok { return v } - resp, err := tx.kv.Get(tx.ctx, k, etcd.WithLimit(1), - etcd.WithSerializable()) + resp, err := tx.kv.Get(tx.ctx, k, etcd.WithLimit(1)) if err != nil { panic(fmt.Errorf("get %v: %s", k, err)) } @@ -85,7 +84,7 @@ func (tx *etcdTxn) gets(keys ...[]byte) [][]byte { } ops := make([]etcd.Op, len(keys)) for i, key := range keys { - ops[i] = etcd.OpGet(string(key), etcd.WithSerializable()) + ops[i] = etcd.OpGet(string(key)) } r, err := tx.kv.Do(tx.ctx, etcd.OpTxn(nil, ops, nil)) if err != nil { From 3ac39bdc8671472cb9cbfcc61182387c6d16e0a2 Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Wed, 30 Mar 2022 16:02:26 +0800 Subject: [PATCH 11/19] use 2389 for etcd --- .travis.yml | 4 ++-- pkg/meta/tkv_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8f0eddf2b1ac..b84c1fbdaa9a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,12 +49,12 @@ before_script: - sudo chmod 777 /usr/local/maven-3.6.3/conf/settings.xml - sudo sed -i "s??/home/travis/.m2/repository?" /usr/local/maven-3.6.3/conf/settings.xml - docker run -d --name sftp -p 2222:22 juicedata/ci-sftp - - /home/travis/.m2/etcd-v3.5.2-linux-amd64/etcd --unsafe-no-fsync & + - /home/travis/.m2/etcd-v3.5.2-linux-amd64/etcd --unsafe-no-fsync --listen-client-urls http://localhost:2389 & - make - sudo make -C fstests setup - chmod +x travis-setup-hdfs.sh - ./travis-setup-hdfs.sh - - for i in {2222,3306,5432,8020,9000,9007,2379} ; do nc -z localhost $i;if [ $? != 0 ];then sleep 5; else continue; fi; nc -z localhost $i; if [ $? != 0 ];then echo "service not ready, port:"$i; exit 1;fi; done + - for i in {2222,3306,5432,8020,9000,9007,2379,2389} ; do nc -z localhost $i;if [ $? != 0 ];then sleep 5; else continue; fi; nc -z localhost $i; if [ $? != 0 ];then echo "service not ready, port:"$i; exit 1;fi; done - nc -z localhost 2379;if [ $? != 0 ];then echo "tikv is not ready";cat output.log;exit 1; fi script: - sudo chmod 777 /var/jfsCache diff --git a/pkg/meta/tkv_test.go b/pkg/meta/tkv_test.go index ecbbbc6342e6..b4b9faf9966c 100644 --- a/pkg/meta/tkv_test.go +++ b/pkg/meta/tkv_test.go @@ -49,7 +49,7 @@ func TestBadgerClient(t *testing.T) { } func TestEtcdClient(t *testing.T) { - m, err := newKVMeta("etcd", "localhost:2379", &Config{}) + m, err := newKVMeta("etcd", "localhost:2389", &Config{}) if err != nil { t.Fatalf("create meta: %s", err) } @@ -186,7 +186,7 @@ func TestBadgerKV(t *testing.T) { } func TestEtcd(t *testing.T) { - c, err := newEtcdClient("localhost:2379/jfs") + c, err := newEtcdClient("localhost:2389/jfs") if err != nil { t.Fatal(err) } From 0545d4d4cd1c7219475f1abedf181220b33d9224 Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Wed, 30 Mar 2022 17:13:13 +0800 Subject: [PATCH 12/19] read from write buffer --- pkg/meta/tkv_etcd.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/meta/tkv_etcd.go b/pkg/meta/tkv_etcd.go index 780a359e5ba6..8c376c587bab 100644 --- a/pkg/meta/tkv_etcd.go +++ b/pkg/meta/tkv_etcd.go @@ -101,6 +101,10 @@ func (tx *etcdTxn) gets(keys ...[]byte) [][]byte { values := make([][]byte, len(keys)) for i, key := range keys { k := string(key) + if v, ok := tx.buffer[k]; ok { + values[i] = v + continue + } values[i] = rs[k] if len(values[i]) == 0 { tx.observed[k] = 0 From 98c55beeda813ac39944586eebfb374b9f4fcf5c Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Fri, 1 Apr 2022 12:40:21 +0800 Subject: [PATCH 13/19] try travis --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index b84c1fbdaa9a..1cfdbfff2bd2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,6 +49,8 @@ before_script: - sudo chmod 777 /usr/local/maven-3.6.3/conf/settings.xml - sudo sed -i "s??/home/travis/.m2/repository?" /usr/local/maven-3.6.3/conf/settings.xml - docker run -d --name sftp -p 2222:22 juicedata/ci-sftp + - ls -l /home/travis/.m2 + - ls -l /home/travis/.m2/etcd-v3.5.2-linux-amd64/ - /home/travis/.m2/etcd-v3.5.2-linux-amd64/etcd --unsafe-no-fsync --listen-client-urls http://localhost:2389 & - make - sudo make -C fstests setup From c57b6b9ba84a6c72198ffefd341b185d62b3ef3e Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Fri, 1 Apr 2022 13:10:12 +0800 Subject: [PATCH 14/19] fix travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1cfdbfff2bd2..fff94c971c78 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,7 +40,7 @@ before_script: - if [ ! -f "/home/travis/.m2/install.sh" ];then wget -O /home/travis/.m2/install.sh https://tiup-mirrors.pingcap.com/install.sh; fi - if [ ! -f "/home/travis/.m2/rclone-v1.57.0-linux-amd64.zip" ];then wget -O /home/travis/.m2/rclone-v1.57.0-linux-amd64.zip --no-check-certificate https://downloads.rclone.org/v1.57.0/rclone-v1.57.0-linux-amd64.zip && unzip /home/travis/.m2/rclone-v1.57.0-linux-amd64.zip -d /home/travis/.m2; fi - if [ ! -f "/home/travis/.m2/litmus-0.13.tar.g" ];then wget -O /home/travis/.m2/litmus-0.13.tar.gz http://www.webdav.org/neon/litmus/litmus-0.13.tar.gz; tar -zxvf /home/travis/.m2/litmus-0.13.tar.gz -C /home/travis/.m2;cd /home/travis/.m2/litmus-0.13;./configure;make;cd -; fi - - if [ ! -f "/home/travis/.m2/etcd-v3.5.2-linux-amd64.tar.gz"];then wget -O /home/travis/.m2/etcd-v3.5.2-linux-amd64.tar.gz https://github.com/etcd-io/etcd/releases/download/v3.5.2/etcd-v3.5.2-linux-amd64.tar.gz; tar -zxf /home/travis/.m2/etcd-v3.5.2-linux-amd64.tar.gz -C /home/travis/.m2; fi + - if [ ! -f "/home/travis/.m2/etcd-v3.5.2-linux-amd64.tar.gz" ];then wget -O /home/travis/.m2/etcd-v3.5.2-linux-amd64.tar.gz https://github.com/etcd-io/etcd/releases/download/v3.5.2/etcd-v3.5.2-linux-amd64.tar.gz; tar -zxf /home/travis/.m2/etcd-v3.5.2-linux-amd64.tar.gz -C /home/travis/.m2; fi - docker images - sh /home/travis/.m2/install.sh && source ~/.bash_profile && nohup tiup playground --mode tikv-slim >> output.log 2>&1 & - docker run -d -p 9000:9000 -p 9001:9001 -e "MINIO_ROOT_USER=testUser" -e "MINIO_ROOT_PASSWORD=testUserPassword" quay.io/minio/minio:RELEASE.2022-01-25T19-56-04Z server /data --console-address ":9001" From 78037393150737ed3a15811968ac86dcdb7c1b40 Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Fri, 1 Apr 2022 19:43:48 +0800 Subject: [PATCH 15/19] fix travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1d4f5ce762a1..2e771615a243 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,7 +51,7 @@ before_script: - docker run -d --name sftp -p 2222:22 juicedata/ci-sftp - ls -l /home/travis/.m2 - ls -l /home/travis/.m2/etcd-v3.5.2-linux-amd64/ - - /home/travis/.m2/etcd-v3.5.2-linux-amd64/etcd --unsafe-no-fsync --listen-client-urls http://localhost:2389 & + - /home/travis/.m2/etcd-v3.5.2-linux-amd64/etcd --unsafe-no-fsync --listen-client-urls http://localhost:2389 --advertise-client-urls http://localhost:2389 & - make - sudo make -C fstests setup - chmod +x travis-setup-hdfs.sh From fad1ff375d5eef69949197c59659c287aa8eaf6f Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Fri, 1 Apr 2022 21:15:23 +0800 Subject: [PATCH 16/19] fix travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2e771615a243..778c01ce687d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,8 +49,8 @@ before_script: - sudo chmod 777 /usr/local/maven-3.6.3/conf/settings.xml - sudo sed -i "s??/home/travis/.m2/repository?" /usr/local/maven-3.6.3/conf/settings.xml - docker run -d --name sftp -p 2222:22 juicedata/ci-sftp - - ls -l /home/travis/.m2 - ls -l /home/travis/.m2/etcd-v3.5.2-linux-amd64/ + - /home/travis/.m2/etcd-v3.5.2-linux-amd64/etcd -h - /home/travis/.m2/etcd-v3.5.2-linux-amd64/etcd --unsafe-no-fsync --listen-client-urls http://localhost:2389 --advertise-client-urls http://localhost:2389 & - make - sudo make -C fstests setup From 91c44df00fe8bf3ec5e305bc8b35b9b6c1747183 Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Sat, 2 Apr 2022 09:52:53 +0800 Subject: [PATCH 17/19] try --- .travis.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 778c01ce687d..feffed468f0a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,17 +46,15 @@ before_script: - docker run -d -p 9000:9000 -p 9001:9001 -e "MINIO_ROOT_USER=testUser" -e "MINIO_ROOT_PASSWORD=testUserPassword" quay.io/minio/minio:RELEASE.2022-01-25T19-56-04Z server /data --console-address ":9001" - go install github.com/minio/mc@RELEASE.2022-01-07T06-01-38Z && mc config host add local http://127.0.0.1:9000 testUser testUserPassword && mc mb local/testbucket - nohup /home/travis/.m2/rclone-v1.57.0-linux-amd64/rclone serve webdav local --addr 127.0.0.1:9007 >> rclone.log 2>&1 & + - # nohup /home/travis/.m2/etcd-v3.5.2-linux-amd64/etcd --unsafe-no-fsync --listen-client-urls http://127.0.0.1:2389 --advertise-client-urls http://127.0.0.1:2389 & - sudo chmod 777 /usr/local/maven-3.6.3/conf/settings.xml - sudo sed -i "s??/home/travis/.m2/repository?" /usr/local/maven-3.6.3/conf/settings.xml - docker run -d --name sftp -p 2222:22 juicedata/ci-sftp - - ls -l /home/travis/.m2/etcd-v3.5.2-linux-amd64/ - - /home/travis/.m2/etcd-v3.5.2-linux-amd64/etcd -h - - /home/travis/.m2/etcd-v3.5.2-linux-amd64/etcd --unsafe-no-fsync --listen-client-urls http://localhost:2389 --advertise-client-urls http://localhost:2389 & - make - sudo make -C fstests setup - chmod +x travis-setup-hdfs.sh - ./travis-setup-hdfs.sh - - for i in {2222,3306,5432,8020,9000,9007,2379,2389} ; do echo "lsof port:"$i;sudo lsof -i :$i;if [ $? != 0 ];then sleep 5; else continue; fi;sudo lsof -i :$i; if [ $? != 0 ];then echo "service not ready, port:"$i; exit 1;fi; done + - for i in {2222,3306,5432,8020,9000,9007,2379} ; do echo "lsof port:"$i;sudo lsof -i :$i;if [ $? != 0 ];then sleep 5; else continue; fi;sudo lsof -i :$i; if [ $? != 0 ];then echo "service not ready, port:"$i; exit 1;fi; done - sudo lsof -i :2379;if [ $? != 0 ];then echo "tikv is not ready";cat output.log;exit 1; fi script: - sudo chmod 777 /var/jfsCache From 7a7ee628b26a7a05bc806b0b07259603e6bdaab9 Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Sat, 2 Apr 2022 10:20:11 +0800 Subject: [PATCH 18/19] test with pd --- pkg/meta/tkv_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/meta/tkv_test.go b/pkg/meta/tkv_test.go index b4b9faf9966c..ecbbbc6342e6 100644 --- a/pkg/meta/tkv_test.go +++ b/pkg/meta/tkv_test.go @@ -49,7 +49,7 @@ func TestBadgerClient(t *testing.T) { } func TestEtcdClient(t *testing.T) { - m, err := newKVMeta("etcd", "localhost:2389", &Config{}) + m, err := newKVMeta("etcd", "localhost:2379", &Config{}) if err != nil { t.Fatalf("create meta: %s", err) } @@ -186,7 +186,7 @@ func TestBadgerKV(t *testing.T) { } func TestEtcd(t *testing.T) { - c, err := newEtcdClient("localhost:2389/jfs") + c, err := newEtcdClient("localhost:2379/jfs") if err != nil { t.Fatal(err) } From 2755e16da88391e93622bafaa534eb92d6f6191c Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Sat, 2 Apr 2022 12:43:57 +0800 Subject: [PATCH 19/19] scan keys in lineariable --- pkg/meta/tkv_etcd.go | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/pkg/meta/tkv_etcd.go b/pkg/meta/tkv_etcd.go index 8c376c587bab..de2b272eecbd 100644 --- a/pkg/meta/tkv_etcd.go +++ b/pkg/meta/tkv_etcd.go @@ -114,10 +114,7 @@ func (tx *etcdTxn) gets(keys ...[]byte) [][]byte { } func (tx *etcdTxn) scanRange(begin_, end_ []byte) map[string][]byte { - resp, err := tx.kv.Get(tx.ctx, - string(begin_), - etcd.WithRange(string(end_)), - etcd.WithSerializable()) + resp, err := tx.kv.Get(tx.ctx, string(begin_), etcd.WithRange(string(end_))) if err != nil { panic(fmt.Errorf("get range [%v-%v): %s", string(begin_), string(end_), err)) } @@ -145,11 +142,7 @@ func (tx *etcdTxn) scan(prefix []byte, handler func(key []byte, value []byte)) { } func (tx *etcdTxn) scanKeys(prefix []byte) [][]byte { - resp, err := tx.kv.Get(tx.ctx, - string(prefix), - etcd.WithPrefix(), - etcd.WithKeysOnly(), - etcd.WithSerializable()) + resp, err := tx.kv.Get(tx.ctx, string(prefix), etcd.WithPrefix(), etcd.WithKeysOnly()) if err != nil { panic(fmt.Errorf("get prefix %v with keys only: %s", string(prefix), err)) } @@ -165,7 +158,7 @@ func (tx *etcdTxn) scanValues(prefix []byte, limit int, filter func(k, v []byte) if limit == 0 { return nil } - resp, err := tx.kv.Get(tx.ctx, string(prefix), etcd.WithPrefix(), etcd.WithSerializable()) + resp, err := tx.kv.Get(tx.ctx, string(prefix), etcd.WithPrefix()) if err != nil { panic(fmt.Errorf("get prefix %s: %s", string(prefix), err)) } @@ -184,8 +177,7 @@ func (tx *etcdTxn) scanValues(prefix []byte, limit int, filter func(k, v []byte) } func (tx *etcdTxn) exist(prefix []byte) bool { - resp, err := tx.kv.Get(tx.ctx, string(prefix), etcd.WithPrefix(), - etcd.WithCountOnly(), etcd.WithSerializable()) + resp, err := tx.kv.Get(tx.ctx, string(prefix), etcd.WithPrefix(), etcd.WithCountOnly()) if err != nil { panic(fmt.Errorf("get prefix %v with count only: %s", string(prefix), err)) }