From ea0b04ab3a4ba0ae085ba2c1f104b2f9c25e2171 Mon Sep 17 00:00:00 2001 From: Ashish Goswami Date: Mon, 11 Nov 2019 15:09:06 +0530 Subject: [PATCH 1/6] Import latest Badger and update import paths --- conn/node.go | 4 ++-- conn/node_test.go | 2 +- conn/pool.go | 2 +- dgraph/cmd/alpha/run.go | 2 +- dgraph/cmd/bulk/count_index.go | 4 ++-- dgraph/cmd/bulk/loader.go | 4 ++-- dgraph/cmd/bulk/reduce.go | 8 ++++---- dgraph/cmd/bulk/schema.go | 2 +- dgraph/cmd/debug/run.go | 4 ++-- dgraph/cmd/debug/wal.go | 2 +- dgraph/cmd/live/batch.go | 2 +- dgraph/cmd/live/run.go | 4 ++-- dgraph/cmd/zero/license.go | 2 +- dgraph/cmd/zero/license_ee.go | 2 +- dgraph/cmd/zero/oracle.go | 4 ++-- dgraph/cmd/zero/raft.go | 2 +- dgraph/cmd/zero/run.go | 4 ++-- dgraph/cmd/zero/zero.go | 2 +- edgraph/access.go | 2 +- edgraph/access_ee.go | 2 +- edgraph/server.go | 4 ++-- ee/backup/backup.go | 4 ++-- ee/backup/restore.go | 6 +++--- go.mod | 3 ++- go.sum | 12 ++++++++---- posting/index.go | 4 ++-- posting/index_test.go | 2 +- posting/list.go | 2 +- posting/list_test.go | 4 ++-- posting/lists.go | 4 ++-- posting/mvcc.go | 2 +- posting/writer.go | 4 ++-- protos/pb/pb.pb.go | 2 +- raftwal/storage.go | 2 +- raftwal/storage_test.go | 2 +- schema/parse_test.go | 2 +- schema/schema.go | 2 +- testutil/backup.go | 6 +++--- worker/draft.go | 6 +++--- worker/draft_test.go | 2 +- worker/export.go | 4 ++-- worker/groups.go | 4 ++-- worker/mutation.go | 2 +- worker/predicate_move.go | 4 ++-- worker/predicate_test.go | 2 +- worker/snapshot.go | 4 ++-- worker/sort.go | 2 +- worker/task.go | 2 +- worker/tokens.go | 2 +- worker/worker.go | 2 +- worker/worker_test.go | 2 +- xidmap/xidmap.go | 2 +- xidmap/xidmap_test.go | 2 +- 53 files changed, 87 insertions(+), 82 deletions(-) diff --git a/conn/node.go b/conn/node.go index 5ec79005f7d..6c7d5d4c8cf 100644 --- a/conn/node.go +++ b/conn/node.go @@ -26,7 +26,7 @@ import ( "sync/atomic" "time" - "github.com/dgraph-io/badger/y" + "github.com/dgraph-io/badger/v2/y" "github.com/dgraph-io/dgo/v2/protos/api" "github.com/dgraph-io/dgraph/protos/pb" "github.com/dgraph-io/dgraph/raftwal" @@ -137,7 +137,7 @@ func NewNode(rc *pb.RaftContext, store *raftwal.DiskStorage) *Node { peers: make(map[uint64]string), requestCh: make(chan linReadReq, 100), } - n.Applied.Init(nil) + n.Applied.Init(nil, true) // This should match up to the Applied index set above. n.Applied.SetDoneUntil(n.Cfg.Applied) glog.Infof("Setting raft.Config to: %+v\n", n.Cfg) diff --git a/conn/node_test.go b/conn/node_test.go index 1d569c14fa6..3a3cbf61d23 100644 --- a/conn/node_test.go +++ b/conn/node_test.go @@ -25,7 +25,7 @@ import ( "testing" "time" - "github.com/dgraph-io/badger" + "github.com/dgraph-io/badger/v2" "github.com/dgraph-io/dgraph/protos/pb" "github.com/dgraph-io/dgraph/raftwal" "github.com/stretchr/testify/require" diff --git a/conn/pool.go b/conn/pool.go index 526459b92fd..325ae3734e6 100644 --- a/conn/pool.go +++ b/conn/pool.go @@ -21,7 +21,7 @@ import ( "sync" "time" - "github.com/dgraph-io/badger/y" + "github.com/dgraph-io/badger/v2/y" "github.com/dgraph-io/dgo/v2/protos/api" "github.com/dgraph-io/dgraph/protos/pb" "github.com/dgraph-io/dgraph/x" diff --git a/dgraph/cmd/alpha/run.go b/dgraph/cmd/alpha/run.go index 19a6be474d9..59761e03116 100644 --- a/dgraph/cmd/alpha/run.go +++ b/dgraph/cmd/alpha/run.go @@ -33,7 +33,7 @@ import ( "syscall" "time" - "github.com/dgraph-io/badger/y" + "github.com/dgraph-io/badger/v2/y" "github.com/dgraph-io/dgo/v2/protos/api" "github.com/dgraph-io/dgraph/edgraph" "github.com/dgraph-io/dgraph/posting" diff --git a/dgraph/cmd/bulk/count_index.go b/dgraph/cmd/bulk/count_index.go index ec76710d384..b09617b4206 100644 --- a/dgraph/cmd/bulk/count_index.go +++ b/dgraph/cmd/bulk/count_index.go @@ -24,8 +24,8 @@ import ( "sync" "sync/atomic" - "github.com/dgraph-io/badger" - bpb "github.com/dgraph-io/badger/pb" + "github.com/dgraph-io/badger/v2" + bpb "github.com/dgraph-io/badger/v2/pb" "github.com/dgraph-io/dgraph/codec" "github.com/dgraph-io/dgraph/posting" "github.com/dgraph-io/dgraph/protos/pb" diff --git a/dgraph/cmd/bulk/loader.go b/dgraph/cmd/bulk/loader.go index 6b725b5bcbc..3f686dda515 100644 --- a/dgraph/cmd/bulk/loader.go +++ b/dgraph/cmd/bulk/loader.go @@ -29,8 +29,8 @@ import ( "sync" "time" - "github.com/dgraph-io/badger" - "github.com/dgraph-io/badger/y" + "github.com/dgraph-io/badger/v2" + "github.com/dgraph-io/badger/v2/y" "github.com/dgraph-io/dgraph/chunker" "github.com/dgraph-io/dgraph/protos/pb" diff --git a/dgraph/cmd/bulk/reduce.go b/dgraph/cmd/bulk/reduce.go index 9c931a8954b..1f15d98ad9f 100644 --- a/dgraph/cmd/bulk/reduce.go +++ b/dgraph/cmd/bulk/reduce.go @@ -29,10 +29,10 @@ import ( "path/filepath" "sync/atomic" - "github.com/dgraph-io/badger" - bo "github.com/dgraph-io/badger/options" - bpb "github.com/dgraph-io/badger/pb" - "github.com/dgraph-io/badger/y" + "github.com/dgraph-io/badger/v2" + bo "github.com/dgraph-io/badger/v2/options" + bpb "github.com/dgraph-io/badger/v2/pb" + "github.com/dgraph-io/badger/v2/y" "github.com/dgraph-io/dgraph/codec" "github.com/dgraph-io/dgraph/posting" "github.com/dgraph-io/dgraph/protos/pb" diff --git a/dgraph/cmd/bulk/schema.go b/dgraph/cmd/bulk/schema.go index 53051e416bf..e6b25f353c7 100644 --- a/dgraph/cmd/bulk/schema.go +++ b/dgraph/cmd/bulk/schema.go @@ -22,7 +22,7 @@ import ( "math" "sync" - "github.com/dgraph-io/badger" + "github.com/dgraph-io/badger/v2" "github.com/dgraph-io/dgraph/posting" "github.com/dgraph-io/dgraph/protos/pb" "github.com/dgraph-io/dgraph/schema" diff --git a/dgraph/cmd/debug/run.go b/dgraph/cmd/debug/run.go index 0d3e374eb99..31e4d232520 100644 --- a/dgraph/cmd/debug/run.go +++ b/dgraph/cmd/debug/run.go @@ -27,8 +27,8 @@ import ( "strconv" "strings" - "github.com/dgraph-io/badger" - "github.com/dgraph-io/badger/options" + "github.com/dgraph-io/badger/v2" + "github.com/dgraph-io/badger/v2/options" "github.com/dgraph-io/dgraph/codec" "github.com/dgraph-io/dgraph/posting" "github.com/dgraph-io/dgraph/protos/pb" diff --git a/dgraph/cmd/debug/wal.go b/dgraph/cmd/debug/wal.go index 1627dc81e29..98c9b524902 100644 --- a/dgraph/cmd/debug/wal.go +++ b/dgraph/cmd/debug/wal.go @@ -24,7 +24,7 @@ import ( "strconv" "strings" - "github.com/dgraph-io/badger" + "github.com/dgraph-io/badger/v2" "github.com/dgraph-io/dgraph/protos/pb" "github.com/dgraph-io/dgraph/raftwal" "github.com/dgraph-io/dgraph/x" diff --git a/dgraph/cmd/live/batch.go b/dgraph/cmd/live/batch.go index 3cb6787dd55..4ab2aac20c7 100644 --- a/dgraph/cmd/live/batch.go +++ b/dgraph/cmd/live/batch.go @@ -29,7 +29,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "github.com/dgraph-io/badger" + "github.com/dgraph-io/badger/v2" "github.com/dgraph-io/dgo/v2" "github.com/dgraph-io/dgo/v2/protos/api" "github.com/dgraph-io/dgraph/dgraph/cmd/zero" diff --git a/dgraph/cmd/live/run.go b/dgraph/cmd/live/run.go index 9f5be92f8b0..e0e1c312d96 100644 --- a/dgraph/cmd/live/run.go +++ b/dgraph/cmd/live/run.go @@ -35,8 +35,8 @@ import ( "google.golang.org/grpc/metadata" - "github.com/dgraph-io/badger" - bopt "github.com/dgraph-io/badger/options" + "github.com/dgraph-io/badger/v2" + bopt "github.com/dgraph-io/badger/v2/options" "github.com/dgraph-io/dgo/v2" "github.com/dgraph-io/dgo/v2/protos/api" diff --git a/dgraph/cmd/zero/license.go b/dgraph/cmd/zero/license.go index 018bdc7345e..a591df9e078 100644 --- a/dgraph/cmd/zero/license.go +++ b/dgraph/cmd/zero/license.go @@ -21,7 +21,7 @@ package zero import ( "net/http" - "github.com/dgraph-io/badger/y" + "github.com/dgraph-io/badger/v2/y" ) // dummy function as enterprise features are not available in oss binary. diff --git a/dgraph/cmd/zero/license_ee.go b/dgraph/cmd/zero/license_ee.go index 80cc773bba3..7abe2338795 100644 --- a/dgraph/cmd/zero/license_ee.go +++ b/dgraph/cmd/zero/license_ee.go @@ -19,7 +19,7 @@ import ( "net/http" "time" - "github.com/dgraph-io/badger/y" + "github.com/dgraph-io/badger/v2/y" "github.com/dgraph-io/dgraph/protos/pb" "github.com/dgraph-io/dgraph/x" humanize "github.com/dustin/go-humanize" diff --git a/dgraph/cmd/zero/oracle.go b/dgraph/cmd/zero/oracle.go index b35d83ec1b3..b96c5cf86cf 100644 --- a/dgraph/cmd/zero/oracle.go +++ b/dgraph/cmd/zero/oracle.go @@ -22,7 +22,7 @@ import ( "strings" "time" - "github.com/dgraph-io/badger/y" + "github.com/dgraph-io/badger/v2/y" "github.com/dgraph-io/dgo/v2/protos/api" "github.com/dgraph-io/dgraph/protos/pb" "github.com/dgraph-io/dgraph/x" @@ -61,7 +61,7 @@ func (o *Oracle) Init() { o.keyCommit = make(map[string]uint64) o.subscribers = make(map[int]chan pb.OracleDelta) o.updates = make(chan *pb.OracleDelta, 100000) // Keeping 1 second worth of updates. - o.doneUntil.Init(nil) + o.doneUntil.Init(nil, true) go o.sendDeltasToSubscribers() } diff --git a/dgraph/cmd/zero/raft.go b/dgraph/cmd/zero/raft.go index 53dedaf35a2..541534bd9d0 100644 --- a/dgraph/cmd/zero/raft.go +++ b/dgraph/cmd/zero/raft.go @@ -27,7 +27,7 @@ import ( otrace "go.opencensus.io/trace" - "github.com/dgraph-io/badger/y" + "github.com/dgraph-io/badger/v2/y" "github.com/dgraph-io/dgraph/conn" "github.com/dgraph-io/dgraph/protos/pb" "github.com/dgraph-io/dgraph/x" diff --git a/dgraph/cmd/zero/run.go b/dgraph/cmd/zero/run.go index d0407c4f8bb..5dd95c8ee18 100644 --- a/dgraph/cmd/zero/run.go +++ b/dgraph/cmd/zero/run.go @@ -33,8 +33,8 @@ import ( "golang.org/x/net/trace" "google.golang.org/grpc" - "github.com/dgraph-io/badger" - "github.com/dgraph-io/badger/y" + "github.com/dgraph-io/badger/v2" + "github.com/dgraph-io/badger/v2/y" "github.com/dgraph-io/dgraph/conn" "github.com/dgraph-io/dgraph/protos/pb" "github.com/dgraph-io/dgraph/raftwal" diff --git a/dgraph/cmd/zero/zero.go b/dgraph/cmd/zero/zero.go index 51fdf7d02f1..44e1abf7d80 100644 --- a/dgraph/cmd/zero/zero.go +++ b/dgraph/cmd/zero/zero.go @@ -26,7 +26,7 @@ import ( otrace "go.opencensus.io/trace" "golang.org/x/net/context" - "github.com/dgraph-io/badger/y" + "github.com/dgraph-io/badger/v2/y" "github.com/dgraph-io/dgo/v2/protos/api" "github.com/dgraph-io/dgraph/conn" "github.com/dgraph-io/dgraph/protos/pb" diff --git a/edgraph/access.go b/edgraph/access.go index 892bd4803b6..859cf50f321 100644 --- a/edgraph/access.go +++ b/edgraph/access.go @@ -21,7 +21,7 @@ package edgraph import ( "context" - "github.com/dgraph-io/badger/y" + "github.com/dgraph-io/badger/v2/y" "github.com/dgraph-io/dgo/v2/protos/api" "github.com/dgraph-io/dgraph/gql" "github.com/dgraph-io/dgraph/x" diff --git a/edgraph/access_ee.go b/edgraph/access_ee.go index c598ccd3da9..1c70feb2b2e 100644 --- a/edgraph/access_ee.go +++ b/edgraph/access_ee.go @@ -21,7 +21,7 @@ import ( "github.com/pkg/errors" - "github.com/dgraph-io/badger/y" + "github.com/dgraph-io/badger/v2/y" "github.com/dgraph-io/dgo/v2/protos/api" "github.com/dgraph-io/dgraph/ee/acl" diff --git a/edgraph/server.go b/edgraph/server.go index df0c6af008b..1d1436b8b71 100644 --- a/edgraph/server.go +++ b/edgraph/server.go @@ -30,8 +30,8 @@ import ( "time" "unicode" - "github.com/dgraph-io/badger" - "github.com/dgraph-io/badger/options" + "github.com/dgraph-io/badger/v2" + "github.com/dgraph-io/badger/v2/options" "github.com/dgraph-io/dgo/v2" "github.com/dgraph-io/dgo/v2/protos/api" diff --git a/ee/backup/backup.go b/ee/backup/backup.go index 7fc5c9649d2..947a88c75ec 100644 --- a/ee/backup/backup.go +++ b/ee/backup/backup.go @@ -23,8 +23,8 @@ import ( "net/url" "sync" - "github.com/dgraph-io/badger" - bpb "github.com/dgraph-io/badger/pb" + "github.com/dgraph-io/badger/v2" + bpb "github.com/dgraph-io/badger/v2/pb" "github.com/golang/glog" "github.com/pkg/errors" diff --git a/ee/backup/restore.go b/ee/backup/restore.go index e6a8ff12a32..d2fea6de885 100644 --- a/ee/backup/restore.go +++ b/ee/backup/restore.go @@ -22,9 +22,9 @@ import ( "math" "path/filepath" - "github.com/dgraph-io/badger" - "github.com/dgraph-io/badger/options" - bpb "github.com/dgraph-io/badger/pb" + "github.com/dgraph-io/badger/v2" + "github.com/dgraph-io/badger/v2/options" + bpb "github.com/dgraph-io/badger/v2/pb" "github.com/pkg/errors" "github.com/dgraph-io/dgraph/posting" diff --git a/go.mod b/go.mod index 54ce62debd6..22c6832163c 100644 --- a/go.mod +++ b/go.mod @@ -15,8 +15,8 @@ require ( github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd github.com/d4l3k/messagediff v1.2.1 // indirect github.com/dgraph-io/badger v0.0.0-20190917133922-cbdef65095c7 + github.com/dgraph-io/badger/v2 v2.0.0-rc4.0.20191104090122-cbbaefa2c523 github.com/dgraph-io/dgo/v2 v2.1.1-0.20191021171913-3efa60e5593b - github.com/dgraph-io/ristretto v0.0.0-20191010170704-2ba187ef9534 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 github.com/dgryski/go-groupvarint v0.0.0-20190318181831-5ce5df8ca4e1 @@ -56,6 +56,7 @@ require ( golang.org/x/text v0.3.2 google.golang.org/genproto v0.0.0-20190516172635-bb713bdc0e52 // indirect google.golang.org/grpc v1.23.0 + gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect gopkg.in/DataDog/dd-trace-go.v1 v1.13.1 // indirect gopkg.in/ini.v1 v1.48.0 // indirect gopkg.in/yaml.v2 v2.2.2 diff --git a/go.sum b/go.sum index 9e22e378e0a..f9c277f8cce 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ github.com/DataDog/datadog-go v0.0.0-20190425163447-40bafcb5f6c1 h1:fSu93OUqfEko github.com/DataDog/datadog-go v0.0.0-20190425163447-40bafcb5f6c1/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20190503082300-0f32ad59ab08 h1:5btKvK+N+FpW0EEgvxq7LWcUEwIRLsL4IwIo0u+Qlhs= github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20190503082300-0f32ad59ab08/go.mod h1:gMGUEe16aZh0QN941HgDjwrdjU4iTthPoz2/AtDRADE= +github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= +github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI= github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= @@ -56,14 +58,14 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgraph-io/badger v0.0.0-20190917133922-cbdef65095c7 h1:sXNo2Xtte2hOGeq21YyV3qjsyOScoQqWvLdBKInkQS8= github.com/dgraph-io/badger v0.0.0-20190917133922-cbdef65095c7/go.mod h1:V7oadAIwiqodXo5jT04lP+8ysOAuMFZO5x20VSI94Uc= -github.com/dgraph-io/dgo/v2 v2.1.1-0.20191011032519-062f5605f6da h1:JAadBE19xm15ZsBZ9HOSuQvMiaCKu+NsV5O5Mfb/62M= -github.com/dgraph-io/dgo/v2 v2.1.1-0.20191011032519-062f5605f6da/go.mod h1:R/MTZMGhTo60XSziuKpLXzI1OnVWQCZS5oJjxA8Q1bo= +github.com/dgraph-io/badger/v2 v2.0.0-rc4.0.20191104090122-cbbaefa2c523 h1:RjjP2rcz6qqBugT3TKzkbHpxrjPkNkIDljzw5LHDajc= +github.com/dgraph-io/badger/v2 v2.0.0-rc4.0.20191104090122-cbbaefa2c523/go.mod h1:YoRSIp1LmAJ7zH7tZwRvjNMUYLxB4wl3ebYkaIruZ04= github.com/dgraph-io/dgo/v2 v2.1.1-0.20191021171913-3efa60e5593b h1:p3gcEqQqQBp4jPIYO5x05sZbY0n8WVp+V1F/EnbJ1oo= github.com/dgraph-io/dgo/v2 v2.1.1-0.20191021171913-3efa60e5593b/go.mod h1:LJCkLxm5fUMcU+yb8gHFjHt7ChgNuz3YnQQ6MQkmscI= github.com/dgraph-io/ristretto v0.0.0-20190903064322-eb48d2f7ca30 h1:FkdGlqxPjfHKdVUzS5dOSMeOkGjEG7PnR1RLbcEqw88= github.com/dgraph-io/ristretto v0.0.0-20190903064322-eb48d2f7ca30/go.mod h1:UvZmzj8odp3S1nli6yEb1vLME8iJFBrRcw8rAJEiu9Q= -github.com/dgraph-io/ristretto v0.0.0-20191010170704-2ba187ef9534 h1:9G6fVccQriMJu4nXwpwLDoy9y31t/KUSLAbPcoBgv+4= -github.com/dgraph-io/ristretto v0.0.0-20191010170704-2ba187ef9534/go.mod h1:edzKIzGvqUCMzhTVWbiTSe75zD9Xxq0GtSBtFmaUTZs= +github.com/dgraph-io/ristretto v0.0.0-20191025175511-c1f00be0418e h1:aeUNgwup7PnDOBAD1BOKAqzb/W/NksOj6r3dwKKuqfg= +github.com/dgraph-io/ristretto v0.0.0-20191025175511-c1f00be0418e/go.mod h1:edzKIzGvqUCMzhTVWbiTSe75zD9Xxq0GtSBtFmaUTZs= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= @@ -331,6 +333,8 @@ google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 h1:FVCohIoYO7IJoDDVpV2pdq7SgrMH6wHnuTyrdrxJNoY= +gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0/go.mod h1:OdE7CF6DbADk7lN8LIKRzRJTTZXIjtWgA5THM5lhBAw= gopkg.in/DataDog/dd-trace-go.v1 v1.13.1 h1:oTzOClfuudNhW9Skkp2jxjqYO92uDKXqKLbiuPA13Rk= gopkg.in/DataDog/dd-trace-go.v1 v1.13.1/go.mod h1:DVp8HmDh8PuTu2Z0fVVlBsyWaC++fzwVCaGWylTe3tg= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= diff --git a/posting/index.go b/posting/index.go index ec30ece18e5..f7757c4b9a3 100644 --- a/posting/index.go +++ b/posting/index.go @@ -28,8 +28,8 @@ import ( ostats "go.opencensus.io/stats" otrace "go.opencensus.io/trace" - "github.com/dgraph-io/badger" - bpb "github.com/dgraph-io/badger/pb" + "github.com/dgraph-io/badger/v2" + bpb "github.com/dgraph-io/badger/v2/pb" "github.com/dgraph-io/dgraph/protos/pb" "github.com/dgraph-io/dgraph/schema" "github.com/dgraph-io/dgraph/tok" diff --git a/posting/index_test.go b/posting/index_test.go index dc7d04fa8c6..de149b96397 100644 --- a/posting/index_test.go +++ b/posting/index_test.go @@ -23,7 +23,7 @@ import ( "testing" "time" - "github.com/dgraph-io/badger" + "github.com/dgraph-io/badger/v2" "github.com/stretchr/testify/require" "github.com/dgraph-io/dgraph/protos/pb" diff --git a/posting/list.go b/posting/list.go index 64b505f7428..c029721f167 100644 --- a/posting/list.go +++ b/posting/list.go @@ -26,7 +26,7 @@ import ( "github.com/dgryski/go-farm" - bpb "github.com/dgraph-io/badger/pb" + bpb "github.com/dgraph-io/badger/v2/pb" "github.com/dgraph-io/dgo/v2/protos/api" "github.com/dgraph-io/dgraph/algo" "github.com/dgraph-io/dgraph/codec" diff --git a/posting/list_test.go b/posting/list_test.go index 95fe7881af2..dc1cad8cda5 100644 --- a/posting/list_test.go +++ b/posting/list_test.go @@ -26,8 +26,8 @@ import ( "strconv" "testing" - "github.com/dgraph-io/badger" - bpb "github.com/dgraph-io/badger/pb" + "github.com/dgraph-io/badger/v2" + bpb "github.com/dgraph-io/badger/v2/pb" "github.com/stretchr/testify/require" "github.com/dgraph-io/dgraph/protos/pb" diff --git a/posting/lists.go b/posting/lists.go index 2ab1ba57432..9f8ec721301 100644 --- a/posting/lists.go +++ b/posting/lists.go @@ -30,8 +30,8 @@ import ( ostats "go.opencensus.io/stats" - "github.com/dgraph-io/badger" - "github.com/dgraph-io/badger/y" + "github.com/dgraph-io/badger/v2" + "github.com/dgraph-io/badger/v2/y" "github.com/dgraph-io/dgo/v2/protos/api" "github.com/golang/glog" diff --git a/posting/mvcc.go b/posting/mvcc.go index 449415f3e77..33432535101 100644 --- a/posting/mvcc.go +++ b/posting/mvcc.go @@ -23,7 +23,7 @@ import ( "strconv" "sync/atomic" - "github.com/dgraph-io/badger" + "github.com/dgraph-io/badger/v2" "github.com/dgraph-io/dgo/v2/protos/api" "github.com/dgraph-io/dgraph/protos/pb" "github.com/dgraph-io/dgraph/x" diff --git a/posting/writer.go b/posting/writer.go index 6f0eb34b0f5..bdd512fc57f 100644 --- a/posting/writer.go +++ b/posting/writer.go @@ -20,8 +20,8 @@ import ( "math" "sync" - "github.com/dgraph-io/badger" - "github.com/dgraph-io/badger/pb" + "github.com/dgraph-io/badger/v2" + "github.com/dgraph-io/badger/v2/pb" "github.com/golang/glog" ) diff --git a/protos/pb/pb.pb.go b/protos/pb/pb.pb.go index 0eaafc25af9..d73ad68bc76 100644 --- a/protos/pb/pb.pb.go +++ b/protos/pb/pb.pb.go @@ -7,7 +7,7 @@ import ( context "context" encoding_binary "encoding/binary" fmt "fmt" - pb "github.com/dgraph-io/badger/pb" + pb "github.com/dgraph-io/badger/v2/pb" api "github.com/dgraph-io/dgo/v2/protos/api" proto "github.com/golang/protobuf/proto" grpc "google.golang.org/grpc" diff --git a/raftwal/storage.go b/raftwal/storage.go index c0587218c1f..4a745d3e3bb 100644 --- a/raftwal/storage.go +++ b/raftwal/storage.go @@ -22,7 +22,7 @@ import ( "math" "sync" - "github.com/dgraph-io/badger" + "github.com/dgraph-io/badger/v2" "github.com/dgraph-io/dgraph/protos/pb" "github.com/dgraph-io/dgraph/x" "github.com/gogo/protobuf/proto" diff --git a/raftwal/storage_test.go b/raftwal/storage_test.go index 46eb96659a0..b7c97759c72 100644 --- a/raftwal/storage_test.go +++ b/raftwal/storage_test.go @@ -39,7 +39,7 @@ import ( "reflect" "testing" - "github.com/dgraph-io/badger" + "github.com/dgraph-io/badger/v2" "github.com/stretchr/testify/require" "go.etcd.io/etcd/raft" pb "go.etcd.io/etcd/raft/raftpb" diff --git a/schema/parse_test.go b/schema/parse_test.go index dfdc5cffa71..40c959a103c 100644 --- a/schema/parse_test.go +++ b/schema/parse_test.go @@ -21,7 +21,7 @@ import ( "os" "testing" - "github.com/dgraph-io/badger" + "github.com/dgraph-io/badger/v2" "github.com/stretchr/testify/require" "github.com/dgraph-io/dgraph/protos/pb" diff --git a/schema/schema.go b/schema/schema.go index 007e18ae525..ae2ada83f8c 100644 --- a/schema/schema.go +++ b/schema/schema.go @@ -22,7 +22,7 @@ import ( "fmt" "sync" - "github.com/dgraph-io/badger" + "github.com/dgraph-io/badger/v2" "github.com/golang/glog" "github.com/golang/protobuf/proto" "golang.org/x/net/trace" diff --git a/testutil/backup.go b/testutil/backup.go index 843ddd686c4..cfb3e48cb03 100644 --- a/testutil/backup.go +++ b/testutil/backup.go @@ -21,9 +21,9 @@ import ( "fmt" "math" - "github.com/dgraph-io/badger" - "github.com/dgraph-io/badger/options" - bpb "github.com/dgraph-io/badger/pb" + "github.com/dgraph-io/badger/v2" + "github.com/dgraph-io/badger/v2/options" + bpb "github.com/dgraph-io/badger/v2/pb" "github.com/dgraph-io/dgraph/posting" "github.com/dgraph-io/dgraph/protos/pb" "github.com/dgraph-io/dgraph/types" diff --git a/worker/draft.go b/worker/draft.go index 8f25b255cd3..a33279ea4ed 100644 --- a/worker/draft.go +++ b/worker/draft.go @@ -33,9 +33,9 @@ import ( "go.opencensus.io/tag" otrace "go.opencensus.io/trace" - "github.com/dgraph-io/badger" - bpb "github.com/dgraph-io/badger/pb" - "github.com/dgraph-io/badger/y" + "github.com/dgraph-io/badger/v2" + bpb "github.com/dgraph-io/badger/v2/pb" + "github.com/dgraph-io/badger/v2/y" "github.com/dgraph-io/dgraph/conn" "github.com/dgraph-io/dgraph/dgraph/cmd/zero" "github.com/dgraph-io/dgraph/posting" diff --git a/worker/draft_test.go b/worker/draft_test.go index 26b66bc6a89..c4c0ce2c530 100644 --- a/worker/draft_test.go +++ b/worker/draft_test.go @@ -21,7 +21,7 @@ import ( "os" "testing" - "github.com/dgraph-io/badger" + "github.com/dgraph-io/badger/v2" "github.com/dgraph-io/dgraph/posting" "github.com/dgraph-io/dgraph/protos/pb" "github.com/dgraph-io/dgraph/raftwal" diff --git a/worker/export.go b/worker/export.go index d87c81cfa24..d55b1d7d4ea 100644 --- a/worker/export.go +++ b/worker/export.go @@ -32,8 +32,8 @@ import ( "github.com/pkg/errors" "golang.org/x/net/context" - "github.com/dgraph-io/badger" - bpb "github.com/dgraph-io/badger/pb" + "github.com/dgraph-io/badger/v2" + bpb "github.com/dgraph-io/badger/v2/pb" "github.com/dgraph-io/dgo/v2/protos/api" diff --git a/worker/groups.go b/worker/groups.go index 26c1c9f0377..0a26804b472 100644 --- a/worker/groups.go +++ b/worker/groups.go @@ -26,8 +26,8 @@ import ( "golang.org/x/net/context" - "github.com/dgraph-io/badger" - "github.com/dgraph-io/badger/y" + "github.com/dgraph-io/badger/v2" + "github.com/dgraph-io/badger/v2/y" "github.com/dgraph-io/dgo/v2/protos/api" "github.com/dgraph-io/dgraph/conn" "github.com/dgraph-io/dgraph/protos/pb" diff --git a/worker/mutation.go b/worker/mutation.go index ea13f787297..25e02160a84 100644 --- a/worker/mutation.go +++ b/worker/mutation.go @@ -21,7 +21,7 @@ import ( "math" "time" - "github.com/dgraph-io/badger" + "github.com/dgraph-io/badger/v2" "github.com/dgraph-io/dgo/v2" "github.com/dgraph-io/dgo/v2/protos/api" diff --git a/worker/predicate_move.go b/worker/predicate_move.go index 9bb21c7fa83..168bb7bfa64 100644 --- a/worker/predicate_move.go +++ b/worker/predicate_move.go @@ -26,8 +26,8 @@ import ( otrace "go.opencensus.io/trace" "golang.org/x/net/context" - "github.com/dgraph-io/badger" - bpb "github.com/dgraph-io/badger/pb" + "github.com/dgraph-io/badger/v2" + bpb "github.com/dgraph-io/badger/v2/pb" "github.com/dgraph-io/dgo/v2/protos/api" "github.com/dgraph-io/dgraph/posting" "github.com/dgraph-io/dgraph/protos/pb" diff --git a/worker/predicate_test.go b/worker/predicate_test.go index 7c8c014b7a3..ded1fa12f05 100644 --- a/worker/predicate_test.go +++ b/worker/predicate_test.go @@ -24,7 +24,7 @@ import ( "sync/atomic" "testing" - "github.com/dgraph-io/badger" + "github.com/dgraph-io/badger/v2" "github.com/golang/glog" "github.com/stretchr/testify/require" "google.golang.org/grpc" diff --git a/worker/snapshot.go b/worker/snapshot.go index 7cb6e010aed..f45380ca0ba 100644 --- a/worker/snapshot.go +++ b/worker/snapshot.go @@ -19,11 +19,11 @@ package worker import ( "sync/atomic" - bpb "github.com/dgraph-io/badger/pb" + bpb "github.com/dgraph-io/badger/v2/pb" "github.com/golang/glog" "go.etcd.io/etcd/raft" - "github.com/dgraph-io/badger" + "github.com/dgraph-io/badger/v2" "github.com/dgraph-io/dgraph/conn" "github.com/dgraph-io/dgraph/posting" "github.com/dgraph-io/dgraph/protos/pb" diff --git a/worker/sort.go b/worker/sort.go index b6deb083ba2..6a9862603c0 100644 --- a/worker/sort.go +++ b/worker/sort.go @@ -22,7 +22,7 @@ import ( "strings" "time" - "github.com/dgraph-io/badger" + "github.com/dgraph-io/badger/v2" "github.com/golang/glog" "github.com/pkg/errors" otrace "go.opencensus.io/trace" diff --git a/worker/task.go b/worker/task.go index baf11f25b7f..c1602a74c2f 100644 --- a/worker/task.go +++ b/worker/task.go @@ -23,7 +23,7 @@ import ( "strings" "time" - "github.com/dgraph-io/badger" + "github.com/dgraph-io/badger/v2" "github.com/dgraph-io/dgo/v2/protos/api" "github.com/dgraph-io/dgraph/algo" "github.com/dgraph-io/dgraph/conn" diff --git a/worker/tokens.go b/worker/tokens.go index 49236f636b3..a8095e2edf9 100644 --- a/worker/tokens.go +++ b/worker/tokens.go @@ -17,7 +17,7 @@ package worker import ( - "github.com/dgraph-io/badger" + "github.com/dgraph-io/badger/v2" "bytes" diff --git a/worker/worker.go b/worker/worker.go index 96641ddcf3b..9dc6e0342b6 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -25,7 +25,7 @@ import ( "net" "sync" - "github.com/dgraph-io/badger" + "github.com/dgraph-io/badger/v2" "github.com/dgraph-io/dgraph/conn" "github.com/dgraph-io/dgraph/protos/pb" "github.com/dgraph-io/dgraph/x" diff --git a/worker/worker_test.go b/worker/worker_test.go index 676b4ee65fa..8e7a4db9ad8 100644 --- a/worker/worker_test.go +++ b/worker/worker_test.go @@ -27,7 +27,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/dgraph-io/badger" + "github.com/dgraph-io/badger/v2" "github.com/dgraph-io/dgo/v2" "github.com/dgraph-io/dgo/v2/protos/api" diff --git a/xidmap/xidmap.go b/xidmap/xidmap.go index f9ffa7b7973..56638ac68ec 100644 --- a/xidmap/xidmap.go +++ b/xidmap/xidmap.go @@ -26,7 +26,7 @@ import ( "google.golang.org/grpc" - "github.com/dgraph-io/badger" + "github.com/dgraph-io/badger/v2" "github.com/dgraph-io/dgraph/protos/pb" "github.com/dgraph-io/dgraph/x" farm "github.com/dgryski/go-farm" diff --git a/xidmap/xidmap_test.go b/xidmap/xidmap_test.go index 63b5ca4cdf0..8fa10e98b93 100644 --- a/xidmap/xidmap_test.go +++ b/xidmap/xidmap_test.go @@ -10,7 +10,7 @@ import ( "testing" "time" - "github.com/dgraph-io/badger" + "github.com/dgraph-io/badger/v2" "github.com/dgraph-io/dgraph/testutil" "github.com/dgraph-io/dgraph/x" "github.com/stretchr/testify/require" From 166a3578bc0450b67f2cc3ef4b7f47eb55d7c5f5 Mon Sep 17 00:00:00 2001 From: Ashish Goswami Date: Mon, 11 Nov 2019 16:22:00 +0530 Subject: [PATCH 2/6] Introduce StreamDone --- dgraph/cmd/bulk/reduce.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/dgraph/cmd/bulk/reduce.go b/dgraph/cmd/bulk/reduce.go index 1f15d98ad9f..c6d44b2ee54 100644 --- a/dgraph/cmd/bulk/reduce.go +++ b/dgraph/cmd/bulk/reduce.go @@ -165,13 +165,32 @@ func (r *reducer) encodeAndWrite( kv.StreamId = streamId } + addDone := func(l *bpb.KVList, doneSteams []uint32) *bpb.KVList { + for _, SID := range doneSteams { + l.Kv = append(l.Kv, &bpb.KV{StreamId: SID, StreamDone: true}) + } + return l + } + + var doneStreams []uint32 + var prevSID uint32 + updateDoneStreams := func(curSID uint32) { + if prevSID != 0 && (prevSID != curSID) { + doneStreams = append(doneStreams, prevSID) + } + } + for batch := range entryCh { listSize += r.toList(batch, list) if listSize > 4<<20 { for _, kv := range list.Kv { setStreamId(kv) + updateDoneStreams(kv.StreamId) + prevSID = kv.StreamId } + list = addDone(list, doneStreams) x.Check(writer.Write(list)) + doneStreams = doneStreams[:0] list = &bpb.KVList{} listSize = 0 } From 9ab49765e2d41c96a4413a3705cd2394cd95e24b Mon Sep 17 00:00:00 2001 From: Ashish Goswami Date: Fri, 22 Nov 2019 14:27:09 +0530 Subject: [PATCH 3/6] Address review comments --- dgraph/cmd/bulk/reduce.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/dgraph/cmd/bulk/reduce.go b/dgraph/cmd/bulk/reduce.go index c6d44b2ee54..c23fa242bc4 100644 --- a/dgraph/cmd/bulk/reduce.go +++ b/dgraph/cmd/bulk/reduce.go @@ -159,17 +159,20 @@ func (r *reducer) encodeAndWrite( streamId = atomic.AddUint32(&r.streamId, 1) preds[pk.Attr] = streamId } - // TODO: Having many stream ids can cause memory issues with StreamWriter. So, we - // should build a way in StreamWriter to indicate that the stream is over, so the - // table for that stream can be flushed and memory released. + kv.StreamId = streamId } - addDone := func(l *bpb.KVList, doneSteams []uint32) *bpb.KVList { + // Once we have processed all records from single stream, we can mark that stream as done. + // This will close underlying table builder in Badger for stream. Since we preallocate 1 MB + // of memory for each table builder, this can result in memory save in case we have large + // number of streams. + // This change limits maximum number of open streams to number of streams created in a single + // write call. This can also be optimised if required. + addDone := func(doneSteams []uint32, l *bpb.KVList) { for _, SID := range doneSteams { l.Kv = append(l.Kv, &bpb.KV{StreamId: SID, StreamDone: true}) } - return l } var doneStreams []uint32 @@ -188,7 +191,7 @@ func (r *reducer) encodeAndWrite( updateDoneStreams(kv.StreamId) prevSID = kv.StreamId } - list = addDone(list, doneStreams) + addDone(doneStreams, list) x.Check(writer.Write(list)) doneStreams = doneStreams[:0] list = &bpb.KVList{} From c7a59615873e2cd0ea284ac4a129ccfb8597660c Mon Sep 17 00:00:00 2001 From: Ashish Goswami Date: Mon, 25 Nov 2019 17:36:14 +0530 Subject: [PATCH 4/6] Set Badger cache size to 1MB --- dgraph/cmd/bulk/reduce.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dgraph/cmd/bulk/reduce.go b/dgraph/cmd/bulk/reduce.go index c23fa242bc4..6b92bba21ca 100644 --- a/dgraph/cmd/bulk/reduce.go +++ b/dgraph/cmd/bulk/reduce.go @@ -90,7 +90,7 @@ func (r *reducer) run() error { func (r *reducer) createBadger(i int) *badger.DB { opt := badger.DefaultOptions(r.opt.shardOutputDirs[i]).WithSyncWrites(false). WithTableLoadingMode(bo.MemoryMap).WithValueThreshold(1 << 10 /* 1 KB */). - WithLogger(nil) + WithLogger(nil).WithMaxCacheSize(1 << 20) db, err := badger.OpenManaged(opt) x.Check(err) r.dbs = append(r.dbs, db) From e2eefa95d964320f7036d1a6d6ee0bd36f2e3e67 Mon Sep 17 00:00:00 2001 From: Ashish Goswami Date: Mon, 25 Nov 2019 17:48:02 +0530 Subject: [PATCH 5/6] Minor fix --- dgraph/cmd/bulk/reduce.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dgraph/cmd/bulk/reduce.go b/dgraph/cmd/bulk/reduce.go index 6b92bba21ca..eca824fb0c9 100644 --- a/dgraph/cmd/bulk/reduce.go +++ b/dgraph/cmd/bulk/reduce.go @@ -165,7 +165,7 @@ func (r *reducer) encodeAndWrite( // Once we have processed all records from single stream, we can mark that stream as done. // This will close underlying table builder in Badger for stream. Since we preallocate 1 MB - // of memory for each table builder, this can result in memory save in case we have large + // of memory for each table builder, this can result in memory saving in case we have large // number of streams. // This change limits maximum number of open streams to number of streams created in a single // write call. This can also be optimised if required. From e46f17edccf838e4c97aa20a5982f11f46c1eaf9 Mon Sep 17 00:00:00 2001 From: Ashish Goswami Date: Tue, 26 Nov 2019 15:17:47 +0530 Subject: [PATCH 6/6] Address review comments --- dgraph/cmd/bulk/reduce.go | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/dgraph/cmd/bulk/reduce.go b/dgraph/cmd/bulk/reduce.go index eca824fb0c9..ec750271659 100644 --- a/dgraph/cmd/bulk/reduce.go +++ b/dgraph/cmd/bulk/reduce.go @@ -170,25 +170,21 @@ func (r *reducer) encodeAndWrite( // This change limits maximum number of open streams to number of streams created in a single // write call. This can also be optimised if required. addDone := func(doneSteams []uint32, l *bpb.KVList) { - for _, SID := range doneSteams { - l.Kv = append(l.Kv, &bpb.KV{StreamId: SID, StreamDone: true}) + for _, streamId := range doneSteams { + l.Kv = append(l.Kv, &bpb.KV{StreamId: streamId, StreamDone: true}) } } var doneStreams []uint32 var prevSID uint32 - updateDoneStreams := func(curSID uint32) { - if prevSID != 0 && (prevSID != curSID) { - doneStreams = append(doneStreams, prevSID) - } - } - for batch := range entryCh { listSize += r.toList(batch, list) if listSize > 4<<20 { for _, kv := range list.Kv { setStreamId(kv) - updateDoneStreams(kv.StreamId) + if prevSID != 0 && (prevSID != kv.StreamId) { + doneStreams = append(doneStreams, prevSID) + } prevSID = kv.StreamId } addDone(doneStreams, list)