From e5726c0066ccdd299a2ec9262f93c7896cdfcd87 Mon Sep 17 00:00:00 2001 From: Ismail Khoffi Date: Wed, 15 Aug 2018 02:19:42 +0100 Subject: [PATCH] Release 0.10.0 (#100) * dep: Change tendermint dep to be ^v0.22.0 (#91) * Mutable/Immutable refactor and GetImmutable snapshots (#92) * Release 0.10.0: Update Changelog and bump version (#99) See changelog: https://github.com/tendermint/iavl/blob/develop/CHANGELOG.md#0100 --- CHANGELOG.md | 24 +- Gopkg.lock | 72 ++- Gopkg.toml | 2 +- basic_test.go | 35 +- benchmarks/README.md | 16 +- benchmarks/bench_test.go | 18 +- ...al-ocean-64gb-fullbench-memory-8f19f23.txt | 149 ++++++ ...al-ocean-64gb-fullbench-memory-c1f6d4e.txt | 149 ++++++ benchmarks/setup/INSTALL_ROOT.sh | 8 +- benchmarks/setup/INSTALL_USER.sh | 26 - benchmarks/setup/RUN_BENCHMARKS.sh | 26 +- doc.go | 4 +- tree.go => immutable_tree.go | 99 +--- mutable_tree.go | 469 ++++++++++++++++++ node.go | 213 +------- nodedb.go | 4 + orphaning_tree.go | 74 --- proof.go | 4 +- proof_range.go | 26 +- proof_test.go | 15 +- testutils_test.go | 6 +- tree_dotgraph.go | 2 +- tree_dotgraph_test.go | 6 +- tree_fuzz_test.go | 8 +- tree_test.go | 152 ++++-- util.go | 4 +- version.go | 2 +- versioned_tree.go | 197 -------- 28 files changed, 1101 insertions(+), 709 deletions(-) create mode 100644 benchmarks/results/digital-ocean-64gb-fullbench-memory-8f19f23.txt create mode 100644 benchmarks/results/digital-ocean-64gb-fullbench-memory-c1f6d4e.txt delete mode 100755 benchmarks/setup/INSTALL_USER.sh rename tree.go => immutable_tree.go (55%) create mode 100644 mutable_tree.go delete mode 100644 orphaning_tree.go delete mode 100644 versioned_tree.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ace77964..46e3f5ca4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,30 @@ # Changelog +## 0.10.0 + +BREAKING CHANGES + +- refactored API for clean separation of [mutable][1] and [immutable][2] tree (#92, #88); +with possibility to: + - load read-only snapshots at previous versions on demand + - load mutable trees at the most recently saved tree + +[1]: https://github.com/tendermint/iavl/blob/9e62436856efa94c1223043be36ebda01ae0b6fc/mutable_tree.go#L14-L21 +[2]: https://github.com/tendermint/iavl/blob/9e62436856efa94c1223043be36ebda01ae0b6fc/immutable_tree.go#L10-L17 + +BUG FIXES + +- remove memory leaks (#92) + +IMPROVEMENTS + +- Change tendermint dep to ^v0.22.0 (#91) + ## 0.9.2 (July 3, 2018) -IMPROVEMTS +IMPROVEMENTS -- some minor changes: mainly lints, updated parts of documentation, unexported some helpers (#80) +- some minor changes: mainly lints, updated parts of documentation, unexported some helpers (#80) ## 0.9.1 (July 1, 2018) diff --git a/Gopkg.lock b/Gopkg.lock index 9a4174900..4ebc929aa 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -2,80 +2,115 @@ [[projects]] + digest = "1:a2c1d0e43bd3baaa071d1b9ed72c27d78169b2b269f71c105ac4ba34b1be4a39" name = "github.com/davecgh/go-spew" packages = ["spew"] + pruneopts = "UT" revision = "346938d642f2ec3594ed81d874461961cd0faa76" version = "v1.1.0" [[projects]] + digest = "1:d0309acc14b09c713e2fd1283be859c8e03f5f40c69e221410a0f652742b139c" name = "github.com/go-kit/kit" packages = [ "log", "log/level", - "log/term" + "log/term", ] + pruneopts = "UT" revision = "4dc7be5d2d12881735283bcab7352178e190fc71" version = "v0.6.0" [[projects]] + digest = "1:31a18dae27a29aa074515e43a443abfd2ba6deb6d69309d8d7ce789c45f34659" name = "github.com/go-logfmt/logfmt" packages = ["."] + pruneopts = "UT" revision = "390ab7935ee28ec6b286364bba9b4dd6410cb3d5" version = "v0.3.0" [[projects]] + digest = "1:c4a2528ccbcabf90f9f3c464a5fc9e302d592861bbfd0b7135a7de8a943d0406" name = "github.com/go-stack/stack" packages = ["."] + pruneopts = "UT" revision = "259ab82a6cad3992b4e21ff5cac294ccb06474bc" version = "v1.7.0" [[projects]] + digest = "1:0cff52d4289e3d31494fb4e63d00c93be0f9d07ad1e3447d1dc300add4e32224" + name = "github.com/gogo/protobuf" + packages = [ + "gogoproto", + "proto", + "protoc-gen-gogo/descriptor", + ] + pruneopts = "UT" + revision = "636bf0302bc95575d69441b25a2603156ffdddf1" + version = "v1.1.1" + +[[projects]] + digest = "1:15042ad3498153684d09f393bbaec6b216c8eec6d61f63dff711de7d64ed8861" name = "github.com/golang/protobuf" packages = ["proto"] - revision = "925541529c1fa6821df4e44ce2723319eb2be768" - version = "v1.0.0" + pruneopts = "UT" + revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265" + version = "v1.1.0" [[projects]] branch = "master" + digest = "1:4a0c6bb4805508a6287675fac876be2ac1182539ca8a32468d8128882e9d5009" name = "github.com/golang/snappy" packages = ["."] + pruneopts = "UT" revision = "2e65f85255dbc3072edf28d6b5b8efc472979f5a" [[projects]] branch = "master" + digest = "1:39b27d1381a30421f9813967a5866fba35dc1d4df43a6eefe3b7a5444cb07214" name = "github.com/jmhodges/levigo" packages = ["."] + pruneopts = "UT" revision = "c42d9e0ca023e2198120196f842701bb4c55d7b9" [[projects]] branch = "master" + digest = "1:a64e323dc06b73892e5bb5d040ced475c4645d456038333883f58934abbf6f72" name = "github.com/kr/logfmt" packages = ["."] + pruneopts = "UT" revision = "b84e30acd515aadc4b783ad4ff83aff3299bdfe0" [[projects]] + digest = "1:40e195917a951a8bf867cd05de2a46aaf1806c50cf92eebf4c16f78cd196f747" name = "github.com/pkg/errors" packages = ["."] + pruneopts = "UT" revision = "645ef00459ed84a119197bfb8d8205042c6df63d" version = "v0.8.0" [[projects]] + digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe" name = "github.com/pmezard/go-difflib" packages = ["difflib"] + pruneopts = "UT" revision = "792786c7400a136282c1664665ae0a8db921c6c2" version = "v1.0.0" [[projects]] + digest = "1:befd7181c2f92c9f05abf17cd7e15538919f19cf7871d794cacc45a4fb99c135" name = "github.com/stretchr/testify" packages = [ "assert", - "require" + "require", ] + pruneopts = "UT" revision = "f35b8ab0b5a2cef36673838d662e249dd9c94686" version = "v1.2.2" [[projects]] branch = "master" + digest = "1:bd62f27525a36697564991b8e6071ff56afa99d3235261924a0212db5ce780bd" name = "github.com/syndtr/goleveldb" packages = [ "leveldb", @@ -89,40 +124,57 @@ "leveldb/opt", "leveldb/storage", "leveldb/table", - "leveldb/util" + "leveldb/util", ] + pruneopts = "UT" revision = "0d5a0ceb10cf9ab89fdd744cc8c50a83134f6697" [[projects]] + digest = "1:e9113641c839c21d8eaeb2c907c7276af1eddeed988df8322168c56b7e06e0e1" name = "github.com/tendermint/go-amino" packages = ["."] + pruneopts = "UT" revision = "2106ca61d91029c931fd54968c2bb02dc96b1412" version = "0.10.1" [[projects]] + digest = "1:3502c7cf4ada0e74d6a21e524fdd8ad8741b0a8da5219b9c38e3890d5ebbad1c" name = "github.com/tendermint/tendermint" packages = [ "crypto/tmhash", "libs/common", "libs/db", "libs/log", - "libs/test" + "libs/test", ] - revision = "5923b6288fe8ce9581936ee97c2bf9cf9c02c2f4" - version = "v0.22.0-rc2" + pruneopts = "UT" + revision = "5fdbcd70df57b71ffba71e1ff5f00d617852a9c0" + version = "v0.22.6" [[projects]] branch = "master" + digest = "1:ac66d893cc5a3e78b27affceb96cdcc284318ae8f8aa86f9f6f51322d7d744f5" name = "golang.org/x/crypto" packages = [ "ripemd160", - "sha3" + "sha3", ] + pruneopts = "UT" revision = "a49355c7e3f8fe157a85be2f77e6e269a0f89602" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "06b259c645a1a884f870ef81ab220a7302b156f6fc1f1500779d540a7eed0d49" + input-imports = [ + "github.com/stretchr/testify/assert", + "github.com/stretchr/testify/require", + "github.com/tendermint/go-amino", + "github.com/tendermint/tendermint/crypto/tmhash", + "github.com/tendermint/tendermint/libs/common", + "github.com/tendermint/tendermint/libs/db", + "github.com/tendermint/tendermint/libs/test", + "golang.org/x/crypto/ripemd160", + "golang.org/x/crypto/sha3", + ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index b00a0c141..39510f840 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -7,7 +7,7 @@ name = "github.com/tendermint/go-amino" [[constraint]] - version = "=0.22.0-rc2" + version = "^0.22.0" name = "github.com/tendermint/tendermint" [prune] diff --git a/basic_test.go b/basic_test.go index 5dca7cb20..07e4cb9fa 100644 --- a/basic_test.go +++ b/basic_test.go @@ -12,7 +12,7 @@ import ( ) func TestBasic(t *testing.T) { - tree := NewTree(nil, 0) + tree := NewMutableTree(db.NewMemDB(), 0) up := tree.Set([]byte("1"), []byte("one")) if up { t.Error("Did not expect an update (should have been create)") @@ -103,7 +103,7 @@ func TestBasic(t *testing.T) { func TestUnit(t *testing.T) { - expectHash := func(tree *Tree, hashCount int64) { + expectHash := func(tree *ImmutableTree, hashCount int64) { // ensure number of new hash calculations is as expected. hash, count := tree.hashWithCount() if count != hashCount { @@ -121,7 +121,7 @@ func TestUnit(t *testing.T) { } } - expectSet := func(tree *Tree, i int, repr string, hashCount int64) { + expectSet := func(tree *MutableTree, i int, repr string, hashCount int64) { origNode := tree.root updated := tree.Set(i2b(i), []byte{}) // ensure node was added & structure is as expected. @@ -130,11 +130,11 @@ func TestUnit(t *testing.T) { i, P(origNode), repr, P(tree.root), updated) } // ensure hash calculation requirements - expectHash(tree, hashCount) + expectHash(tree.ImmutableTree, hashCount) tree.root = origNode } - expectRemove := func(tree *Tree, i int, repr string, hashCount int64) { + expectRemove := func(tree *MutableTree, i int, repr string, hashCount int64) { origNode := tree.root value, removed := tree.Remove(i2b(i)) // ensure node was added & structure is as expected. @@ -143,7 +143,7 @@ func TestUnit(t *testing.T) { i, P(origNode), repr, P(tree.root), value, removed) } // ensure hash calculation requirements - expectHash(tree, hashCount) + expectHash(tree.ImmutableTree, hashCount) tree.root = origNode } @@ -190,7 +190,7 @@ func TestRemove(t *testing.T) { d := db.NewDB("test", "memdb", "") defer d.Close() - t1 := NewVersionedTree(d, size) + t1 := NewMutableTree(d, size) // insert a bunch of random nodes keys := make([][]byte, size) @@ -220,7 +220,7 @@ func TestIntegration(t *testing.T) { } records := make([]*record, 400) - tree := NewTree(nil, 0) + tree := NewMutableTree(db.NewMemDB(), 0) randomRecord := func() *record { return &record{randstr(20), randstr(20)} @@ -302,7 +302,7 @@ func TestIterateRange(t *testing.T) { } sort.Strings(keys) - tree := NewTree(nil, 0) + tree := NewMutableTree(db.NewMemDB(), 0) // insert all the data for _, r := range records { @@ -372,14 +372,14 @@ func TestPersistence(t *testing.T) { } // Construct some tree and save it - t1 := NewVersionedTree(db, 0) + t1 := NewMutableTree(db, 0) for key, value := range records { t1.Set([]byte(key), []byte(value)) } t1.SaveVersion() // Load a tree - t2 := NewVersionedTree(db, 0) + t2 := NewMutableTree(db, 0) t2.Load() for key, value := range records { _, t2value := t2.Get64([]byte(key)) @@ -390,12 +390,11 @@ func TestPersistence(t *testing.T) { } func TestProof(t *testing.T) { - t.Skipf("This test has a race condition causing it to occasionally panic.") // Construct some random tree db := db.NewMemDB() - tree := NewVersionedTree(db, 100) - for i := 0; i < 1000; i++ { + tree := NewMutableTree(db, 100) + for i := 0; i < 10; i++ { key, value := randstr(20), randstr(20) tree.Set([]byte(key), []byte(value)) } @@ -404,7 +403,7 @@ func TestProof(t *testing.T) { tree.SaveVersion() // Add more items so it's not all persisted - for i := 0; i < 100; i++ { + for i := 0; i < 10; i++ { key, value := randstr(20), randstr(20) tree.Set([]byte(key), []byte(value)) } @@ -415,7 +414,7 @@ func TestProof(t *testing.T) { assert.NoError(t, err) assert.Equal(t, value, value2) if assert.NotNil(t, proof) { - verifyProof(t, proof, tree.Hash()) + verifyProof(t, proof, tree.WorkingHash()) } return false }) @@ -423,7 +422,7 @@ func TestProof(t *testing.T) { func TestTreeProof(t *testing.T) { db := db.NewMemDB() - tree := NewTree(db, 100) + tree := NewMutableTree(db, 100) // should get false for proof with nil root _, _, err := tree.GetWithProof([]byte("foo")) @@ -442,7 +441,7 @@ func TestTreeProof(t *testing.T) { assert.NoError(t, err) // valid proof for real keys - root := tree.Hash() + root := tree.WorkingHash() for _, key := range keys { value, proof, err := tree.GetWithProof(key) if assert.NoError(t, err) { diff --git a/benchmarks/README.md b/benchmarks/README.md index be253d827..a6295385c 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -2,7 +2,7 @@ These instructions are mainly for running the benchmarks on an cloud instance that is intended to be thrown away, not on a dev machine. Be careful with the install scripts locally. -This has only been tested on Ubuntu 16.04. It *should* work on Ubuntu 14.04 as well. It *may* work on Debian, but has never been tested. +This has only been tested on Ubuntu 16.04 and 18.04. It *should* work on Ubuntu 14.04 as well. It *may* work on Debian, but has never been tested. ## Setting up the machine @@ -14,34 +14,26 @@ scp -r setup user@host: ssh user@host ``` -Run the install scripts (once per machine) +Run the install script (once per machine) ``` cd setup chmod +x * sudo ./INSTALL_ROOT.sh -./INSTALL_USER.sh ``` ## Running the tests -Make sure the hostname is set to a good value for recording: - -``` -hostname -s -sudo hostname -``` - Run the benchmarks in a screen: ``` screen -~/RUN_BENCHMARKS.sh +./RUN_BENCHMARKS.sh ``` Copy them back from your local machine: ``` -scp user@host:go/src/github.com/tendermint/merkleeyes/iavl/benchmarks/results/* results +scp user@host:go/src/github.com/tendermint/iavl/results.txt results.txt git add results ``` diff --git a/benchmarks/bench_test.go b/benchmarks/bench_test.go index 820ba8db3..9284a7a2e 100644 --- a/benchmarks/bench_test.go +++ b/benchmarks/bench_test.go @@ -20,8 +20,8 @@ func randBytes(length int) []byte { return key } -func prepareTree(b *testing.B, db db.DB, size, keyLen, dataLen int) (*iavl.VersionedTree, [][]byte) { - t := iavl.NewVersionedTree(db, size) +func prepareTree(b *testing.B, db db.DB, size, keyLen, dataLen int) (*iavl.MutableTree, [][]byte) { + t := iavl.NewMutableTree(db, size) keys := make([][]byte, size) for i := 0; i < size; i++ { @@ -35,7 +35,7 @@ func prepareTree(b *testing.B, db db.DB, size, keyLen, dataLen int) (*iavl.Versi } // commit tree saves a new version and deletes and old one... -func commitTree(b *testing.B, t *iavl.VersionedTree) { +func commitTree(b *testing.B, t *iavl.MutableTree) { t.Hash() _, version, err := t.SaveVersion() if err != nil { @@ -49,14 +49,14 @@ func commitTree(b *testing.B, t *iavl.VersionedTree) { } } -func runQueries(b *testing.B, t *iavl.VersionedTree, keyLen int) { +func runQueries(b *testing.B, t *iavl.MutableTree, keyLen int) { for i := 0; i < b.N; i++ { q := randBytes(keyLen) t.Get(q) } } -func runKnownQueries(b *testing.B, t *iavl.VersionedTree, keys [][]byte) { +func runKnownQueries(b *testing.B, t *iavl.MutableTree, keys [][]byte) { l := int32(len(keys)) for i := 0; i < b.N; i++ { q := keys[rand.Int31n(l)] @@ -64,7 +64,7 @@ func runKnownQueries(b *testing.B, t *iavl.VersionedTree, keys [][]byte) { } } -func runInsert(b *testing.B, t *iavl.VersionedTree, keyLen, dataLen, blockSize int) *iavl.VersionedTree { +func runInsert(b *testing.B, t *iavl.MutableTree, keyLen, dataLen, blockSize int) *iavl.MutableTree { for i := 1; i <= b.N; i++ { t.Set(randBytes(keyLen), randBytes(dataLen)) if i%blockSize == 0 { @@ -75,7 +75,7 @@ func runInsert(b *testing.B, t *iavl.VersionedTree, keyLen, dataLen, blockSize i return t } -func runUpdate(b *testing.B, t *iavl.VersionedTree, dataLen, blockSize int, keys [][]byte) *iavl.VersionedTree { +func runUpdate(b *testing.B, t *iavl.MutableTree, dataLen, blockSize int, keys [][]byte) *iavl.MutableTree { l := int32(len(keys)) for i := 1; i <= b.N; i++ { key := keys[rand.Int31n(l)] @@ -87,7 +87,7 @@ func runUpdate(b *testing.B, t *iavl.VersionedTree, dataLen, blockSize int, keys return t } -func runDelete(b *testing.B, t *iavl.VersionedTree, blockSize int, keys [][]byte) *iavl.VersionedTree { +func runDelete(b *testing.B, t *iavl.MutableTree, blockSize int, keys [][]byte) *iavl.MutableTree { var key []byte l := int32(len(keys)) for i := 1; i <= b.N; i++ { @@ -103,7 +103,7 @@ func runDelete(b *testing.B, t *iavl.VersionedTree, blockSize int, keys [][]byte } // runBlock measures time for an entire block, not just one tx -func runBlock(b *testing.B, t *iavl.VersionedTree, keyLen, dataLen, blockSize int, keys [][]byte) *iavl.VersionedTree { +func runBlock(b *testing.B, t *iavl.MutableTree, keyLen, dataLen, blockSize int, keys [][]byte) *iavl.MutableTree { l := int32(len(keys)) // XXX: This was adapted to work with VersionedTree but needs to be re-thought. diff --git a/benchmarks/results/digital-ocean-64gb-fullbench-memory-8f19f23.txt b/benchmarks/results/digital-ocean-64gb-fullbench-memory-8f19f23.txt new file mode 100644 index 000000000..ca6a7c618 --- /dev/null +++ b/benchmarks/results/digital-ocean-64gb-fullbench-memory-8f19f23.txt @@ -0,0 +1,149 @@ +cd benchmarks && \ + go test -bench=RandomBytes . -benchmem && \ + go test -bench=Small . -benchmem && \ + go test -bench=Medium . -benchmem && \ + go test -timeout=30m -bench=Large . -benchmem && \ + go test -bench=Mem . -benchmem && \ + go test -timeout=60m -bench=LevelDB . -benchmem +goos: linux +goarch: amd64 +pkg: github.com/tendermint/iavl/benchmarks +BenchmarkRandomBytes/random-4-24 20000000 72.0 ns/op 4 B/op 1 allocs/op +BenchmarkRandomBytes/random-16-24 20000000 118 ns/op 16 B/op 1 allocs/op +BenchmarkRandomBytes/random-32-24 10000000 156 ns/op 32 B/op 1 allocs/op +BenchmarkRandomBytes/random-100-24 5000000 370 ns/op 112 B/op 1 allocs/op +BenchmarkRandomBytes/random-1000-24 500000 2958 ns/op 1024 B/op 1 allocs/op +PASS +ok github.com/tendermint/iavl/benchmarks 9.506s +Init Tree took 0.91 MB +goos: linux +goarch: amd64 +pkg: github.com/tendermint/iavl/benchmarks +BenchmarkSmall/memdb-1000-100-4-10/query-miss-24 500000 4420 ns/op 435 B/op 9 allocs/op +BenchmarkSmall/memdb-1000-100-4-10/query-hits-24 200000 5272 ns/op 634 B/op 12 allocs/op +BenchmarkSmall/memdb-1000-100-4-10/update-24 10000 159192 ns/op 42771 B/op 764 allocs/op +BenchmarkSmall/memdb-1000-100-4-10/block-24 100 24824001 ns/op 6620344 B/op 120912 allocs/op +Init Tree took 0.49 MB +BenchmarkSmall/goleveldb-1000-100-4-10/query-miss-24 300000 6368 ns/op 649 B/op 13 allocs/op +BenchmarkSmall/goleveldb-1000-100-4-10/query-hits-24 200000 8741 ns/op 918 B/op 18 allocs/op +BenchmarkSmall/goleveldb-1000-100-4-10/update-24 10000 109113 ns/op 22326 B/op 254 allocs/op +BenchmarkSmall/goleveldb-1000-100-4-10/block-24 100 17755475 ns/op 3487423 B/op 39158 allocs/op +Init Tree took 0.49 MB +BenchmarkSmall/leveldb-1000-100-4-10/query-miss-24 300000 6584 ns/op 651 B/op 14 allocs/op +BenchmarkSmall/leveldb-1000-100-4-10/query-hits-24 200000 7898 ns/op 918 B/op 18 allocs/op +BenchmarkSmall/leveldb-1000-100-4-10/update-24 10000 102305 ns/op 22352 B/op 254 allocs/op +BenchmarkSmall/leveldb-1000-100-4-10/block-24 100 16187510 ns/op 3461634 B/op 39097 allocs/op +PASS +ok github.com/tendermint/iavl/benchmarks 20.641s +Init Tree took 85.10 MB +goos: linux +goarch: amd64 +pkg: github.com/tendermint/iavl/benchmarks +BenchmarkMedium/memdb-100000-100-16-40/query-miss-24 200000 9595 ns/op 513 B/op 10 allocs/op +BenchmarkMedium/memdb-100000-100-16-40/query-hits-24 200000 10834 ns/op 676 B/op 12 allocs/op +BenchmarkMedium/memdb-100000-100-16-40/update-24 3000 1261394 ns/op 246830 B/op 4746 allocs/op +BenchmarkMedium/memdb-100000-100-16-40/block-24 10 198937291 ns/op 40016809 B/op 795945 allocs/op +Init Tree took 47.42 MB +BenchmarkMedium/goleveldb-100000-100-16-40/query-miss-24 50000 22673 ns/op 1594 B/op 27 allocs/op +BenchmarkMedium/goleveldb-100000-100-16-40/query-hits-24 50000 28135 ns/op 2144 B/op 35 allocs/op +BenchmarkMedium/goleveldb-100000-100-16-40/update-24 10000 294852 ns/op 53115 B/op 592 allocs/op +BenchmarkMedium/goleveldb-100000-100-16-40/block-24 50 34807659 ns/op 5968256 B/op 67622 allocs/op +Init Tree took 47.80 MB +BenchmarkMedium/leveldb-100000-100-16-40/query-miss-24 50000 22686 ns/op 1532 B/op 25 allocs/op +BenchmarkMedium/leveldb-100000-100-16-40/query-hits-24 50000 27668 ns/op 2159 B/op 35 allocs/op +BenchmarkMedium/leveldb-100000-100-16-40/update-24 10000 287108 ns/op 53026 B/op 593 allocs/op +BenchmarkMedium/leveldb-100000-100-16-40/block-24 30 35596044 ns/op 6206475 B/op 67118 allocs/op +PASS +ok github.com/tendermint/iavl/benchmarks 38.966s +Init Tree took 917.91 MB +goos: linux +goarch: amd64 +pkg: github.com/tendermint/iavl/benchmarks +BenchmarkLarge/memdb-1000000-100-16-40/query-miss-24 100000 15911 ns/op 1061 B/op 20 allocs/op +BenchmarkLarge/memdb-1000000-100-16-40/query-hits-24 100000 15748 ns/op 829 B/op 15 allocs/op +BenchmarkLarge/memdb-1000000-100-16-40/update-24 300 5161409 ns/op 994056 B/op 20570 allocs/op +BenchmarkLarge/memdb-1000000-100-16-40/block-24 2 514923704 ns/op 100084344 B/op 2069282 allocs/op +Init Tree took 416.94 MB +BenchmarkLarge/goleveldb-1000000-100-16-40/query-miss-24 20000 60866 ns/op 4902 B/op 82 allocs/op +BenchmarkLarge/goleveldb-1000000-100-16-40/query-hits-24 30000 49123 ns/op 3745 B/op 62 allocs/op +BenchmarkLarge/goleveldb-1000000-100-16-40/update-24 10000 478900 ns/op 81376 B/op 836 allocs/op +BenchmarkLarge/goleveldb-1000000-100-16-40/block-24 20 53327657 ns/op 10397199 B/op 98883 allocs/op +Init Tree took 404.64 MB +BenchmarkLarge/leveldb-1000000-100-16-40/query-miss-24 20000 57290 ns/op 4590 B/op 76 allocs/op +BenchmarkLarge/leveldb-1000000-100-16-40/query-hits-24 30000 48691 ns/op 3640 B/op 60 allocs/op +BenchmarkLarge/leveldb-1000000-100-16-40/update-24 5000 381530 ns/op 69914 B/op 736 allocs/op +BenchmarkLarge/leveldb-1000000-100-16-40/block-24 30 56220875 ns/op 10656291 B/op 101281 allocs/op +Init Tree took 24.87 MB +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100/query-miss-24 50000 23263 ns/op 1745 B/op 27 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100/query-hits-24 50000 25201 ns/op 2238 B/op 34 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100/update-24 10000 283769 ns/op 52149 B/op 563 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100/block-24 50 33455540 ns/op 5749534 B/op 64612 allocs/op +Init Tree took 39.73 MB +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-1000/query-miss-24 50000 23683 ns/op 2716 B/op 28 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-1000/query-hits-24 50000 32174 ns/op 3716 B/op 37 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-1000/update-24 10000 317281 ns/op 64047 B/op 616 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-1000/block-24 50 44231625 ns/op 7399545 B/op 73233 allocs/op +Init Tree took 264.19 MB +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-10000/query-miss-24 30000 35564 ns/op 11875 B/op 30 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-10000/query-hits-24 30000 45093 ns/op 16763 B/op 39 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-10000/update-24 10000 574978 ns/op 207200 B/op 752 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-10000/block-24 20 76693083 ns/op 28530852 B/op 92942 allocs/op +Init Tree took 2676.96 MB +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100000/query-miss-24 10000 147724 ns/op 256119 B/op 64 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100000/query-hits-24 10000 169279 ns/op 307666 B/op 70 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100000/update-24 1000 3882880 ns/op 2601694 B/op 639 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100000/block-24 10 520453137 ns/op 431258147 B/op 102744 allocs/op +PASS +ok github.com/tendermint/iavl/benchmarks 448.959s +PASS +ok github.com/tendermint/iavl/benchmarks 0.008s +Init Tree took 47.64 MB +goos: linux +goarch: amd64 +pkg: github.com/tendermint/iavl/benchmarks +BenchmarkLevelDBBatchSizes/goleveldb-100000-5-16-40/query-miss-24 50000 22097 ns/op 1601 B/op 27 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-5-16-40/query-hits-24 50000 27734 ns/op 2157 B/op 35 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-5-16-40/update-24 10000 511317 ns/op 78187 B/op 842 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-5-16-40/block-24 500 2681891 ns/op 398000 B/op 4350 allocs/op +Init Tree took 47.60 MB +BenchmarkLevelDBBatchSizes/goleveldb-100000-25-16-40/query-miss-24 50000 23397 ns/op 1536 B/op 26 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-25-16-40/query-hits-24 50000 28137 ns/op 2163 B/op 35 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-25-16-40/update-24 10000 349234 ns/op 64083 B/op 704 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-25-16-40/block-24 200 9405218 ns/op 1702630 B/op 18575 allocs/op +Init Tree took 35.87 MB +BenchmarkLevelDBBatchSizes/goleveldb-100000-100-16-40/query-miss-24 50000 22684 ns/op 1609 B/op 27 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-100-16-40/query-hits-24 50000 28836 ns/op 2156 B/op 35 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-100-16-40/update-24 10000 279935 ns/op 52609 B/op 595 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-100-16-40/block-24 50 34924252 ns/op 5994318 B/op 67455 allocs/op +Init Tree took 40.94 MB +BenchmarkLevelDBBatchSizes/goleveldb-100000-400-16-40/query-miss-24 50000 21974 ns/op 1568 B/op 26 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-400-16-40/query-hits-24 50000 29079 ns/op 2212 B/op 36 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-400-16-40/update-24 10000 200319 ns/op 40108 B/op 454 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-400-16-40/block-24 10 143250792 ns/op 23631405 B/op 273493 allocs/op +Init Tree took 38.19 MB +BenchmarkLevelDBBatchSizes/goleveldb-100000-2000-16-40/query-miss-24 50000 22070 ns/op 1543 B/op 26 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-2000-16-40/query-hits-24 50000 28305 ns/op 2150 B/op 35 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-2000-16-40/update-24 10000 150990 ns/op 31752 B/op 326 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-2000-16-40/block-24 3 416917935 ns/op 78008320 B/op 824207 allocs/op +Init Tree took 23.60 MB +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100/query-miss-24 100000 19841 ns/op 1591 B/op 24 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100/query-hits-24 50000 25320 ns/op 2237 B/op 34 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100/update-24 10000 310418 ns/op 51051 B/op 564 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100/block-24 50 33174907 ns/op 5796150 B/op 64856 allocs/op +Init Tree took 39.77 MB +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-1000/query-miss-24 100000 23820 ns/op 2638 B/op 27 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-1000/query-hits-24 50000 28973 ns/op 3720 B/op 37 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-1000/update-24 10000 315650 ns/op 64815 B/op 618 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-1000/block-24 50 41392373 ns/op 7424452 B/op 74022 allocs/op +Init Tree took 266.07 MB +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-10000/query-miss-24 30000 33961 ns/op 11884 B/op 30 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-10000/query-hits-24 30000 45861 ns/op 16535 B/op 39 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-10000/update-24 10000 594813 ns/op 213379 B/op 760 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-10000/block-24 20 77458855 ns/op 28589214 B/op 92146 allocs/op +Init Tree took 2677.35 MB +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100000/query-miss-24 10000 148305 ns/op 252620 B/op 63 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100000/query-hits-24 10000 169529 ns/op 310811 B/op 71 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100000/update-24 1000 3540659 ns/op 2632158 B/op 640 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100000/block-24 10 515252342 ns/op 440797695 B/op 102385 allocs/op +PASS +ok github.com/tendermint/iavl/benchmarks 243.566s diff --git a/benchmarks/results/digital-ocean-64gb-fullbench-memory-c1f6d4e.txt b/benchmarks/results/digital-ocean-64gb-fullbench-memory-c1f6d4e.txt new file mode 100644 index 000000000..5c43e9624 --- /dev/null +++ b/benchmarks/results/digital-ocean-64gb-fullbench-memory-c1f6d4e.txt @@ -0,0 +1,149 @@ +cd benchmarks && \ + go test -bench=RandomBytes . -benchmem && \ + go test -bench=Small . -benchmem && \ + go test -bench=Medium . -benchmem && \ + go test -timeout=30m -bench=Large . -benchmem && \ + go test -bench=Mem . -benchmem && \ + go test -timeout=60m -bench=LevelDB . -benchmem +goos: linux +goarch: amd64 +pkg: github.com/tendermint/iavl/benchmarks +BenchmarkRandomBytes/random-4-24 20000000 74.8 ns/op 4 B/op 1 allocs/op +BenchmarkRandomBytes/random-16-24 20000000 121 ns/op 16 B/op 1 allocs/op +BenchmarkRandomBytes/random-32-24 10000000 166 ns/op 32 B/op 1 allocs/op +BenchmarkRandomBytes/random-100-24 5000000 376 ns/op 112 B/op 1 allocs/op +BenchmarkRandomBytes/random-1000-24 500000 2943 ns/op 1024 B/op 1 allocs/op +PASS +ok github.com/tendermint/iavl/benchmarks 9.769s +Init Tree took 0.91 MB +goos: linux +goarch: amd64 +pkg: github.com/tendermint/iavl/benchmarks +BenchmarkSmall/memdb-1000-100-4-10/query-miss-24 300000 4081 ns/op 434 B/op 9 allocs/op +BenchmarkSmall/memdb-1000-100-4-10/query-hits-24 300000 5112 ns/op 633 B/op 12 allocs/op +BenchmarkSmall/memdb-1000-100-4-10/update-24 10000 157089 ns/op 42683 B/op 763 allocs/op +BenchmarkSmall/memdb-1000-100-4-10/block-24 100 25654741 ns/op 6619026 B/op 120940 allocs/op +Init Tree took 0.49 MB +BenchmarkSmall/goleveldb-1000-100-4-10/query-miss-24 200000 7127 ns/op 637 B/op 13 allocs/op +BenchmarkSmall/goleveldb-1000-100-4-10/query-hits-24 200000 8713 ns/op 918 B/op 18 allocs/op +BenchmarkSmall/goleveldb-1000-100-4-10/update-24 10000 103509 ns/op 22232 B/op 253 allocs/op +BenchmarkSmall/goleveldb-1000-100-4-10/block-24 100 17394312 ns/op 3483478 B/op 39338 allocs/op +Init Tree took 0.49 MB +BenchmarkSmall/leveldb-1000-100-4-10/query-miss-24 300000 6214 ns/op 646 B/op 13 allocs/op +BenchmarkSmall/leveldb-1000-100-4-10/query-hits-24 200000 8254 ns/op 919 B/op 18 allocs/op +BenchmarkSmall/leveldb-1000-100-4-10/update-24 10000 107058 ns/op 22312 B/op 254 allocs/op +BenchmarkSmall/leveldb-1000-100-4-10/block-24 100 17031744 ns/op 3482495 B/op 39144 allocs/op +PASS +ok github.com/tendermint/iavl/benchmarks 19.874s +Init Tree took 85.10 MB +goos: linux +goarch: amd64 +pkg: github.com/tendermint/iavl/benchmarks +BenchmarkMedium/memdb-100000-100-16-40/query-miss-24 200000 10064 ns/op 513 B/op 10 allocs/op +BenchmarkMedium/memdb-100000-100-16-40/query-hits-24 200000 11143 ns/op 676 B/op 12 allocs/op +BenchmarkMedium/memdb-100000-100-16-40/update-24 3000 1303374 ns/op 246834 B/op 4746 allocs/op +BenchmarkMedium/memdb-100000-100-16-40/block-24 10 190258294 ns/op 40016520 B/op 795943 allocs/op +Init Tree took 47.63 MB +BenchmarkMedium/goleveldb-100000-100-16-40/query-miss-24 50000 22452 ns/op 1539 B/op 26 allocs/op +BenchmarkMedium/goleveldb-100000-100-16-40/query-hits-24 50000 28301 ns/op 2148 B/op 35 allocs/op +BenchmarkMedium/goleveldb-100000-100-16-40/update-24 10000 296013 ns/op 52887 B/op 594 allocs/op +BenchmarkMedium/goleveldb-100000-100-16-40/block-24 30 35855483 ns/op 6213133 B/op 67658 allocs/op +Init Tree took 42.26 MB +BenchmarkMedium/leveldb-100000-100-16-40/query-miss-24 50000 22802 ns/op 1595 B/op 27 allocs/op +BenchmarkMedium/leveldb-100000-100-16-40/query-hits-24 50000 31757 ns/op 2147 B/op 35 allocs/op +BenchmarkMedium/leveldb-100000-100-16-40/update-24 10000 297615 ns/op 52713 B/op 594 allocs/op +BenchmarkMedium/leveldb-100000-100-16-40/block-24 30 36791150 ns/op 6289507 B/op 67963 allocs/op +PASS +ok github.com/tendermint/iavl/benchmarks 38.736s +Init Tree took 917.92 MB +goos: linux +goarch: amd64 +pkg: github.com/tendermint/iavl/benchmarks +BenchmarkLarge/memdb-1000000-100-16-40/query-miss-24 100000 15781 ns/op 1061 B/op 20 allocs/op +BenchmarkLarge/memdb-1000000-100-16-40/query-hits-24 100000 15750 ns/op 829 B/op 15 allocs/op +BenchmarkLarge/memdb-1000000-100-16-40/update-24 300 5256647 ns/op 994044 B/op 20570 allocs/op +BenchmarkLarge/memdb-1000000-100-16-40/block-24 2 534785952 ns/op 100083320 B/op 2069277 allocs/op +Init Tree took 416.96 MB +BenchmarkLarge/goleveldb-1000000-100-16-40/query-miss-24 20000 59997 ns/op 4900 B/op 82 allocs/op +BenchmarkLarge/goleveldb-1000000-100-16-40/query-hits-24 30000 51637 ns/op 3748 B/op 62 allocs/op +BenchmarkLarge/goleveldb-1000000-100-16-40/update-24 10000 476117 ns/op 81887 B/op 838 allocs/op +BenchmarkLarge/goleveldb-1000000-100-16-40/block-24 30 56340657 ns/op 10034120 B/op 95529 allocs/op +Init Tree took 404.27 MB +BenchmarkLarge/leveldb-1000000-100-16-40/query-miss-24 20000 62528 ns/op 5003 B/op 81 allocs/op +BenchmarkLarge/leveldb-1000000-100-16-40/query-hits-24 30000 50966 ns/op 3701 B/op 61 allocs/op +BenchmarkLarge/leveldb-1000000-100-16-40/update-24 10000 456299 ns/op 86644 B/op 841 allocs/op +BenchmarkLarge/leveldb-1000000-100-16-40/block-24 30 58929008 ns/op 12193146 B/op 100887 allocs/op +Init Tree took 25.20 MB +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100/query-miss-24 50000 20237 ns/op 1659 B/op 26 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100/query-hits-24 50000 26440 ns/op 2248 B/op 34 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100/update-24 10000 276849 ns/op 52649 B/op 565 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100/block-24 50 33371134 ns/op 5881967 B/op 65264 allocs/op +Init Tree took 39.72 MB +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-1000/query-miss-24 50000 23127 ns/op 2732 B/op 28 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-1000/query-hits-24 50000 30518 ns/op 3739 B/op 37 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-1000/update-24 10000 317968 ns/op 63822 B/op 616 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-1000/block-24 50 40372117 ns/op 7424951 B/op 73717 allocs/op +Init Tree took 264.06 MB +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-10000/query-miss-24 30000 34792 ns/op 11953 B/op 30 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-10000/query-hits-24 30000 45317 ns/op 16693 B/op 39 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-10000/update-24 5000 521571 ns/op 189390 B/op 695 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-10000/block-24 20 78074233 ns/op 26628664 B/op 93850 allocs/op +Init Tree took 2676.68 MB +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100000/query-miss-24 10000 160833 ns/op 257161 B/op 65 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100000/query-hits-24 10000 172494 ns/op 312459 B/op 71 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100000/update-24 1000 3285298 ns/op 2404346 B/op 621 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100000/block-24 5 482757364 ns/op 364325902 B/op 84213 allocs/op +PASS +ok github.com/tendermint/iavl/benchmarks 441.311s +PASS +ok github.com/tendermint/iavl/benchmarks 0.008s +Init Tree took 47.04 MB +goos: linux +goarch: amd64 +pkg: github.com/tendermint/iavl/benchmarks +BenchmarkLevelDBBatchSizes/goleveldb-100000-5-16-40/query-miss-24 50000 22448 ns/op 1530 B/op 25 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-5-16-40/query-hits-24 50000 32738 ns/op 2172 B/op 35 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-5-16-40/update-24 10000 515236 ns/op 78273 B/op 843 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-5-16-40/block-24 500 2702588 ns/op 402931 B/op 4385 allocs/op +Init Tree took 47.13 MB +BenchmarkLevelDBBatchSizes/goleveldb-100000-25-16-40/query-miss-24 50000 23648 ns/op 1598 B/op 27 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-25-16-40/query-hits-24 50000 28103 ns/op 2160 B/op 35 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-25-16-40/update-24 10000 351358 ns/op 65065 B/op 703 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-25-16-40/block-24 200 9918965 ns/op 1711946 B/op 18689 allocs/op +Init Tree took 42.02 MB +BenchmarkLevelDBBatchSizes/goleveldb-100000-100-16-40/query-miss-24 50000 24165 ns/op 1620 B/op 27 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-100-16-40/query-hits-24 50000 29367 ns/op 2154 B/op 35 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-100-16-40/update-24 10000 291198 ns/op 53061 B/op 594 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-100-16-40/block-24 30 35941442 ns/op 6034383 B/op 67049 allocs/op +Init Tree took 45.32 MB +BenchmarkLevelDBBatchSizes/goleveldb-100000-400-16-40/query-miss-24 50000 22196 ns/op 1579 B/op 26 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-400-16-40/query-hits-24 50000 28725 ns/op 2154 B/op 35 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-400-16-40/update-24 10000 200024 ns/op 40108 B/op 448 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-400-16-40/block-24 10 152324790 ns/op 23616529 B/op 273242 allocs/op +Init Tree took 38.47 MB +BenchmarkLevelDBBatchSizes/goleveldb-100000-2000-16-40/query-miss-24 50000 22555 ns/op 1538 B/op 26 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-2000-16-40/query-hits-24 50000 28427 ns/op 2159 B/op 35 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-2000-16-40/update-24 10000 143810 ns/op 30211 B/op 320 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-2000-16-40/block-24 3 408154340 ns/op 78403509 B/op 815927 allocs/op +Init Tree took 27.54 MB +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100/query-miss-24 50000 20109 ns/op 1668 B/op 26 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100/query-hits-24 50000 25620 ns/op 2243 B/op 34 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100/update-24 10000 276745 ns/op 52750 B/op 566 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100/block-24 50 33831408 ns/op 5826982 B/op 64847 allocs/op +Init Tree took 39.80 MB +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-1000/query-miss-24 50000 23057 ns/op 2722 B/op 28 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-1000/query-hits-24 50000 28850 ns/op 3742 B/op 37 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-1000/update-24 10000 310652 ns/op 63335 B/op 617 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-1000/block-24 30 40865126 ns/op 7547879 B/op 74541 allocs/op +Init Tree took 266.72 MB +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-10000/query-miss-24 30000 33848 ns/op 12084 B/op 30 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-10000/query-hits-24 30000 43692 ns/op 16592 B/op 39 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-10000/update-24 10000 588636 ns/op 211307 B/op 754 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-10000/block-24 20 82922740 ns/op 29007420 B/op 93883 allocs/op +Init Tree took 2675.64 MB +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100000/query-miss-24 10000 147862 ns/op 250440 B/op 63 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100000/query-hits-24 10000 178420 ns/op 313624 B/op 72 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100000/update-24 1000 3606703 ns/op 2569932 B/op 640 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100000/block-24 5 459105626 ns/op 401755606 B/op 89339 allocs/op +PASS +ok github.com/tendermint/iavl/benchmarks 241.593s diff --git a/benchmarks/setup/INSTALL_ROOT.sh b/benchmarks/setup/INSTALL_ROOT.sh index 632a90272..14b37b03d 100755 --- a/benchmarks/setup/INSTALL_ROOT.sh +++ b/benchmarks/setup/INSTALL_ROOT.sh @@ -1,12 +1,10 @@ -#!/bin/bash +#!/bin/sh apt-get update apt-get -y upgrade apt-get -y install make screen -GOFILE=go1.7.linux-amd64.tar.gz +GOFILE=go1.10.linux-amd64.tar.gz wget https://storage.googleapis.com/golang/${GOFILE} -tar xzf ${GOFILE} -mv go /usr/local/go1.7 -ln -s /usr/local/go1.7 /usr/local/go +tar -C /usr/local -xzf ${GOFILE} diff --git a/benchmarks/setup/INSTALL_USER.sh b/benchmarks/setup/INSTALL_USER.sh deleted file mode 100755 index 4695a610a..000000000 --- a/benchmarks/setup/INSTALL_USER.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -# This runs benchmarks, by default from develop branch of -# github.com/tendermint/merkleeyes/iavl -# You can customize this by optional command line args -# -# INSTALL_USER.sh [branch] [repouser] -# -# set repouser as your username to time your fork - -BRANCH=${1:-develop} -REPOUSER=${2:-tendermint} - -cat <<'EOF' > ~/.goenv -export GOROOT=/usr/local/go -export GOPATH=$HOME/go -export PATH=$GOPATH/bin:$GOROOT/bin:$PATH -EOF - -. ~/.goenv - -mkdir -p $GOPATH/src/github.com/tendermint -MERKLE=$GOPATH/src/github.com/tendermint/merkleeyes/iavl -git clone https://github.com/${REPOUSER}/merkleeyes/iavl.git $MERKLE -cd $MERKLE -git checkout ${BRANCH} diff --git a/benchmarks/setup/RUN_BENCHMARKS.sh b/benchmarks/setup/RUN_BENCHMARKS.sh index 6753f8e35..2bed5c1e2 100755 --- a/benchmarks/setup/RUN_BENCHMARKS.sh +++ b/benchmarks/setup/RUN_BENCHMARKS.sh @@ -1,10 +1,24 @@ #!/bin/sh -. ~/.goenv +# This runs benchmarks, by default from develop branch of +# github.com/tendermint/iavl +# You can customize this by optional command line args +# +# INSTALL_USER.sh [branch] [repouser] +# +# set repouser as your username to time your fork -MERKLE=$GOPATH/src/github.com/tendermint/merkleeyes/iavl -cd $MERKLE -git pull +BRANCH=${1:-develop} +REPOUSER=${2:-tendermint} + +export PATH=$PATH:/usr/local/go/bin +export PATH=$PATH:$HOME/go/bin +export GOPATH=$HOME/go + +go get -u github.com/${REPOUSER}/iavl +cd ~/go/src/github.com/${REPOUSER}/iavl +git checkout ${BRANCH} + +make get_vendor_deps +make bench > results.txt -make get_deps -make record diff --git a/doc.go b/doc.go index 7e4891bcb..7751bccad 100644 --- a/doc.go +++ b/doc.go @@ -2,13 +2,13 @@ // for persisting key-value pairs. // // -// Basic usage of VersionedTree. +// Basic usage of MutableTree. // // import "github.com/tendermint/iavl" // import "github.com/tendermint/tendermint/libs/db" // ... // -// tree := iavl.NewVersionedTree(db.NewMemDB(), 128) +// tree := iavl.NewMutableTree(db.NewMemDB(), 128) // // tree.IsEmpty() // true // diff --git a/tree.go b/immutable_tree.go similarity index 55% rename from tree.go rename to immutable_tree.go index c92901532..e334ef52b 100644 --- a/tree.go +++ b/immutable_tree.go @@ -7,29 +7,29 @@ import ( dbm "github.com/tendermint/tendermint/libs/db" ) -// Tree is a container for an immutable AVL+ Tree. Changes are performed by +// ImmutableTree is a container for an immutable AVL+ ImmutableTree. Changes are performed by // swapping the internal root with a new one, while the container is mutable. // Note that this tree is not thread-safe. -type Tree struct { +type ImmutableTree struct { root *Node ndb *nodeDB version int64 } -// NewTree creates both in-memory and persistent instances -func NewTree(db dbm.DB, cacheSize int) *Tree { +// NewImmutableTree creates both in-memory and persistent instances +func NewImmutableTree(db dbm.DB, cacheSize int) *ImmutableTree { if db == nil { // In-memory Tree. - return &Tree{} + return &ImmutableTree{} } - return &Tree{ + return &ImmutableTree{ // NodeDB-backed Tree. ndb: newNodeDB(db, cacheSize), } } // String returns a string representation of Tree. -func (t *Tree) String() string { +func (t *ImmutableTree) String() string { leaves := []string{} t.Iterate(func(key []byte, val []byte) (stop bool) { leaves = append(leaves, fmt.Sprintf("%x: %x", key, val)) @@ -39,11 +39,11 @@ func (t *Tree) String() string { } // Size returns the number of leaf nodes in the tree. -func (t *Tree) Size() int { +func (t *ImmutableTree) Size() int { return int(t.Size64()) } -func (t *Tree) Size64() int64 { +func (t *ImmutableTree) Size64() int64 { if t.root == nil { return 0 } @@ -51,20 +51,20 @@ func (t *Tree) Size64() int64 { } // Version returns the version of the tree. -func (t *Tree) Version() int { +func (t *ImmutableTree) Version() int { return int(t.Version64()) } -func (t *Tree) Version64() int64 { +func (t *ImmutableTree) Version64() int64 { return t.version } // Height returns the height of the tree. -func (t *Tree) Height() int { +func (t *ImmutableTree) Height() int { return int(t.Height8()) } -func (t *Tree) Height8() int8 { +func (t *ImmutableTree) Height8() int8 { if t.root == nil { return 0 } @@ -72,34 +72,15 @@ func (t *Tree) Height8() int8 { } // Has returns whether or not a key exists. -func (t *Tree) Has(key []byte) bool { +func (t *ImmutableTree) Has(key []byte) bool { if t.root == nil { return false } return t.root.has(t, key) } -// Set a key. Nil values are not supported. -func (t *Tree) Set(key []byte, value []byte) (updated bool) { - _, updated = t.set(key, value) - return updated -} - -func (t *Tree) set(key []byte, value []byte) (orphaned []*Node, updated bool) { - if value == nil { - panic(fmt.Sprintf("Attempt to store nil value at key '%s'", key)) - } - if t.root == nil { - t.root = NewNode(key, value, t.version+1) - return nil, false - } - t.root, updated, orphaned = t.root.set(t, key, value) - - return orphaned, updated -} - // Hash returns the root hash. -func (t *Tree) Hash() []byte { +func (t *ImmutableTree) Hash() []byte { if t.root == nil { return nil } @@ -108,7 +89,7 @@ func (t *Tree) Hash() []byte { } // hashWithCount returns the root hash and hash count. -func (t *Tree) hashWithCount() ([]byte, int64) { +func (t *ImmutableTree) hashWithCount() ([]byte, int64) { if t.root == nil { return nil, 0 } @@ -117,12 +98,12 @@ func (t *Tree) hashWithCount() ([]byte, int64) { // Get returns the index and value of the specified key if it exists, or nil // and the next index, if it doesn't. -func (t *Tree) Get(key []byte) (index int, value []byte) { +func (t *ImmutableTree) Get(key []byte) (index int, value []byte) { index64, value := t.Get64(key) return int(index64), value } -func (t *Tree) Get64(key []byte) (index int64, value []byte) { +func (t *ImmutableTree) Get64(key []byte) (index int64, value []byte) { if t.root == nil { return 0, nil } @@ -130,45 +111,19 @@ func (t *Tree) Get64(key []byte) (index int64, value []byte) { } // GetByIndex gets the key and value at the specified index. -func (t *Tree) GetByIndex(index int) (key []byte, value []byte) { +func (t *ImmutableTree) GetByIndex(index int) (key []byte, value []byte) { return t.GetByIndex64(int64(index)) } -func (t *Tree) GetByIndex64(index int64) (key []byte, value []byte) { +func (t *ImmutableTree) GetByIndex64(index int64) (key []byte, value []byte) { if t.root == nil { return nil, nil } return t.root.getByIndex(t, index) } -// Remove tries to remove a key from the tree and if removed, returns its -// value, and 'true'. -func (t *Tree) Remove(key []byte) ([]byte, bool) { - value, _, removed := t.remove(key) - return value, removed -} - -// remove tries to remove a key from the tree and if removed, returns its -// value, nodes orphaned and 'true'. -func (t *Tree) remove(key []byte) (value []byte, orphans []*Node, removed bool) { - if t.root == nil { - return nil, nil, false - } - newRootHash, newRoot, _, value, orphaned := t.root.remove(t, key) - if len(orphaned) == 0 { - return nil, nil, false - } - - if newRoot == nil && newRootHash != nil { - t.root = t.ndb.GetNode(newRootHash) - } else { - t.root = newRoot - } - return value, orphaned, true -} - // Iterate iterates over all keys of the tree, in order. -func (t *Tree) Iterate(fn func(key []byte, value []byte) bool) (stopped bool) { +func (t *ImmutableTree) Iterate(fn func(key []byte, value []byte) bool) (stopped bool) { if t.root == nil { return false } @@ -182,7 +137,7 @@ func (t *Tree) Iterate(fn func(key []byte, value []byte) bool) (stopped bool) { // IterateRange makes a callback for all nodes with key between start and end non-inclusive. // If either are nil, then it is open on that side (nil, nil is the same as Iterate) -func (t *Tree) IterateRange(start, end []byte, ascending bool, fn func(key []byte, value []byte) bool) (stopped bool) { +func (t *ImmutableTree) IterateRange(start, end []byte, ascending bool, fn func(key []byte, value []byte) bool) (stopped bool) { if t.root == nil { return false } @@ -196,7 +151,7 @@ func (t *Tree) IterateRange(start, end []byte, ascending bool, fn func(key []byt // IterateRangeInclusive makes a callback for all nodes with key between start and end inclusive. // If either are nil, then it is open on that side (nil, nil is the same as Iterate) -func (t *Tree) IterateRangeInclusive(start, end []byte, ascending bool, fn func(key, value []byte, version int64) bool) (stopped bool) { +func (t *ImmutableTree) IterateRangeInclusive(start, end []byte, ascending bool, fn func(key, value []byte, version int64) bool) (stopped bool) { if t.root == nil { return false } @@ -209,9 +164,9 @@ func (t *Tree) IterateRangeInclusive(start, end []byte, ascending bool, fn func( } // Clone creates a clone of the tree. -// Used internally by VersionedTree. -func (t *Tree) clone() *Tree { - return &Tree{ +// Used internally by MutableTree. +func (t *ImmutableTree) clone() *ImmutableTree { + return &ImmutableTree{ root: t.root, ndb: t.ndb, version: t.version, @@ -219,7 +174,7 @@ func (t *Tree) clone() *Tree { } // nodeSize is like Size, but includes inner nodes too. -func (t *Tree) nodeSize() int { +func (t *ImmutableTree) nodeSize() int { size := 0 t.root.traverse(t, true, func(n *Node) bool { size++ diff --git a/mutable_tree.go b/mutable_tree.go new file mode 100644 index 000000000..8ad073345 --- /dev/null +++ b/mutable_tree.go @@ -0,0 +1,469 @@ +package iavl + +import ( + "bytes" + "fmt" + + cmn "github.com/tendermint/tendermint/libs/common" + dbm "github.com/tendermint/tendermint/libs/db" +) + +// ErrVersionDoesNotExist is returned if a requested version does not exist. +var ErrVersionDoesNotExist = fmt.Errorf("version does not exist") + +// MutableTree is a persistent tree which keeps track of versions. +type MutableTree struct { + *ImmutableTree // The current, working tree. + lastSaved *ImmutableTree // The most recently saved tree. + orphans map[string]int64 // Nodes removed by changes to working tree. + versions map[int64]bool // The previous, saved versions of the tree. + ndb *nodeDB +} + +// NewMutableTree returns a new tree with the specified cache size and datastore. +func NewMutableTree(db dbm.DB, cacheSize int) *MutableTree { + ndb := newNodeDB(db, cacheSize) + head := &ImmutableTree{ndb: ndb} + + return &MutableTree{ + ImmutableTree: head, + lastSaved: head.clone(), + orphans: map[string]int64{}, + versions: map[int64]bool{}, + ndb: ndb, + } +} + +// IsEmpty returns whether or not the tree has any keys. Only trees that are +// not empty can be saved. +func (tree *MutableTree) IsEmpty() bool { + return tree.ImmutableTree.Size() == 0 +} + +// VersionExists returns whether or not a version exists. +func (tree *MutableTree) VersionExists(version int64) bool { + return tree.versions[version] +} + +// Hash returns the hash of the latest saved version of the tree, as returned +// by SaveVersion. If no versions have been saved, Hash returns nil. +func (tree *MutableTree) Hash() []byte { + if tree.version > 0 { + return tree.lastSaved.Hash() + } + return nil +} + +// WorkingHash returns the hash of the current working tree. +func (tree *MutableTree) WorkingHash() []byte { + return tree.ImmutableTree.Hash() +} + +// String returns a string representation of the tree. +func (tree *MutableTree) String() string { + return tree.ndb.String() +} + +// Set sets a key in the working tree. Nil values are not supported. +func (tree *MutableTree) Set(key, value []byte) bool { + orphaned, updated := tree.set(key, value) + tree.addOrphans(orphaned) + return updated +} + +func (tree *MutableTree) set(key []byte, value []byte) (orphaned []*Node, updated bool) { + if value == nil { + panic(fmt.Sprintf("Attempt to store nil value at key '%s'", key)) + } + if tree.ImmutableTree.root == nil { + tree.ImmutableTree.root = NewNode(key, value, tree.version+1) + return nil, false + } + tree.ImmutableTree.root, updated, orphaned = tree.recursiveSet(tree.ImmutableTree.root, key, value) + + return orphaned, updated +} + +func (tree *MutableTree) recursiveSet(node *Node, key []byte, value []byte) ( + newSelf *Node, updated bool, orphaned []*Node, +) { + version := tree.version + 1 + + if node.isLeaf() { + switch bytes.Compare(key, node.key) { + case -1: + return &Node{ + key: node.key, + height: 1, + size: 2, + leftNode: NewNode(key, value, version), + rightNode: node, + version: version, + }, false, []*Node{} + case 1: + return &Node{ + key: key, + height: 1, + size: 2, + leftNode: node, + rightNode: NewNode(key, value, version), + version: version, + }, false, []*Node{} + default: + return NewNode(key, value, version), true, []*Node{node} + } + } else { + orphaned = append(orphaned, node) + node = node.clone(version) + + if bytes.Compare(key, node.key) < 0 { + var leftOrphaned []*Node + node.leftNode, updated, leftOrphaned = tree.recursiveSet(node.getLeftNode(tree.ImmutableTree), key, value) + node.leftHash = nil // leftHash is yet unknown + orphaned = append(orphaned, leftOrphaned...) + } else { + var rightOrphaned []*Node + node.rightNode, updated, rightOrphaned = tree.recursiveSet(node.getRightNode(tree.ImmutableTree), key, value) + node.rightHash = nil // rightHash is yet unknown + orphaned = append(orphaned, rightOrphaned...) + } + + if updated { + return node, updated, orphaned + } + node.calcHeightAndSize(tree.ImmutableTree) + newNode, balanceOrphaned := tree.balance(node) + return newNode, updated, append(orphaned, balanceOrphaned...) + } +} + +// Remove removes a key from the working tree. +func (tree *MutableTree) Remove(key []byte) ([]byte, bool) { + val, orphaned, removed := tree.remove(key) + tree.addOrphans(orphaned) + return val, removed +} + +// remove tries to remove a key from the tree and if removed, returns its +// value, nodes orphaned and 'true'. +func (tree *MutableTree) remove(key []byte) (value []byte, orphans []*Node, removed bool) { + if tree.root == nil { + return nil, nil, false + } + newRootHash, newRoot, _, value, orphaned := tree.recursiveRemove(tree.root, key) + if len(orphaned) == 0 { + return nil, nil, false + } + + if newRoot == nil && newRootHash != nil { + tree.root = tree.ndb.GetNode(newRootHash) + } else { + tree.root = newRoot + } + return value, orphaned, true +} + +// removes the node corresponding to the passed key and balances the tree. +// It returns: +// - the hash of the new node (or nil if the node is the one removed) +// - the node that replaces the orig. node after remove +// - new leftmost leaf key for tree after successfully removing 'key' if changed. +// - the removed value +// - the orphaned nodes. +func (tree *MutableTree) recursiveRemove(node *Node, key []byte) ([]byte, *Node, []byte, []byte, []*Node) { + version := tree.version + 1 + + if node.isLeaf() { + if bytes.Equal(key, node.key) { + return nil, nil, nil, node.value, []*Node{node} + } + return node.hash, node, nil, nil, nil + } + + // node.key < key; we go to the left to find the key: + if bytes.Compare(key, node.key) < 0 { + newLeftHash, newLeftNode, newKey, value, orphaned := tree.recursiveRemove(node.getLeftNode(tree.ImmutableTree), key) + + if len(orphaned) == 0 { + return node.hash, node, nil, value, orphaned + } else if newLeftHash == nil && newLeftNode == nil { // left node held value, was removed + return node.rightHash, node.rightNode, node.key, value, orphaned + } + orphaned = append(orphaned, node) + + newNode := node.clone(version) + newNode.leftHash, newNode.leftNode = newLeftHash, newLeftNode + newNode.calcHeightAndSize(tree.ImmutableTree) + newNode, balanceOrphaned := tree.balance(newNode) + + return newNode.hash, newNode, newKey, value, append(orphaned, balanceOrphaned...) + } + // node.key >= key; either found or look to the right: + newRightHash, newRightNode, newKey, value, orphaned := tree.recursiveRemove(node.getRightNode(tree.ImmutableTree), key) + + if len(orphaned) == 0 { + return node.hash, node, nil, value, orphaned + } else if newRightHash == nil && newRightNode == nil { // right node held value, was removed + return node.leftHash, node.leftNode, nil, value, orphaned + } + orphaned = append(orphaned, node) + + newNode := node.clone(version) + newNode.rightHash, newNode.rightNode = newRightHash, newRightNode + if newKey != nil { + newNode.key = newKey + } + newNode.calcHeightAndSize(tree.ImmutableTree) + newNode, balanceOrphaned := tree.balance(newNode) + + return newNode.hash, newNode, nil, value, append(orphaned, balanceOrphaned...) +} + +// Load the latest versioned tree from disk. +func (tree *MutableTree) Load() (int64, error) { + return tree.LoadVersion(int64(0)) +} + +// Returns the version number of the latest version found +func (tree *MutableTree) LoadVersion(targetVersion int64) (int64, error) { + roots, err := tree.ndb.getRoots() + if err != nil { + return 0, err + } + if len(roots) == 0 { + return 0, nil + } + latestVersion := int64(0) + var latestRoot []byte + for version, r := range roots { + tree.versions[version] = true + if version > latestVersion && + (targetVersion == 0 || version <= targetVersion) { + latestVersion = version + latestRoot = r + } + } + + if !(targetVersion == 0 || latestVersion == targetVersion) { + return latestVersion, fmt.Errorf("wanted to load target %v but only found up to %v", + targetVersion, latestVersion) + } + + t := &ImmutableTree{ + ndb: tree.ndb, + version: latestVersion, + } + if len(latestRoot) != 0 { + t.root = tree.ndb.GetNode(latestRoot) + } + + tree.orphans = map[string]int64{} + tree.ImmutableTree = t + tree.lastSaved = t.clone() + return latestVersion, nil +} + +// GetImmutable loads an ImmutableTree at a given version for querying +func (tree *MutableTree) GetImmutable(version int64) (*ImmutableTree, error) { + rootHash := tree.ndb.getRoot(version) + if rootHash == nil { + return nil, ErrVersionDoesNotExist + } else if len(rootHash) == 0 { + return &ImmutableTree{ + ndb: tree.ndb, + version: version, + }, nil + } + return &ImmutableTree{ + root: tree.ndb.GetNode(rootHash), + ndb: tree.ndb, + version: version, + }, nil +} + +// Rollback resets the working tree to the latest saved version, discarding +// any unsaved modifications. +func (tree *MutableTree) Rollback() { + if tree.version > 0 { + tree.ImmutableTree = tree.lastSaved.clone() + } else { + tree.ImmutableTree = &ImmutableTree{ndb: tree.ndb, version: 0} + } + tree.orphans = map[string]int64{} +} + +// GetVersioned gets the value at the specified key and version. +func (tree *MutableTree) GetVersioned(key []byte, version int64) ( + index int, value []byte, +) { + if tree.versions[version] { + t, err := tree.GetImmutable(version) + if err != nil { + return -1, nil + } + return t.Get(key) + } + return -1, nil +} + +// SaveVersion saves a new tree version to disk, based on the current state of +// the tree. Returns the hash and new version number. +func (tree *MutableTree) SaveVersion() ([]byte, int64, error) { + version := tree.version + 1 + + if tree.versions[version] { + //version already exists, throw an error if attempting to overwrite + // Same hash means idempotent. Return success. + existingHash := tree.ndb.getRoot(version) + var newHash = tree.WorkingHash() + if bytes.Equal(existingHash, newHash) { + tree.version = version + tree.ImmutableTree = tree.ImmutableTree.clone() + tree.lastSaved = tree.ImmutableTree.clone() + tree.orphans = map[string]int64{} + return existingHash, version, nil + } + return nil, version, fmt.Errorf("version %d was already saved to different hash %X (existing hash %X)", + version, newHash, existingHash) + } + + if tree.root == nil { + // There can still be orphans, for example if the root is the node being + // removed. + debug("SAVE EMPTY TREE %v\n", version) + tree.ndb.SaveOrphans(version, tree.orphans) + tree.ndb.SaveEmptyRoot(version) + } else { + debug("SAVE TREE %v\n", version) + // Save the current tree. + tree.ndb.SaveBranch(tree.root) + tree.ndb.SaveOrphans(version, tree.orphans) + tree.ndb.SaveRoot(tree.root, version) + } + tree.ndb.Commit() + tree.version = version + tree.versions[version] = true + + // Set new working tree. + tree.ImmutableTree = tree.ImmutableTree.clone() + tree.lastSaved = tree.ImmutableTree.clone() + tree.orphans = map[string]int64{} + + return tree.Hash(), version, nil +} + +// DeleteVersion deletes a tree version from disk. The version can then no +// longer be accessed. +func (tree *MutableTree) DeleteVersion(version int64) error { + if version == 0 { + return cmn.NewError("version must be greater than 0") + } + if version == tree.version { + return cmn.NewError("cannot delete latest saved version (%d)", version) + } + if _, ok := tree.versions[version]; !ok { + return cmn.ErrorWrap(ErrVersionDoesNotExist, "") + } + + tree.ndb.DeleteVersion(version) + tree.ndb.Commit() + + delete(tree.versions, version) + + return nil +} + +// Rotate right and return the new node and orphan. +func (tree *MutableTree) rotateRight(node *Node) (*Node, *Node) { + version := tree.version + 1 + + // TODO: optimize balance & rotate. + node = node.clone(version) + orphaned := node.getLeftNode(tree.ImmutableTree) + newNode := orphaned.clone(version) + + newNoderHash, newNoderCached := newNode.rightHash, newNode.rightNode + newNode.rightHash, newNode.rightNode = node.hash, node + node.leftHash, node.leftNode = newNoderHash, newNoderCached + + node.calcHeightAndSize(tree.ImmutableTree) + newNode.calcHeightAndSize(tree.ImmutableTree) + + return newNode, orphaned +} + +// Rotate left and return the new node and orphan. +func (tree *MutableTree) rotateLeft(node *Node) (*Node, *Node) { + version := tree.version + 1 + + // TODO: optimize balance & rotate. + node = node.clone(version) + orphaned := node.getRightNode(tree.ImmutableTree) + newNode := orphaned.clone(version) + + newNodelHash, newNodelCached := newNode.leftHash, newNode.leftNode + newNode.leftHash, newNode.leftNode = node.hash, node + node.rightHash, node.rightNode = newNodelHash, newNodelCached + + node.calcHeightAndSize(tree.ImmutableTree) + newNode.calcHeightAndSize(tree.ImmutableTree) + + return newNode, orphaned +} + +// NOTE: assumes that node can be modified +// TODO: optimize balance & rotate +func (tree *MutableTree) balance(node *Node) (newSelf *Node, orphaned []*Node) { + if node.persisted { + panic("Unexpected balance() call on persisted node") + } + balance := node.calcBalance(tree.ImmutableTree) + + if balance > 1 { + if node.getLeftNode(tree.ImmutableTree).calcBalance(tree.ImmutableTree) >= 0 { + // Left Left Case + newNode, orphaned := tree.rotateRight(node) + return newNode, []*Node{orphaned} + } + // Left Right Case + var leftOrphaned *Node + + left := node.getLeftNode(tree.ImmutableTree) + node.leftHash = nil + node.leftNode, leftOrphaned = tree.rotateLeft(left) + newNode, rightOrphaned := tree.rotateRight(node) + + return newNode, []*Node{left, leftOrphaned, rightOrphaned} + } + if balance < -1 { + if node.getRightNode(tree.ImmutableTree).calcBalance(tree.ImmutableTree) <= 0 { + // Right Right Case + newNode, orphaned := tree.rotateLeft(node) + return newNode, []*Node{orphaned} + } + // Right Left Case + var rightOrphaned *Node + + right := node.getRightNode(tree.ImmutableTree) + node.rightHash = nil + node.rightNode, rightOrphaned = tree.rotateRight(right) + newNode, leftOrphaned := tree.rotateLeft(node) + + return newNode, []*Node{right, leftOrphaned, rightOrphaned} + } + // Nothing changed + return node, []*Node{} +} + +func (tree *MutableTree) addOrphans(orphans []*Node) { + for _, node := range orphans { + if !node.persisted { + // We don't need to orphan nodes that were never persisted. + continue + } + if len(node.hash) == 0 { + panic("Expected to find node hash, but was empty") + } + tree.orphans[string(node.hash)] = node.version + } +} diff --git a/node.go b/node.go index 307412c33..863751b6a 100644 --- a/node.go +++ b/node.go @@ -140,7 +140,7 @@ func (node *Node) isLeaf() bool { } // Check if the node has a descendant with the given key. -func (node *Node) has(t *Tree, key []byte) (has bool) { +func (node *Node) has(t *ImmutableTree, key []byte) (has bool) { if bytes.Equal(node.key, key) { return true } @@ -154,7 +154,7 @@ func (node *Node) has(t *Tree, key []byte) (has bool) { } // Get a key under the node. -func (node *Node) get(t *Tree, key []byte) (index int64, value []byte) { +func (node *Node) get(t *ImmutableTree, key []byte) (index int64, value []byte) { if node.isLeaf() { switch bytes.Compare(node.key, key) { case -1: @@ -175,7 +175,7 @@ func (node *Node) get(t *Tree, key []byte) (index int64, value []byte) { return index, value } -func (node *Node) getByIndex(t *Tree, index int64) (key []byte, value []byte) { +func (node *Node) getByIndex(t *ImmutableTree, index int64) (key []byte, value []byte) { if node.isLeaf() { if index == 0 { return node.key, node.value @@ -341,233 +341,42 @@ func (node *Node) writeBytes(w io.Writer) cmn.Error { return nil } -func (node *Node) set(t *Tree, key []byte, value []byte) ( - newSelf *Node, updated bool, orphaned []*Node, -) { - version := t.version + 1 - - if node.isLeaf() { - switch bytes.Compare(key, node.key) { - case -1: - return &Node{ - key: node.key, - height: 1, - size: 2, - leftNode: NewNode(key, value, version), - rightNode: node, - version: version, - }, false, []*Node{} - case 1: - return &Node{ - key: key, - height: 1, - size: 2, - leftNode: node, - rightNode: NewNode(key, value, version), - version: version, - }, false, []*Node{} - default: - return NewNode(key, value, version), true, []*Node{node} - } - } else { - orphaned = append(orphaned, node) - node = node.clone(version) - - if bytes.Compare(key, node.key) < 0 { - var leftOrphaned []*Node - node.leftNode, updated, leftOrphaned = node.getLeftNode(t).set(t, key, value) - node.leftHash = nil // leftHash is yet unknown - orphaned = append(orphaned, leftOrphaned...) - } else { - var rightOrphaned []*Node - node.rightNode, updated, rightOrphaned = node.getRightNode(t).set(t, key, value) - node.rightHash = nil // rightHash is yet unknown - orphaned = append(orphaned, rightOrphaned...) - } - - if updated { - return node, updated, orphaned - } - node.calcHeightAndSize(t) - newNode, balanceOrphaned := node.balance(t) - return newNode, updated, append(orphaned, balanceOrphaned...) - } -} - -// removes the node corresponding to the passed key and balances the tree. -// It returns: -// - the hash of the new node (or nil if the node is the one removed) -// - the node that replaces the orig. node after remove -// - new leftmost leaf key for tree after successfully removing 'key' if changed. -// - the removed value -// - the orphaned nodes. -func (node *Node) remove(t *Tree, key []byte) ([]byte, *Node, []byte, []byte, []*Node) { - version := t.version + 1 - - if node.isLeaf() { - if bytes.Equal(key, node.key) { - return nil, nil, nil, node.value, []*Node{node} - } - return node.hash, node, nil, nil, nil - } - - // node.key < key; we go to the left to find the key: - if bytes.Compare(key, node.key) < 0 { - newLeftHash, newLeftNode, newKey, value, orphaned := node.getLeftNode(t).remove(t, key) - - if len(orphaned) == 0 { - return node.hash, node, nil, value, orphaned - } else if newLeftHash == nil && newLeftNode == nil { // left node held value, was removed - return node.rightHash, node.rightNode, node.key, value, orphaned - } - orphaned = append(orphaned, node) - - newNode := node.clone(version) - newNode.leftHash, newNode.leftNode = newLeftHash, newLeftNode - newNode.calcHeightAndSize(t) - newNode, balanceOrphaned := newNode.balance(t) - - return newNode.hash, newNode, newKey, value, append(orphaned, balanceOrphaned...) - } - // node.key >= key; either found or look to the right: - newRightHash, newRightNode, newKey, value, orphaned := node.getRightNode(t).remove(t, key) - - if len(orphaned) == 0 { - return node.hash, node, nil, value, orphaned - } else if newRightHash == nil && newRightNode == nil { // right node held value, was removed - return node.leftHash, node.leftNode, nil, value, orphaned - } - orphaned = append(orphaned, node) - - newNode := node.clone(version) - newNode.rightHash, newNode.rightNode = newRightHash, newRightNode - if newKey != nil { - newNode.key = newKey - } - newNode.calcHeightAndSize(t) - newNode, balanceOrphaned := newNode.balance(t) - - return newNode.hash, newNode, nil, value, append(orphaned, balanceOrphaned...) -} - -func (node *Node) getLeftNode(t *Tree) *Node { +func (node *Node) getLeftNode(t *ImmutableTree) *Node { if node.leftNode != nil { return node.leftNode } return t.ndb.GetNode(node.leftHash) } -func (node *Node) getRightNode(t *Tree) *Node { +func (node *Node) getRightNode(t *ImmutableTree) *Node { if node.rightNode != nil { return node.rightNode } return t.ndb.GetNode(node.rightHash) } -// Rotate right and return the new node and orphan. -func (node *Node) rotateRight(t *Tree) (newNode *Node, orphan *Node) { - version := t.version + 1 - - // TODO: optimize balance & rotate. - node = node.clone(version) - l := node.getLeftNode(t) - _l := l.clone(version) - - _lrHash, _lrCached := _l.rightHash, _l.rightNode - _l.rightHash, _l.rightNode = node.hash, node - node.leftHash, node.leftNode = _lrHash, _lrCached - - node.calcHeightAndSize(t) - _l.calcHeightAndSize(t) - - return _l, l -} - -// Rotate left and return the new node and orphan. -func (node *Node) rotateLeft(t *Tree) (newNode *Node, orphan *Node) { - version := t.version + 1 - - // TODO: optimize balance & rotate. - node = node.clone(version) - r := node.getRightNode(t) - _r := r.clone(version) - - _rlHash, _rlCached := _r.leftHash, _r.leftNode - _r.leftHash, _r.leftNode = node.hash, node - node.rightHash, node.rightNode = _rlHash, _rlCached - - node.calcHeightAndSize(t) - _r.calcHeightAndSize(t) - - return _r, r -} - // NOTE: mutates height and size -func (node *Node) calcHeightAndSize(t *Tree) { +func (node *Node) calcHeightAndSize(t *ImmutableTree) { node.height = maxInt8(node.getLeftNode(t).height, node.getRightNode(t).height) + 1 node.size = node.getLeftNode(t).size + node.getRightNode(t).size } -func (node *Node) calcBalance(t *Tree) int { +func (node *Node) calcBalance(t *ImmutableTree) int { return int(node.getLeftNode(t).height) - int(node.getRightNode(t).height) } -// NOTE: assumes that node can be modified -// TODO: optimize balance & rotate -func (node *Node) balance(t *Tree) (newSelf *Node, orphaned []*Node) { - if node.persisted { - panic("Unexpected balance() call on persisted node") - } - balance := node.calcBalance(t) - - if balance > 1 { - if node.getLeftNode(t).calcBalance(t) >= 0 { - // Left Left Case - newNode, orphaned := node.rotateRight(t) - return newNode, []*Node{orphaned} - } - // Left Right Case - var leftOrphaned *Node - - left := node.getLeftNode(t) - node.leftHash = nil - node.leftNode, leftOrphaned = left.rotateLeft(t) - newNode, rightOrphaned := node.rotateRight(t) - - return newNode, []*Node{left, leftOrphaned, rightOrphaned} - } - if balance < -1 { - if node.getRightNode(t).calcBalance(t) <= 0 { - // Right Right Case - newNode, orphaned := node.rotateLeft(t) - return newNode, []*Node{orphaned} - } - // Right Left Case - var rightOrphaned *Node - - right := node.getRightNode(t) - node.rightHash = nil - node.rightNode, rightOrphaned = right.rotateRight(t) - newNode, leftOrphaned := node.rotateLeft(t) - - return newNode, []*Node{right, leftOrphaned, rightOrphaned} - } - // Nothing changed - return node, []*Node{} -} - // traverse is a wrapper over traverseInRange when we want the whole tree -func (node *Node) traverse(t *Tree, ascending bool, cb func(*Node) bool) bool { +func (node *Node) traverse(t *ImmutableTree, ascending bool, cb func(*Node) bool) bool { return node.traverseInRange(t, nil, nil, ascending, false, 0, func(node *Node, depth uint8) bool { return cb(node) }) } -func (node *Node) traverseWithDepth(t *Tree, ascending bool, cb func(*Node, uint8) bool) bool { +func (node *Node) traverseWithDepth(t *ImmutableTree, ascending bool, cb func(*Node, uint8) bool) bool { return node.traverseInRange(t, nil, nil, ascending, false, 0, cb) } -func (node *Node) traverseInRange(t *Tree, start, end []byte, ascending bool, inclusive bool, depth uint8, cb func(*Node, uint8) bool) bool { +func (node *Node) traverseInRange(t *ImmutableTree, start, end []byte, ascending bool, inclusive bool, depth uint8, cb func(*Node, uint8) bool) bool { afterStart := start == nil || bytes.Compare(start, node.key) < 0 startOrAfter := start == nil || bytes.Compare(start, node.key) <= 0 beforeEnd := end == nil || bytes.Compare(node.key, end) < 0 @@ -615,7 +424,7 @@ func (node *Node) traverseInRange(t *Tree, start, end []byte, ascending bool, in } // Only used in testing... -func (node *Node) lmd(t *Tree) *Node { +func (node *Node) lmd(t *ImmutableTree) *Node { if node.isLeaf() { return node } diff --git a/nodedb.go b/nodedb.go index d0d3df19c..a933b2662 100644 --- a/nodedb.go +++ b/nodedb.go @@ -332,6 +332,10 @@ func (ndb *nodeDB) Commit() { ndb.batch = ndb.db.NewBatch() } +func (ndb *nodeDB) getRoot(version int64) []byte { + return ndb.db.Get(ndb.rootKey(version)) +} + func (ndb *nodeDB) getRoots() (map[int64][]byte, error) { roots := map[int64][]byte{} diff --git a/orphaning_tree.go b/orphaning_tree.go deleted file mode 100644 index fb7493f28..000000000 --- a/orphaning_tree.go +++ /dev/null @@ -1,74 +0,0 @@ -package iavl - -import ( - "fmt" -) - -// orphaningTree is a tree which keeps track of orphaned nodes. -type orphaningTree struct { - *Tree - - // A map of orphan hash to orphan version. - // The version stored here is the one at which the orphan's lifetime - // begins. - orphans map[string]int64 -} - -// newOrphaningTree creates a new orphaning tree from the given *Tree. -func newOrphaningTree(t *Tree) *orphaningTree { - return &orphaningTree{ - Tree: t, - orphans: map[string]int64{}, - } -} - -// Set a key on the underlying tree while storing the orphaned nodes. -func (tree *orphaningTree) Set(key, value []byte) bool { - orphaned, updated := tree.Tree.set(key, value) - tree.addOrphans(orphaned) - return updated -} - -// Remove a key from the underlying tree while storing the orphaned nodes. -func (tree *orphaningTree) Remove(key []byte) ([]byte, bool) { - val, orphaned, removed := tree.Tree.remove(key) - tree.addOrphans(orphaned) - return val, removed -} - -// SaveAs saves the underlying Tree and assigns it a new version. -// Saves orphans too. -func (tree *orphaningTree) SaveAs(version int64) { - if version != tree.version+1 { - panic(fmt.Sprintf("Expected to save version %d but tried to save %d", tree.version+1, version)) - } - if tree.root == nil { - // There can still be orphans, for example if the root is the node being - // removed. - debug("SAVE EMPTY TREE %v\n", version) - tree.ndb.SaveOrphans(version, tree.orphans) - tree.ndb.SaveEmptyRoot(version) - } else { - debug("SAVE TREE %v\n", version) - // Save the current tree. - tree.ndb.SaveBranch(tree.root) - tree.ndb.SaveOrphans(version, tree.orphans) - tree.ndb.SaveRoot(tree.root, version) - } - tree.ndb.Commit() - tree.version = version -} - -// Add orphans to the orphan list. Doesn't write to disk. -func (tree *orphaningTree) addOrphans(orphans []*Node) { - for _, node := range orphans { - if !node.persisted { - // We don't need to orphan nodes that were never persisted. - continue - } - if len(node.hash) == 0 { - panic("Expected to find node hash, but was empty") - } - tree.orphans[string(node.hash)] = node.version - } -} diff --git a/proof.go b/proof.go index a87877048..b68293ccc 100644 --- a/proof.go +++ b/proof.go @@ -142,7 +142,7 @@ func (pln proofLeafNode) Hash() []byte { // If the key does not exist, returns the path to the next leaf left of key (w/ // path), except when key is less than the least item, in which case it returns // a path to the least item. -func (node *Node) PathToLeaf(t *Tree, key []byte) (PathToLeaf, *Node, error) { +func (node *Node) PathToLeaf(t *ImmutableTree, key []byte) (PathToLeaf, *Node, error) { path := new(PathToLeaf) val, err := node.pathToLeaf(t, key, path) return *path, val, err @@ -151,7 +151,7 @@ func (node *Node) PathToLeaf(t *Tree, key []byte) (PathToLeaf, *Node, error) { // pathToLeaf is a helper which recursively constructs the PathToLeaf. // As an optimization the already constructed path is passed in as an argument // and is shared among recursive calls. -func (node *Node) pathToLeaf(t *Tree, key []byte, path *PathToLeaf) (*Node, error) { +func (node *Node) pathToLeaf(t *ImmutableTree, key []byte, path *PathToLeaf) (*Node, error) { if node.height == 0 { if bytes.Equal(node.key, key) { return node, nil diff --git a/proof_range.go b/proof_range.go index cc12618f9..ec438515e 100644 --- a/proof_range.go +++ b/proof_range.go @@ -26,7 +26,7 @@ type RangeProof struct { // Keys returns all the keys in the RangeProof. NOTE: The keys here may // include more keys than provided by tree.GetRangeWithProof or -// VersionedTree.GetVersionedRangeWithProof. The keys returned there are only +// MutableTree.GetVersionedRangeWithProof. The keys returned there are only // in the provided [startKey,endKey){limit} range. The keys returned here may // include extra keys, such as: // - the key before startKey if startKey is provided and doesn't exist; @@ -308,7 +308,7 @@ func (proof *RangeProof) _computeRootHash() (rootHash []byte, treeEnd bool, err // If keyEnd-1 exists, no later leaves will be included. // If keyStart >= keyEnd and both not nil, panics. // Limit is never exceeded. -func (t *Tree) getRangeProof(keyStart, keyEnd []byte, limit int) (*RangeProof, [][]byte, [][]byte, error) { +func (t *ImmutableTree) getRangeProof(keyStart, keyEnd []byte, limit int) (*RangeProof, [][]byte, [][]byte, error) { if keyStart != nil && keyEnd != nil && bytes.Compare(keyStart, keyEnd) >= 0 { panic("if keyStart and keyEnd are present, need keyStart < keyEnd.") } @@ -365,7 +365,6 @@ func (t *Tree) getRangeProof(keyStart, keyEnd []byte, limit int) (*RangeProof, [ // nolint var innersq = []PathToLeaf(nil) var inners = PathToLeaf(nil) - var lastDepth uint8 = 0 var leafCount = 1 // from left above. var pathCount = 0 // var keys, values [][]byte defined as function outs. @@ -435,7 +434,6 @@ func (t *Tree) getRangeProof(keyStart, keyEnd []byte, limit int) (*RangeProof, [ }) } } - lastDepth = depth return false }, ) @@ -451,7 +449,7 @@ func (t *Tree) getRangeProof(keyStart, keyEnd []byte, limit int) (*RangeProof, [ // GetWithProof gets the value under the key if it exists, or returns nil. // A proof of existence or absence is returned alongside the value. -func (t *Tree) GetWithProof(key []byte) (value []byte, proof *RangeProof, err error) { +func (t *ImmutableTree) GetWithProof(key []byte) (value []byte, proof *RangeProof, err error) { proof, _, values, err := t.getRangeProof(key, cpIncr(key), 2) if err == nil { if len(values) > 0 { @@ -466,15 +464,19 @@ func (t *Tree) GetWithProof(key []byte) (value []byte, proof *RangeProof, err er } // GetRangeWithProof gets key/value pairs within the specified range and limit. -func (t *Tree) GetRangeWithProof(startKey []byte, endKey []byte, limit int) (keys, values [][]byte, proof *RangeProof, err error) { +func (t *ImmutableTree) GetRangeWithProof(startKey []byte, endKey []byte, limit int) (keys, values [][]byte, proof *RangeProof, err error) { proof, keys, values, err = t.getRangeProof(startKey, endKey, limit) return } // GetVersionedWithProof gets the value under the key at the specified version // if it exists, or returns nil. -func (tree *VersionedTree) GetVersionedWithProof(key []byte, version int64) ([]byte, *RangeProof, error) { - if t, ok := tree.versions[version]; ok { +func (tree *MutableTree) GetVersionedWithProof(key []byte, version int64) ([]byte, *RangeProof, error) { + if tree.versions[version] { + t, err := tree.GetImmutable(version) + if err != nil { + return nil, nil, err + } return t.GetWithProof(key) } return nil, nil, cmn.ErrorWrap(ErrVersionDoesNotExist, "") @@ -482,10 +484,14 @@ func (tree *VersionedTree) GetVersionedWithProof(key []byte, version int64) ([]b // GetVersionedRangeWithProof gets key/value pairs within the specified range // and limit. -func (tree *VersionedTree) GetVersionedRangeWithProof(startKey, endKey []byte, limit int, version int64) ( +func (tree *MutableTree) GetVersionedRangeWithProof(startKey, endKey []byte, limit int, version int64) ( keys, values [][]byte, proof *RangeProof, err error) { - if t, ok := tree.versions[version]; ok { + if tree.versions[version] { + t, err := tree.GetImmutable(version) + if err != nil { + return nil, nil, nil, err + } return t.GetRangeWithProof(startKey, endKey, limit) } return nil, nil, nil, cmn.ErrorWrap(ErrVersionDoesNotExist, "") diff --git a/proof_test.go b/proof_test.go index 53cc5c977..b408a0d9f 100644 --- a/proof_test.go +++ b/proof_test.go @@ -8,17 +8,18 @@ import ( "github.com/stretchr/testify/require" "github.com/tendermint/go-amino" + "github.com/tendermint/tendermint/libs/db" "github.com/tendermint/tendermint/libs/test" ) func TestTreeGetWithProof(t *testing.T) { - tree := NewTree(nil, 0) + tree := NewMutableTree(db.NewMemDB(), 0) require := require.New(t) for _, ikey := range []byte{0x11, 0x32, 0x50, 0x72, 0x99} { key := []byte{ikey} tree.Set(key, []byte(rand.Str(8))) } - root := tree.Hash() + root := tree.WorkingHash() key := []byte{0x32} val, proof, err := tree.GetWithProof(key) @@ -46,8 +47,8 @@ func TestTreeGetWithProof(t *testing.T) { } func TestTreeKeyExistsProof(t *testing.T) { - tree := NewTree(nil, 0) - root := tree.Hash() + tree := NewMutableTree(db.NewMemDB(), 0) + root := tree.WorkingHash() // should get false for proof with nil root proof, _, _, err := tree.getRangeProof([]byte("foo"), nil, 1) @@ -63,7 +64,7 @@ func TestTreeKeyExistsProof(t *testing.T) { allkeys[i] = []byte(key) } sortByteSlices(allkeys) // Sort all keys - root = tree.Hash() + root = tree.WorkingHash() // query random key fails proof, _, _, err = tree.getRangeProof([]byte("foo"), nil, 2) @@ -109,14 +110,14 @@ func TestTreeKeyExistsProof(t *testing.T) { } func TestTreeKeyInRangeProofs(t *testing.T) { - tree := NewTree(nil, 0) + tree := NewMutableTree(db.NewMemDB(), 0) require := require.New(t) keys := []byte{0x0a, 0x11, 0x2e, 0x32, 0x50, 0x72, 0x99, 0xa1, 0xe4, 0xf7} // 10 total. for _, ikey := range keys { key := []byte{ikey} tree.Set(key, key) } - root := tree.Hash() + root := tree.WorkingHash() // For spacing: T := 10 diff --git a/testutils_test.go b/testutils_test.go index 1e727fb52..1a76d60b9 100644 --- a/testutils_test.go +++ b/testutils_test.go @@ -53,9 +53,9 @@ func N(l, r interface{}) *Node { } // Setup a deep node -func T(n *Node) *Tree { +func T(n *Node) *MutableTree { d := db.NewDB("test", db.MemDBBackend, "") - t := NewTree(d, 0) + t := NewMutableTree(d, 0) n.hashWithCount() t.root = n @@ -114,7 +114,7 @@ func benchmarkImmutableAvlTreeWithDB(b *testing.B, db db.DB) { b.StopTimer() - t := NewVersionedTree(db, 100000) + t := NewMutableTree(db, 100000) for i := 0; i < 1000000; i++ { t.Set(i2b(int(cmn.RandInt32())), nil) if i > 990000 && i%1000 == 999 { diff --git a/tree_dotgraph.go b/tree_dotgraph.go index 14294851a..c6f50374d 100644 --- a/tree_dotgraph.go +++ b/tree_dotgraph.go @@ -41,7 +41,7 @@ var defaultGraphNodeAttrs = map[string]string{ "shape": "circle", } -func WriteDOTGraph(w io.Writer, tree *Tree, paths []PathToLeaf) { +func WriteDOTGraph(w io.Writer, tree *ImmutableTree, paths []PathToLeaf) { ctx := &graphContext{} tree.root.hashWithCount() diff --git a/tree_dotgraph_test.go b/tree_dotgraph_test.go index 02b245c5d..cec1475bd 100644 --- a/tree_dotgraph_test.go +++ b/tree_dotgraph_test.go @@ -3,15 +3,17 @@ package iavl import ( "io/ioutil" "testing" + + "github.com/tendermint/tendermint/libs/db" ) func TestWriteDOTGraph(t *testing.T) { - tree := NewTree(nil, 0) + tree := NewMutableTree(db.NewMemDB(), 0) for _, ikey := range []byte{ 0x0a, 0x11, 0x2e, 0x32, 0x50, 0x72, 0x99, 0xa1, 0xe4, 0xf7, } { key := []byte{ikey} tree.Set(key, key) } - WriteDOTGraph(ioutil.Discard, tree, []PathToLeaf{}) + WriteDOTGraph(ioutil.Discard, tree.ImmutableTree, []PathToLeaf{}) } diff --git a/tree_fuzz_test.go b/tree_fuzz_test.go index 59d693153..d5e0bcaf9 100644 --- a/tree_fuzz_test.go +++ b/tree_fuzz_test.go @@ -16,7 +16,7 @@ type program struct { instructions []instruction } -func (p *program) Execute(tree *VersionedTree) (err error) { +func (p *program) Execute(tree *MutableTree) (err error) { var errLine int defer func() { @@ -55,7 +55,7 @@ type instruction struct { version int64 } -func (i instruction) Execute(tree *VersionedTree) { +func (i instruction) Execute(tree *MutableTree) { switch i.op { case "SET": tree.Set(i.k, i.v) @@ -103,14 +103,14 @@ func genRandomProgram(size int) *program { } // Generate many programs and run them. -func TestVersionedTreeFuzz(t *testing.T) { +func TestMutableTreeFuzz(t *testing.T) { maxIterations := testFuzzIterations progsPerIteration := 100000 iterations := 0 for size := 5; iterations < maxIterations; size++ { for i := 0; i < progsPerIteration/size; i++ { - tree := NewVersionedTree(db.NewMemDB(), 0) + tree := NewMutableTree(db.NewMemDB(), 0) program := genRandomProgram(size) err := program.Execute(tree) if err != nil { diff --git a/tree_test.go b/tree_test.go index d4d0ff0d3..c12eb7116 100644 --- a/tree_test.go +++ b/tree_test.go @@ -3,6 +3,7 @@ package iavl import ( "bytes" "flag" + "fmt" "os" "runtime" "testing" @@ -10,6 +11,8 @@ import ( "github.com/stretchr/testify/require" "github.com/tendermint/tendermint/libs/db" + mathrand "math/rand" + cmn "github.com/tendermint/tendermint/libs/common" ) @@ -45,7 +48,7 @@ func TestVersionedRandomTree(t *testing.T) { d, closeDB := getTestDB() defer closeDB() - tree := NewVersionedTree(d, 100) + tree := NewMutableTree(d, 100) versions := 50 keysPerVersion := 30 @@ -70,7 +73,9 @@ func TestVersionedRandomTree(t *testing.T) { } require.Len(tree.versions, 1, "tree must have one version left") - require.Equal(tree.versions[int64(versions)].root, tree.root) + tr, err := tree.GetImmutable(int64(versions)) + require.NoError(err, "GetImmutable should not error for version %d", versions) + require.Equal(tr.root, tree.root) // After cleaning up all previous versions, we should have as many nodes // in the db as in the current tree version. @@ -84,8 +89,8 @@ func TestVersionedRandomTreeSmallKeys(t *testing.T) { d, closeDB := getTestDB() defer closeDB() - tree := NewVersionedTree(d, 100) - singleVersionTree := NewVersionedTree(db.NewMemDB(), 0) + tree := NewMutableTree(d, 100) + singleVersionTree := NewMutableTree(db.NewMemDB(), 0) versions := 20 keysPerVersion := 50 @@ -125,8 +130,8 @@ func TestVersionedRandomTreeSmallKeysRandomDeletes(t *testing.T) { d, closeDB := getTestDB() defer closeDB() - tree := NewVersionedTree(d, 100) - singleVersionTree := NewVersionedTree(db.NewMemDB(), 0) + tree := NewMutableTree(d, 100) + singleVersionTree := NewMutableTree(db.NewMemDB(), 0) versions := 30 keysPerVersion := 50 @@ -162,7 +167,7 @@ func TestVersionedRandomTreeSmallKeysRandomDeletes(t *testing.T) { } func TestVersionedTreeSpecial1(t *testing.T) { - tree := NewVersionedTree(db.NewMemDB(), 100) + tree := NewMutableTree(db.NewMemDB(), 100) tree.Set([]byte("C"), []byte("so43QQFN")) tree.SaveVersion() @@ -185,7 +190,7 @@ func TestVersionedTreeSpecial1(t *testing.T) { func TestVersionedRandomTreeSpecial2(t *testing.T) { require := require.New(t) - tree := NewVersionedTree(db.NewMemDB(), 100) + tree := NewMutableTree(db.NewMemDB(), 100) tree.Set([]byte("OFMe2Yvm"), []byte("ez2OtQtE")) tree.Set([]byte("WEN4iN7Y"), []byte("kQNyUalI")) @@ -204,7 +209,7 @@ func TestVersionedEmptyTree(t *testing.T) { d, closeDB := getTestDB() defer closeDB() - tree := NewVersionedTree(d, 0) + tree := NewMutableTree(d, 0) hash, v, err := tree.SaveVersion() require.Nil(hash) @@ -242,14 +247,17 @@ func TestVersionedEmptyTree(t *testing.T) { // Now reload the tree. - tree = NewVersionedTree(d, 0) + tree = NewMutableTree(d, 0) tree.Load() require.False(tree.VersionExists(1)) require.True(tree.VersionExists(2)) require.False(tree.VersionExists(3)) - require.Empty(tree.versions[2].root) + t2, err := tree.GetImmutable(2) + require.NoError(err, "GetImmutable should not fail for version 2") + + require.Empty(t2.root) } func TestVersionedTree(t *testing.T) { @@ -257,7 +265,7 @@ func TestVersionedTree(t *testing.T) { d, closeDB := getTestDB() defer closeDB() - tree := NewVersionedTree(d, 0) + tree := NewMutableTree(d, 0) // We start with zero keys in the databse. require.Equal(0, tree.ndb.size()) @@ -301,7 +309,7 @@ func TestVersionedTree(t *testing.T) { // Recreate a new tree and load it, to make sure it works in this // scenario. - tree = NewVersionedTree(d, 100) + tree = NewMutableTree(d, 100) _, err = tree.Load() require.NoError(err) @@ -347,7 +355,7 @@ func TestVersionedTree(t *testing.T) { require.EqualValues(hash3, hash4) require.NotNil(hash4) - tree = NewVersionedTree(d, 100) + tree = NewMutableTree(d, 100) _, err = tree.Load() require.NoError(err) @@ -442,7 +450,7 @@ func TestVersionedTreeVersionDeletingEfficiency(t *testing.T) { d, closeDB := getTestDB() defer closeDB() - tree := NewVersionedTree(d, 0) + tree := NewMutableTree(d, 0) tree.Set([]byte("key0"), []byte("val0")) tree.Set([]byte("key1"), []byte("val0")) @@ -473,7 +481,7 @@ func TestVersionedTreeVersionDeletingEfficiency(t *testing.T) { require.Len(t, tree.ndb.leafNodes(), 3) - tree2 := NewVersionedTree(db.NewMemDB(), 0) + tree2 := NewMutableTree(db.NewMemDB(), 0) tree2.Set([]byte("key0"), []byte("val2")) tree2.Set([]byte("key2"), []byte("val2")) tree2.Set([]byte("key3"), []byte("val1")) @@ -484,7 +492,7 @@ func TestVersionedTreeVersionDeletingEfficiency(t *testing.T) { func TestVersionedTreeOrphanDeleting(t *testing.T) { mdb := db.NewMemDB() - tree := NewVersionedTree(mdb, 0) + tree := NewMutableTree(mdb, 0) tree.Set([]byte("key0"), []byte("val0")) tree.Set([]byte("key1"), []byte("val0")) @@ -522,7 +530,7 @@ func TestVersionedTreeOrphanDeleting(t *testing.T) { func TestVersionedTreeSpecialCase(t *testing.T) { require := require.New(t) - tree := NewVersionedTree(db.NewMemDB(), 100) + tree := NewMutableTree(db.NewMemDB(), 100) tree.Set([]byte("key1"), []byte("val0")) tree.Set([]byte("key2"), []byte("val0")) @@ -545,7 +553,7 @@ func TestVersionedTreeSpecialCase2(t *testing.T) { require := require.New(t) d := db.NewMemDB() - tree := NewVersionedTree(d, 100) + tree := NewMutableTree(d, 100) tree.Set([]byte("key1"), []byte("val0")) tree.Set([]byte("key2"), []byte("val0")) @@ -558,7 +566,7 @@ func TestVersionedTreeSpecialCase2(t *testing.T) { tree.Set([]byte("key2"), []byte("val2")) tree.SaveVersion() - tree = NewVersionedTree(d, 100) + tree = NewMutableTree(d, 100) _, err := tree.Load() require.NoError(err) @@ -570,7 +578,7 @@ func TestVersionedTreeSpecialCase2(t *testing.T) { func TestVersionedTreeSpecialCase3(t *testing.T) { require := require.New(t) - tree := NewVersionedTree(db.NewMemDB(), 0) + tree := NewMutableTree(db.NewMemDB(), 0) tree.Set([]byte("m"), []byte("liWT0U6G")) tree.Set([]byte("G"), []byte("7PxRXwUA")) @@ -599,7 +607,7 @@ func TestVersionedTreeSpecialCase3(t *testing.T) { func TestVersionedTreeSaveAndLoad(t *testing.T) { require := require.New(t) d := db.NewMemDB() - tree := NewVersionedTree(d, 0) + tree := NewMutableTree(d, 0) // Loading with an empty root is a no-op. tree.Load() @@ -623,7 +631,7 @@ func TestVersionedTreeSaveAndLoad(t *testing.T) { require.Equal(6, tree.Version()) // Reload the tree, to test that roots and orphans are properly loaded. - ntree := NewVersionedTree(d, 0) + ntree := NewMutableTree(d, 0) ntree.Load() require.False(ntree.IsEmpty()) @@ -649,7 +657,7 @@ func TestVersionedTreeSaveAndLoad(t *testing.T) { func TestVersionedTreeErrors(t *testing.T) { require := require.New(t) - tree := NewVersionedTree(db.NewMemDB(), 100) + tree := NewMutableTree(db.NewMemDB(), 100) // Can't delete non-existent versions. require.Error(tree.DeleteVersion(1)) @@ -681,7 +689,7 @@ func TestVersionedCheckpoints(t *testing.T) { d, closeDB := getTestDB() defer closeDB() - tree := NewVersionedTree(d, 100) + tree := NewMutableTree(d, 100) versions := 50 keysPerVersion := 10 versionsPerCheckpoint := 5 @@ -734,7 +742,7 @@ func TestVersionedCheckpoints(t *testing.T) { func TestVersionedCheckpointsSpecialCase(t *testing.T) { require := require.New(t) - tree := NewVersionedTree(db.NewMemDB(), 0) + tree := NewMutableTree(db.NewMemDB(), 0) key := []byte("k") tree.Set(key, []byte("val1")) @@ -759,7 +767,7 @@ func TestVersionedCheckpointsSpecialCase(t *testing.T) { } func TestVersionedCheckpointsSpecialCase2(t *testing.T) { - tree := NewVersionedTree(db.NewMemDB(), 0) + tree := NewMutableTree(db.NewMemDB(), 0) tree.Set([]byte("U"), []byte("XamDUtiJ")) tree.Set([]byte("A"), []byte("UkZBuYIU")) @@ -779,7 +787,7 @@ func TestVersionedCheckpointsSpecialCase2(t *testing.T) { } func TestVersionedCheckpointsSpecialCase3(t *testing.T) { - tree := NewVersionedTree(db.NewMemDB(), 0) + tree := NewMutableTree(db.NewMemDB(), 0) tree.Set([]byte("n"), []byte("2wUCUs8q")) tree.Set([]byte("l"), []byte("WQ7mvMbc")) @@ -799,7 +807,7 @@ func TestVersionedCheckpointsSpecialCase3(t *testing.T) { } func TestVersionedCheckpointsSpecialCase4(t *testing.T) { - tree := NewVersionedTree(db.NewMemDB(), 0) + tree := NewMutableTree(db.NewMemDB(), 0) tree.Set([]byte("U"), []byte("XamDUtiJ")) tree.Set([]byte("A"), []byte("UkZBuYIU")) @@ -831,7 +839,7 @@ func TestVersionedCheckpointsSpecialCase4(t *testing.T) { } func TestVersionedCheckpointsSpecialCase5(t *testing.T) { - tree := NewVersionedTree(db.NewMemDB(), 0) + tree := NewMutableTree(db.NewMemDB(), 0) tree.Set([]byte("R"), []byte("ygZlIzeW")) tree.SaveVersion() @@ -848,7 +856,7 @@ func TestVersionedCheckpointsSpecialCase5(t *testing.T) { } func TestVersionedCheckpointsSpecialCase6(t *testing.T) { - tree := NewVersionedTree(db.NewMemDB(), 0) + tree := NewMutableTree(db.NewMemDB(), 0) tree.Set([]byte("Y"), []byte("MW79JQeV")) tree.Set([]byte("7"), []byte("Kp0ToUJB")) @@ -880,7 +888,7 @@ func TestVersionedCheckpointsSpecialCase6(t *testing.T) { } func TestVersionedCheckpointsSpecialCase7(t *testing.T) { - tree := NewVersionedTree(db.NewMemDB(), 100) + tree := NewMutableTree(db.NewMemDB(), 100) tree.Set([]byte("n"), []byte("OtqD3nyn")) tree.Set([]byte("W"), []byte("kMdhJjF5")) @@ -914,7 +922,7 @@ func TestVersionedCheckpointsSpecialCase7(t *testing.T) { func TestVersionedTreeEfficiency(t *testing.T) { require := require.New(t) - tree := NewVersionedTree(db.NewMemDB(), 0) + tree := NewMutableTree(db.NewMemDB(), 0) versions := 20 keysPerVersion := 100 keysAddedPerVersion := map[int]int{} @@ -949,7 +957,7 @@ func TestVersionedTreeEfficiency(t *testing.T) { func TestVersionedTreeProofs(t *testing.T) { require := require.New(t) - tree := NewVersionedTree(db.NewMemDB(), 0) + tree := NewMutableTree(db.NewMemDB(), 0) tree.Set([]byte("k1"), []byte("v1")) tree.Set([]byte("k2"), []byte("v1")) @@ -1017,9 +1025,41 @@ func TestVersionedTreeProofs(t *testing.T) { require.Error(proof.Verify(root2)) } +func TestOrphans(t *testing.T) { + //If you create a sequence of saved versions + //Then randomly delete versions other than the first and last until only those two remain + //Any remaining orphan nodes should be constrained to just the first version + require := require.New(t) + tree := NewMutableTree(db.NewMemDB(), 100) + + NUMVERSIONS := 100 + NUMUPDATES := 100 + + for i := 0; i < NUMVERSIONS; i++ { + for j := 1; j < NUMUPDATES; j++ { + tree.Set(randBytes(2), randBytes(2)) + } + _, _, err := tree.SaveVersion() + require.NoError(err, "SaveVersion should not error") + } + + idx := mathrand.Perm(NUMVERSIONS - 2) + for i := range idx { + err := tree.DeleteVersion(int64(i + 2)) + require.NoError(err, "DeleteVersion should not error") + } + + tree.ndb.traverseOrphans(func(k, v []byte) { + var fromVersion, toVersion int64 + fmt.Sscanf(string(k), orphanKeyFmt, &toVersion, &fromVersion) + require.Equal(fromVersion, int64(1), "fromVersion should be 1") + require.Equal(toVersion, int64(1), "toVersion should be 1") + }) +} + func TestVersionedTreeHash(t *testing.T) { require := require.New(t) - tree := NewVersionedTree(db.NewMemDB(), 0) + tree := NewMutableTree(db.NewMemDB(), 0) require.Nil(tree.Hash()) tree.Set([]byte("I"), []byte("D")) @@ -1041,7 +1081,7 @@ func TestVersionedTreeHash(t *testing.T) { func TestNilValueSemantics(t *testing.T) { require := require.New(t) - tree := NewVersionedTree(db.NewMemDB(), 0) + tree := NewMutableTree(db.NewMemDB(), 0) require.Panics(func() { tree.Set([]byte("k"), nil) @@ -1051,7 +1091,7 @@ func TestNilValueSemantics(t *testing.T) { func TestCopyValueSemantics(t *testing.T) { require := require.New(t) - tree := NewVersionedTree(db.NewMemDB(), 0) + tree := NewMutableTree(db.NewMemDB(), 0) val := []byte("v1") @@ -1068,7 +1108,7 @@ func TestCopyValueSemantics(t *testing.T) { func TestRollback(t *testing.T) { require := require.New(t) - tree := NewVersionedTree(db.NewMemDB(), 0) + tree := NewMutableTree(db.NewMemDB(), 0) tree.Set([]byte("k"), []byte("v")) tree.SaveVersion() @@ -1094,6 +1134,38 @@ func TestRollback(t *testing.T) { require.Equal([]byte("v"), val) } +func TestOverwrite(t *testing.T) { + require := require.New(t) + + mdb := db.NewMemDB() + tree := NewMutableTree(mdb, 0) + + // Set one kv pair and save version 1 + tree.Set([]byte("key1"), []byte("value1")) + _, _, err := tree.SaveVersion() + require.NoError(err, "SaveVersion should not fail") + + // Set another kv pair and save version 2 + tree.Set([]byte("key2"), []byte("value2")) + _, _, err = tree.SaveVersion() + require.NoError(err, "SaveVersion should not fail") + + // Reload tree at version 1 + tree = NewMutableTree(mdb, 0) + _, err = tree.LoadVersion(int64(1)) + require.NoError(err, "LoadVersion should not fail") + + // Attempt to put a different kv pair into the tree and save + tree.Set([]byte("key2"), []byte("different value 2")) + _, _, err = tree.SaveVersion() + require.Error(err, "SaveVersion should fail because of changed value") + + // Replay the original transition from version 1 to version 2 and attempt to save + tree.Set([]byte("key2"), []byte("value2")) + _, _, err = tree.SaveVersion() + require.NoError(err, "SaveVersion should not fail, overwrite was idempotent") +} + //////////////////////////// BENCHMARKS /////////////////////////////////////// func BenchmarkTreeLoadAndDelete(b *testing.B) { @@ -1107,7 +1179,7 @@ func BenchmarkTreeLoadAndDelete(b *testing.B) { defer d.Close() defer os.RemoveAll("./bench.db") - tree := NewVersionedTree(d, 0) + tree := NewMutableTree(d, 0) for v := 1; v < numVersions; v++ { for i := 0; i < numKeysPerVersion; i++ { tree.Set([]byte(rand.Str(16)), rand.Bytes(32)) @@ -1118,7 +1190,7 @@ func BenchmarkTreeLoadAndDelete(b *testing.B) { b.Run("LoadAndDelete", func(b *testing.B) { for n := 0; n < b.N; n++ { b.StopTimer() - tree = NewVersionedTree(d, 0) + tree = NewMutableTree(d, 0) runtime.GC() b.StartTimer() diff --git a/util.go b/util.go index 96f754189..dc9f92515 100644 --- a/util.go +++ b/util.go @@ -7,7 +7,7 @@ import ( ) // PrintTree prints the whole tree in an indented form. -func PrintTree(tree *Tree) { +func PrintTree(tree *ImmutableTree) { ndb, root := tree.ndb, tree.root printNode(ndb, root, 0) } @@ -71,8 +71,6 @@ func cpIncr(bz []byte) (ret []byte) { ret[i] = byte(0x00) if i == 0 { return append(ret, 0x00) - // Overflow - return nil } } return []byte{0x00} diff --git a/version.go b/version.go index 9efd11145..f88f00abc 100644 --- a/version.go +++ b/version.go @@ -1,4 +1,4 @@ package iavl // Version of iavl. -const Version = "0.9.2" +const Version = "0.10.0" diff --git a/versioned_tree.go b/versioned_tree.go deleted file mode 100644 index 7d8108ea8..000000000 --- a/versioned_tree.go +++ /dev/null @@ -1,197 +0,0 @@ -package iavl - -import ( - "bytes" - "fmt" - - cmn "github.com/tendermint/tendermint/libs/common" - dbm "github.com/tendermint/tendermint/libs/db" -) - -// ErrVersionDoesNotExist is returned if a requested version does not exist. -var ErrVersionDoesNotExist = fmt.Errorf("version does not exist") - -// VersionedTree is a persistent tree which keeps track of versions. -type VersionedTree struct { - *orphaningTree // The current, working tree. - versions map[int64]*Tree // The previous, saved versions of the tree. - ndb *nodeDB -} - -// NewVersionedTree returns a new tree with the specified cache size and datastore. -func NewVersionedTree(db dbm.DB, cacheSize int) *VersionedTree { - ndb := newNodeDB(db, cacheSize) - head := &Tree{ndb: ndb} - - return &VersionedTree{ - orphaningTree: newOrphaningTree(head), - versions: map[int64]*Tree{}, - ndb: ndb, - } -} - -// IsEmpty returns whether or not the tree has any keys. Only trees that are -// not empty can be saved. -func (tree *VersionedTree) IsEmpty() bool { - return tree.orphaningTree.Size() == 0 -} - -// VersionExists returns whether or not a version exists. -func (tree *VersionedTree) VersionExists(version int64) bool { - _, ok := tree.versions[version] - return ok -} - -// Tree returns the current working tree. -func (tree *VersionedTree) Tree() *Tree { - return tree.orphaningTree.Tree -} - -// Hash returns the hash of the latest saved version of the tree, as returned -// by SaveVersion. If no versions have been saved, Hash returns nil. -func (tree *VersionedTree) Hash() []byte { - if tree.version > 0 { - return tree.versions[tree.version].Hash() - } - return nil -} - -// String returns a string representation of the tree. -func (tree *VersionedTree) String() string { - return tree.ndb.String() -} - -// Set sets a key in the working tree. Nil values are not supported. -func (tree *VersionedTree) Set(key, val []byte) bool { - return tree.orphaningTree.Set(key, val) -} - -// Remove removes a key from the working tree. -func (tree *VersionedTree) Remove(key []byte) ([]byte, bool) { - return tree.orphaningTree.Remove(key) -} - -// Load the latest versioned tree from disk. -// -// Returns the version number of the latest version found -func (tree *VersionedTree) Load() (int64, error) { - return tree.LoadVersion(0) -} - -// Load a versioned tree from disk. -// -// If version is 0, the latest version is loaded. -// -// Returns the version number of the latest version found -func (tree *VersionedTree) LoadVersion(targetVersion int64) (int64, error) { - roots, err := tree.ndb.getRoots() - if err != nil { - return 0, err - } - if len(roots) == 0 { - return 0, nil - } - - // Load all roots from the database. - latestVersion := int64(0) - for version, root := range roots { - - // Construct a tree manually. - t := &Tree{} - t.ndb = tree.ndb - t.version = version - if len(root) != 0 { - t.root = tree.ndb.GetNode(root) - } - tree.versions[version] = t - - if version > latestVersion && - (targetVersion == 0 || version <= targetVersion) { - - latestVersion = version - } - } - - // Validate latestVersion - if !(targetVersion == 0 || latestVersion == targetVersion) { - return latestVersion, fmt.Errorf("Wanted to load target %v but only found up to %v", - targetVersion, latestVersion) - } - - // Set the working tree to a copy of the latest. - tree.orphaningTree = newOrphaningTree( - tree.versions[latestVersion].clone(), - ) - - return latestVersion, nil -} - -// Rollback resets the working tree to the latest saved version, discarding -// any unsaved modifications. -func (tree *VersionedTree) Rollback() { - if tree.version > 0 { - tree.orphaningTree = newOrphaningTree( - tree.versions[tree.version].clone(), - ) - } else { - tree.orphaningTree = newOrphaningTree(&Tree{ndb: tree.ndb, version: 0}) - } -} - -// GetVersioned gets the value at the specified key and version. -func (tree *VersionedTree) GetVersioned(key []byte, version int64) ( - index int, value []byte, -) { - if t, ok := tree.versions[version]; ok { - return t.Get(key) - } - return -1, nil -} - -// SaveVersion saves a new tree version to disk, based on the current state of -// the tree. Returns the hash and new version number. -func (tree *VersionedTree) SaveVersion() ([]byte, int64, error) { - version := tree.version + 1 - - if _, ok := tree.versions[version]; ok { - // Same hash means idempotent. Return success. - var existingHash = tree.versions[version].Hash() - var newHash = tree.orphaningTree.Hash() - if bytes.Equal(existingHash, newHash) { - tree.orphaningTree = newOrphaningTree(tree.versions[version].clone()) - return existingHash, version, nil - } - return nil, version, fmt.Errorf("version %d was already saved to different hash %X (existing hash %X)", - version, newHash, existingHash) - } - - // Persist version and stash to .versions. - tree.orphaningTree.SaveAs(version) - tree.versions[version] = tree.orphaningTree.Tree - - // Set new working tree. - tree.orphaningTree = newOrphaningTree(tree.orphaningTree.clone()) - - return tree.Hash(), version, nil -} - -// DeleteVersion deletes a tree version from disk. The version can then no -// longer be accessed. -func (tree *VersionedTree) DeleteVersion(version int64) error { - if version == 0 { - return cmn.NewError("version must be greater than 0") - } - if version == tree.version { - return cmn.NewError("cannot delete latest saved version (%d)", version) - } - if _, ok := tree.versions[version]; !ok { - return cmn.ErrorWrap(ErrVersionDoesNotExist, "") - } - - tree.ndb.DeleteVersion(version) - tree.ndb.Commit() - - delete(tree.versions, version) - - return nil -}