From be26364c73155fb049b0459d7557010ad4ac46e6 Mon Sep 17 00:00:00 2001 From: Cool Developer Date: Tue, 17 Jan 2023 15:47:39 -0500 Subject: [PATCH 1/7] refactor the export traversal order as pre-order --- export.go | 4 +- export_test.go | 8 ++-- import.go | 127 +++++++++++++++++++++++-------------------------- import_test.go | 2 + node.go | 9 +--- node_test.go | 4 +- 6 files changed, 70 insertions(+), 84 deletions(-) diff --git a/export.go b/export.go index e19998d9e..04b31909c 100644 --- a/export.go +++ b/export.go @@ -28,7 +28,7 @@ type ExportNode struct { // Exporter exports nodes from an ImmutableTree. It is created by ImmutableTree.Export(). // // Exported nodes can be imported into an empty tree with MutableTree.Import(). Nodes are exported -// depth-first post-order (LRN), this order must be preserved when importing in order to recreate +// depth-first pre-order (NLR), this order must be preserved when importing in order to recreate // the same tree structure. type Exporter struct { tree *ImmutableTree @@ -61,7 +61,7 @@ func newExporter(tree *ImmutableTree) (*Exporter, error) { // export exports nodes func (e *Exporter) export(ctx context.Context) { - e.tree.root.traversePost(e.tree, true, func(node *Node) bool { + e.tree.root.traverse(e.tree, true, func(node *Node) bool { exportNode := &ExportNode{ Key: node.key, Value: node.value, diff --git a/export_test.go b/export_test.go index 1294f71b1..489877d5a 100644 --- a/export_test.go +++ b/export_test.go @@ -161,15 +161,15 @@ func TestExporter(t *testing.T) { tree := setupExportTreeBasic(t) expect := []*ExportNode{ + {Key: []byte("d"), Value: nil, Version: 3, Height: 3}, + {Key: []byte("c"), Value: nil, Version: 3, Height: 2}, + {Key: []byte("b"), Value: nil, Version: 3, Height: 1}, {Key: []byte("a"), Value: []byte{1}, Version: 1, Height: 0}, {Key: []byte("b"), Value: []byte{2}, Version: 3, Height: 0}, - {Key: []byte("b"), Value: nil, Version: 3, Height: 1}, {Key: []byte("c"), Value: []byte{3}, Version: 3, Height: 0}, - {Key: []byte("c"), Value: nil, Version: 3, Height: 2}, + {Key: []byte("e"), Value: nil, Version: 3, Height: 1}, {Key: []byte("d"), Value: []byte{4}, Version: 2, Height: 0}, {Key: []byte("e"), Value: []byte{5}, Version: 3, Height: 0}, - {Key: []byte("e"), Value: nil, Version: 3, Height: 1}, - {Key: []byte("d"), Value: nil, Version: 3, Height: 3}, } actual := make([]*ExportNode, 0, len(expect)) diff --git a/import.go b/import.go index 550593b41..130a31262 100644 --- a/import.go +++ b/import.go @@ -17,7 +17,7 @@ var ErrNoImport = errors.New("no import in progress") // Importer imports data into an empty MutableTree. It is created by MutableTree.Import(). Users // must call Close() when done. // -// ExportNodes must be imported in the order returned by Exporter, i.e. depth-first post-order (LRN). +// ExportNodes must be imported in the order returned by Exporter, i.e. depth-first pre-order (NLR). // // Importer is not concurrency-safe, it is the caller's responsibility to ensure the tree is not // modified while performing an import. @@ -52,6 +52,43 @@ func newImporter(tree *MutableTree, version int64) (*Importer, error) { }, nil } +// writeNode writes the node content to the storage. +func (i *Importer) writeNode(node *Node) error { + if _, err := node._hash(); err != nil { + return err + } + if err := node.validate(); err != nil { + return err + } + + buf := bufPool.Get().(*bytes.Buffer) + buf.Reset() + defer bufPool.Put(buf) + + if err := node.writeBytes(buf); err != nil { + return err + } + + bytesCopy := make([]byte, buf.Len()) + copy(bytesCopy, buf.Bytes()) + + if err := i.batch.Set(i.tree.ndb.nodeKey(node.hash), bytesCopy); err != nil { + return err + } + + i.batchSize++ + if i.batchSize >= maxBatchSize { + if err := i.batch.Write(); err != nil { + return err + } + i.batch.Close() + i.batch = i.tree.ndb.db.NewBatch() + i.batchSize = 0 + } + + return nil +} + // Close frees all resources. It is safe to call multiple times. Uncommitted nodes may already have // been flushed to the database, but will not be visible. func (i *Importer) Close() { @@ -63,7 +100,7 @@ func (i *Importer) Close() { } // Add adds an ExportNode to the import. ExportNodes must be added in the order returned by -// Exporter, i.e. depth-first post-order (LRN). Nodes are periodically flushed to the database, +// Exporter, i.e. depth-first pre-order (NLR). Nodes are periodically flushed to the database, // but the imported version is not visible until Commit() is called. func (i *Importer) Add(exportNode *ExportNode) error { if i.tree == nil { @@ -84,80 +121,31 @@ func (i *Importer) Add(exportNode *ExportNode) error { subtreeHeight: exportNode.Height, } - // We build the tree from the bottom-left up. The stack is used to store unresolved left + // We build the tree from the bottom-left up. The stack is used to store resolved left // children while constructing right children. When all children are built, the parent can - // be constructed and the resolved children can be discarded from the stack. Using a stack - // ensures that we can handle additional unresolved left children while building a right branch. - // - // We don't modify the stack until we've verified the built node, to avoid leaving the - // importer in an inconsistent state when we return an error. - stackSize := len(i.stack) - switch { - case stackSize >= 2 && i.stack[stackSize-1].subtreeHeight < node.subtreeHeight && i.stack[stackSize-2].subtreeHeight < node.subtreeHeight: - node.leftNode = i.stack[stackSize-2] - node.leftHash = node.leftNode.hash - node.rightNode = i.stack[stackSize-1] - node.rightHash = node.rightNode.hash - case stackSize >= 1 && i.stack[stackSize-1].subtreeHeight < node.subtreeHeight: - node.leftNode = i.stack[stackSize-1] - node.leftHash = node.leftNode.hash - } + // be constructed and the resolved children can be discarded from the stack. if node.subtreeHeight == 0 { node.size = 1 } - if node.leftNode != nil { - node.size += node.leftNode.size - } - if node.rightNode != nil { - node.size += node.rightNode.size - } - - _, err := node._hash() - if err != nil { - return err - } - - err = node.validate() - if err != nil { - return err - } - - buf := bufPool.Get().(*bytes.Buffer) - buf.Reset() - defer bufPool.Put(buf) - - if err = node.writeBytes(buf); err != nil { - return err - } - - bytesCopy := make([]byte, buf.Len()) - copy(bytesCopy, buf.Bytes()) - - if err = i.batch.Set(i.tree.ndb.nodeKey(node.hash), bytesCopy); err != nil { - return err - } - - i.batchSize++ - if i.batchSize >= maxBatchSize { - err = i.batch.Write() - if err != nil { + // just append to the stack + i.stack = append(i.stack, node) + stackSize := len(i.stack) + // if the last two nodes of the stack are resolved, it means we can resolve the parent node + for stackSize > 2 && i.stack[stackSize-1].size > 0 && i.stack[stackSize-2].size > 0 { + right, left, parent := i.stack[stackSize-1], i.stack[stackSize-2], i.stack[stackSize-3] + if err := i.writeNode(right); err != nil { return err } - i.batch.Close() - i.batch = i.tree.ndb.db.NewBatch() - i.batchSize = 0 - } - - // Update the stack now that we know there were no errors - switch { - case node.leftHash != nil && node.rightHash != nil: - i.stack = i.stack[:stackSize-2] - case node.leftHash != nil || node.rightHash != nil: - i.stack = i.stack[:stackSize-1] + if err := i.writeNode(left); err != nil { + return err + } + parent.leftHash = left.hash + parent.rightHash = right.hash + parent.size = left.size + right.size + stackSize -= 2 + i.stack = i.stack[:stackSize] } - // Only hash\height\size of the node will be used after it be pushed into the stack. - i.stack = append(i.stack, &Node{hash: node.hash, subtreeHeight: node.subtreeHeight, size: node.size}) return nil } @@ -176,6 +164,9 @@ func (i *Importer) Commit() error { return err } case 1: + if err := i.writeNode(i.stack[0]); err != nil { + return err + } if err := i.batch.Set(i.tree.ndb.rootKey(i.version), i.stack[0].hash); err != nil { return err } diff --git a/import_test.go b/import_test.go index eaceff335..45fb0c36e 100644 --- a/import_test.go +++ b/import_test.go @@ -148,6 +148,8 @@ func TestImporter_Add(t *testing.T) { require.NoError(t, err) defer importer.Close() + require.NoError(t, importer.Add(&ExportNode{Key: k, Value: v, Version: 1, Height: 1})) + require.NoError(t, importer.Add(&ExportNode{Key: k, Value: v, Version: 1, Height: 0})) err = importer.Add(tc.node) if tc.valid { require.NoError(t, err) diff --git a/node.go b/node.go index f138d51ed..5f79288d5 100644 --- a/node.go +++ b/node.go @@ -321,7 +321,7 @@ func (node *Node) validate() error { if node.value != nil { return errors.New("value must be nil for non-leaf node") } - if node.leftHash == nil && node.rightHash == nil { + if node.leftHash == nil || node.rightHash == nil { return errors.New("inner node must have children") } } @@ -527,13 +527,6 @@ func (node *Node) traverse(t *ImmutableTree, ascending bool, cb func(*Node) bool }) } -// traversePost is a wrapper over traverseInRange when we want the whole tree post-order -func (node *Node) traversePost(t *ImmutableTree, ascending bool, cb func(*Node) bool) bool { - return node.traverseInRange(t, nil, nil, ascending, false, true, func(node *Node) bool { - return cb(node) - }) -} - func (node *Node) traverseInRange(tree *ImmutableTree, start, end []byte, ascending bool, inclusive bool, post bool, cb func(*Node) bool) bool { stop := false t := node.newTraversal(tree, start, end, ascending, inclusive, post) diff --git a/node_test.go b/node_test.go index 8837c4337..0623ea31a 100644 --- a/node_test.go +++ b/node_test.go @@ -114,8 +114,8 @@ func TestNode_validate(t *testing.T) { "inner with nil key": {&Node{key: nil, value: v, version: 1, size: 1, subtreeHeight: 1, leftHash: h, rightHash: h}, false}, "inner with value": {&Node{key: k, value: v, version: 1, size: 1, subtreeHeight: 1, leftHash: h, rightHash: h}, false}, "inner with empty value": {&Node{key: k, value: []byte{}, version: 1, size: 1, subtreeHeight: 1, leftHash: h, rightHash: h}, false}, - "inner with left child": {&Node{key: k, version: 1, size: 1, subtreeHeight: 1, leftHash: h}, true}, - "inner with right child": {&Node{key: k, version: 1, size: 1, subtreeHeight: 1, rightHash: h}, true}, + "inner with left child": {&Node{key: k, version: 1, size: 1, subtreeHeight: 1, leftHash: h}, false}, + "inner with right child": {&Node{key: k, version: 1, size: 1, subtreeHeight: 1, rightHash: h}, false}, "inner with no child": {&Node{key: k, version: 1, size: 1, subtreeHeight: 1}, false}, "inner with height 0": {&Node{key: k, version: 1, size: 1, subtreeHeight: 0, leftHash: h, rightHash: h}, false}, } From 58fab7e0f78b89a9f98c88fd1c4b17145c16643b Mon Sep 17 00:00:00 2001 From: Cool Developer Date: Tue, 17 Jan 2023 16:00:09 -0500 Subject: [PATCH 2/7] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b42bde17..309cfd017 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - [#640](https://github.com/cosmos/iavl/pull/640) commit `NodeDB` batch in `LoadVersionForOverwriting`. - [#636](https://github.com/cosmos/iavl/pull/636) Speed up rollback method: `LoadVersionForOverwriting`. - [#654](https://github.com/cosmos/iavl/pull/654) Add API `TraverseStateChanges` to extract state changes from iavl versions. +- [#662](https://github.com/cosmos/iavl/pull/662) Refactor the traversal order of `Export` from `post-order` to `pre-order`. ## 0.19.4 (October 28, 2022) From 18f4491681b925567b0cee96b06b4be0b8e35dbc Mon Sep 17 00:00:00 2001 From: Cool Developer Date: Thu, 26 Jan 2023 15:08:02 -0500 Subject: [PATCH 3/7] add flag --- benchmarks/cosmos-exim/main.go | 4 +- export.go | 80 ++++++++++----- export_test.go | 180 +++++++++++++++++++-------------- immutable_tree.go | 4 +- import.go | 94 +++++++++++------ import_test.go | 119 ++++++++++++---------- mutable_tree.go | 4 +- node.go | 7 ++ 8 files changed, 302 insertions(+), 190 deletions(-) diff --git a/benchmarks/cosmos-exim/main.go b/benchmarks/cosmos-exim/main.go index b6e3023fd..940144027 100644 --- a/benchmarks/cosmos-exim/main.go +++ b/benchmarks/cosmos-exim/main.go @@ -127,7 +127,7 @@ func runExport(dbPath string) (int64, map[string][]*iavl.ExportNode, error) { return 0, nil, err } start := time.Now().UTC() - exporter, err := itree.Export() + exporter, err := itree.Export(iavl.PreOrderTraverse) if err != nil { return 0, nil, err } @@ -177,7 +177,7 @@ func runImport(version int64, exports map[string][]*iavl.ExportNode) error { if err != nil { return err } - importer, err := newTree.Import(version) + importer, err := newTree.Import(version, iavl.PreOrderTraverse) if err != nil { return err } diff --git a/export.go b/export.go index 04b31909c..b64b20fa9 100644 --- a/export.go +++ b/export.go @@ -6,6 +6,14 @@ import ( "fmt" ) +type OrderType int + +// OrderTraverse is the type of traversal order to use when exporting and importing. +const ( + PreOrderTraverse OrderType = iota + PostOrderTraverse +) + // exportBufferSize is the number of nodes to buffer in the exporter. It improves throughput by // processing multiple nodes per context switch, but take care to avoid excessive memory usage, // especially since callers may export several IAVL stores in parallel (e.g. the Cosmos SDK). @@ -27,17 +35,17 @@ type ExportNode struct { // Exporter exports nodes from an ImmutableTree. It is created by ImmutableTree.Export(). // -// Exported nodes can be imported into an empty tree with MutableTree.Import(). Nodes are exported -// depth-first pre-order (NLR), this order must be preserved when importing in order to recreate -// the same tree structure. +// Exported nodes can be imported into an empty tree with MutableTree.Import(). Nodes are exported in traverseOrder, +// this order must be preserved when importing in order to recreate the same tree structure. type Exporter struct { - tree *ImmutableTree - ch chan *ExportNode - cancel context.CancelFunc + tree *ImmutableTree + traverseOrder OrderType + ch chan *ExportNode + cancel context.CancelFunc } // NewExporter creates a new Exporter. Callers must call Close() when done. -func newExporter(tree *ImmutableTree) (*Exporter, error) { +func newExporter(tree *ImmutableTree, traverseOrder OrderType) (*Exporter, error) { if tree == nil { return nil, fmt.Errorf("tree is nil: %w", ErrNotInitalizedTree) } @@ -48,9 +56,10 @@ func newExporter(tree *ImmutableTree) (*Exporter, error) { ctx, cancel := context.WithCancel(context.Background()) exporter := &Exporter{ - tree: tree, - ch: make(chan *ExportNode, exportBufferSize), - cancel: cancel, + tree: tree, + traverseOrder: traverseOrder, + ch: make(chan *ExportNode, exportBufferSize), + cancel: cancel, } tree.ndb.incrVersionReaders(tree.version) @@ -61,22 +70,41 @@ func newExporter(tree *ImmutableTree) (*Exporter, error) { // export exports nodes func (e *Exporter) export(ctx context.Context) { - e.tree.root.traverse(e.tree, true, func(node *Node) bool { - exportNode := &ExportNode{ - Key: node.key, - Value: node.value, - Version: node.version, - Height: node.subtreeHeight, - } - - select { - case e.ch <- exportNode: - return false - case <-ctx.Done(): - return true - } - }) - close(e.ch) + defer close(e.ch) + switch e.traverseOrder { + case PreOrderTraverse: + e.tree.root.traverse(e.tree, true, func(node *Node) bool { + exportNode := &ExportNode{ + Key: node.key, + Value: node.value, + Version: node.version, + Height: node.subtreeHeight, + } + + select { + case e.ch <- exportNode: + return false + case <-ctx.Done(): + return true + } + }) + case PostOrderTraverse: + e.tree.root.traversePost(e.tree, true, func(node *Node) bool { + exportNode := &ExportNode{ + Key: node.key, + Value: node.value, + Version: node.version, + Height: node.subtreeHeight, + } + + select { + case e.ch <- exportNode: + return false + case <-ctx.Done(): + return true + } + }) + } } // Next fetches the next exported node, or returns ExportDone when done. diff --git a/export_test.go b/export_test.go index 489877d5a..85ee35ce6 100644 --- a/export_test.go +++ b/export_test.go @@ -160,32 +160,46 @@ func setupExportTreeSized(t require.TestingT, treeSize int) *ImmutableTree { //n func TestExporter(t *testing.T) { tree := setupExportTreeBasic(t) - expect := []*ExportNode{ - {Key: []byte("d"), Value: nil, Version: 3, Height: 3}, - {Key: []byte("c"), Value: nil, Version: 3, Height: 2}, - {Key: []byte("b"), Value: nil, Version: 3, Height: 1}, - {Key: []byte("a"), Value: []byte{1}, Version: 1, Height: 0}, - {Key: []byte("b"), Value: []byte{2}, Version: 3, Height: 0}, - {Key: []byte("c"), Value: []byte{3}, Version: 3, Height: 0}, - {Key: []byte("e"), Value: nil, Version: 3, Height: 1}, - {Key: []byte("d"), Value: []byte{4}, Version: 2, Height: 0}, - {Key: []byte("e"), Value: []byte{5}, Version: 3, Height: 0}, + expects := map[OrderType][]*ExportNode{ + PreOrderTraverse: { + {Key: []byte("d"), Value: nil, Version: 3, Height: 3}, + {Key: []byte("c"), Value: nil, Version: 3, Height: 2}, + {Key: []byte("b"), Value: nil, Version: 3, Height: 1}, + {Key: []byte("a"), Value: []byte{1}, Version: 1, Height: 0}, + {Key: []byte("b"), Value: []byte{2}, Version: 3, Height: 0}, + {Key: []byte("c"), Value: []byte{3}, Version: 3, Height: 0}, + {Key: []byte("e"), Value: nil, Version: 3, Height: 1}, + {Key: []byte("d"), Value: []byte{4}, Version: 2, Height: 0}, + {Key: []byte("e"), Value: []byte{5}, Version: 3, Height: 0}, + }, + PostOrderTraverse: { + {Key: []byte("a"), Value: []byte{1}, Version: 1, Height: 0}, + {Key: []byte("b"), Value: []byte{2}, Version: 3, Height: 0}, + {Key: []byte("b"), Value: nil, Version: 3, Height: 1}, + {Key: []byte("c"), Value: []byte{3}, Version: 3, Height: 0}, + {Key: []byte("c"), Value: nil, Version: 3, Height: 2}, + {Key: []byte("d"), Value: []byte{4}, Version: 2, Height: 0}, + {Key: []byte("e"), Value: []byte{5}, Version: 3, Height: 0}, + {Key: []byte("e"), Value: nil, Version: 3, Height: 1}, + {Key: []byte("d"), Value: nil, Version: 3, Height: 3}, + }, } - - actual := make([]*ExportNode, 0, len(expect)) - exporter, err := tree.Export() - require.NoError(t, err) - defer exporter.Close() - for { - node, err := exporter.Next() - if err == ErrorExportDone { - break - } + for orderType, expect := range expects { + actual := make([]*ExportNode, 0, len(expect)) + exporter, err := tree.Export(orderType) require.NoError(t, err) - actual = append(actual, node) - } + defer exporter.Close() + for { + node, err := exporter.Next() + if err == ErrorExportDone { + break + } + require.NoError(t, err) + actual = append(actual, node) + } - assert.Equal(t, expect, actual) + assert.Equal(t, expect, actual) + } } func TestExporter_Import(t *testing.T) { @@ -197,59 +211,60 @@ func TestExporter_Import(t *testing.T) { testcases["sized tree"] = setupExportTreeSized(t, 4096) testcases["random tree"] = setupExportTreeRandom(t) } + for _, orderType := range []OrderType{PostOrderTraverse, PreOrderTraverse} { + for desc, tree := range testcases { + tree := tree + t.Run(desc, func(t *testing.T) { + // t.Parallel() - for desc, tree := range testcases { - tree := tree - t.Run(desc, func(t *testing.T) { - t.Parallel() - - exporter, err := tree.Export() - require.NoError(t, err) - defer exporter.Close() - - newTree, err := NewMutableTree(db.NewMemDB(), 0, false) - require.NoError(t, err) - importer, err := newTree.Import(tree.Version()) - require.NoError(t, err) - defer importer.Close() + exporter, err := tree.Export(orderType) + require.NoError(t, err) + defer exporter.Close() - for { - item, err := exporter.Next() - if err == ErrorExportDone { - err = importer.Commit() + newTree, err := NewMutableTree(db.NewMemDB(), 0, false) + require.NoError(t, err) + importer, err := newTree.Import(tree.Version(), orderType) + require.NoError(t, err) + defer importer.Close() + + for { + item, err := exporter.Next() + if err == ErrorExportDone { + err = importer.Commit() + require.NoError(t, err) + break + } + require.NoError(t, err) + err = importer.Add(item) require.NoError(t, err) - break } + + treeHash, err := tree.Hash() require.NoError(t, err) - err = importer.Add(item) + newTreeHash, err := newTree.Hash() require.NoError(t, err) - } - - treeHash, err := tree.Hash() - require.NoError(t, err) - newTreeHash, err := newTree.Hash() - require.NoError(t, err) - require.Equal(t, treeHash, newTreeHash, "Tree hash mismatch") - require.Equal(t, tree.Size(), newTree.Size(), "Tree size mismatch") - require.Equal(t, tree.Version(), newTree.Version(), "Tree version mismatch") + require.Equal(t, treeHash, newTreeHash, "Tree hash mismatch") + require.Equal(t, tree.Size(), newTree.Size(), "Tree size mismatch") + require.Equal(t, tree.Version(), newTree.Version(), "Tree version mismatch") - tree.Iterate(func(key, value []byte) bool { //nolint:errcheck - index, _, err := tree.GetWithIndex(key) - require.NoError(t, err) - newIndex, newValue, err := newTree.GetWithIndex(key) - require.NoError(t, err) - require.Equal(t, index, newIndex, "Index mismatch for key %v", key) - require.Equal(t, value, newValue, "Value mismatch for key %v", key) - return false + tree.Iterate(func(key, value []byte) bool { //nolint:errcheck + index, _, err := tree.GetWithIndex(key) + require.NoError(t, err) + newIndex, newValue, err := newTree.GetWithIndex(key) + require.NoError(t, err) + require.Equal(t, index, newIndex, "Index mismatch for key %v", key) + require.Equal(t, value, newValue, "Value mismatch for key %v", key) + return false + }) }) - }) + } } } func TestExporter_Close(t *testing.T) { tree := setupExportTreeSized(t, 4096) - exporter, err := tree.Export() + exporter, err := tree.Export(PreOrderTraverse) require.NoError(t, err) node, err := exporter.Next() @@ -292,7 +307,7 @@ func TestExporter_DeleteVersionErrors(t *testing.T) { itree, err := tree.GetImmutable(2) require.NoError(t, err) - exporter, err := itree.Export() + exporter, err := itree.Export(PostOrderTraverse) require.NoError(t, err) defer exporter.Close() @@ -310,17 +325,34 @@ func BenchmarkExport(b *testing.B) { b.StopTimer() tree := setupExportTreeSized(b, 4096) b.StartTimer() - for n := 0; n < b.N; n++ { - exporter, err := tree.Export() - require.NoError(b, err) - for { - _, err := exporter.Next() - if err == ErrorExportDone { - break - } else if err != nil { - b.Error(err) + b.Run("post order export", func(sub *testing.B) { + for n := 0; n < sub.N; n++ { + exporter, err := tree.Export(PostOrderTraverse) + require.NoError(sub, err) + for { + _, err := exporter.Next() + if err == ErrorExportDone { + break + } else if err != nil { + sub.Error(err) + } } + exporter.Close() } - exporter.Close() - } + }) + b.Run("pre order export", func(sub *testing.B) { + for n := 0; n < sub.N; n++ { + exporter, err := tree.Export(PreOrderTraverse) + require.NoError(sub, err) + for { + _, err := exporter.Next() + if err == ErrorExportDone { + break + } else if err != nil { + sub.Error(err) + } + } + exporter.Close() + } + }) } diff --git a/immutable_tree.go b/immutable_tree.go index 734cd7e9b..4eec71ee3 100644 --- a/immutable_tree.go +++ b/immutable_tree.go @@ -155,8 +155,8 @@ func (t *ImmutableTree) Hash() ([]byte, error) { // Export returns an iterator that exports tree nodes as ExportNodes. These nodes can be // imported with MutableTree.Import() to recreate an identical tree. -func (t *ImmutableTree) Export() (*Exporter, error) { - return newExporter(t) +func (t *ImmutableTree) Export(traverseOrder OrderType) (*Exporter, error) { + return newExporter(t, traverseOrder) } // GetWithIndex returns the index and value of the specified key if it exists, or nil and the next index diff --git a/import.go b/import.go index 130a31262..5c318ba11 100644 --- a/import.go +++ b/import.go @@ -17,23 +17,24 @@ var ErrNoImport = errors.New("no import in progress") // Importer imports data into an empty MutableTree. It is created by MutableTree.Import(). Users // must call Close() when done. // -// ExportNodes must be imported in the order returned by Exporter, i.e. depth-first pre-order (NLR). +// ExportNodes must be imported in the order returned by Exporter, i.e. depth-first pre-order (NLR) in case of PreOrderType. // // Importer is not concurrency-safe, it is the caller's responsibility to ensure the tree is not // modified while performing an import. type Importer struct { - tree *MutableTree - version int64 - batch db.Batch - batchSize uint32 - stack []*Node + tree *MutableTree + traverseOrder OrderType + version int64 + batch db.Batch + batchSize uint32 + stack []*Node } // newImporter creates a new Importer for an empty MutableTree. // // version should correspond to the version that was initially exported. It must be greater than // or equal to the highest ExportNode version number given. -func newImporter(tree *MutableTree, version int64) (*Importer, error) { +func newImporter(tree *MutableTree, version int64, traverseOrder OrderType) (*Importer, error) { if version < 0 { return nil, errors.New("imported version cannot be negative") } @@ -45,10 +46,11 @@ func newImporter(tree *MutableTree, version int64) (*Importer, error) { } return &Importer{ - tree: tree, - version: version, - batch: tree.ndb.db.NewBatch(), - stack: make([]*Node, 0, 8), + tree: tree, + traverseOrder: traverseOrder, + version: version, + batch: tree.ndb.db.NewBatch(), + stack: make([]*Node, 0, 8), }, nil } @@ -120,33 +122,57 @@ func (i *Importer) Add(exportNode *ExportNode) error { version: exportNode.Version, subtreeHeight: exportNode.Height, } - - // We build the tree from the bottom-left up. The stack is used to store resolved left - // children while constructing right children. When all children are built, the parent can - // be constructed and the resolved children can be discarded from the stack. - if node.subtreeHeight == 0 { node.size = 1 } - // just append to the stack - i.stack = append(i.stack, node) - stackSize := len(i.stack) - // if the last two nodes of the stack are resolved, it means we can resolve the parent node - for stackSize > 2 && i.stack[stackSize-1].size > 0 && i.stack[stackSize-2].size > 0 { - right, left, parent := i.stack[stackSize-1], i.stack[stackSize-2], i.stack[stackSize-3] - if err := i.writeNode(right); err != nil { - return err + // We build the tree from the bottom-left up. The stack is used to store unresolved left + // children while constructing right children. When all children are built, the parent can + // be constructed and the resolved children can be discarded from the stack. Using a stack + // ensures that we can handle additional unresolved left children while building a right branch. + switch i.traverseOrder { + case PreOrderTraverse: + // just append to the stack + i.stack = append(i.stack, node) + stackSize := len(i.stack) + // if the last two nodes of the stack are resolved, it means we can resolve the parent node + for stackSize > 2 && i.stack[stackSize-1].size > 0 && i.stack[stackSize-2].size > 0 { + right, left, parent := i.stack[stackSize-1], i.stack[stackSize-2], i.stack[stackSize-3] + if err := i.writeNode(right); err != nil { + return err + } + if err := i.writeNode(left); err != nil { + return err + } + parent.leftHash = left.hash + parent.rightHash = right.hash + parent.size = left.size + right.size + stackSize -= 2 + i.stack = i.stack[:stackSize] } - if err := i.writeNode(left); err != nil { + case PostOrderTraverse: + // We don't modify the stack until we've verified the built node, to avoid leaving the + // importer in an inconsistent state when we return an error. + stackSize := len(i.stack) + if stackSize >= 2 && i.stack[stackSize-1].subtreeHeight < node.subtreeHeight && i.stack[stackSize-2].subtreeHeight < node.subtreeHeight { + node.leftNode = i.stack[stackSize-2] + node.leftHash = node.leftNode.hash + node.rightNode = i.stack[stackSize-1] + node.rightHash = node.rightNode.hash + node.size = node.leftNode.size + node.rightNode.size + } + if err := i.writeNode(node); err != nil { return err } - parent.leftHash = left.hash - parent.rightHash = right.hash - parent.size = left.size + right.size - stackSize -= 2 - i.stack = i.stack[:stackSize] + // Update the stack now that we know there were no errors + switch { + case node.leftHash != nil && node.rightHash != nil: + i.stack = i.stack[:stackSize-2] + case node.leftHash != nil || node.rightHash != nil: + i.stack = i.stack[:stackSize-1] + } + // Only hash\height\size of the node will be used after it be pushed into the stack. + i.stack = append(i.stack, &Node{hash: node.hash, subtreeHeight: node.subtreeHeight, size: node.size}) } - return nil } @@ -164,8 +190,10 @@ func (i *Importer) Commit() error { return err } case 1: - if err := i.writeNode(i.stack[0]); err != nil { - return err + if i.traverseOrder == PreOrderTraverse { + if err := i.writeNode(i.stack[0]); err != nil { + return err + } } if err := i.batch.Set(i.tree.ndb.rootKey(i.version), i.stack[0].hash); err != nil { return err diff --git a/import_test.go b/import_test.go index 45fb0c36e..f8a4de549 100644 --- a/import_test.go +++ b/import_test.go @@ -1,6 +1,7 @@ package iavl import ( + "fmt" "testing" "github.com/stretchr/testify/assert" @@ -37,7 +38,7 @@ func ExampleImporter() { if err != nil { // handle err } - exporter, err := itree.Export() + exporter, err := itree.Export(PostOrderTraverse) if err != nil { // handle err } @@ -58,7 +59,7 @@ func ExampleImporter() { if err != nil { // handle err } - importer, err := newTree.Import(version) + importer, err := newTree.Import(version, PostOrderTraverse) if err != nil { // handle err } @@ -78,7 +79,7 @@ func ExampleImporter() { func TestImporter_NegativeVersion(t *testing.T) { tree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(t, err) - _, err = tree.Import(-1) + _, err = tree.Import(-1, PostOrderTraverse) require.Error(t, err) } @@ -90,7 +91,7 @@ func TestImporter_NotEmpty(t *testing.T) { _, _, err = tree.SaveVersion() require.NoError(t, err) - _, err = tree.Import(1) + _, err = tree.Import(1, PostOrderTraverse) require.Error(t, err) } @@ -109,7 +110,7 @@ func TestImporter_NotEmptyDatabase(t *testing.T) { _, err = tree.Load() require.NoError(t, err) - _, err = tree.Import(1) + _, err = tree.Import(1, PostOrderTraverse) require.Error(t, err) } @@ -119,7 +120,7 @@ func TestImporter_NotEmptyUnsaved(t *testing.T) { _, err = tree.Set([]byte("a"), []byte{1}) require.NoError(t, err) - _, err = tree.Import(1) + _, err = tree.Import(1, PostOrderTraverse) require.Error(t, err) } @@ -142,19 +143,23 @@ func TestImporter_Add(t *testing.T) { for desc, tc := range testcases { tc := tc // appease scopelint t.Run(desc, func(t *testing.T) { - tree, err := NewMutableTree(db.NewMemDB(), 0, false) - require.NoError(t, err) - importer, err := tree.Import(1) - require.NoError(t, err) - defer importer.Close() - - require.NoError(t, importer.Add(&ExportNode{Key: k, Value: v, Version: 1, Height: 1})) - require.NoError(t, importer.Add(&ExportNode{Key: k, Value: v, Version: 1, Height: 0})) - err = importer.Add(tc.node) - if tc.valid { + for _, orderType := range []OrderType{PreOrderTraverse, PostOrderTraverse} { + tree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(t, err) - } else { - require.Error(t, err) + var importer *Importer + importer, err = tree.Import(1, orderType) + require.NoError(t, err) + defer importer.Close() + if orderType == PreOrderTraverse { + require.NoError(t, importer.Add(&ExportNode{Key: k, Value: v, Version: 1, Height: 1})) + require.NoError(t, importer.Add(&ExportNode{Key: k, Value: v, Version: 1, Height: 0})) + } + err = importer.Add(tc.node) + if tc.valid { + require.NoError(t, err) + } else { + require.Error(t, err) + } } }) } @@ -163,7 +168,7 @@ func TestImporter_Add(t *testing.T) { func TestImporter_Add_Closed(t *testing.T) { tree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(t, err) - importer, err := tree.Import(1) + importer, err := tree.Import(1, PostOrderTraverse) require.NoError(t, err) importer.Close() @@ -175,7 +180,7 @@ func TestImporter_Add_Closed(t *testing.T) { func TestImporter_Close(t *testing.T) { tree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(t, err) - importer, err := tree.Import(1) + importer, err := tree.Import(1, PostOrderTraverse) require.NoError(t, err) err = importer.Add(&ExportNode{Key: []byte("key"), Value: []byte("value"), Version: 1, Height: 0}) @@ -192,7 +197,7 @@ func TestImporter_Close(t *testing.T) { func TestImporter_Commit(t *testing.T) { tree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(t, err) - importer, err := tree.Import(1) + importer, err := tree.Import(1, PostOrderTraverse) require.NoError(t, err) err = importer.Add(&ExportNode{Key: []byte("key"), Value: []byte("value"), Version: 1, Height: 0}) @@ -208,7 +213,7 @@ func TestImporter_Commit(t *testing.T) { func TestImporter_Commit_Closed(t *testing.T) { tree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(t, err) - importer, err := tree.Import(1) + importer, err := tree.Import(1, PreOrderTraverse) require.NoError(t, err) err = importer.Add(&ExportNode{Key: []byte("key"), Value: []byte("value"), Version: 1, Height: 0}) @@ -223,7 +228,7 @@ func TestImporter_Commit_Closed(t *testing.T) { func TestImporter_Commit_Empty(t *testing.T) { tree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(t, err) - importer, err := tree.Import(3) + importer, err := tree.Import(3, PostOrderTraverse) require.NoError(t, err) defer importer.Close() @@ -233,35 +238,47 @@ func TestImporter_Commit_Empty(t *testing.T) { } func BenchmarkImport(b *testing.B) { - b.StopTimer() tree := setupExportTreeSized(b, 4096) - exported := make([]*ExportNode, 0, 4096) - exporter, err := tree.Export() - require.NoError(b, err) - for { - item, err := exporter.Next() - if err == ErrorExportDone { - break - } else if err != nil { - b.Error(err) - } - exported = append(exported, item) - } - exporter.Close() - b.StartTimer() - - for n := 0; n < b.N; n++ { - newTree, err := NewMutableTree(db.NewMemDB(), 0, false) - require.NoError(b, err) - importer, err := newTree.Import(tree.Version()) - require.NoError(b, err) - for _, item := range exported { - err = importer.Add(item) - if err != nil { - b.Error(err) + b.ResetTimer() + for _, desc := range []string{"Pre Order", "Post Order"} { + b.Run(fmt.Sprintf("%s Import", desc), func(sub *testing.B) { + sub.StopTimer() + var orderType OrderType + switch desc { + case "Pre Order": + orderType = PreOrderTraverse + case "Post Order": + orderType = PostOrderTraverse } - } - err = importer.Commit() - require.NoError(b, err) + exported := make([]*ExportNode, 0, 4096) + exporter, err := tree.Export(orderType) + require.NoError(sub, err) + for { + item, err := exporter.Next() + if err == ErrorExportDone { + break + } else if err != nil { + sub.Error(err) + } + exported = append(exported, item) + } + exporter.Close() + sub.StartTimer() + + for n := 0; n < sub.N; n++ { + newTree, err := NewMutableTree(db.NewMemDB(), 0, false) + require.NoError(sub, err) + importer, err := newTree.Import(tree.Version(), orderType) + require.NoError(sub, err) + for _, item := range exported { + err = importer.Add(item) + if err != nil { + sub.Error(err) + } + } + err = importer.Commit() + require.NoError(sub, err) + } + }) } } diff --git a/mutable_tree.go b/mutable_tree.go index 238c49969..aacbc11b7 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -172,8 +172,8 @@ func (tree *MutableTree) Get(key []byte) ([]byte, error) { // // Import can only be called on an empty tree. It is the callers responsibility that no other // modifications are made to the tree while importing. -func (tree *MutableTree) Import(version int64) (*Importer, error) { - return newImporter(tree, version) +func (tree *MutableTree) Import(version int64, traverseOrder OrderType) (*Importer, error) { + return newImporter(tree, version, traverseOrder) } // Iterate iterates over all keys of the tree. The keys and values must not be modified, diff --git a/node.go b/node.go index 5f79288d5..f2c14ce24 100644 --- a/node.go +++ b/node.go @@ -527,6 +527,13 @@ func (node *Node) traverse(t *ImmutableTree, ascending bool, cb func(*Node) bool }) } +// traversePost is a wrapper over traverseInRange when we want the whole tree post-order +func (node *Node) traversePost(t *ImmutableTree, ascending bool, cb func(*Node) bool) bool { + return node.traverseInRange(t, nil, nil, ascending, false, true, func(node *Node) bool { + return cb(node) + }) +} + func (node *Node) traverseInRange(tree *ImmutableTree, start, end []byte, ascending bool, inclusive bool, post bool, cb func(*Node) bool) bool { stop := false t := node.newTraversal(tree, start, end, ascending, inclusive, post) From 7ac76aa4e5c1e8ee266e96691b6218d1bcd49660 Mon Sep 17 00:00:00 2001 From: cool-developer <51834436+cool-develope@users.noreply.github.com> Date: Thu, 26 Jan 2023 15:12:37 -0500 Subject: [PATCH 4/7] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3eacd2be..7be86eb6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,8 +12,8 @@ - [#640](https://github.com/cosmos/iavl/pull/640) commit `NodeDB` batch in `LoadVersionForOverwriting`. - [#636](https://github.com/cosmos/iavl/pull/636) Speed up rollback method: `LoadVersionForOverwriting`. - [#654](https://github.com/cosmos/iavl/pull/654) Add API `TraverseStateChanges` to extract state changes from iavl versions. -- [#662](https://github.com/cosmos/iavl/pull/662) Refactor the traversal order of `Export` from `post-order` to `pre-order`. - [#638](https://github.com/cosmos/iavl/pull/638) Make LazyLoadVersion check the opts.InitialVersion, add API `LazyLoadVersionForOverwriting`. +- [#662](https://github.com/cosmos/iavl/pull/662) Refactor `Export` and `Import` to provide both `post-order` and `pre-order`. ## 0.19.4 (October 28, 2022) From 58bbcb08d731ac83210279c17832615cb35571af Mon Sep 17 00:00:00 2001 From: Cool Developer Date: Thu, 26 Jan 2023 15:14:56 -0500 Subject: [PATCH 5/7] small fix --- export_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/export_test.go b/export_test.go index f6300bbaf..ad00b7333 100644 --- a/export_test.go +++ b/export_test.go @@ -215,7 +215,7 @@ func TestExporter_Import(t *testing.T) { for desc, tree := range testcases { tree := tree t.Run(desc, func(t *testing.T) { - // t.Parallel() + t.Parallel() exporter, err := tree.Export(orderType) require.NoError(t, err) From 0a4235eb02062122016fcb94ad9bcb7ccc3a1375 Mon Sep 17 00:00:00 2001 From: Cool Developer Date: Tue, 31 Jan 2023 10:35:23 -0500 Subject: [PATCH 6/7] godoc --- benchmarks/cosmos-exim/main.go | 4 +-- export.go | 1 + export_test.go | 35 ++++++++++++++++++++------ immutable_tree.go | 12 +++++++-- import_test.go | 45 +++++++++++++++++++++++----------- mutable_tree.go | 10 ++++++-- 6 files changed, 80 insertions(+), 27 deletions(-) diff --git a/benchmarks/cosmos-exim/main.go b/benchmarks/cosmos-exim/main.go index 940144027..2ad6abbe8 100644 --- a/benchmarks/cosmos-exim/main.go +++ b/benchmarks/cosmos-exim/main.go @@ -127,7 +127,7 @@ func runExport(dbPath string) (int64, map[string][]*iavl.ExportNode, error) { return 0, nil, err } start := time.Now().UTC() - exporter, err := itree.Export(iavl.PreOrderTraverse) + exporter, err := itree.Export() if err != nil { return 0, nil, err } @@ -177,7 +177,7 @@ func runImport(version int64, exports map[string][]*iavl.ExportNode) error { if err != nil { return err } - importer, err := newTree.Import(version, iavl.PreOrderTraverse) + importer, err := newTree.ImportPreOrder(version) if err != nil { return err } diff --git a/export.go b/export.go index b64b20fa9..a49972554 100644 --- a/export.go +++ b/export.go @@ -9,6 +9,7 @@ import ( type OrderType int // OrderTraverse is the type of traversal order to use when exporting and importing. +// PreOrder is needed for the new node-key refactoring. The default is PostOrder. const ( PreOrderTraverse OrderType = iota PostOrderTraverse diff --git a/export_test.go b/export_test.go index ad00b7333..c1bd1eb81 100644 --- a/export_test.go +++ b/export_test.go @@ -186,7 +186,15 @@ func TestExporter(t *testing.T) { } for orderType, expect := range expects { actual := make([]*ExportNode, 0, len(expect)) - exporter, err := tree.Export(orderType) + var ( + exporter *Exporter + err error + ) + if orderType == PreOrderTraverse { + exporter, err = tree.ExportPreOrder() + } else { + exporter, err = tree.Export() + } require.NoError(t, err) defer exporter.Close() for { @@ -217,13 +225,26 @@ func TestExporter_Import(t *testing.T) { t.Run(desc, func(t *testing.T) { t.Parallel() - exporter, err := tree.Export(orderType) + var ( + exporter *Exporter + importer *Importer + err error + ) + if orderType == PreOrderTraverse { + exporter, err = tree.ExportPreOrder() + } else { + exporter, err = tree.Export() + } require.NoError(t, err) defer exporter.Close() newTree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(t, err) - importer, err := newTree.Import(tree.Version(), orderType) + if orderType == PreOrderTraverse { + importer, err = newTree.ImportPreOrder(tree.Version()) + } else { + importer, err = newTree.Import(tree.Version()) + } require.NoError(t, err) defer importer.Close() @@ -264,7 +285,7 @@ func TestExporter_Import(t *testing.T) { func TestExporter_Close(t *testing.T) { tree := setupExportTreeSized(t, 4096) - exporter, err := tree.Export(PreOrderTraverse) + exporter, err := tree.ExportPreOrder() require.NoError(t, err) node, err := exporter.Next() @@ -307,7 +328,7 @@ func TestExporter_DeleteVersionErrors(t *testing.T) { itree, err := tree.GetImmutable(2) require.NoError(t, err) - exporter, err := itree.Export(PostOrderTraverse) + exporter, err := itree.Export() require.NoError(t, err) defer exporter.Close() @@ -328,7 +349,7 @@ func BenchmarkExport(b *testing.B) { b.StartTimer() b.Run("post order export", func(sub *testing.B) { for n := 0; n < sub.N; n++ { - exporter, err := tree.Export(PostOrderTraverse) + exporter, err := tree.Export() require.NoError(sub, err) for { _, err := exporter.Next() @@ -343,7 +364,7 @@ func BenchmarkExport(b *testing.B) { }) b.Run("pre order export", func(sub *testing.B) { for n := 0; n < sub.N; n++ { - exporter, err := tree.Export(PreOrderTraverse) + exporter, err := tree.ExportPreOrder() require.NoError(sub, err) for { _, err := exporter.Next() diff --git a/immutable_tree.go b/immutable_tree.go index 4eec71ee3..8e2d31562 100644 --- a/immutable_tree.go +++ b/immutable_tree.go @@ -155,8 +155,16 @@ func (t *ImmutableTree) Hash() ([]byte, error) { // Export returns an iterator that exports tree nodes as ExportNodes. These nodes can be // imported with MutableTree.Import() to recreate an identical tree. -func (t *ImmutableTree) Export(traverseOrder OrderType) (*Exporter, error) { - return newExporter(t, traverseOrder) +// The default export order is PostOrderTraverse. +func (t *ImmutableTree) Export() (*Exporter, error) { + return newExporter(t, PostOrderTraverse) +} + +// Export returns an iterator that exports tree nodes as ExportNodes with pre-order traversal. +// This api is for new node-key refactoring since we need to build a new tree with pre-order traversal +// to construct the new node-key (path) +func (t *ImmutableTree) ExportPreOrder() (*Exporter, error) { + return newExporter(t, PreOrderTraverse) } // GetWithIndex returns the index and value of the specified key if it exists, or nil and the next index diff --git a/import_test.go b/import_test.go index f8a4de549..05407c3c4 100644 --- a/import_test.go +++ b/import_test.go @@ -38,7 +38,7 @@ func ExampleImporter() { if err != nil { // handle err } - exporter, err := itree.Export(PostOrderTraverse) + exporter, err := itree.Export() if err != nil { // handle err } @@ -59,7 +59,7 @@ func ExampleImporter() { if err != nil { // handle err } - importer, err := newTree.Import(version, PostOrderTraverse) + importer, err := newTree.Import(version) if err != nil { // handle err } @@ -79,7 +79,7 @@ func ExampleImporter() { func TestImporter_NegativeVersion(t *testing.T) { tree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(t, err) - _, err = tree.Import(-1, PostOrderTraverse) + _, err = tree.Import(-1) require.Error(t, err) } @@ -91,7 +91,7 @@ func TestImporter_NotEmpty(t *testing.T) { _, _, err = tree.SaveVersion() require.NoError(t, err) - _, err = tree.Import(1, PostOrderTraverse) + _, err = tree.Import(1) require.Error(t, err) } @@ -110,7 +110,7 @@ func TestImporter_NotEmptyDatabase(t *testing.T) { _, err = tree.Load() require.NoError(t, err) - _, err = tree.Import(1, PostOrderTraverse) + _, err = tree.Import(1) require.Error(t, err) } @@ -120,7 +120,7 @@ func TestImporter_NotEmptyUnsaved(t *testing.T) { _, err = tree.Set([]byte("a"), []byte{1}) require.NoError(t, err) - _, err = tree.Import(1, PostOrderTraverse) + _, err = tree.Import(1) require.Error(t, err) } @@ -147,7 +147,11 @@ func TestImporter_Add(t *testing.T) { tree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(t, err) var importer *Importer - importer, err = tree.Import(1, orderType) + if orderType == PreOrderTraverse { + importer, err = tree.ImportPreOrder(1) + } else { + importer, err = tree.Import(1) + } require.NoError(t, err) defer importer.Close() if orderType == PreOrderTraverse { @@ -168,7 +172,7 @@ func TestImporter_Add(t *testing.T) { func TestImporter_Add_Closed(t *testing.T) { tree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(t, err) - importer, err := tree.Import(1, PostOrderTraverse) + importer, err := tree.Import(1) require.NoError(t, err) importer.Close() @@ -180,7 +184,7 @@ func TestImporter_Add_Closed(t *testing.T) { func TestImporter_Close(t *testing.T) { tree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(t, err) - importer, err := tree.Import(1, PostOrderTraverse) + importer, err := tree.Import(1) require.NoError(t, err) err = importer.Add(&ExportNode{Key: []byte("key"), Value: []byte("value"), Version: 1, Height: 0}) @@ -197,7 +201,7 @@ func TestImporter_Close(t *testing.T) { func TestImporter_Commit(t *testing.T) { tree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(t, err) - importer, err := tree.Import(1, PostOrderTraverse) + importer, err := tree.Import(1) require.NoError(t, err) err = importer.Add(&ExportNode{Key: []byte("key"), Value: []byte("value"), Version: 1, Height: 0}) @@ -213,7 +217,7 @@ func TestImporter_Commit(t *testing.T) { func TestImporter_Commit_Closed(t *testing.T) { tree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(t, err) - importer, err := tree.Import(1, PreOrderTraverse) + importer, err := tree.ImportPreOrder(1) require.NoError(t, err) err = importer.Add(&ExportNode{Key: []byte("key"), Value: []byte("value"), Version: 1, Height: 0}) @@ -228,7 +232,7 @@ func TestImporter_Commit_Closed(t *testing.T) { func TestImporter_Commit_Empty(t *testing.T) { tree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(t, err) - importer, err := tree.Import(3, PostOrderTraverse) + importer, err := tree.Import(3) require.NoError(t, err) defer importer.Close() @@ -251,7 +255,15 @@ func BenchmarkImport(b *testing.B) { orderType = PostOrderTraverse } exported := make([]*ExportNode, 0, 4096) - exporter, err := tree.Export(orderType) + var ( + exporter *Exporter + err error + ) + if orderType == PreOrderTraverse { + exporter, err = tree.ExportPreOrder() + } else { + exporter, err = tree.Export() + } require.NoError(sub, err) for { item, err := exporter.Next() @@ -268,7 +280,12 @@ func BenchmarkImport(b *testing.B) { for n := 0; n < sub.N; n++ { newTree, err := NewMutableTree(db.NewMemDB(), 0, false) require.NoError(sub, err) - importer, err := newTree.Import(tree.Version(), orderType) + var importer *Importer + if orderType == PreOrderTraverse { + importer, err = newTree.ImportPreOrder(tree.Version()) + } else { + importer, err = newTree.Import(tree.Version()) + } require.NoError(sub, err) for _, item := range exported { err = importer.Add(item) diff --git a/mutable_tree.go b/mutable_tree.go index 8d103ec10..b11d4a7cb 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -156,8 +156,14 @@ func (tree *MutableTree) Get(key []byte) ([]byte, error) { // // Import can only be called on an empty tree. It is the callers responsibility that no other // modifications are made to the tree while importing. -func (tree *MutableTree) Import(version int64, traverseOrder OrderType) (*Importer, error) { - return newImporter(tree, version, traverseOrder) +func (tree *MutableTree) Import(version int64) (*Importer, error) { + return newImporter(tree, version, PostOrderTraverse) +} + +// ImportPreOrder is a convenience function that calls Import with traverseOrder = PreOrder. +// This is for the new updates of node-key refactoring. +func (tree *MutableTree) ImportPreOrder(version int64) (*Importer, error) { + return newImporter(tree, version, PreOrderTraverse) } // Iterate iterates over all keys of the tree. The keys and values must not be modified, From 82e1ae11503cc748b818771c9b60e28f8cf67faf Mon Sep 17 00:00:00 2001 From: Cool Developer Date: Tue, 31 Jan 2023 15:45:19 -0500 Subject: [PATCH 7/7] comments --- import.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/import.go b/import.go index 5c318ba11..0dc1fa4ff 100644 --- a/import.go +++ b/import.go @@ -101,8 +101,8 @@ func (i *Importer) Close() { i.tree = nil } -// Add adds an ExportNode to the import. ExportNodes must be added in the order returned by -// Exporter, i.e. depth-first pre-order (NLR). Nodes are periodically flushed to the database, +// Add adds an ExportNode to the import. ExportNodes must be added in the same order returned by +// Exporter. Nodes are periodically flushed to the database, // but the imported version is not visible until Commit() is called. func (i *Importer) Add(exportNode *ExportNode) error { if i.tree == nil { @@ -122,6 +122,7 @@ func (i *Importer) Add(exportNode *ExportNode) error { version: exportNode.Version, subtreeHeight: exportNode.Height, } + // set the subtree size as 1 for the leaf nodes if node.subtreeHeight == 0 { node.size = 1 }