From e096dc14192ae4f430cdb0887ca53fc653853571 Mon Sep 17 00:00:00 2001 From: Rodrigo Villar Date: Tue, 2 Dec 2025 10:22:34 -0500 Subject: [PATCH 1/6] refactor: block export --- Taskfile.yml | 2 +- tests/reexecute/blockexport/main.go | 62 ++++++++++++++++++++++++++ tests/reexecute/c/vm_reexecute_test.go | 37 +-------------- tests/reexecute/db.go | 9 ++-- 4 files changed, 67 insertions(+), 43 deletions(-) create mode 100644 tests/reexecute/blockexport/main.go diff --git a/Taskfile.yml b/Taskfile.yml index 7f127831eb1e..230daca54dd0 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -107,7 +107,7 @@ tasks: START_BLOCK: '{{.START_BLOCK}}' END_BLOCK: '{{.END_BLOCK}}' cmds: - - cmd: go test -timeout=0 -run=TestExportBlockRange github.com/ava-labs/avalanchego/tests/reexecute/c --block-dir-src={{.BLOCK_DIR_SRC}} --block-dir-dst={{.BLOCK_DIR_DST}} --start-block={{.START_BLOCK}} --end-block={{.END_BLOCK}} + - cmd: go run github.com/ava-labs/avalanchego/tests/reexecute/blockexport --block-dir-src={{.BLOCK_DIR_SRC}} --block-dir-dst={{.BLOCK_DIR_DST}} --start-block={{.START_BLOCK}} --end-block={{.END_BLOCK}} export-dir-to-s3: desc: Copies a directory to s3 diff --git a/tests/reexecute/blockexport/main.go b/tests/reexecute/blockexport/main.go new file mode 100644 index 000000000000..201f22713c58 --- /dev/null +++ b/tests/reexecute/blockexport/main.go @@ -0,0 +1,62 @@ +// Copyright (C) 2019-2025, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package main + +import ( + "flag" + + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/require" + + "github.com/ava-labs/avalanchego/database/leveldb" + "github.com/ava-labs/avalanchego/tests" + "github.com/ava-labs/avalanchego/tests/reexecute" + "github.com/ava-labs/avalanchego/utils/logging" + "github.com/ava-labs/avalanchego/utils/units" +) + +var ( + blockDirSrcArg string + blockDirDstArg string + startBlockArg uint64 + endBlockArg uint64 + chanSizeArg int +) + +func init() { + flag.StringVar(&blockDirSrcArg, "block-dir-src", blockDirSrcArg, "Source block directory to copy from when running TestExportBlockRange.") + flag.StringVar(&blockDirDstArg, "block-dir-dst", blockDirDstArg, "Destination block directory to write blocks into when executing TestExportBlockRange.") + flag.Uint64Var(&startBlockArg, "start-block", 101, "Start block to begin execution (exclusive).") + flag.Uint64Var(&endBlockArg, "end-block", 200, "End block to end execution (inclusive).") + flag.IntVar(&chanSizeArg, "chan-size", 100, "Size of the channel to use for block processing.") + + flag.Parse() +} + +func main() { + tc := tests.NewTestContext(tests.NewDefaultLogger("")) + defer tc.RecoverAndExit() + + r := require.New(tc) + blockChan, err := reexecute.CreateBlockChanFromLevelDB(blockDirSrcArg, startBlockArg, endBlockArg, chanSizeArg, tc.DeferCleanup) + r.NoError(err) + + db, err := leveldb.New(blockDirDstArg, nil, logging.NoLog{}, prometheus.NewRegistry()) + r.NoError(err) + tc.DeferCleanup(func() { + r.NoError(db.Close()) + }) + + batch := db.NewBatch() + for blkResult := range blockChan { + r.NoError(batch.Put(reexecute.BlockKey(blkResult.Height), blkResult.BlockBytes)) + + if batch.Size() > 10*units.MiB { + r.NoError(batch.Write()) + batch = db.NewBatch() + } + } + + r.NoError(batch.Write()) +} diff --git a/tests/reexecute/c/vm_reexecute_test.go b/tests/reexecute/c/vm_reexecute_test.go index 5efe7a8446fa..b17b62771c32 100644 --- a/tests/reexecute/c/vm_reexecute_test.go +++ b/tests/reexecute/c/vm_reexecute_test.go @@ -42,7 +42,6 @@ import ( "github.com/ava-labs/avalanchego/utils/crypto/bls/signer/localsigner" "github.com/ava-labs/avalanchego/utils/logging" "github.com/ava-labs/avalanchego/utils/timer" - "github.com/ava-labs/avalanchego/utils/units" "github.com/ava-labs/avalanchego/vms/metervm" "github.com/ava-labs/avalanchego/vms/platformvm/warp" ) @@ -55,8 +54,6 @@ var ( var ( blockDirArg string - blockDirSrcArg string - blockDirDstArg string currentStateDirArg string startBlockArg uint64 endBlockArg uint64 @@ -122,10 +119,6 @@ func TestMain(m *testing.M) { flag.StringVar(&configNameArg, configKey, defaultConfigKey, fmt.Sprintf("Specifies the predefined config to use for the VM. Options include %s.", predefinedConfigOptionsStr)) flag.StringVar(&runnerNameArg, "runner", "dev", "Name of the runner executing this test. Added as a metric label and to the sub-benchmark's name to differentiate results on the runner key.") - // Flags specific to TestExportBlockRange. - flag.StringVar(&blockDirSrcArg, "block-dir-src", blockDirSrcArg, "Source block directory to copy from when running TestExportBlockRange.") - flag.StringVar(&blockDirDstArg, "block-dir-dst", blockDirDstArg, "Destination block directory to write blocks into when executing TestExportBlockRange.") - flag.Parse() if metricsCollectorEnabledArg { @@ -232,7 +225,7 @@ func benchmarkReexecuteRange( zap.Int("chan-size", chanSize), ) - blockChan, err := reexecute.CreateBlockChanFromLevelDB(b, blockDir, startBlock, endBlock, chanSize) + blockChan, err := reexecute.CreateBlockChanFromLevelDB(blockDir, startBlock, endBlock, chanSize, b.Cleanup) r.NoError(err) dbLogger := tests.NewDefaultLogger("db") @@ -477,34 +470,6 @@ func (e *vmExecutor) executeSequence(ctx context.Context, blkChan <-chan reexecu return nil } -func TestExportBlockRange(t *testing.T) { - exportBlockRange(t, blockDirSrcArg, blockDirDstArg, startBlockArg, endBlockArg, chanSizeArg) -} - -func exportBlockRange(tb testing.TB, blockDirSrc string, blockDirDst string, startBlock, endBlock uint64, chanSize int) { - r := require.New(tb) - blockChan, err := reexecute.CreateBlockChanFromLevelDB(tb, blockDirSrc, startBlock, endBlock, chanSize) - r.NoError(err) - - db, err := leveldb.New(blockDirDst, nil, logging.NoLog{}, prometheus.NewRegistry()) - r.NoError(err) - tb.Cleanup(func() { - r.NoError(db.Close()) - }) - - batch := db.NewBatch() - for blkResult := range blockChan { - r.NoError(batch.Put(reexecute.BlockKey(blkResult.Height), blkResult.BlockBytes)) - - if batch.Size() > 10*units.MiB { - r.NoError(batch.Write()) - batch = db.NewBatch() - } - } - - r.NoError(batch.Write()) -} - type consensusMetrics struct { lastAcceptedHeight prometheus.Gauge } diff --git a/tests/reexecute/db.go b/tests/reexecute/db.go index 04353db62d82..8432a65b5707 100644 --- a/tests/reexecute/db.go +++ b/tests/reexecute/db.go @@ -6,10 +6,8 @@ package reexecute import ( "encoding/binary" "fmt" - "testing" "github.com/prometheus/client_golang/prometheus" - "github.com/stretchr/testify/require" "github.com/ava-labs/avalanchego/database" "github.com/ava-labs/avalanchego/database/leveldb" @@ -29,16 +27,15 @@ type BlockResult struct { // Blocks are read sequentially and sent to the returned channel as BlockResult values. // // Any validation errors or iteration errors are sent as BlockResult with Err set, then the channel is closed. -func CreateBlockChanFromLevelDB(tb testing.TB, sourceDir string, startBlock, endBlock uint64, chanSize int) (<-chan BlockResult, error) { - r := require.New(tb) +func CreateBlockChanFromLevelDB(sourceDir string, startBlock, endBlock uint64, chanSize int, cleanup func(func())) (<-chan BlockResult, error) { ch := make(chan BlockResult, chanSize) db, err := leveldb.New(sourceDir, nil, logging.NoLog{}, prometheus.NewRegistry()) if err != nil { return nil, fmt.Errorf("failed to create leveldb database from %q: %w", sourceDir, err) } - tb.Cleanup(func() { - r.NoError(db.Close()) + cleanup(func() { + db.Close() }) go func() { From fa99ce5c1d7f29f3094fbf8bd0cfd2aadd1f5232 Mon Sep 17 00:00:00 2001 From: Rodrigo Villar Date: Wed, 3 Dec 2025 17:55:05 -0500 Subject: [PATCH 2/6] chore: nits --- tests/reexecute/blockexport/main.go | 9 ++++++++- tests/reexecute/c/vm_reexecute_test.go | 2 +- tests/reexecute/db.go | 6 ++++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/reexecute/blockexport/main.go b/tests/reexecute/blockexport/main.go index 201f22713c58..d00385cd9ac0 100644 --- a/tests/reexecute/blockexport/main.go +++ b/tests/reexecute/blockexport/main.go @@ -39,7 +39,14 @@ func main() { defer tc.RecoverAndExit() r := require.New(tc) - blockChan, err := reexecute.CreateBlockChanFromLevelDB(blockDirSrcArg, startBlockArg, endBlockArg, chanSizeArg, tc.DeferCleanup) + blockChan, err := reexecute.CreateBlockChanFromLevelDB( + tc, + blockDirSrcArg, + startBlockArg, + endBlockArg, + chanSizeArg, + tc.DeferCleanup, + ) r.NoError(err) db, err := leveldb.New(blockDirDstArg, nil, logging.NoLog{}, prometheus.NewRegistry()) diff --git a/tests/reexecute/c/vm_reexecute_test.go b/tests/reexecute/c/vm_reexecute_test.go index b17b62771c32..b5247d3239d7 100644 --- a/tests/reexecute/c/vm_reexecute_test.go +++ b/tests/reexecute/c/vm_reexecute_test.go @@ -225,7 +225,7 @@ func benchmarkReexecuteRange( zap.Int("chan-size", chanSize), ) - blockChan, err := reexecute.CreateBlockChanFromLevelDB(blockDir, startBlock, endBlock, chanSize, b.Cleanup) + blockChan, err := reexecute.CreateBlockChanFromLevelDB(b, blockDir, startBlock, endBlock, chanSize, b.Cleanup) r.NoError(err) dbLogger := tests.NewDefaultLogger("db") diff --git a/tests/reexecute/db.go b/tests/reexecute/db.go index 8432a65b5707..348ddfd35f51 100644 --- a/tests/reexecute/db.go +++ b/tests/reexecute/db.go @@ -8,6 +8,7 @@ import ( "fmt" "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/require" "github.com/ava-labs/avalanchego/database" "github.com/ava-labs/avalanchego/database/leveldb" @@ -27,7 +28,8 @@ type BlockResult struct { // Blocks are read sequentially and sent to the returned channel as BlockResult values. // // Any validation errors or iteration errors are sent as BlockResult with Err set, then the channel is closed. -func CreateBlockChanFromLevelDB(sourceDir string, startBlock, endBlock uint64, chanSize int, cleanup func(func())) (<-chan BlockResult, error) { +func CreateBlockChanFromLevelDB(t require.TestingT, sourceDir string, startBlock, endBlock uint64, chanSize int, cleanup func(func())) (<-chan BlockResult, error) { + r := require.New(t) ch := make(chan BlockResult, chanSize) db, err := leveldb.New(sourceDir, nil, logging.NoLog{}, prometheus.NewRegistry()) @@ -35,7 +37,7 @@ func CreateBlockChanFromLevelDB(sourceDir string, startBlock, endBlock uint64, c return nil, fmt.Errorf("failed to create leveldb database from %q: %w", sourceDir, err) } cleanup(func() { - db.Close() + r.NoError(db.Close()) }) go func() { From 7244405f1318a60939e8297a8d098f8f55c74723 Mon Sep 17 00:00:00 2001 From: Rodrigo Villar Date: Wed, 3 Dec 2025 18:53:18 -0500 Subject: [PATCH 3/6] docs: update outdated command info --- tests/reexecute/blockexport/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/reexecute/blockexport/main.go b/tests/reexecute/blockexport/main.go index d00385cd9ac0..5ac9116e2577 100644 --- a/tests/reexecute/blockexport/main.go +++ b/tests/reexecute/blockexport/main.go @@ -25,8 +25,8 @@ var ( ) func init() { - flag.StringVar(&blockDirSrcArg, "block-dir-src", blockDirSrcArg, "Source block directory to copy from when running TestExportBlockRange.") - flag.StringVar(&blockDirDstArg, "block-dir-dst", blockDirDstArg, "Destination block directory to write blocks into when executing TestExportBlockRange.") + flag.StringVar(&blockDirSrcArg, "block-dir-src", blockDirSrcArg, "Source block directory to copy from.") + flag.StringVar(&blockDirDstArg, "block-dir-dst", blockDirDstArg, "Destination block directory to write blocks into.") flag.Uint64Var(&startBlockArg, "start-block", 101, "Start block to begin execution (exclusive).") flag.Uint64Var(&endBlockArg, "end-block", 200, "End block to end execution (inclusive).") flag.IntVar(&chanSizeArg, "chan-size", 100, "Size of the channel to use for block processing.") From e555764eef5e79d8496872f1416858014dc10a90 Mon Sep 17 00:00:00 2001 From: Rodrigo Villar Date: Thu, 4 Dec 2025 09:56:47 -0500 Subject: [PATCH 4/6] chore: remove unnecessary chan param --- tests/reexecute/blockexport/main.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/reexecute/blockexport/main.go b/tests/reexecute/blockexport/main.go index 5ac9116e2577..9d392bf30d29 100644 --- a/tests/reexecute/blockexport/main.go +++ b/tests/reexecute/blockexport/main.go @@ -21,7 +21,6 @@ var ( blockDirDstArg string startBlockArg uint64 endBlockArg uint64 - chanSizeArg int ) func init() { @@ -29,7 +28,6 @@ func init() { flag.StringVar(&blockDirDstArg, "block-dir-dst", blockDirDstArg, "Destination block directory to write blocks into.") flag.Uint64Var(&startBlockArg, "start-block", 101, "Start block to begin execution (exclusive).") flag.Uint64Var(&endBlockArg, "end-block", 200, "End block to end execution (inclusive).") - flag.IntVar(&chanSizeArg, "chan-size", 100, "Size of the channel to use for block processing.") flag.Parse() } @@ -39,12 +37,14 @@ func main() { defer tc.RecoverAndExit() r := require.New(tc) + + chanSize := 100 blockChan, err := reexecute.CreateBlockChanFromLevelDB( tc, blockDirSrcArg, startBlockArg, endBlockArg, - chanSizeArg, + chanSize, tc.DeferCleanup, ) r.NoError(err) From 3394dd90e1b6f59dc02ed132746cf567341d4adb Mon Sep 17 00:00:00 2001 From: Rodrigo Villar Date: Thu, 4 Dec 2025 13:09:05 -0500 Subject: [PATCH 5/6] docs: clarify cleanup parameter --- tests/reexecute/db.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/reexecute/db.go b/tests/reexecute/db.go index 348ddfd35f51..89ce315c2f2c 100644 --- a/tests/reexecute/db.go +++ b/tests/reexecute/db.go @@ -27,6 +27,9 @@ type BlockResult struct { // It opens the database at sourceDir and iterates through blocks from startBlock to endBlock (inclusive). // Blocks are read sequentially and sent to the returned channel as BlockResult values. // +// cleanup is a function that registers cleanup callbacks and is used to ensure +// that the database being read from is properly closed prior to the test terminating. +// // Any validation errors or iteration errors are sent as BlockResult with Err set, then the channel is closed. func CreateBlockChanFromLevelDB(t require.TestingT, sourceDir string, startBlock, endBlock uint64, chanSize int, cleanup func(func())) (<-chan BlockResult, error) { r := require.New(t) From ad91c27a613aa866d8b6f0aeef44d1c361b92e0c Mon Sep 17 00:00:00 2001 From: Rodrigo Villar Date: Tue, 9 Dec 2025 10:39:48 -0500 Subject: [PATCH 6/6] chore: require blk result is nil --- tests/reexecute/blockexport/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/reexecute/blockexport/main.go b/tests/reexecute/blockexport/main.go index 9d392bf30d29..7c070fd5f185 100644 --- a/tests/reexecute/blockexport/main.go +++ b/tests/reexecute/blockexport/main.go @@ -57,6 +57,7 @@ func main() { batch := db.NewBatch() for blkResult := range blockChan { + r.NoError(blkResult.Err) r.NoError(batch.Put(reexecute.BlockKey(blkResult.Height), blkResult.BlockBytes)) if batch.Size() > 10*units.MiB {