[vm] Block Pruning (Default to Only Keeping Last 768 Blocks)

README.md

@@ -74,15 +74,30 @@ to that team for all the work they put into researching this approach.
 Instead of requiring nodes to execute all previous transactions when joining
 any `hyperchain` (which may not be possible if there is very high throughput on a Subnet),
 the `hypersdk` just syncs the most recent state from the network. To avoid falling
-behind the network while syncing this state, the `hypersdk` acts as an Avalanche Light
+behind the network while syncing this state, the `hypersdk` acts as an Avalanche Lite
 Client and performs consensus on newly processed blocks without verifying them (updating its
 state sync target whenever a new block is accepted).
 
 The `hypersdk` relies on [`x/sync`](https://github.com/ava-labs/avalanchego/tree/master/x/sync),
 a bandwidth-aware dynamic sync implementation provided by `avalanchego`, to
 sync to the tip of any `hyperchain`.
 
-#### Pebble as Default
+#### Block Pruning
+By default, the `hypersdk` only stores what is necessary to build/verfiy the next block
+and to help new nodes sync the current state. This means the `hypersdk` only needs to store
+the last accepted block, the genesis block, and the last 256 revisions of the current
+state (the ProposerVM is configured to store the last 256 accepted blocks as well).
+
+If the `hypersdk` did not do this, the storage requirements for validators
+would grow at an alarming rate each day (making running any `hypervm` impractical).
+Consider the simple example where we process 25k transactions per second (assume each
+transaction is ~400 bytes). This would would require the `hypersdk` to store 10MB per
+second (not including any overhead in the database for doing so). This works out to
+864GB per day or 20.7TB per year.
+
+_The 256 block history constant referenced above is tunable by any `hypervm`._
+
+#### PebbleDB
 Instead of employing [`goleveldb`](https://github.com/syndtr/goleveldb), the
 `hypersdk` uses CockroachDB's [`pebble`](https://github.com/cockroachdb/pebble) database for
 on-disk storage. This database is inspired by LevelDB/RocksDB but offers [a few
@@ -129,7 +144,8 @@ All `hypersdk` blocks include a state root to support dynamic state sync. In dyn
 state sync, the state target is updated to the root of the last accepted block while
 the sync is ongoing instead of staying pinned to the last accepted root when the sync
 started. Root block inclusion means consensus can be used to select the next state
-target to sync to instead of using some less secure, out-of-consensus mechanism.
+target to sync to instead of using some less secure, out-of-consensus mechanism (i.e.
+Avalanche Lite Client).
 
 Dynamic state sync is required for high-throughput blockchains because it relieves
 the nodes that serve state sync queries from storing all historical state revisions

chain/block.go

@@ -728,9 +728,6 @@ func (b *StatelessBlock) Accept(ctx context.Context) error {
 // SetLastAccepted is called during [Accept] and at the start and end of state
 // sync.
 func (b *StatelessBlock) SetLastAccepted(ctx context.Context) error {
-	if err := b.vm.SetLastAccepted(b); err != nil {
-		return err
-	}
 	b.st = choices.Accepted
 	b.txsSet = nil // only used for replay protection when processing
 

config/config.go

@@ -34,6 +34,8 @@ func (c *Config) GetMempoolPayerSize() int               { return 32 }
 func (c *Config) GetMempoolExemptPayers() [][]byte       { return nil }
 func (c *Config) GetStreamingBacklogSize() int           { return 1024 }
 func (c *Config) GetStateHistoryLength() int             { return 256 }
+func (c *Config) GetParsedBlockCacheSize() int           { return 128 }
+func (c *Config) GetAcceptedBlockCacheSize() int         { return 256 }
 func (c *Config) GetStateEvictionBatchSize() int         { return 4 * units.MiB }
 func (c *Config) GetIntermediateNodeCacheSize() int      { return 2 * units.GiB }
 func (c *Config) GetValueNodeCacheSize() int             { return 2 * units.GiB }
@@ -42,8 +44,6 @@ func (c *Config) GetTraceConfig() *trace.Config          { return &trace.Config{
 func (c *Config) GetStateSyncParallelism() int           { return 4 }
 func (c *Config) GetStateSyncMinBlocks() uint64          { return 256 }
 func (c *Config) GetStateSyncServerDelay() time.Duration { return 0 } // used for testing
-func (c *Config) GetParsedBlockCacheSize() int           { return 128 }
-func (c *Config) GetAcceptedBlockCacheSize() int         { return 128 }
 
 func (c *Config) GetContinuousProfilerConfig() *profiler.Config {
 	return &profiler.Config{Enabled: false}

examples/morpheusvm/scripts/run.sh

@@ -17,17 +17,20 @@ if ! [[ "$0" =~ scripts/run.sh ]]; then
   exit 255
 fi
 
-VERSION=2eabd228952b6b7c9075bc45653f70643d9a5a7c
+# VERSION=2eabd228952b6b7c9075bc45653f70643d9a5a7c
+VERSION=f502b7909ea4114d9a2acb384229723a12b6b49f
 MODE=${MODE:-run}
 LOGLEVEL=${LOGLEVEL:-info}
 STATESYNC_DELAY=${STATESYNC_DELAY:-0}
 MIN_BLOCK_GAP=${MIN_BLOCK_GAP:-100}
 CREATE_TARGET=${CREATE_TARGET:-75000}
+STORE_TXS=${STORE_TXS:-false}
 if [[ ${MODE} != "run" ]]; then
   LOGLEVEL=debug
   STATESYNC_DELAY=100000000 # 100ms
   MIN_BLOCK_GAP=250 #ms
   CREATE_TARGET=100000000 # 4M accounts (we send to random addresses)
+  STORE_TXS=true
 fi
 
 echo "Running with:"
@@ -36,6 +39,7 @@ echo MODE: ${MODE}
 echo LOG LEVEL: ${LOGLEVEL}
 echo STATESYNC_DELAY \(ns\): ${STATESYNC_DELAY}
 echo MIN_BLOCK_GAP \(ms\): ${MIN_BLOCK_GAP}
+echo STORE_TXS: ${STORE_TXS}
 
 ############################
 # build avalanchego
@@ -132,7 +136,7 @@ cat <<EOF > ${TMPDIR}/morpheusvm.config
   "mempoolExemptPayers":["morpheus1rvzhmceq997zntgvravfagsks6w0ryud3rylh4cdvayry0dl97nsp30ucp"],
   "parallelism": 5,
   "verifySignatures":true,
-  "storeTransactions":true,
+  "storeTransactions": ${STORE_TXS},
   "streamingBacklogSize": 10000000,
   "logLevel": "${LOGLEVEL}",
   "stateSyncServerDelay": ${STATESYNC_DELAY}
@@ -150,7 +154,8 @@ echo "creating subnet config"
 rm -f ${TMPDIR}/morpheusvm.subnet
 cat <<EOF > ${TMPDIR}/morpheusvm.subnet
 {
-  "proposerMinBlockDelay": 0
+  "proposerMinBlockDelay": 0,
+  "proposerNumHistoricalBlocks": 256
 }
 EOF
 

examples/morpheusvm/tests/e2e/e2e_test.go

@@ -222,6 +222,7 @@ var _ = ginkgo.BeforeSuite(func() {
 				"log-level":"info",
 				"log-display-level":"info",
 				"proposervm-use-current-height":true,
+				"bootstrap-retry-enabled":false,
 				"throttler-inbound-validator-alloc-size":"10737418240",
 				"throttler-inbound-at-large-alloc-size":"10737418240",
 				"throttler-inbound-node-max-processing-msgs":"100000",
@@ -520,8 +521,8 @@ var _ = ginkgo.Describe("[Test]", func() {
 	// blocks)
 	//
 	// We do 1024 so that there are a number of ranges of data to fetch.
-	ginkgo.It("supports issuance of at least 1024 more blocks", func() {
-		count += generateBlocks(context.Background(), count, 1024, instances, true)
+	ginkgo.It("supports issuance of at least 256 more blocks", func() {
+		count += generateBlocks(context.Background(), count, 256, instances, true)
 		// TODO: verify all roots are equal
 	})
 

examples/morpheusvm/tests/integration/integration_test.go

@@ -110,6 +110,7 @@ var (
 	// when used with embedded VMs
 	genesisBytes []byte
 	instances    []instance
+	blocks       []snowman.Block
 
 	networkID uint32
 	gen       *genesis.Genesis
@@ -263,6 +264,7 @@ var _ = ginkgo.BeforeSuite(func() {
 			csupply += alloc.Balance
 		}
 	}
+	blocks = []snowman.Block{}
 
 	app.instances = instances
 	color.Blue("created %d VMs", vms)
@@ -401,6 +403,7 @@ var _ = ginkgo.Describe("[Tx Processing]", func() {
 
 			gomega.Ω(blk.Accept(ctx)).To(gomega.BeNil())
 			gomega.Ω(blk.Status()).To(gomega.Equal(choices.Accepted))
+			blocks = append(blocks, blk)
 
 			lastAccepted, err := instances[1].vm.LastAccepted(ctx)
 			gomega.Ω(err).To(gomega.BeNil())
@@ -454,7 +457,7 @@ var _ = ginkgo.Describe("[Tx Processing]", func() {
 			gomega.Ω(err).Should(gomega.BeNil())
 			gomega.Ω(submit(context.Background())).Should(gomega.BeNil())
 			accept := expectBlk(instances[1])
-			results := accept()
+			results := accept(true)
 			gomega.Ω(results).Should(gomega.HaveLen(1))
 			gomega.Ω(results[0].Success).Should(gomega.BeTrue())
 
@@ -532,7 +535,7 @@ var _ = ginkgo.Describe("[Tx Processing]", func() {
 			gomega.Ω(submit(context.Background())).Should(gomega.BeNil())
 
 			accept := expectBlk(instances[1])
-			results := accept()
+			results := accept(true)
 
 			// Check results
 			gomega.Ω(results).Should(gomega.HaveLen(4))
@@ -608,7 +611,7 @@ var _ = ginkgo.Describe("[Tx Processing]", func() {
 	})
 
 	ginkgo.It("Test processing block handling", func() {
-		var accept, accept2 func() []*chain.Result
+		var accept, accept2 func(bool) []*chain.Result
 
 		ginkgo.By("create processing tip", func() {
 			parser, err := instances[1].lcli.Parser(context.Background())
@@ -643,10 +646,10 @@ var _ = ginkgo.Describe("[Tx Processing]", func() {
 		})
 
 		ginkgo.By("clear processing tip", func() {
-			results := accept()
+			results := accept(true)
 			gomega.Ω(results).Should(gomega.HaveLen(1))
 			gomega.Ω(results[0].Success).Should(gomega.BeTrue())
-			results = accept2()
+			results = accept2(true)
 			gomega.Ω(results).Should(gomega.HaveLen(1))
 			gomega.Ω(results[0].Success).Should(gomega.BeTrue())
 		})
@@ -680,30 +683,19 @@ var _ = ginkgo.Describe("[Tx Processing]", func() {
 	ginkgo.It("ensure unprocessed tip works", func() {
 		ginkgo.By("import accepted blocks to instance 2", func() {
 			ctx := context.TODO()
-			o := instances[1]
-			blks := []snowman.Block{}
-			next, err := o.vm.LastAccepted(ctx)
-			gomega.Ω(err).Should(gomega.BeNil())
-			for {
-				blk, err := o.vm.GetBlock(ctx, next)
-				gomega.Ω(err).Should(gomega.BeNil())
-				blks = append([]snowman.Block{blk}, blks...)
-				if blk.Height() == 1 {
-					break
-				}
-				next = blk.Parent()
-			}
+
+			gomega.Ω(blocks[0].Height()).Should(gomega.Equal(uint64(1)))
 
 			n := instances[2]
-			blk1, err := n.vm.ParseBlock(ctx, blks[0].Bytes())
+			blk1, err := n.vm.ParseBlock(ctx, blocks[0].Bytes())
 			gomega.Ω(err).Should(gomega.BeNil())
 			err = blk1.Verify(ctx)
 			gomega.Ω(err).Should(gomega.BeNil())
 
 			// Parse tip
-			blk2, err := n.vm.ParseBlock(ctx, blks[1].Bytes())
+			blk2, err := n.vm.ParseBlock(ctx, blocks[1].Bytes())
 			gomega.Ω(err).Should(gomega.BeNil())
-			blk3, err := n.vm.ParseBlock(ctx, blks[2].Bytes())
+			blk3, err := n.vm.ParseBlock(ctx, blocks[2].Bytes())
 			gomega.Ω(err).Should(gomega.BeNil())
 
 			// Verify tip
@@ -721,7 +713,7 @@ var _ = ginkgo.Describe("[Tx Processing]", func() {
 			gomega.Ω(err).Should(gomega.BeNil())
 
 			// Parse another
-			blk4, err := n.vm.ParseBlock(ctx, blks[3].Bytes())
+			blk4, err := n.vm.ParseBlock(ctx, blocks[3].Bytes())
 			gomega.Ω(err).Should(gomega.BeNil())
 			err = blk4.Verify(ctx)
 			gomega.Ω(err).Should(gomega.BeNil())
@@ -734,7 +726,7 @@ var _ = ginkgo.Describe("[Tx Processing]", func() {
 	ginkgo.It("processes valid index transactions (w/block listening)", func() {
 		// Clear previous txs on instance 0
 		accept := expectBlk(instances[0])
-		accept() // don't care about results
+		accept(false) // don't care about results
 
 		// Subscribe to blocks
 		cli, err := rpc.NewWebSocketClient(instances[0].WebSocketServer.URL, rpc.DefaultHandshakeTimeout, pubsub.MaxPendingMessages, pubsub.MaxReadMessageSize)
@@ -770,7 +762,7 @@ var _ = ginkgo.Describe("[Tx Processing]", func() {
 
 		gomega.Ω(err).Should(gomega.BeNil())
 		accept = expectBlk(instances[0])
-		results := accept()
+		results := accept(false)
 		gomega.Ω(results).Should(gomega.HaveLen(1))
 		gomega.Ω(results[0].Success).Should(gomega.BeTrue())
 
@@ -829,7 +821,7 @@ var _ = ginkgo.Describe("[Tx Processing]", func() {
 		}
 		gomega.Ω(err).Should(gomega.BeNil())
 		accept := expectBlk(instances[0])
-		results := accept()
+		results := accept(false)
 		gomega.Ω(results).Should(gomega.HaveLen(1))
 		gomega.Ω(results[0].Success).Should(gomega.BeTrue())
 
@@ -846,7 +838,7 @@ var _ = ginkgo.Describe("[Tx Processing]", func() {
 	})
 })
 
-func expectBlk(i instance) func() []*chain.Result {
+func expectBlk(i instance) func(bool) []*chain.Result {
 	ctx := context.TODO()
 
 	// manually signal ready
@@ -867,10 +859,14 @@ func expectBlk(i instance) func() []*chain.Result {
 	err = i.vm.SetPreference(ctx, blk.ID())
 	gomega.Ω(err).To(gomega.BeNil())
 
-	return func() []*chain.Result {
+	return func(add bool) []*chain.Result {
 		gomega.Ω(blk.Accept(ctx)).To(gomega.BeNil())
 		gomega.Ω(blk.Status()).To(gomega.Equal(choices.Accepted))
 
+		if add {
+			blocks = append(blocks, blk)
+		}
+
 		lastAccepted, err := i.vm.LastAccepted(ctx)
 		gomega.Ω(err).To(gomega.BeNil())
 		gomega.Ω(lastAccepted).To(gomega.Equal(blk.ID()))

examples/tokenvm/scripts/run.sh

@@ -23,18 +23,21 @@ LOGLEVEL=${LOGLEVEL:-info}
 STATESYNC_DELAY=${STATESYNC_DELAY:-0}
 MIN_BLOCK_GAP=${MIN_BLOCK_GAP:-100}
 CREATE_TARGET=${CREATE_TARGET:-75000}
+STORE_TXS=${STORE_TXS:-false}
 if [[ ${MODE} != "run" && ${MODE} != "run-single" ]]; then
   LOGLEVEL=debug
   STATESYNC_DELAY=100000000 # 100ms
   MIN_BLOCK_GAP=250 #ms
   CREATE_TARGET=100000000 # 4M accounts (we send to random addresses)
+  STORE_TXS=true
 fi
 
 echo "Running with:"
 echo VERSION: ${VERSION}
 echo MODE: ${MODE}
 echo STATESYNC_DELAY \(ns\): ${STATESYNC_DELAY}
 echo MIN_BLOCK_GAP \(ms\): ${MIN_BLOCK_GAP}
+echo STORE_TXS: ${STORE_TXS}
 
 ############################
 # build avalanchego
@@ -134,7 +137,7 @@ cat <<EOF > ${TMPDIR}/tokenvm.config
   "mempoolExemptPayers":["token1rvzhmceq997zntgvravfagsks6w0ryud3rylh4cdvayry0dl97nsjzf3yp"],
   "parallelism": 5,
   "verifySignatures":true,
-  "storeTransactions":true,
+  "storeTransactions": ${STORE_TXS},
   "streamingBacklogSize": 10000000,
   "trackedPairs":["*"],
   "logLevel": "${LOGLEVEL}",
@@ -153,7 +156,8 @@ echo "creating subnet config"
 rm -f ${TMPDIR}/tokenvm.subnet
 cat <<EOF > ${TMPDIR}/tokenvm.subnet
 {
-  "proposerMinBlockDelay": 0
+  "proposerMinBlockDelay": 0,
+  "proposerNumHistoricalBlocks": 256
 }
 EOF