feat(blockdb): add lru cache for block entries (#4425)

Co-authored-by: yacovm <[email protected]>

Author: DracoLi

x/blockdb/README.md

@@ -10,6 +10,7 @@ BlockDB is a specialized database optimized for blockchain blocks.
 - **Configurable Durability**: Optional `syncToDisk` mode guarantees immediate recoverability
 - **Automatic Recovery**: Detects and recovers unindexed blocks after unclean shutdowns
 - **Block Compression**: zstd compression for block data
+- **In-Memory Cache**: LRU cache for recently accessed blocks
 
 ## Design
 
@@ -167,7 +168,6 @@ if err != nil {
 
 ## TODO
 
-- Implement a block cache for recently accessed blocks
 - Use a buffered pool to avoid allocations on reads and writes
 - Add performance benchmarks
 - Consider supporting missing data files (currently we error if any data files are missing)

x/blockdb/cache_db.go

@@ -0,0 +1,101 @@
+// Copyright (C) 2019-2025, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package blockdb
+
+import (
+	"slices"
+	"sync"
+
+	"go.uber.org/zap"
+
+	"github.com/ava-labs/avalanchego/cache/lru"
+	"github.com/ava-labs/avalanchego/database"
+)
+
+var _ database.HeightIndex = (*cacheDB)(nil)
+
+// cacheDB caches data from the underlying [Database].
+//
+// Operations (Get, Has, Put) are not atomic with the underlying database.
+// Concurrent writes to the same height can result in cache inconsistencies where
+// the cache and database contain different values. This limitation is acceptable
+// because concurrent writes to the same height are not an intended use case.
+type cacheDB struct {
+	db    *Database
+	cache *lru.Cache[BlockHeight, BlockData]
+
+	closeMu sync.RWMutex
+	closed  bool
+}
+
+func newCacheDB(db *Database, size uint16) *cacheDB {
+	return &cacheDB{
+		db:    db,
+		cache: lru.NewCache[BlockHeight, BlockData](int(size)),
+	}
+}
+
+func (c *cacheDB) Get(height BlockHeight) (BlockData, error) {
+	c.closeMu.RLock()
+	defer c.closeMu.RUnlock()
+
+	if c.closed {
+		c.db.log.Error("Failed Get: database closed", zap.Uint64("height", height))
+		return nil, database.ErrClosed
+	}
+
+	if cached, ok := c.cache.Get(height); ok {
+		return slices.Clone(cached), nil
+	}
+	data, err := c.db.Get(height)
+	if err != nil {
+		return nil, err
+	}
+	c.cache.Put(height, slices.Clone(data))
+	return data, nil
+}
+
+func (c *cacheDB) Put(height BlockHeight, data BlockData) error {
+	c.closeMu.RLock()
+	defer c.closeMu.RUnlock()
+
+	if c.closed {
+		c.db.log.Error("Failed Put: database closed", zap.Uint64("height", height))
+		return database.ErrClosed
+	}
+
+	if err := c.db.Put(height, data); err != nil {
+		return err
+	}
+
+	c.cache.Put(height, slices.Clone(data))
+	return nil
+}
+
+func (c *cacheDB) Has(height BlockHeight) (bool, error) {
+	c.closeMu.RLock()
+	defer c.closeMu.RUnlock()
+
+	if c.closed {
+		c.db.log.Error("Failed Has: database closed", zap.Uint64("height", height))
+		return false, database.ErrClosed
+	}
+
+	if _, ok := c.cache.Get(height); ok {
+		return true, nil
+	}
+	return c.db.Has(height)
+}
+
+func (c *cacheDB) Close() error {
+	c.closeMu.Lock()
+	defer c.closeMu.Unlock()
+
+	if c.closed {
+		return database.ErrClosed
+	}
+	c.closed = true
+	c.cache.Flush()
+	return c.db.Close()
+}

x/blockdb/cache_db_test.go

@@ -0,0 +1,144 @@
+// Copyright (C) 2019-2025, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package blockdb
+
+import (
+	"slices"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/ava-labs/avalanchego/database"
+)
+
+func TestCacheOnMiss(t *testing.T) {
+	db := newCacheDatabase(t, DefaultConfig())
+	height := uint64(20)
+	block := randomBlock(t)
+	require.NoError(t, db.Put(height, block))
+
+	// Evict the entry from cache to simulate a cache miss
+	db.cache.Evict(height)
+
+	// Read the block - should populate the cache on cache miss
+	_, err := db.Get(height)
+	require.NoError(t, err)
+
+	_, ok := db.cache.Get(height)
+	require.True(t, ok)
+}
+
+func TestCacheGet(t *testing.T) {
+	db := newCacheDatabase(t, DefaultConfig())
+	height := uint64(30)
+	block := randomBlock(t)
+
+	// Populate cache directly without writing to database
+	db.cache.Put(height, block)
+
+	// Get should return the block from cache
+	data, err := db.Get(height)
+	require.NoError(t, err)
+	require.Equal(t, block, data)
+}
+
+func TestCacheHas(t *testing.T) {
+	db := newCacheDatabase(t, DefaultConfig())
+	height := uint64(40)
+	block := randomBlock(t)
+
+	// Populate cache directly without writing to database
+	db.cache.Put(height, block)
+
+	// Has should return true from cache even though block is not in database
+	has, err := db.Has(height)
+	require.NoError(t, err)
+	require.True(t, has)
+}
+
+func TestCachePutStoresClone(t *testing.T) {
+	db := newCacheDatabase(t, DefaultConfig())
+	height := uint64(40)
+	block := randomBlock(t)
+	clone := slices.Clone(block)
+	require.NoError(t, db.Put(height, clone))
+
+	// Modify the original block after Put
+	clone[0] = 99
+
+	// Cache should have the original unmodified data
+	cached, ok := db.cache.Get(height)
+	require.True(t, ok)
+	require.Equal(t, block, cached)
+}
+
+func TestCacheGetReturnsClone(t *testing.T) {
+	db := newCacheDatabase(t, DefaultConfig())
+	height := uint64(50)
+	block := randomBlock(t)
+	require.NoError(t, db.Put(height, block))
+
+	// Get the block and modify the returned data
+	data, err := db.Get(height)
+	require.NoError(t, err)
+	data[0] = 99
+
+	// Cache should still have the original unmodified data
+	cached, ok := db.cache.Get(height)
+	require.True(t, ok)
+	require.Equal(t, block, cached)
+
+	// Second Get should also return original data
+	data, err = db.Get(height)
+	require.NoError(t, err)
+	require.Equal(t, block, data)
+}
+
+func TestCachePutOverridesSameHeight(t *testing.T) {
+	db := newCacheDatabase(t, DefaultConfig())
+	height := uint64(60)
+	b1 := randomBlock(t)
+	require.NoError(t, db.Put(height, b1))
+
+	// Verify first block is in cache
+	cached, ok := db.cache.Get(height)
+	require.True(t, ok)
+	require.Equal(t, b1, cached)
+
+	// Put second block at same height and verify it overrides the first one
+	b2 := randomBlock(t)
+	require.NoError(t, db.Put(height, b2))
+	cached, ok = db.cache.Get(height)
+	require.True(t, ok)
+	require.Equal(t, b2, cached)
+
+	// Get should also return the new block
+	data, err := db.Get(height)
+	require.NoError(t, err)
+	require.Equal(t, b2, data)
+}
+
+func TestCacheClose(t *testing.T) {
+	db := newCacheDatabase(t, DefaultConfig())
+	height := uint64(70)
+	block := randomBlock(t)
+	require.NoError(t, db.Put(height, block))
+
+	_, ok := db.cache.Get(height)
+	require.True(t, ok)
+	require.NoError(t, db.Close())
+
+	// cache is flushed
+	require.Zero(t, db.cache.Len())
+
+	// db operations now fails
+	_, err := db.Get(height)
+	require.ErrorIs(t, err, database.ErrClosed)
+	_, err = db.Has(height)
+	require.ErrorIs(t, err, database.ErrClosed)
+	err = db.Put(height+1, block)
+	require.ErrorIs(t, err, database.ErrClosed)
+	err = db.Close()
+	require.ErrorIs(t, err, database.ErrClosed)
+}

x/blockdb/config.go

@@ -11,6 +11,9 @@ const DefaultMaxDataFileSize = 500 * 1024 * 1024 * 1024
 // DefaultMaxDataFiles is the default maximum number of data files descriptors cached.
 const DefaultMaxDataFiles = 10
 
+// DefaultBlockCacheSize is the default size of the block cache.
+const DefaultBlockCacheSize uint16 = 256
+
 // DatabaseConfig contains configuration parameters for BlockDB.
 type DatabaseConfig struct {
 	// IndexDir is the directory where the index file is stored.
@@ -28,6 +31,9 @@ type DatabaseConfig struct {
 	// MaxDataFiles is the maximum number of data files descriptors cached.
 	MaxDataFiles int
 
+	// BlockCacheSize is the size of the block cache (default: 256).
+	BlockCacheSize uint16
+
 	// CheckpointInterval defines how frequently (in blocks) the index file header is updated (default: 1024).
 	CheckpointInterval uint64
 
@@ -43,6 +49,7 @@ func DefaultConfig() DatabaseConfig {
 		MinimumHeight:      0,
 		MaxDataFileSize:    DefaultMaxDataFileSize,
 		MaxDataFiles:       DefaultMaxDataFiles,
+		BlockCacheSize:     DefaultBlockCacheSize,
 		CheckpointInterval: 1024,
 		SyncToDisk:         true,
 	}
@@ -91,6 +98,12 @@ func (c DatabaseConfig) WithMaxDataFiles(maxFiles int) DatabaseConfig {
 	return c
 }
 
+// WithBlockCacheSize returns a copy of the config with BlockCacheSize set to the given value.
+func (c DatabaseConfig) WithBlockCacheSize(size uint16) DatabaseConfig {
+	c.BlockCacheSize = size
+	return c
+}
+
 // WithCheckpointInterval returns a copy of the config with CheckpointInterval set to the given value.
 func (c DatabaseConfig) WithCheckpointInterval(interval uint64) DatabaseConfig {
 	c.CheckpointInterval = interval

x/blockdb/database.go

@@ -197,7 +197,7 @@ type Database struct {
 // Parameters:
 //   - config: Configuration parameters
 //   - log: Logger instance for structured logging
-func New(config DatabaseConfig, log logging.Logger) (*Database, error) {
+func New(config DatabaseConfig, log logging.Logger) (database.HeightIndex, error) {
 	if err := config.Validate(); err != nil {
 		return nil, err
 	}
@@ -231,6 +231,7 @@ func New(config DatabaseConfig, log logging.Logger) (*Database, error) {
 		zap.String("dataDir", config.DataDir),
 		zap.Uint64("maxDataFileSize", config.MaxDataFileSize),
 		zap.Int("maxDataFiles", config.MaxDataFiles),
+		zap.Uint16("blockCacheSize", config.BlockCacheSize),
 	)
 
 	if err := s.openAndInitializeIndex(); err != nil {
@@ -256,6 +257,9 @@ func New(config DatabaseConfig, log logging.Logger) (*Database, error) {
 		zap.Uint64("maxBlockHeight", maxHeight),
 	)
 
+	if config.BlockCacheSize > 0 {
+		return newCacheDB(s, config.BlockCacheSize), nil
+	}
 	return s, nil
 }
 
@@ -286,9 +290,7 @@ func (s *Database) Put(height BlockHeight, block BlockData) error {
 	defer s.closeMu.RUnlock()
 
 	if s.closed {
-		s.log.Error("Failed to write block: database is closed",
-			zap.Uint64("height", height),
-		)
+		s.log.Error("Failed Put: database closed", zap.Uint64("height", height))
 		return database.ErrClosed
 	}
 
@@ -385,12 +387,6 @@ func (s *Database) Put(height BlockHeight, block BlockData) error {
 // It returns database.ErrNotFound if the block does not exist.
 func (s *Database) readBlockIndex(height BlockHeight) (indexEntry, error) {
 	var entry indexEntry
-	if s.closed {
-		s.log.Error("Failed to read block index: database is closed",
-			zap.Uint64("height", height),
-		)
-		return entry, database.ErrClosed
-	}
 
 	// Skip the index entry read if we know the block is past the max height.
 	maxHeight := s.maxBlockHeight.Load()
@@ -436,6 +432,11 @@ func (s *Database) Get(height BlockHeight) (BlockData, error) {
 	s.closeMu.RLock()
 	defer s.closeMu.RUnlock()
 
+	if s.closed {
+		s.log.Error("Failed Get: database closed", zap.Uint64("height", height))
+		return nil, database.ErrClosed
+	}
+
 	indexEntry, err := s.readBlockIndex(height)
 	if err != nil {
 		return nil, err
@@ -494,6 +495,15 @@ func (s *Database) Has(height BlockHeight) (bool, error) {
 	s.closeMu.RLock()
 	defer s.closeMu.RUnlock()
 
+	if s.closed {
+		s.log.Error("Failed Has: database closed", zap.Uint64("height", height))
+		return false, database.ErrClosed
+	}
+
+	return s.hasWithoutLock(height)
+}
+
+func (s *Database) hasWithoutLock(height BlockHeight) (bool, error) {
 	_, err := s.readBlockIndex(height)
 	if err != nil {
 		if errors.Is(err, database.ErrNotFound) || errors.Is(err, ErrInvalidBlockHeight) {