From 86290b1d214d2a2c024e87c6c52d8fdd3446ba65 Mon Sep 17 00:00:00 2001 From: Josh Bowen Date: Thu, 7 Oct 2021 14:22:29 -0600 Subject: [PATCH 1/6] add PrefixKV matching node/node.go to da/mock.go and da/mock_test.go --- da/da.go | 2 +- da/mock/mock.go | 5 ++++- da/mock/mock_test.go | 9 +++++++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/da/da.go b/da/da.go index 769f343af6..560bfa7c81 100644 --- a/da/da.go +++ b/da/da.go @@ -54,7 +54,7 @@ type ResultRetrieveBlock struct { // It also contains life-cycle methods. type DataAvailabilityLayerClient interface { // Init is called once to allow DA client to read configuration and initialize resources. - Init(config []byte, kvStore store.KVStore, logger log.Logger) error + Init(config []byte, kvStore *store.PrefixKV, logger log.Logger) error Start() error Stop() error diff --git a/da/mock/mock.go b/da/mock/mock.go index 1c706f5588..2fd9825300 100644 --- a/da/mock/mock.go +++ b/da/mock/mock.go @@ -14,16 +14,19 @@ type MockDataAvailabilityLayerClient struct { Blocks map[[32]byte]*types.Block BlockIndex map[uint64][32]byte + + dalcKV *store.PrefixKV } var _ da.DataAvailabilityLayerClient = &MockDataAvailabilityLayerClient{} var _ da.BlockRetriever = &MockDataAvailabilityLayerClient{} // Init is called once to allow DA client to read configuration and initialize resources. -func (m *MockDataAvailabilityLayerClient) Init(config []byte, kvStore store.KVStore, logger log.Logger) error { +func (m *MockDataAvailabilityLayerClient) Init(config []byte, dalcKV *store.PrefixKV, logger log.Logger) error { m.logger = logger m.Blocks = make(map[[32]byte]*types.Block) m.BlockIndex = make(map[uint64][32]byte) + m.dalcKV = dalcKV return nil } diff --git a/da/mock/mock_test.go b/da/mock/mock_test.go index 79e1862c90..bf3d3dfd3b 100644 --- a/da/mock/mock_test.go +++ b/da/mock/mock_test.go @@ -9,15 +9,20 @@ import ( "github.com/celestiaorg/optimint/da" "github.com/celestiaorg/optimint/log/test" + "github.com/celestiaorg/optimint/store" "github.com/celestiaorg/optimint/types" ) +var dalcPrefix = []byte{1} +var baseKV store.KVStore = store.NewInMemoryKVStore() +var dalcKV *store.PrefixKV = store.NewPrefixKV(baseKV, dalcPrefix) + func TestLifecycle(t *testing.T) { var da da.DataAvailabilityLayerClient = &MockDataAvailabilityLayerClient{} require := require.New(t) - err := da.Init([]byte{}, nil, &test.TestLogger{T: t}) + err := da.Init([]byte{}, dalcKV, &test.TestLogger{T: t}) require.NoError(err) err = da.Start() @@ -33,7 +38,7 @@ func TestMockDALC(t *testing.T) { require := require.New(t) assert := assert.New(t) - err := dalc.Init([]byte{}, nil, &test.TestLogger{T: t}) + err := dalc.Init([]byte{}, dalcKV, &test.TestLogger{T: t}) require.NoError(err) err = dalc.Start() From c98cbb70182ea4f0232b6bedc5066408f3b70d9a Mon Sep 17 00:00:00 2001 From: Josh Bowen Date: Thu, 7 Oct 2021 18:42:27 -0600 Subject: [PATCH 2/6] fix some tests, add some TODOs --- da/celestia/celestia.go | 4 ++-- da/celestia/celestia_test.go | 3 +++ da/mock/mock.go | 4 ++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/da/celestia/celestia.go b/da/celestia/celestia.go index fe38aa3807..b7a6aac708 100644 --- a/da/celestia/celestia.go +++ b/da/celestia/celestia.go @@ -51,7 +51,7 @@ type Config struct { // It use celestia-app via gRPC. type Celestia struct { config Config - kvStore store.KVStore + kvStore *store.PrefixKV logger log.Logger keyring keyring.Keyring @@ -62,7 +62,7 @@ type Celestia struct { var _ da.DataAvailabilityLayerClient = &Celestia{} // Init is called once to allow DA client to read configuration and initialize resources. -func (ll *Celestia) Init(config []byte, kvStore store.KVStore, logger log.Logger) error { +func (ll *Celestia) Init(config []byte, kvStore *store.PrefixKV, logger log.Logger) error { ll.logger = logger ll.kvStore = kvStore err := toml.Unmarshal(config, &ll.config) diff --git a/da/celestia/celestia_test.go b/da/celestia/celestia_test.go index d115ff0fc3..3eae353c91 100644 --- a/da/celestia/celestia_test.go +++ b/da/celestia/celestia_test.go @@ -30,6 +30,8 @@ func TestConfiguration(t *testing.T) { t.Run(c.name, func(t *testing.T) { assert := assert.New(t) ll := &Celestia{} + + // TODO(jbowen93): This is where we need to pass a test kvStore err := ll.Init(c.input, nil, nil) if c.err != nil { @@ -56,6 +58,7 @@ func TestSubmission(t *testing.T) { key, err := kr.Key("test-account") require.NoError(err) conf := testConfig(key) + // TODO(jbowen93): This is where we need to pass a test kvStore err = ll.Init([]byte(conf), nil, nil) require.NoError(err) ll.keyring = kr diff --git a/da/mock/mock.go b/da/mock/mock.go index 2fd9825300..3c049e5ed0 100644 --- a/da/mock/mock.go +++ b/da/mock/mock.go @@ -42,6 +42,8 @@ func (m *MockDataAvailabilityLayerClient) Stop() error { return nil } +// TODO(jbowen93): This is where storage to the kvStore needs to take place + // SubmitBlock submits the passed in block to the DA layer. // This should create a transaction which (potentially) // triggers a state transition in the DA layer. @@ -66,6 +68,8 @@ func (m *MockDataAvailabilityLayerClient) CheckBlockAvailability(header *types.H return da.ResultCheckBlock{DAResult: da.DAResult{Code: da.StatusSuccess}, DataAvailable: ok} } +// TODO(jbowen93): This is where retrieval from the kvStore needs to take place + // RetrieveBlock returns block at given height from data availability layer. func (m *MockDataAvailabilityLayerClient) RetrieveBlock(height uint64) da.ResultRetrieveBlock { hash := m.BlockIndex[height] From 02f6568094c17464ebac7d73d5077b8151fb474a Mon Sep 17 00:00:00 2001 From: Josh Bowen Date: Fri, 8 Oct 2021 15:50:47 -0600 Subject: [PATCH 3/6] update mock.go to use kvStore --- da/celestia/celestia.go | 4 ++-- da/celestia/celestia_test.go | 3 --- da/da.go | 2 +- da/mock/mock.go | 46 +++++++++++++++++++++++++++++------- da/mock/mock_test.go | 16 +++++++++---- types/serialization_test.go | 1 - 6 files changed, 53 insertions(+), 19 deletions(-) diff --git a/da/celestia/celestia.go b/da/celestia/celestia.go index b7a6aac708..fe38aa3807 100644 --- a/da/celestia/celestia.go +++ b/da/celestia/celestia.go @@ -51,7 +51,7 @@ type Config struct { // It use celestia-app via gRPC. type Celestia struct { config Config - kvStore *store.PrefixKV + kvStore store.KVStore logger log.Logger keyring keyring.Keyring @@ -62,7 +62,7 @@ type Celestia struct { var _ da.DataAvailabilityLayerClient = &Celestia{} // Init is called once to allow DA client to read configuration and initialize resources. -func (ll *Celestia) Init(config []byte, kvStore *store.PrefixKV, logger log.Logger) error { +func (ll *Celestia) Init(config []byte, kvStore store.KVStore, logger log.Logger) error { ll.logger = logger ll.kvStore = kvStore err := toml.Unmarshal(config, &ll.config) diff --git a/da/celestia/celestia_test.go b/da/celestia/celestia_test.go index 3eae353c91..d115ff0fc3 100644 --- a/da/celestia/celestia_test.go +++ b/da/celestia/celestia_test.go @@ -30,8 +30,6 @@ func TestConfiguration(t *testing.T) { t.Run(c.name, func(t *testing.T) { assert := assert.New(t) ll := &Celestia{} - - // TODO(jbowen93): This is where we need to pass a test kvStore err := ll.Init(c.input, nil, nil) if c.err != nil { @@ -58,7 +56,6 @@ func TestSubmission(t *testing.T) { key, err := kr.Key("test-account") require.NoError(err) conf := testConfig(key) - // TODO(jbowen93): This is where we need to pass a test kvStore err = ll.Init([]byte(conf), nil, nil) require.NoError(err) ll.keyring = kr diff --git a/da/da.go b/da/da.go index 560bfa7c81..769f343af6 100644 --- a/da/da.go +++ b/da/da.go @@ -54,7 +54,7 @@ type ResultRetrieveBlock struct { // It also contains life-cycle methods. type DataAvailabilityLayerClient interface { // Init is called once to allow DA client to read configuration and initialize resources. - Init(config []byte, kvStore *store.PrefixKV, logger log.Logger) error + Init(config []byte, kvStore store.KVStore, logger log.Logger) error Start() error Stop() error diff --git a/da/mock/mock.go b/da/mock/mock.go index 3c049e5ed0..eff27dd6f0 100644 --- a/da/mock/mock.go +++ b/da/mock/mock.go @@ -1,6 +1,8 @@ package mock import ( + "encoding/binary" + "github.com/celestiaorg/optimint/da" "github.com/celestiaorg/optimint/log" "github.com/celestiaorg/optimint/store" @@ -15,14 +17,14 @@ type MockDataAvailabilityLayerClient struct { Blocks map[[32]byte]*types.Block BlockIndex map[uint64][32]byte - dalcKV *store.PrefixKV + dalcKV store.KVStore } var _ da.DataAvailabilityLayerClient = &MockDataAvailabilityLayerClient{} var _ da.BlockRetriever = &MockDataAvailabilityLayerClient{} // Init is called once to allow DA client to read configuration and initialize resources. -func (m *MockDataAvailabilityLayerClient) Init(config []byte, dalcKV *store.PrefixKV, logger log.Logger) error { +func (m *MockDataAvailabilityLayerClient) Init(config []byte, dalcKV store.KVStore, logger log.Logger) error { m.logger = logger m.Blocks = make(map[[32]byte]*types.Block) m.BlockIndex = make(map[uint64][32]byte) @@ -42,8 +44,6 @@ func (m *MockDataAvailabilityLayerClient) Stop() error { return nil } -// TODO(jbowen93): This is where storage to the kvStore needs to take place - // SubmitBlock submits the passed in block to the DA layer. // This should create a transaction which (potentially) // triggers a state transition in the DA layer. @@ -51,6 +51,22 @@ func (m *MockDataAvailabilityLayerClient) SubmitBlock(block *types.Block) da.Res m.logger.Debug("Submitting block to DA layer!", "height", block.Header.Height) hash := block.Header.Hash() + blob, err := block.MarshalBinary() + if err != nil { + return da.ResultSubmitBlock{ + DAResult: da.DAResult{ + Code: da.StatusError, + Message: err.Error(), + }, + } + } + + b := make([]byte, 8) + binary.LittleEndian.PutUint64(b, block.Header.Height) + + m.dalcKV.Set(b, hash[:]) + m.dalcKV.Set(hash[:], blob) + m.Blocks[hash] = block m.BlockIndex[block.Header.Height] = hash @@ -68,10 +84,24 @@ func (m *MockDataAvailabilityLayerClient) CheckBlockAvailability(header *types.H return da.ResultCheckBlock{DAResult: da.DAResult{Code: da.StatusSuccess}, DataAvailable: ok} } -// TODO(jbowen93): This is where retrieval from the kvStore needs to take place - // RetrieveBlock returns block at given height from data availability layer. func (m *MockDataAvailabilityLayerClient) RetrieveBlock(height uint64) da.ResultRetrieveBlock { - hash := m.BlockIndex[height] - return da.ResultRetrieveBlock{DAResult: da.DAResult{Code: da.StatusSuccess}, Block: m.Blocks[hash]} + b := make([]byte, 8) + binary.LittleEndian.PutUint64(b, height) + hash, err := m.dalcKV.Get(b) + if err != nil { + return da.ResultRetrieveBlock{DAResult: da.DAResult{Code: da.StatusError, Message: err.Error()}} + } + blob, err := m.dalcKV.Get(hash) + if err != nil { + return da.ResultRetrieveBlock{DAResult: da.DAResult{Code: da.StatusError, Message: err.Error()}} + } + + block := &types.Block{} + err = block.UnmarshalBinary(blob) + if err != nil { + return da.ResultRetrieveBlock{DAResult: da.DAResult{Code: da.StatusError, Message: err.Error()}} + } + + return da.ResultRetrieveBlock{DAResult: da.DAResult{Code: da.StatusSuccess}, Block: block} } diff --git a/da/mock/mock_test.go b/da/mock/mock_test.go index bf3d3dfd3b..405957e67f 100644 --- a/da/mock/mock_test.go +++ b/da/mock/mock_test.go @@ -13,9 +13,10 @@ import ( "github.com/celestiaorg/optimint/types" ) -var dalcPrefix = []byte{1} -var baseKV store.KVStore = store.NewInMemoryKVStore() -var dalcKV *store.PrefixKV = store.NewPrefixKV(baseKV, dalcPrefix) +var ( + dalcPrefix = []byte{1} + dalcKV *store.PrefixKV = store.NewPrefixKV(store.NewInMemoryKVStore(), dalcPrefix) +) func TestLifecycle(t *testing.T) { var da da.DataAvailabilityLayerClient = &MockDataAvailabilityLayerClient{} @@ -77,7 +78,7 @@ func TestRetrieve(t *testing.T) { require := require.New(t) assert := assert.New(t) - err := dalc.Init([]byte{}, nil, &test.TestLogger{T: t}) + err := dalc.Init([]byte{}, dalcKV, &test.TestLogger{T: t}) require.NoError(err) err = dalc.Start() @@ -114,6 +115,13 @@ func getRandomBlock(height uint64, nTxs int) *types.Block { block.Data.IntermediateStateRoots.RawRootsList[i] = getRandomBytes(32) } + // TODO(https://github.com/celestiaorg/optimint/issues/143) + // This is a hack to get around equality checks on serialization round trips + if nTxs == 0 { + block.Data.Txs = nil + block.Data.IntermediateStateRoots.RawRootsList = nil + } + return block } diff --git a/types/serialization_test.go b/types/serialization_test.go index cc6f0306a5..13eb62d1c0 100644 --- a/types/serialization_test.go +++ b/types/serialization_test.go @@ -69,7 +69,6 @@ func TestBlockSerializationRoundTrip(t *testing.T) { deserialized := &Block{} err = deserialized.UnmarshalBinary(blob) assert.NoError(err) - assert.Equal(c.input, deserialized) }) } From 16dd787028e72cfdb0619a957f2436f57ffa5ea8 Mon Sep 17 00:00:00 2001 From: Josh Bowen Date: Fri, 8 Oct 2021 15:53:53 -0600 Subject: [PATCH 4/6] fix whitespace --- types/serialization_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/types/serialization_test.go b/types/serialization_test.go index 13eb62d1c0..cc6f0306a5 100644 --- a/types/serialization_test.go +++ b/types/serialization_test.go @@ -69,6 +69,7 @@ func TestBlockSerializationRoundTrip(t *testing.T) { deserialized := &Block{} err = deserialized.UnmarshalBinary(blob) assert.NoError(err) + assert.Equal(c.input, deserialized) }) } From d5e6a7d25074dbd1d09da0d4785c1f6ea4b9f9d2 Mon Sep 17 00:00:00 2001 From: Josh Bowen Date: Fri, 8 Oct 2021 16:09:52 -0600 Subject: [PATCH 5/6] Update mock CheckBlockAvailability function --- da/mock/mock.go | 24 +++++++----------------- da/mock/mock_test.go | 2 +- 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/da/mock/mock.go b/da/mock/mock.go index eff27dd6f0..a087c47a16 100644 --- a/da/mock/mock.go +++ b/da/mock/mock.go @@ -13,10 +13,6 @@ import ( // It does actually ensures DA - it stores data in-memory. type MockDataAvailabilityLayerClient struct { logger log.Logger - - Blocks map[[32]byte]*types.Block - BlockIndex map[uint64][32]byte - dalcKV store.KVStore } @@ -26,8 +22,6 @@ var _ da.BlockRetriever = &MockDataAvailabilityLayerClient{} // Init is called once to allow DA client to read configuration and initialize resources. func (m *MockDataAvailabilityLayerClient) Init(config []byte, dalcKV store.KVStore, logger log.Logger) error { m.logger = logger - m.Blocks = make(map[[32]byte]*types.Block) - m.BlockIndex = make(map[uint64][32]byte) m.dalcKV = dalcKV return nil } @@ -53,12 +47,7 @@ func (m *MockDataAvailabilityLayerClient) SubmitBlock(block *types.Block) da.Res hash := block.Header.Hash() blob, err := block.MarshalBinary() if err != nil { - return da.ResultSubmitBlock{ - DAResult: da.DAResult{ - Code: da.StatusError, - Message: err.Error(), - }, - } + return da.ResultSubmitBlock{DAResult: da.DAResult{Code: da.StatusError, Message: err.Error()}} } b := make([]byte, 8) @@ -67,9 +56,6 @@ func (m *MockDataAvailabilityLayerClient) SubmitBlock(block *types.Block) da.Res m.dalcKV.Set(b, hash[:]) m.dalcKV.Set(hash[:], blob) - m.Blocks[hash] = block - m.BlockIndex[block.Header.Height] = hash - return da.ResultSubmitBlock{ DAResult: da.DAResult{ Code: da.StatusSuccess, @@ -80,8 +66,12 @@ func (m *MockDataAvailabilityLayerClient) SubmitBlock(block *types.Block) da.Res // CheckBlockAvailability queries DA layer to check data availability of block corresponding to given header. func (m *MockDataAvailabilityLayerClient) CheckBlockAvailability(header *types.Header) da.ResultCheckBlock { - _, ok := m.Blocks[header.Hash()] - return da.ResultCheckBlock{DAResult: da.DAResult{Code: da.StatusSuccess}, DataAvailable: ok} + hash := header.Hash() + _, err := m.dalcKV.Get(hash[:]) + if err != nil { + return da.ResultCheckBlock{DAResult: da.DAResult{Code: da.StatusError, Message: err.Error()}} + } + return da.ResultCheckBlock{DAResult: da.DAResult{Code: da.StatusSuccess}, DataAvailable: true} } // RetrieveBlock returns block at given height from data availability layer. diff --git a/da/mock/mock_test.go b/da/mock/mock_test.go index 405957e67f..11430b48a5 100644 --- a/da/mock/mock_test.go +++ b/da/mock/mock_test.go @@ -66,7 +66,7 @@ func TestMockDALC(t *testing.T) { // this block was never submitted to DA check = dalc.CheckBlockAvailability(&b3.Header) - assert.Equal(da.StatusSuccess, check.Code) + assert.Equal(da.StatusError, check.Code) assert.False(check.DataAvailable) } From cfea2eb717c82b885862c05fcdc54e16d1985c05 Mon Sep 17 00:00:00 2001 From: Josh Bowen Date: Sat, 9 Oct 2021 15:04:53 -0600 Subject: [PATCH 6/6] Fix PR comments --- da/mock/mock.go | 13 +++---------- da/mock/mock_test.go | 11 +++++------ types/serialization.go | 8 ++++++++ 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/da/mock/mock.go b/da/mock/mock.go index a087c47a16..c70e9afed9 100644 --- a/da/mock/mock.go +++ b/da/mock/mock.go @@ -1,8 +1,6 @@ package mock import ( - "encoding/binary" - "github.com/celestiaorg/optimint/da" "github.com/celestiaorg/optimint/log" "github.com/celestiaorg/optimint/store" @@ -50,10 +48,7 @@ func (m *MockDataAvailabilityLayerClient) SubmitBlock(block *types.Block) da.Res return da.ResultSubmitBlock{DAResult: da.DAResult{Code: da.StatusError, Message: err.Error()}} } - b := make([]byte, 8) - binary.LittleEndian.PutUint64(b, block.Header.Height) - - m.dalcKV.Set(b, hash[:]) + m.dalcKV.Set(types.Uint64ToByteSlice(block.Header.Height), hash[:]) m.dalcKV.Set(hash[:], blob) return da.ResultSubmitBlock{ @@ -69,16 +64,14 @@ func (m *MockDataAvailabilityLayerClient) CheckBlockAvailability(header *types.H hash := header.Hash() _, err := m.dalcKV.Get(hash[:]) if err != nil { - return da.ResultCheckBlock{DAResult: da.DAResult{Code: da.StatusError, Message: err.Error()}} + return da.ResultCheckBlock{DAResult: da.DAResult{Code: da.StatusSuccess}, DataAvailable: false} } return da.ResultCheckBlock{DAResult: da.DAResult{Code: da.StatusSuccess}, DataAvailable: true} } // RetrieveBlock returns block at given height from data availability layer. func (m *MockDataAvailabilityLayerClient) RetrieveBlock(height uint64) da.ResultRetrieveBlock { - b := make([]byte, 8) - binary.LittleEndian.PutUint64(b, height) - hash, err := m.dalcKV.Get(b) + hash, err := m.dalcKV.Get(types.Uint64ToByteSlice(height)) if err != nil { return da.ResultRetrieveBlock{DAResult: da.DAResult{Code: da.StatusError, Message: err.Error()}} } diff --git a/da/mock/mock_test.go b/da/mock/mock_test.go index 11430b48a5..64a2101873 100644 --- a/da/mock/mock_test.go +++ b/da/mock/mock_test.go @@ -13,13 +13,9 @@ import ( "github.com/celestiaorg/optimint/types" ) -var ( - dalcPrefix = []byte{1} - dalcKV *store.PrefixKV = store.NewPrefixKV(store.NewInMemoryKVStore(), dalcPrefix) -) - func TestLifecycle(t *testing.T) { var da da.DataAvailabilityLayerClient = &MockDataAvailabilityLayerClient{} + dalcKV := store.NewInMemoryKVStore() require := require.New(t) @@ -35,6 +31,7 @@ func TestLifecycle(t *testing.T) { func TestMockDALC(t *testing.T) { var dalc da.DataAvailabilityLayerClient = &MockDataAvailabilityLayerClient{} + dalcKV := store.NewInMemoryKVStore() require := require.New(t) assert := assert.New(t) @@ -66,7 +63,7 @@ func TestMockDALC(t *testing.T) { // this block was never submitted to DA check = dalc.CheckBlockAvailability(&b3.Header) - assert.Equal(da.StatusError, check.Code) + assert.Equal(da.StatusSuccess, check.Code) assert.False(check.DataAvailable) } @@ -75,6 +72,8 @@ func TestRetrieve(t *testing.T) { var dalc da.DataAvailabilityLayerClient = mock var retriever da.BlockRetriever = mock + dalcKV := store.NewInMemoryKVStore() + require := require.New(t) assert := assert.New(t) diff --git a/types/serialization.go b/types/serialization.go index 8f7188ec79..7f68edb368 100644 --- a/types/serialization.go +++ b/types/serialization.go @@ -1,6 +1,7 @@ package types import ( + "encoding/binary" "errors" pb "github.com/celestiaorg/optimint/types/pb/optimint" @@ -182,6 +183,13 @@ func (c *Commit) FromProto(other *pb.Commit) error { return nil } +// Uint64ToByteSlice converts a uint64 into an 8 byte little endian slice +func Uint64ToByteSlice(v uint64) []byte { + b := make([]byte, 8) + binary.LittleEndian.PutUint64(b, v) + return b +} + func txsToByteSlices(txs Txs) [][]byte { bytes := make([][]byte, len(txs)) for i := range txs {