From 77bcc9d839f71b982a56d0f25b89328cb2cee275 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Fri, 24 Jan 2025 14:58:42 +0200 Subject: [PATCH] test(swamp/share): cover share module with swamp tests (#4036) --- .github/workflows/integration-tests.yml | 15 ++ blob/helper.go | 16 ++ nodebuilder/tests/blob_test.go | 42 ++--- nodebuilder/tests/share_test.go | 221 ++++++++++++++++++++++++ share/shwap/sample.go | 23 +++ 5 files changed, 291 insertions(+), 26 deletions(-) create mode 100644 nodebuilder/tests/share_test.go diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 2f5b4a3dbd..ef8657ae3e 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -143,3 +143,18 @@ jobs: - name: run sync tests run: make test-integration SHORT=true TAGS=pruning + + share_tests: + name: Integration Tests Share + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: set up go + uses: actions/setup-go@v5 + with: + go-version: ${{ inputs.go-version }} + + - name: run share tests + run: make test-integration SHORT=true TAGS=share diff --git a/blob/helper.go b/blob/helper.go index d764e827e9..57c3856362 100644 --- a/blob/helper.go +++ b/blob/helper.go @@ -3,6 +3,8 @@ package blob import ( "sort" + "github.com/celestiaorg/go-square/merkle" + "github.com/celestiaorg/go-square/v2/inclusion" libshare "github.com/celestiaorg/go-square/v2/share" ) @@ -32,6 +34,20 @@ func ToLibBlobs(blobs ...*Blob) []*libshare.Blob { return libBlobs } +// ToNodeBlobs converts libshare blob type to the node's specific blob type. +func ToNodeBlobs(blobs ...*libshare.Blob) ([]*Blob, error) { + nodeBlobs := make([]*Blob, len(blobs)) + hashFromByteSlices := merkle.HashFromByteSlices + for i, blob := range blobs { + com, err := inclusion.CreateCommitment(blob, hashFromByteSlices, subtreeRootThreshold) + if err != nil { + return nil, err + } + nodeBlobs[i] = &Blob{Blob: blob, Commitment: com, index: -1} + } + return nodeBlobs, nil +} + func calculateIndex(rowLength, blobIndex int) (row, col int) { row = blobIndex / rowLength col = blobIndex - (row * rowLength) diff --git a/nodebuilder/tests/blob_test.go b/nodebuilder/tests/blob_test.go index 817dd81014..4b5ad52b24 100644 --- a/nodebuilder/tests/blob_test.go +++ b/nodebuilder/tests/blob_test.go @@ -30,13 +30,9 @@ func TestBlobModule(t *testing.T) { require.NoError(t, err) libBlobs1, err := libshare.GenerateV0Blobs([]int{4}, false) require.NoError(t, err) - blobs := make([]*blob.Blob, 0, len(libBlobs0)+len(libBlobs1)) - for _, libBlob := range append(libBlobs0, libBlobs1...) { - blob, err := convert(libBlob) - require.NoError(t, err) - blobs = append(blobs, blob) - } + blobs, err := blob.ToNodeBlobs(append(libBlobs0, libBlobs1...)...) + require.NoError(t, err) bridge := sw.NewBridgeNode() require.NoError(t, bridge.Start(ctx)) @@ -68,9 +64,9 @@ func TestBlobModule(t *testing.T) { ) require.NoError(t, err) - v1, err := convert(v1Blob) + v1, err := blob.ToNodeBlobs(v1Blob) require.NoError(t, err) - blobs = append(blobs, v1) + blobs = append(blobs, v1[0]) height, err := fullClient.Blob.Submit(ctx, blobs, state.NewTxConfig()) require.NoError(t, err) @@ -109,12 +105,12 @@ func TestBlobModule(t *testing.T) { { name: "Get BlobV1", doFn: func(t *testing.T) { - blobV1, err := fullClient.Blob.Get(ctx, height, v1.Namespace(), v1.Commitment) + blobV1, err := fullClient.Blob.Get(ctx, height, v1[0].Namespace(), v1[0].Commitment) require.NoError(t, err) assert.Equal(t, libshare.ShareVersionOne, blobV1.ShareVersion()) - assert.Equal(t, v1.Commitment, blobV1.Commitment) + assert.Equal(t, v1[0].Commitment, blobV1.Commitment) assert.NotNil(t, blobV1.Signer()) - assert.Equal(t, blobV1.Signer(), v1.Signer()) + assert.Equal(t, blobV1.Signer(), v1[0].Signer()) }, }, @@ -140,15 +136,15 @@ func TestBlobModule(t *testing.T) { doFn: func(t *testing.T) { libBlob, err := libshare.GenerateV0Blobs([]int{4}, false) require.NoError(t, err) - newBlob, err := convert(libBlob[0]) + newBlob, err := blob.ToNodeBlobs(libBlob[0]) require.NoError(t, err) - b, err := fullClient.Blob.Get(ctx, height, newBlob.Namespace(), newBlob.Commitment) + b, err := fullClient.Blob.Get(ctx, height, newBlob[0].Namespace(), newBlob[0].Commitment) assert.Nil(t, b) require.Error(t, err) require.ErrorContains(t, err, blob.ErrBlobNotFound.Error()) - blobs, err := fullClient.Blob.GetAll(ctx, height, []libshare.Namespace{newBlob.Namespace()}) + blobs, err := fullClient.Blob.GetAll(ctx, height, []libshare.Namespace{newBlob[0].Namespace()}) require.NoError(t, err) assert.Empty(t, blobs) }, @@ -158,23 +154,23 @@ func TestBlobModule(t *testing.T) { doFn: func(t *testing.T) { libBlob, err := libshare.GenerateV0Blobs([]int{8, 4}, true) require.NoError(t, err) - b, err := convert(libBlob[0]) + b, err := blob.ToNodeBlobs(libBlob[0]) require.NoError(t, err) - height, err := fullClient.Blob.Submit(ctx, []*blob.Blob{b, b}, state.NewTxConfig()) + height, err := fullClient.Blob.Submit(ctx, []*blob.Blob{b[0], b[0]}, state.NewTxConfig()) require.NoError(t, err) _, err = fullClient.Header.WaitForHeight(ctx, height) require.NoError(t, err) - b0, err := fullClient.Blob.Get(ctx, height, b.Namespace(), b.Commitment) + b0, err := fullClient.Blob.Get(ctx, height, b[0].Namespace(), b[0].Commitment) require.NoError(t, err) - require.Equal(t, b.Commitment, b0.Commitment) + require.Equal(t, b[0].Commitment, b0.Commitment) - proof, err := fullClient.Blob.GetProof(ctx, height, b.Namespace(), b.Commitment) + proof, err := fullClient.Blob.GetProof(ctx, height, b[0].Namespace(), b[0].Commitment) require.NoError(t, err) - included, err := fullClient.Blob.Included(ctx, height, b.Namespace(), proof, b.Commitment) + included, err := fullClient.Blob.Included(ctx, height, b[0].Namespace(), proof, b[0].Commitment) require.NoError(t, err) require.True(t, included) }, @@ -212,9 +208,3 @@ func TestBlobModule(t *testing.T) { }) } } - -// convert converts a libshare.Blob to a blob.Blob. -// convert may be deduplicated with convertBlobs from the blob package. -func convert(libBlob *libshare.Blob) (nodeBlob *blob.Blob, err error) { - return blob.NewBlob(libBlob.ShareVersion(), libBlob.Namespace(), libBlob.Data(), libBlob.Signer()) -} diff --git a/nodebuilder/tests/share_test.go b/nodebuilder/tests/share_test.go new file mode 100644 index 0000000000..551b0d9c69 --- /dev/null +++ b/nodebuilder/tests/share_test.go @@ -0,0 +1,221 @@ +//go:build share || integration + +package tests + +import ( + "context" + "testing" + "time" + + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + libshare "github.com/celestiaorg/go-square/v2/share" + + "github.com/celestiaorg/celestia-node/api/rpc/client" + "github.com/celestiaorg/celestia-node/blob" + "github.com/celestiaorg/celestia-node/nodebuilder/node" + "github.com/celestiaorg/celestia-node/nodebuilder/tests/swamp" + "github.com/celestiaorg/celestia-node/share/shwap" + "github.com/celestiaorg/celestia-node/state" +) + +func TestShareModule(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second) + t.Cleanup(cancel) + sw := swamp.NewSwamp(t, swamp.WithBlockTime(time.Second*1)) + blobSize := 128 + libBlob, err := libshare.GenerateV0Blobs([]int{blobSize}, true) + require.NoError(t, err) + + nodeBlob, err := blob.ToNodeBlobs(libBlob[0]) + require.NoError(t, err) + + bridge := sw.NewBridgeNode() + require.NoError(t, bridge.Start(ctx)) + addrs, err := peer.AddrInfoToP2pAddrs(host.InfoFromHost(bridge.Host)) + require.NoError(t, err) + + fullCfg := sw.DefaultTestConfig(node.Full) + fullCfg.Header.TrustedPeers = append(fullCfg.Header.TrustedPeers, addrs[0].String()) + fullNode := sw.NewNodeWithConfig(node.Full, fullCfg) + require.NoError(t, fullNode.Start(ctx)) + + addrsFull, err := peer.AddrInfoToP2pAddrs(host.InfoFromHost(fullNode.Host)) + require.NoError(t, err) + + lightCfg := sw.DefaultTestConfig(node.Light) + lightCfg.Header.TrustedPeers = append(lightCfg.Header.TrustedPeers, addrsFull[0].String()) + lightNode := sw.NewNodeWithConfig(node.Light, lightCfg) + require.NoError(t, lightNode.Start(ctx)) + + bridgeClient := getAdminClient(ctx, bridge, t) + fullClient := getAdminClient(ctx, fullNode, t) + lightClient := getAdminClient(ctx, lightNode, t) + + height, err := fullClient.Blob.Submit(ctx, nodeBlob, state.NewTxConfig()) + require.NoError(t, err) + + _, err = fullClient.Header.WaitForHeight(ctx, height) + require.NoError(t, err) + _, err = lightClient.Header.WaitForHeight(ctx, height) + require.NoError(t, err) + + sampledBlob, err := fullClient.Blob.Get(ctx, height, nodeBlob[0].Namespace(), nodeBlob[0].Commitment) + require.NoError(t, err) + + hdr, err := fullClient.Header.GetByHeight(ctx, height) + require.NoError(t, err) + + coords, err := shwap.SampleCoordsFrom1DIndex(sampledBlob.Index(), len(hdr.DAH.RowRoots)) + require.NoError(t, err) + + blobAsShares, err := blob.BlobsToShares(sampledBlob) + require.NoError(t, err) + // different clients allow to test different getters that are used to get the data. + clients := []*client.Client{lightClient, fullClient, bridgeClient} + + testCases := []struct { + name string + doFn func(t *testing.T) + }{ + { + name: "SharesAvailable", + doFn: func(t *testing.T) { + for _, client := range clients { + err := client.Share.SharesAvailable(ctx, height) + require.NoError(t, err) + } + }, + }, + { + name: "GetShareQ1", + doFn: func(t *testing.T) { + for _, client := range clients { + // compare the share from quadrant1 by its coordinate. + // Additionally check that received share the same as the first share of the blob. + sh, err := client.Share.GetShare(ctx, height, coords.Row, coords.Col) + require.NoError(t, err) + assert.Equal(t, blobAsShares[0], sh) + } + }, + }, + { + name: "GetShareQ4", + doFn: func(t *testing.T) { + for _, client := range clients { + _, err := client.Share.GetShare(ctx, height, len(hdr.DAH.RowRoots)-1, len(hdr.DAH.ColumnRoots)-1) + require.NoError(t, err) + } + }, + }, + { + name: "GetSamplesQ1", + doFn: func(t *testing.T) { + dah := hdr.DAH + requestCoords := []shwap.SampleCoords{coords} + for _, client := range clients { + // request from the first quadrant using the blob coordinates. + samples, err := client.Share.GetSamples(ctx, hdr, requestCoords) + require.NoError(t, err) + err = samples[0].Verify(dah, coords.Row, coords.Col) + require.NoError(t, err) + require.Equal(t, blobAsShares[0], samples[0].Share) + } + }, + }, + { + name: "GetSamplesQ4", + doFn: func(t *testing.T) { + dah := hdr.DAH + coords := shwap.SampleCoords{Row: len(dah.RowRoots) - 1, Col: len(dah.RowRoots) - 1} + requestCoords := []shwap.SampleCoords{coords} + for _, client := range clients { + // getting the last sample from the eds(from quadrant 4). + samples, err := client.Share.GetSamples(ctx, hdr, requestCoords) + require.NoError(t, err) + err = samples[0].Verify(dah, coords.Row, coords.Col) + require.NoError(t, err) + } + }, + }, + { + name: "GetEDS", + doFn: func(t *testing.T) { + for _, client := range clients { + eds, err := client.Share.GetEDS(ctx, height) + require.NoError(t, err) + rawShares := eds.Row(uint(coords.Row)) + sh, err := libshare.FromBytes([][]byte{rawShares[coords.Col]}) + require.NoError(t, err) + assert.Equal(t, blobAsShares[0], sh[0]) + } + }, + }, + { + name: "GetRowQ1", + doFn: func(t *testing.T) { + dah := hdr.DAH + for _, client := range clients { + // request row from the first half of the EDS(using the blob's coordinates). + row, err := client.Share.GetRow(ctx, height, coords.Row) + require.NoError(t, err) + // verify row against the DAH. + err = row.Verify(dah, coords.Row) + require.NoError(t, err) + shrs, err := row.Shares() + require.NoError(t, err) + // additionally compare shares + assert.Equal(t, blobAsShares[0], shrs[coords.Col]) + } + }, + }, + { + name: "GetRowQ4", + doFn: func(t *testing.T) { + dah := hdr.DAH + coords := shwap.SampleCoords{Row: len(dah.RowRoots) - 1, Col: len(dah.RowRoots) - 1} + for _, client := range clients { + // request the last row + row, err := client.Share.GetRow(ctx, height, coords.Row) + require.NoError(t, err) + // verify against DAH + err = row.Verify(dah, coords.Row) + require.NoError(t, err) + } + }, + }, + { + name: "GetNamespaceData", + doFn: func(t *testing.T) { + dah := hdr.DAH + for _, client := range clients { + // request data from the blob's namespace + nsData, err := client.Share.GetNamespaceData(ctx, height, blobAsShares[0].Namespace()) + require.NoError(t, err) + + // verify against the DAH + err = nsData.Verify(dah, blobAsShares[0].Namespace()) + require.NoError(t, err) + + b, err := libshare.ParseBlobs(nsData.Flatten()) + require.NoError(t, err) + + blb, err := blob.ToNodeBlobs(b[0]) + require.NoError(t, err) + // compare commitments + require.Equal(t, nodeBlob[0].Commitment, blb[0].Commitment) + + } + }, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + tt.doFn(t) + }) + } +} diff --git a/share/shwap/sample.go b/share/shwap/sample.go index 9b8e16a93d..0156679be2 100644 --- a/share/shwap/sample.go +++ b/share/shwap/sample.go @@ -1,6 +1,7 @@ package shwap import ( + "encoding/json" "errors" "fmt" @@ -88,6 +89,28 @@ func (s Sample) ToProto() *pb.Sample { } } +// MarshalJSON encodes sample to the json encoded bytes. +func (s Sample) MarshalJSON() ([]byte, error) { + pbSample := s.ToProto() + return json.Marshal(*pbSample) +} + +// UnmarshalJSON decodes bytes to the Sample. +func (s *Sample) UnmarshalJSON(data []byte) error { + var ss pb.Sample + err := json.Unmarshal(data, &ss) + if err != nil { + return err + } + + sample, err := SampleFromProto(&ss) + if err != nil { + return err + } + *s = sample + return nil +} + // IsEmpty reports whether the Sample is empty, i.e. doesn't contain a proof. func (s Sample) IsEmpty() bool { return s.Proof == nil