diff --git a/storagemarket/provider_offline_test.go b/storagemarket/provider_offline_test.go index cb8f0ebc5..defee3a02 100644 --- a/storagemarket/provider_offline_test.go +++ b/storagemarket/provider_offline_test.go @@ -86,7 +86,7 @@ func TestOfflineDealInsufficientProviderFunds(t *testing.T) { require.Contains(t, pi.Reason, "insufficient funds") } -func XTestOfflineDealDataCleanup(t *testing.T) { +func TestOfflineDealDataCleanup(t *testing.T) { ctx := context.Background() for _, delAfterImport := range []bool{true, false} { diff --git a/storagemarket/provider_test.go b/storagemarket/provider_test.go index 2a136570d..3c000f08c 100644 --- a/storagemarket/provider_test.go +++ b/storagemarket/provider_test.go @@ -1620,7 +1620,7 @@ func NewHarness(t *testing.T, opts ...harnessOpt) *ProviderHarness { askStore.SetAsk(pc.price, pc.verifiedPrice, pc.minPieceSize, pc.maxPieceSize) pdctx, cancel := context.WithCancel(context.Background()) - pm := piecedirectory.NewPieceDirectory(bdclientutil.NewTestStore(pdctx), nil, 1) + pm := piecedirectory.NewPieceDirectory(bdclientutil.NewTestStore(pdctx), minerStub.MockPieceReader, 1) pm.Start(pdctx) t.Cleanup(cancel) @@ -1953,7 +1953,7 @@ func (ph *ProviderHarness) newDealBuilder(t *testing.T, seed int, opts ...dealPr sectorId := abi.SectorNumber(rand.Intn(100)) offset := abi.PaddedPieceSize(rand.Intn(100)) - tbuilder.ms = tbuilder.ph.MinerStub.ForDeal(dealParams, publishCid, finalPublishCid, dealId, sectorsStatusDealId, sectorId, offset) + tbuilder.ms = tbuilder.ph.MinerStub.ForDeal(dealParams, publishCid, finalPublishCid, dealId, sectorsStatusDealId, sectorId, offset, carFilePath) tbuilder.td = td return tbuilder } @@ -2301,7 +2301,7 @@ func (td *testDeal) updateWithRestartedProvider(ph *ProviderHarness) *testDealBu td.tBuilder.ph = ph td.tBuilder.td = td - td.tBuilder.ms = ph.MinerStub.ForDeal(td.params, old.PublishCid, old.FinalPublishCid, old.DealID, old.SectorsStatusDealID, old.SectorID, old.Offset) + td.tBuilder.ms = ph.MinerStub.ForDeal(td.params, old.PublishCid, old.FinalPublishCid, old.DealID, old.SectorsStatusDealID, old.SectorID, old.Offset, old.CarFilePath) return td.tBuilder } diff --git a/storagemarket/smtestutil/mocks.go b/storagemarket/smtestutil/mocks.go index 5011a0ab8..60faf58b6 100644 --- a/storagemarket/smtestutil/mocks.go +++ b/storagemarket/smtestutil/mocks.go @@ -2,11 +2,14 @@ package smtestutil import ( "context" + "fmt" "io" "strings" "sync" "github.com/filecoin-project/boost-gfm/storagemarket" + pdtypes "github.com/filecoin-project/boost/piecedirectory/types" + mock_piecedirectory "github.com/filecoin-project/boost/piecedirectory/types/mocks" mock_sealingpipeline "github.com/filecoin-project/boost/storagemarket/sealingpipeline/mock" "github.com/filecoin-project/boost/storagemarket/types" "github.com/filecoin-project/boost/storagemarket/types/mock_types" @@ -19,6 +22,7 @@ import ( "github.com/golang/mock/gomock" "github.com/google/uuid" "github.com/ipfs/go-cid" + "github.com/ipld/go-car/v2" ) type MinerStub struct { @@ -27,6 +31,7 @@ type MinerStub struct { *mock_types.MockPieceAdder *mock_types.MockCommpCalculator *mock_types.MockIndexProvider + *mock_piecedirectory.MockPieceReader *mock_sealingpipeline.MockAPI lk sync.Mutex @@ -44,6 +49,7 @@ func NewMinerStub(ctrl *gomock.Controller) *MinerStub { MockChainDealManager: mock_types.NewMockChainDealManager(ctrl), MockPieceAdder: mock_types.NewMockPieceAdder(ctrl), MockIndexProvider: mock_types.NewMockIndexProvider(ctrl), + MockPieceReader: mock_piecedirectory.NewMockPieceReader(ctrl), MockAPI: mock_sealingpipeline.NewMockAPI(ctrl), unblockCommp: make(map[uuid.UUID]chan struct{}), @@ -81,8 +87,7 @@ func (ms *MinerStub) UnblockAddPiece(id uuid.UUID) { close(ch) } -func (ms *MinerStub) ForDeal(dp *types.DealParams, publishCid, finalPublishCid cid.Cid, dealId abi.DealID, sectorsStatusDealId abi.DealID, sectorId abi.SectorNumber, - offset abi.PaddedPieceSize) *MinerStubBuilder { +func (ms *MinerStub) ForDeal(dp *types.DealParams, publishCid, finalPublishCid cid.Cid, dealId, sectorsStatusDealId abi.DealID, sectorId abi.SectorNumber, offset abi.PaddedPieceSize, carFilePath string) *MinerStubBuilder { return &MinerStubBuilder{ stub: ms, dp: dp, @@ -93,6 +98,7 @@ func (ms *MinerStub) ForDeal(dp *types.DealParams, publishCid, finalPublishCid c sectorsStatusDealId: sectorsStatusDealId, sectorId: sectorId, offset: offset, + carFilePath: carFilePath, } } @@ -105,9 +111,10 @@ type MinerStubBuilder struct { dealId abi.DealID sectorsStatusDealId abi.DealID - sectorId abi.SectorNumber - offset abi.PaddedPieceSize - rb *[]byte + sectorId abi.SectorNumber + offset abi.PaddedPieceSize + carFilePath string + rb *[]byte } func (mb *MinerStubBuilder) SetupNoOp() *MinerStubBuilder { @@ -338,6 +345,21 @@ func (mb *MinerStubBuilder) SetupAnnounce(blocking bool, announce bool) *MinerSt callCount = 1 } + // When boost finishes adding the piece to a sector, it creates an index + // of the piece data and then announces the index. We need to mock a piece + // reader that returns the CAR file. + carReader, err := car.OpenReader(mb.carFilePath) + if err != nil { + panic(fmt.Sprintf("opening car file %s: %s", mb.carFilePath, err)) + } + carv1Reader, err := carReader.DataReader() + if err != nil { + panic(fmt.Sprintf("opening car v1 reader %s: %s", mb.carFilePath, err)) + } + readerWithClose := withClose(carv1Reader) + + mb.stub.MockPieceReader.EXPECT().GetReader(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(readerWithClose, nil) + mb.stub.MockIndexProvider.EXPECT().Enabled().AnyTimes().Return(true) mb.stub.MockIndexProvider.EXPECT().Start(gomock.Any()).AnyTimes() mb.stub.MockIndexProvider.EXPECT().AnnounceBoostDeal(gomock.Any(), gomock.Any()).Times(callCount).DoAndReturn(func(ctx context.Context, _ *types.ProviderDealState) (cid.Cid, error) { @@ -380,6 +402,7 @@ func (mb *MinerStubBuilder) Output() *StubbedMinerOutput { SealedBytes: mb.rb, SectorID: mb.sectorId, Offset: mb.offset, + CarFilePath: mb.carFilePath, } } @@ -391,4 +414,17 @@ type StubbedMinerOutput struct { SealedBytes *[]byte SectorID abi.SectorNumber Offset abi.PaddedPieceSize + CarFilePath string +} + +func withClose(reader car.SectionReader) pdtypes.SectionReader { + return §ionReaderWithClose{SectionReader: reader} +} + +type sectionReaderWithClose struct { + car.SectionReader +} + +func (s *sectionReaderWithClose) Close() error { + return nil }