From 1f90ebd4a5d5cf688e62b5c3c35d27c4f0c48fb2 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 23 May 2023 15:12:39 +0200 Subject: [PATCH 01/41] fix timer.Reset and improve logs --- cmd/boostd/main.go | 2 ++ node/modules/piecedirectory.go | 6 ++-- piecedirectory/doctor.go | 60 +++++++++++++++++++++++++++++----- 3 files changed, 58 insertions(+), 10 deletions(-) diff --git a/cmd/boostd/main.go b/cmd/boostd/main.go index c89e58cb7..0dbb8c468 100644 --- a/cmd/boostd/main.go +++ b/cmd/boostd/main.go @@ -69,6 +69,7 @@ func before(cctx *cli.Context) error { _ = logging.SetLogLevel("piecedir", "INFO") _ = logging.SetLogLevel("index-provider-wrapper", "INFO") _ = logging.SetLogLevel("unsmgr", "INFO") + _ = logging.SetLogLevel("piecedoc", "INFO") if cliutil.IsVeryVerbose { _ = logging.SetLogLevel("boostd", "DEBUG") @@ -83,6 +84,7 @@ func before(cctx *cli.Context) error { _ = logging.SetLogLevel("piecedir", "DEBUG") _ = logging.SetLogLevel("fxlog", "DEBUG") _ = logging.SetLogLevel("unsmgr", "DEBUG") + _ = logging.SetLogLevel("piecedoc", "DEBUG") } return nil diff --git a/node/modules/piecedirectory.go b/node/modules/piecedirectory.go index 24a94fc26..930db8656 100644 --- a/node/modules/piecedirectory.go +++ b/node/modules/piecedirectory.go @@ -22,9 +22,11 @@ import ( "github.com/filecoin-project/dagstore/shard" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-jsonrpc" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/v1api" mktsdagstore "github.com/filecoin-project/lotus/markets/dagstore" "github.com/filecoin-project/lotus/node/modules/dtypes" + lotus_dtypes "github.com/filecoin-project/lotus/node/modules/dtypes" lotus_repo "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/storage/sealer" "github.com/filecoin-project/lotus/storage/sectorblocks" @@ -153,8 +155,8 @@ func NewPieceStore(pm *piecedirectory.PieceDirectory, maddr address.Address) pie return &boostPieceStoreWrapper{piecedirectory: pm, maddr: maddr} } -func NewPieceDoctor(lc fx.Lifecycle, store types.Store, sapi mktsdagstore.SectorAccessor) *piecedirectory.Doctor { - doc := piecedirectory.NewDoctor(store, sapi) +func NewPieceDoctor(lc fx.Lifecycle, store types.Store, sapi mktsdagstore.SectorAccessor, fullnodeApi api.FullNode, maddr lotus_dtypes.MinerAddress) *piecedirectory.Doctor { + doc := piecedirectory.NewDoctor(store, sapi, fullnodeApi, address.Address(maddr)) docctx, cancel := context.WithCancel(context.Background()) lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { diff --git a/piecedirectory/doctor.go b/piecedirectory/doctor.go index ec6cfe77e..c71f60c66 100644 --- a/piecedirectory/doctor.go +++ b/piecedirectory/doctor.go @@ -4,11 +4,14 @@ import ( "context" "errors" "fmt" - "math/rand" "time" "github.com/filecoin-project/boost/piecedirectory/types" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/builtin/v9/miner" + "github.com/filecoin-project/lotus/api" + lotuschaintypes "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" ) @@ -25,18 +28,25 @@ type SealingApi interface { // Note that multiple Doctor processes can run in parallel. The logic for which // pieces to give to the Doctor to check is in the local index directory. type Doctor struct { - store types.Store - sapi SealingApi + store types.Store + sapi SealingApi + fullnodeApi api.FullNode + maddr address.Address + + allSectors map[abi.SectorNumber]*miner.SectorOnChainInfo + activeSectors map[abi.SectorNumber]struct{} } -func NewDoctor(store types.Store, sapi SealingApi) *Doctor { - return &Doctor{store: store, sapi: sapi} +func NewDoctor(store types.Store, sapi SealingApi, fullnodeApi api.FullNode, maddr address.Address) *Doctor { + return &Doctor{store: store, sapi: sapi, fullnodeApi: fullnodeApi, maddr: maddr} } // The average interval between calls to NextPiecesToCheck const avgCheckInterval = 30 * time.Second func (d *Doctor) Run(ctx context.Context) { + doclog.Info("piece doctor: running") + timer := time.NewTimer(0) defer timer.Stop() @@ -47,6 +57,30 @@ func (d *Doctor) Run(ctx context.Context) { case <-timer.C: } + sectors, err := d.fullnodeApi.StateMinerSectors(ctx, d.maddr, nil, lotuschaintypes.EmptyTSK) + if err != nil { + return + } + + d.allSectors = make(map[abi.SectorNumber]*miner.SectorOnChainInfo) + for _, info := range sectors { + d.allSectors[info.SectorNumber] = info + } + + head, err := d.fullnodeApi.ChainHead(ctx) + if err != nil { + return + } + + activeSet, err := d.fullnodeApi.StateMinerActiveSectors(ctx, d.maddr, head.Key()) + if err != nil { + return + } + d.activeSectors = make(map[abi.SectorNumber]struct{}, len(activeSet)) + for _, info := range activeSet { + d.activeSectors[info.SectorNumber] = struct{}{} + } + // Get the next pieces to check (eg pieces that haven't been checked // for a while) from the local index directory pcids, err := d.store.NextPiecesToCheck(ctx) @@ -60,7 +94,7 @@ func (d *Doctor) Run(ctx context.Context) { } // Check each piece for problems - doclog.Debugw("piece doctor: checking pieces", "count", len(pcids)) + doclog.Debugw("piece doctor: checking pieces", "count", len(pcids), "all sectors", len(d.allSectors), "active sectors", len(d.activeSectors)) for _, pcid := range pcids { err := d.checkPiece(ctx, pcid) if err != nil { @@ -75,7 +109,8 @@ func (d *Doctor) Run(ctx context.Context) { // Sleep for a few seconds between ticks. // The time to sleep is randomized, so that if there are multiple doctor // processes they will each process some pieces some of the time. - sleepTime := avgCheckInterval/2 + time.Duration(rand.Intn(int(avgCheckInterval)))*time.Millisecond + sleepTime := avgCheckInterval / 2 + timer.Reset(sleepTime) } } @@ -103,8 +138,17 @@ func (d *Doctor) checkPiece(ctx context.Context, pieceCid cid.Cid) error { // Check if there is an unsealed copy of the piece var hasUnsealedDeal bool + + // Check whether the piece is present in active sector + lacksActiveSector := true dls := md.Deals for _, dl := range dls { + + // check if we have an active sector + if _, ok := d.activeSectors[dl.SectorID]; ok { + lacksActiveSector = false + } + isUnsealedCtx, cancel := context.WithTimeout(ctx, 5*time.Second) isUnsealed, err := d.sapi.IsUnsealed(isUnsealedCtx, dl.SectorID, dl.PieceOffset.Unpadded(), dl.PieceLength.Unpadded()) cancel() @@ -119,7 +163,7 @@ func (d *Doctor) checkPiece(ctx context.Context, pieceCid cid.Cid) error { } } - if !hasUnsealedDeal { + if !hasUnsealedDeal && !lacksActiveSector { err = d.store.FlagPiece(ctx, pieceCid) if err != nil { return fmt.Errorf("failed to flag piece %s with no unsealed deal: %w", pieceCid, err) From 1b2264cb98e5e30eeb5c86ecefba994214ba4008 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 24 May 2023 12:58:47 +0200 Subject: [PATCH 02/41] revert randomization --- piecedirectory/doctor.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piecedirectory/doctor.go b/piecedirectory/doctor.go index c71f60c66..118ef99df 100644 --- a/piecedirectory/doctor.go +++ b/piecedirectory/doctor.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "math/rand" "time" "github.com/filecoin-project/boost/piecedirectory/types" @@ -109,8 +110,7 @@ func (d *Doctor) Run(ctx context.Context) { // Sleep for a few seconds between ticks. // The time to sleep is randomized, so that if there are multiple doctor // processes they will each process some pieces some of the time. - sleepTime := avgCheckInterval / 2 - + sleepTime := avgCheckInterval/2 + time.Duration(rand.Intn(int(avgCheckInterval))) timer.Reset(sleepTime) } } From 8409e5b2d00f7cdb7b6be89549031e0162f54a27 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 24 May 2023 14:16:58 +0200 Subject: [PATCH 03/41] piece doc: handle errors --- cmd/boostd/main.go | 2 + piecedirectory/doctor.go | 83 ++++++++++++++++++++++------------------ 2 files changed, 47 insertions(+), 38 deletions(-) diff --git a/cmd/boostd/main.go b/cmd/boostd/main.go index 0dbb8c468..451c2ad84 100644 --- a/cmd/boostd/main.go +++ b/cmd/boostd/main.go @@ -70,6 +70,7 @@ func before(cctx *cli.Context) error { _ = logging.SetLogLevel("index-provider-wrapper", "INFO") _ = logging.SetLogLevel("unsmgr", "INFO") _ = logging.SetLogLevel("piecedoc", "INFO") + _ = logging.SetLogLevel("piecedirectory", "INFO") if cliutil.IsVeryVerbose { _ = logging.SetLogLevel("boostd", "DEBUG") @@ -85,6 +86,7 @@ func before(cctx *cli.Context) error { _ = logging.SetLogLevel("fxlog", "DEBUG") _ = logging.SetLogLevel("unsmgr", "DEBUG") _ = logging.SetLogLevel("piecedoc", "DEBUG") + _ = logging.SetLogLevel("piecedirectory", "DEBUG") } return nil diff --git a/piecedirectory/doctor.go b/piecedirectory/doctor.go index 118ef99df..8a640fca9 100644 --- a/piecedirectory/doctor.go +++ b/piecedirectory/doctor.go @@ -58,54 +58,61 @@ func (d *Doctor) Run(ctx context.Context) { case <-timer.C: } - sectors, err := d.fullnodeApi.StateMinerSectors(ctx, d.maddr, nil, lotuschaintypes.EmptyTSK) - if err != nil { - return - } + err := func() error { + sectors, err := d.fullnodeApi.StateMinerSectors(ctx, d.maddr, nil, lotuschaintypes.EmptyTSK) + if err != nil { + return err + } - d.allSectors = make(map[abi.SectorNumber]*miner.SectorOnChainInfo) - for _, info := range sectors { - d.allSectors[info.SectorNumber] = info - } + d.allSectors = make(map[abi.SectorNumber]*miner.SectorOnChainInfo) + for _, info := range sectors { + d.allSectors[info.SectorNumber] = info + } - head, err := d.fullnodeApi.ChainHead(ctx) - if err != nil { - return - } + head, err := d.fullnodeApi.ChainHead(ctx) + if err != nil { + return err + } - activeSet, err := d.fullnodeApi.StateMinerActiveSectors(ctx, d.maddr, head.Key()) - if err != nil { - return - } - d.activeSectors = make(map[abi.SectorNumber]struct{}, len(activeSet)) - for _, info := range activeSet { - d.activeSectors[info.SectorNumber] = struct{}{} - } + activeSet, err := d.fullnodeApi.StateMinerActiveSectors(ctx, d.maddr, head.Key()) + if err != nil { + return err + } + d.activeSectors = make(map[abi.SectorNumber]struct{}, len(activeSet)) + for _, info := range activeSet { + d.activeSectors[info.SectorNumber] = struct{}{} + } + + // Get the next pieces to check (eg pieces that haven't been checked + // for a while) from the local index directory + pcids, err := d.store.NextPiecesToCheck(ctx) + if err != nil { + return err + } - // Get the next pieces to check (eg pieces that haven't been checked - // for a while) from the local index directory - pcids, err := d.store.NextPiecesToCheck(ctx) + // Check each piece for problems + doclog.Debugw("piece doctor: checking pieces", "count", len(pcids), "all sectors", len(d.allSectors), "active sectors", len(d.activeSectors)) + for _, pcid := range pcids { + err := d.checkPiece(ctx, pcid) + if err != nil { + if errors.Is(err, context.Canceled) { + return err + } + doclog.Errorw("checking piece", "piece", pcid, "err", err) + } + } + doclog.Debugw("piece doctor: completed checking pieces", "count", len(pcids)) + + return nil + }() if err != nil { if errors.Is(err, context.Canceled) { + doclog.Errorw("piece doctor: context canceled, stopping doctor", "error", err) return } - doclog.Errorw("getting next pieces to check", "err", err) - time.Sleep(time.Minute) - continue - } - // Check each piece for problems - doclog.Debugw("piece doctor: checking pieces", "count", len(pcids), "all sectors", len(d.allSectors), "active sectors", len(d.activeSectors)) - for _, pcid := range pcids { - err := d.checkPiece(ctx, pcid) - if err != nil { - if errors.Is(err, context.Canceled) { - return - } - doclog.Errorw("checking piece", "piece", pcid, "err", err) - } + doclog.Errorw("piece doctor: iteration got error", "error", err) } - doclog.Debugw("piece doctor: completed checking pieces", "count", len(pcids)) // Sleep for a few seconds between ticks. // The time to sleep is randomized, so that if there are multiple doctor From a9a7b0758a32f41928ba47878e91516f8ef10f82 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 24 May 2023 14:21:01 +0200 Subject: [PATCH 04/41] adjust piece check --- extern/boostd-data/yugabyte/piecedoctor.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/extern/boostd-data/yugabyte/piecedoctor.go b/extern/boostd-data/yugabyte/piecedoctor.go index 93a6a9699..5e47272df 100644 --- a/extern/boostd-data/yugabyte/piecedoctor.go +++ b/extern/boostd-data/yugabyte/piecedoctor.go @@ -3,13 +3,14 @@ package yugabyte import ( "context" "fmt" + "time" + "github.com/filecoin-project/boostd-data/model" "github.com/filecoin-project/boostd-data/shared/tracing" "github.com/ipfs/go-cid" "github.com/jackc/pgtype" "go.opentelemetry.io/otel/attribute" "golang.org/x/sync/errgroup" - "time" ) var TrackerCheckBatchSize = 1024 @@ -165,7 +166,7 @@ func (s *Store) execWithConcurrency(ctx context.Context, pcids []pieceCreated, c } // The minimum frequency with which to check pieces for errors (eg bad index) -var MinPieceCheckPeriod = 30 * time.Second +var MinPieceCheckPeriod = 5 * time.Minute // Work out how frequently to check each piece, based on how many pieces // there are: if there are many pieces, each piece will be checked @@ -178,10 +179,10 @@ func (s *Store) getPieceCheckPeriod(ctx context.Context) (time.Duration, error) } // Check period: - // - 1k pieces; every 10s - // - 100k pieces; every 15m - // - 1m pieces; every 2 hours - period := time.Duration(count*10) * time.Millisecond + // - 1k pieces; every 100s (5 minutes because of MinPieceCheckPeriod) + // - 100k pieces; every 150m + // - 1m pieces; every 20 hours + period := time.Duration(count*100) * time.Millisecond if period < MinPieceCheckPeriod { period = MinPieceCheckPeriod } From 3918e94693bb5bebae1c35e490f4149d3261fefd Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 25 May 2023 13:23:45 +0200 Subject: [PATCH 05/41] refactor unsealsectormanager --- indexprovider/unsealedstatemanager_test.go | 522 ------------------ indexprovider/wrapper.go | 10 +- node/builder.go | 2 + .../sectorstatemgr.go | 109 ++-- 4 files changed, 82 insertions(+), 561 deletions(-) delete mode 100644 indexprovider/unsealedstatemanager_test.go rename indexprovider/unsealedstatemanager.go => sectorstatemgr/sectorstatemgr.go (71%) diff --git a/indexprovider/unsealedstatemanager_test.go b/indexprovider/unsealedstatemanager_test.go deleted file mode 100644 index 916f5a351..000000000 --- a/indexprovider/unsealedstatemanager_test.go +++ /dev/null @@ -1,522 +0,0 @@ -package indexprovider - -import ( - "context" - "testing" - - "github.com/filecoin-project/boost-gfm/storagemarket" - "github.com/filecoin-project/boost/db" - "github.com/filecoin-project/boost/db/migrations" - "github.com/filecoin-project/boost/indexprovider/mock" - "github.com/filecoin-project/boost/node/config" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/builtin/v9/market" - "github.com/filecoin-project/lotus/markets/idxprov" - "github.com/filecoin-project/lotus/storage/sealer/storiface" - "github.com/golang/mock/gomock" - "github.com/ipni/index-provider/metadata" - mock_provider "github.com/ipni/index-provider/mock" - "github.com/stretchr/testify/require" -) - -// Empty response from MinerAPI.StorageList() -func TestUnsealedStateManagerEmptyStorageList(t *testing.T) { - usm, legacyStorageProvider, _, _ := setup(t) - legacyStorageProvider.EXPECT().ListLocalDeals().AnyTimes().Return(nil, nil) - - // Check for updates with an empty response from MinerAPI.StorageList() - err := usm.checkForUpdates(context.Background()) - require.NoError(t, err) -} - -// Only announce sectors for deals that are in the boost database or -// legacy datastore -func TestUnsealedStateManagerMatchingDealOnly(t *testing.T) { - ctx := context.Background() - - runTest := func(t *testing.T, usm *UnsealedStateManager, storageMiner *mockApiStorageMiner, prov *mock_provider.MockInterface, provAddr address.Address, sectorNum abi.SectorNumber) { - // Set the response from MinerAPI.StorageList() to be two unsealed sectors - minerID, err := address.IDFromAddress(provAddr) - require.NoError(t, err) - storageMiner.storageList = map[storiface.ID][]storiface.Decl{ - // This sector matches the deal in the database - "uuid-existing-deal": {{ - SectorID: abi.SectorID{Miner: abi.ActorID(minerID), Number: sectorNum}, - SectorFileType: storiface.FTUnsealed, - }}, - // This sector should be ignored because the sector ID doesn't match - // any deal in the database - "uuid-unknown-deal": {{ - SectorID: abi.SectorID{Miner: abi.ActorID(minerID), Number: sectorNum + 1}, - SectorFileType: storiface.FTUnsealed, - }}, - } - - // Expect checkForUpdates to call NotifyPut exactly once, because only - // one sector from the storage list is in the database - prov.EXPECT().NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(1) - - err = usm.checkForUpdates(ctx) - require.NoError(t, err) - } - - t.Run("deal in boost db", func(t *testing.T) { - usm, legacyStorageProvider, storageMiner, prov := setup(t) - legacyStorageProvider.EXPECT().ListLocalDeals().Return(nil, nil) - - // Add a deal to the database - deals, err := db.GenerateNDeals(1) - require.NoError(t, err) - err = usm.dealsDB.Insert(ctx, &deals[0]) - require.NoError(t, err) - - provAddr := deals[0].ClientDealProposal.Proposal.Provider - sectorNum := deals[0].SectorID - runTest(t, usm, storageMiner, prov, provAddr, sectorNum) - }) - - t.Run("deal in legacy datastore", func(t *testing.T) { - usm, legacyStorageProvider, storageMiner, prov := setup(t) - - // Simulate returning a deal from the legacy datastore - boostDeals, err := db.GenerateNDeals(1) - require.NoError(t, err) - - sectorNum := abi.SectorNumber(10) - deals := []storagemarket.MinerDeal{{ - ClientDealProposal: boostDeals[0].ClientDealProposal, - SectorNumber: sectorNum, - }} - legacyStorageProvider.EXPECT().ListLocalDeals().Return(deals, nil) - - provAddr := deals[0].ClientDealProposal.Proposal.Provider - runTest(t, usm, storageMiner, prov, provAddr, sectorNum) - }) -} - -// Tests that various scenarios of sealing state changes produce the expected -// calls to NotifyPut / NotifyRemove -func TestUnsealedStateManagerStateChange(t *testing.T) { - ctx := context.Background() - - testCases := []struct { - name string - storageListResponse1 func(sectorID abi.SectorID) *storiface.Decl - storageListResponse2 func(sectorID abi.SectorID) *storiface.Decl - expect func(*mock_provider.MockInterfaceMockRecorder, market.DealProposal) - }{{ - name: "unsealed -> sealed", - storageListResponse1: func(sectorID abi.SectorID) *storiface.Decl { - return &storiface.Decl{ - SectorID: sectorID, - SectorFileType: storiface.FTUnsealed, - } - }, - storageListResponse2: func(sectorID abi.SectorID) *storiface.Decl { - return &storiface.Decl{ - SectorID: sectorID, - SectorFileType: storiface.FTSealed, - } - }, - expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { - // Expect a call to NotifyPut with fast retrieval = true (unsealed) - prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ - PieceCID: prop.PieceCID, - VerifiedDeal: prop.VerifiedDeal, - FastRetrieval: true, - })).Times(1) - - // Expect a call to NotifyPut with fast retrieval = false (sealed) - prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ - PieceCID: prop.PieceCID, - VerifiedDeal: prop.VerifiedDeal, - FastRetrieval: false, - })).Times(1) - }, - }, { - name: "sealed -> unsealed", - storageListResponse1: func(sectorID abi.SectorID) *storiface.Decl { - return &storiface.Decl{ - SectorID: sectorID, - SectorFileType: storiface.FTSealed, - } - }, - storageListResponse2: func(sectorID abi.SectorID) *storiface.Decl { - return &storiface.Decl{ - SectorID: sectorID, - SectorFileType: storiface.FTUnsealed, - } - }, - expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { - // Expect a call to NotifyPut with fast retrieval = false (sealed) - prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ - PieceCID: prop.PieceCID, - VerifiedDeal: prop.VerifiedDeal, - FastRetrieval: false, - })).Times(1) - - // Expect a call to NotifyPut with fast retrieval = true (unsealed) - prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ - PieceCID: prop.PieceCID, - VerifiedDeal: prop.VerifiedDeal, - FastRetrieval: true, - })).Times(1) - }, - }, { - name: "unsealed -> unsealed (no change)", - storageListResponse1: func(sectorID abi.SectorID) *storiface.Decl { - return &storiface.Decl{ - SectorID: sectorID, - SectorFileType: storiface.FTUnsealed, - } - }, - storageListResponse2: func(sectorID abi.SectorID) *storiface.Decl { - return &storiface.Decl{ - SectorID: sectorID, - SectorFileType: storiface.FTUnsealed, - } - }, - expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { - // Expect only one call to NotifyPut with fast retrieval = true (unsealed) - // because the state of the sector doesn't change on the second call - prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ - PieceCID: prop.PieceCID, - VerifiedDeal: prop.VerifiedDeal, - FastRetrieval: true, - })).Times(1) - }, - }, { - name: "unsealed -> removed", - storageListResponse1: func(sectorID abi.SectorID) *storiface.Decl { - return &storiface.Decl{ - SectorID: sectorID, - SectorFileType: storiface.FTUnsealed, - } - }, - storageListResponse2: func(sectorID abi.SectorID) *storiface.Decl { - return nil - }, - expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { - // Expect a call to NotifyPut with fast retrieval = true (unsealed) - prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ - PieceCID: prop.PieceCID, - VerifiedDeal: prop.VerifiedDeal, - FastRetrieval: true, - })).Times(1) - - // Expect a call to NotifyRemove because the sector is no longer in the list response - prov.NotifyRemove(gomock.Any(), gomock.Any(), gomock.Any()).Times(1) - }, - }, { - name: "sealed -> removed", - storageListResponse1: func(sectorID abi.SectorID) *storiface.Decl { - return &storiface.Decl{ - SectorID: sectorID, - SectorFileType: storiface.FTSealed, - } - }, - storageListResponse2: func(sectorID abi.SectorID) *storiface.Decl { - return nil - }, - expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { - // Expect a call to NotifyPut with fast retrieval = false (sealed) - prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ - PieceCID: prop.PieceCID, - VerifiedDeal: prop.VerifiedDeal, - FastRetrieval: false, - })).Times(1) - - // Expect a call to NotifyRemove because the sector is no longer in the list response - prov.NotifyRemove(gomock.Any(), gomock.Any(), gomock.Any()).Times(1) - }, - }, { - name: "removed -> unsealed", - storageListResponse1: func(sectorID abi.SectorID) *storiface.Decl { - return nil - }, - storageListResponse2: func(sectorID abi.SectorID) *storiface.Decl { - return &storiface.Decl{ - SectorID: sectorID, - SectorFileType: storiface.FTUnsealed, - } - }, - expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { - // Expect a call to NotifyPut with fast retrieval = true (unsealed) - prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ - PieceCID: prop.PieceCID, - VerifiedDeal: prop.VerifiedDeal, - FastRetrieval: true, - })).Times(1) - }, - }, { - name: "unsealed -> cache", - storageListResponse1: func(sectorID abi.SectorID) *storiface.Decl { - return &storiface.Decl{ - SectorID: sectorID, - SectorFileType: storiface.FTUnsealed, - } - }, - storageListResponse2: func(sectorID abi.SectorID) *storiface.Decl { - return &storiface.Decl{ - SectorID: sectorID, - SectorFileType: storiface.FTCache, - } - }, - expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { - // Expect only one call to NotifyPut with fast retrieval = true (unsealed) - // because we ignore a state change to cache - prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ - PieceCID: prop.PieceCID, - VerifiedDeal: prop.VerifiedDeal, - FastRetrieval: true, - })).Times(1) - }, - }, { - name: "cache -> unsealed", - storageListResponse1: func(sectorID abi.SectorID) *storiface.Decl { - return &storiface.Decl{ - SectorID: sectorID, - SectorFileType: storiface.FTCache, - } - }, - storageListResponse2: func(sectorID abi.SectorID) *storiface.Decl { - return &storiface.Decl{ - SectorID: sectorID, - SectorFileType: storiface.FTUnsealed, - } - }, - expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { - // Expect only one call to NotifyPut with fast retrieval = true (unsealed) - // because we ignore a state change to cache - prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ - PieceCID: prop.PieceCID, - VerifiedDeal: prop.VerifiedDeal, - FastRetrieval: true, - })).Times(1) - }, - }, { - name: "cache -> sealed", - storageListResponse1: func(sectorID abi.SectorID) *storiface.Decl { - return &storiface.Decl{ - SectorID: sectorID, - SectorFileType: storiface.FTCache, - } - }, - storageListResponse2: func(sectorID abi.SectorID) *storiface.Decl { - return &storiface.Decl{ - SectorID: sectorID, - SectorFileType: storiface.FTSealed, - } - }, - expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { - // Expect only one call to NotifyPut with fast retrieval = true (unsealed) - // because we ignore a state change to cache - prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ - PieceCID: prop.PieceCID, - VerifiedDeal: prop.VerifiedDeal, - FastRetrieval: false, - })).Times(1) - }, - }} - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - usm, legacyStorageProvider, storageMiner, prov := setup(t) - legacyStorageProvider.EXPECT().ListLocalDeals().AnyTimes().Return(nil, nil) - - // Add a deal to the database - deals, err := db.GenerateNDeals(1) - require.NoError(t, err) - err = usm.dealsDB.Insert(ctx, &deals[0]) - require.NoError(t, err) - - // Set up expectations (automatically verified when the test exits) - prop := deals[0].ClientDealProposal.Proposal - tc.expect(prov.EXPECT(), prop) - - minerID, err := address.IDFromAddress(deals[0].ClientDealProposal.Proposal.Provider) - require.NoError(t, err) - - // Set the first response from MinerAPI.StorageList() - resp1 := tc.storageListResponse1(abi.SectorID{Miner: abi.ActorID(minerID), Number: deals[0].SectorID}) - storageMiner.storageList = map[storiface.ID][]storiface.Decl{} - if resp1 != nil { - storageMiner.storageList["uuid"] = []storiface.Decl{*resp1} - } - - // Trigger check for updates - err = usm.checkForUpdates(ctx) - require.NoError(t, err) - - // Set the second response from MinerAPI.StorageList() - resp2 := tc.storageListResponse2(abi.SectorID{Miner: abi.ActorID(minerID), Number: deals[0].SectorID}) - storageMiner.storageList = map[storiface.ID][]storiface.Decl{} - if resp2 != nil { - storageMiner.storageList["uuid"] = []storiface.Decl{*resp2} - } - - // Trigger check for updates again - err = usm.checkForUpdates(ctx) - require.NoError(t, err) - }) - } -} - -// Verify that multiple storage file types are handled from StorageList correctly -func TestUnsealedStateManagerStorageList(t *testing.T) { - ctx := context.Background() - - testCases := []struct { - name string - storageListResponse func(sectorID abi.SectorID) []storiface.Decl - expect func(*mock_provider.MockInterfaceMockRecorder, market.DealProposal) - }{{ - name: "unsealed and sealed status", - storageListResponse: func(sectorID abi.SectorID) []storiface.Decl { - return []storiface.Decl{ - { - SectorID: sectorID, - SectorFileType: storiface.FTUnsealed, - }, - { - SectorID: sectorID, - SectorFileType: storiface.FTSealed, - }, - } - }, - expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { - // Expect only one call to NotifyPut with fast retrieval = true (unsealed) - // because we ignore a state change to cache - prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ - PieceCID: prop.PieceCID, - VerifiedDeal: prop.VerifiedDeal, - FastRetrieval: true, - })).Times(1) - }, - }, { - name: "unsealed and cached status", - storageListResponse: func(sectorID abi.SectorID) []storiface.Decl { - return []storiface.Decl{ - { - SectorID: sectorID, - SectorFileType: storiface.FTUnsealed, - }, - { - SectorID: sectorID, - SectorFileType: storiface.FTCache, - }, - } - }, - expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { - // Expect only one call to NotifyPut with fast retrieval = true (unsealed) - // because we ignore a state change to cache - prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ - PieceCID: prop.PieceCID, - VerifiedDeal: prop.VerifiedDeal, - FastRetrieval: true, - })).Times(1) - }, - }, { - name: "sealed and cached status", - storageListResponse: func(sectorID abi.SectorID) []storiface.Decl { - return []storiface.Decl{ - { - SectorID: sectorID, - SectorFileType: storiface.FTSealed, - }, - { - SectorID: sectorID, - SectorFileType: storiface.FTCache, - }, - } - }, - expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { - // Expect only one call to NotifyPut with fast retrieval = true (unsealed) - // because we ignore a state change to cache - prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ - PieceCID: prop.PieceCID, - VerifiedDeal: prop.VerifiedDeal, - FastRetrieval: false, - })).Times(1) - }, - }} - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - usm, legacyStorageProvider, storageMiner, prov := setup(t) - legacyStorageProvider.EXPECT().ListLocalDeals().AnyTimes().Return(nil, nil) - - // Add a deal to the database - deals, err := db.GenerateNDeals(1) - require.NoError(t, err) - err = usm.dealsDB.Insert(ctx, &deals[0]) - require.NoError(t, err) - - // Set up expectations (automatically verified when the test exits) - prop := deals[0].ClientDealProposal.Proposal - tc.expect(prov.EXPECT(), prop) - - minerID, err := address.IDFromAddress(deals[0].ClientDealProposal.Proposal.Provider) - require.NoError(t, err) - - // Set the first response from MinerAPI.StorageList() - storageMiner.storageList = map[storiface.ID][]storiface.Decl{} - resp1 := tc.storageListResponse(abi.SectorID{Miner: abi.ActorID(minerID), Number: deals[0].SectorID}) - storageMiner.storageList["uuid"] = resp1 - - // Trigger check for updates - err = usm.checkForUpdates(ctx) - require.NoError(t, err) - }) - } -} - -func setup(t *testing.T) (*UnsealedStateManager, *mock.MockStorageProvider, *mockApiStorageMiner, *mock_provider.MockInterface) { - ctx := context.Background() - ctrl := gomock.NewController(t) - prov := mock_provider.NewMockInterface(ctrl) - - sqldb := db.CreateTestTmpDB(t) - require.NoError(t, db.CreateAllBoostTables(ctx, sqldb, sqldb)) - require.NoError(t, migrations.Migrate(sqldb)) - - dealsDB := db.NewDealsDB(sqldb) - sectorStateDB := db.NewSectorStateDB(sqldb) - storageMiner := &mockApiStorageMiner{} - storageProvider := mock.NewMockStorageProvider(ctrl) - - wrapper := &Wrapper{ - enabled: true, - dealsDB: dealsDB, - prov: prov, - meshCreator: &meshCreatorStub{}, - } - - cfg := config.StorageConfig{} - usm := NewUnsealedStateManager(wrapper, storageProvider, dealsDB, sectorStateDB, storageMiner, cfg) - return usm, storageProvider, storageMiner, prov -} - -type mockApiStorageMiner struct { - storageList map[storiface.ID][]storiface.Decl -} - -var _ ApiStorageMiner = (*mockApiStorageMiner)(nil) - -func (m mockApiStorageMiner) StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error) { - return m.storageList, nil -} - -func (m mockApiStorageMiner) StorageRedeclareLocal(ctx context.Context, id *storiface.ID, dropMissing bool) error { - return nil -} - -type meshCreatorStub struct { -} - -var _ idxprov.MeshCreator = (*meshCreatorStub)(nil) - -func (m *meshCreatorStub) Connect(context.Context) error { - return nil -} diff --git a/indexprovider/wrapper.go b/indexprovider/wrapper.go index a9d73aa98..706bc7c69 100644 --- a/indexprovider/wrapper.go +++ b/indexprovider/wrapper.go @@ -41,7 +41,6 @@ type Wrapper struct { piecedirectory *piecedirectory.PieceDirectory meshCreator idxprov.MeshCreator h host.Host - usm *UnsealedStateManager // bitswapEnabled records whether to announce bitswap as an available // protocol to the network indexer bitswapEnabled bool @@ -78,7 +77,6 @@ func NewWrapper(cfg *config.Boost) func(lc fx.Lifecycle, h host.Host, r repo.Loc piecedirectory: piecedirectory, bitswapEnabled: bitswapEnabled, } - w.usm = NewUnsealedStateManager(w, legacyProv, dealsDB, ssDB, storageService, w.cfg.Storage) return w, nil } } @@ -89,10 +87,6 @@ func (w *Wrapper) Start(ctx context.Context) { runCtx, runCancel := context.WithCancel(context.Background()) w.stop = runCancel - // Watch for changes in sector unseal state and update the - // indexer when there are changes - go w.usm.Run(runCtx) - // Announce all deals on startup in case of a config change go func() { err := w.AnnounceExtendedProviders(runCtx) @@ -342,10 +336,10 @@ func (w *Wrapper) AnnounceBoostDeal(ctx context.Context, deal *types.ProviderDea FastRetrieval: deal.FastRetrieval, VerifiedDeal: deal.ClientDealProposal.Proposal.VerifiedDeal, } - return w.announceBoostDealMetadata(ctx, md, propCid) + return w.AnnounceBoostDealMetadata(ctx, md, propCid) } -func (w *Wrapper) announceBoostDealMetadata(ctx context.Context, md metadata.GraphsyncFilecoinV1, propCid cid.Cid) (cid.Cid, error) { +func (w *Wrapper) AnnounceBoostDealMetadata(ctx context.Context, md metadata.GraphsyncFilecoinV1, propCid cid.Cid) (cid.Cid, error) { if !w.enabled { return cid.Undef, errors.New("cannot announce deal: index provider is disabled") } diff --git a/node/builder.go b/node/builder.go index b14c331ee..7993dceb9 100644 --- a/node/builder.go +++ b/node/builder.go @@ -34,6 +34,7 @@ import ( "github.com/filecoin-project/boost/retrievalmarket/lp2pimpl" "github.com/filecoin-project/boost/retrievalmarket/rtvllog" "github.com/filecoin-project/boost/retrievalmarket/server" + "github.com/filecoin-project/boost/sectorstatemgr" "github.com/filecoin-project/boost/storagemanager" "github.com/filecoin-project/boost/storagemarket" "github.com/filecoin-project/boost/storagemarket/dealfilter" @@ -508,6 +509,7 @@ func ConfigBoost(cfg *config.Boost) Option { // Sealing Pipeline State API Override(new(sealingpipeline.API), From(new(lotus_modules.MinerStorageService))), + Override(new(*sectorstatemgr.SectorStateMgr), sectorstatemgr.NewSectorStateMgr), Override(new(*indexprovider.Wrapper), indexprovider.NewWrapper(cfg)), Override(new(*storagemarket.ChainDealManager), modules.NewChainDealManager), diff --git a/indexprovider/unsealedstatemanager.go b/sectorstatemgr/sectorstatemgr.go similarity index 71% rename from indexprovider/unsealedstatemanager.go rename to sectorstatemgr/sectorstatemgr.go index 93e171944..9421687a2 100644 --- a/indexprovider/unsealedstatemanager.go +++ b/sectorstatemgr/sectorstatemgr.go @@ -1,62 +1,90 @@ -package indexprovider +package sectorstatemgr import ( "context" "errors" "fmt" + "sync" "time" "github.com/filecoin-project/boost-gfm/storagemarket" "github.com/filecoin-project/boost/db" + "github.com/filecoin-project/boost/indexprovider" "github.com/filecoin-project/boost/node/config" "github.com/filecoin-project/go-address" cborutil "github.com/filecoin-project/go-cbor-util" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/storage/sealer/storiface" logging "github.com/ipfs/go-log/v2" provider "github.com/ipni/index-provider" "github.com/ipni/index-provider/metadata" + "go.uber.org/fx" ) -//go:generate go run github.com/golang/mock/mockgen -destination=./mock/mock.go -package=mock github.com/filecoin-project/boost-gfm/storagemarket StorageProvider - -var usmlog = logging.Logger("unsmgr") +var log = logging.Logger("sectorstatemgr") type ApiStorageMiner interface { StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error) StorageRedeclareLocal(ctx context.Context, id *storiface.ID, dropMissing bool) error } -type UnsealedStateManager struct { - idxprov *Wrapper +type SectorStateMgr struct { + sync.Mutex + + cfg config.StorageConfig + fullnodeApi api.FullNode + minerApi ApiStorageMiner + Maddr address.Address + StateUpdates map[abi.SectorID]db.SealState + ActiveSectors map[abi.SectorNumber]struct{} + + idxprov *indexprovider.Wrapper legacyProv storagemarket.StorageProvider dealsDB *db.DealsDB sdb *db.SectorStateDB - api ApiStorageMiner - cfg config.StorageConfig } -func NewUnsealedStateManager(idxprov *Wrapper, legacyProv storagemarket.StorageProvider, dealsDB *db.DealsDB, sdb *db.SectorStateDB, api ApiStorageMiner, cfg config.StorageConfig) *UnsealedStateManager { - return &UnsealedStateManager{ +func NewSectorStateMgr(lc fx.Lifecycle, idxprov *indexprovider.Wrapper, legacyProv storagemarket.StorageProvider, dealsDB *db.DealsDB, sdb *db.SectorStateDB, minerApi ApiStorageMiner, fullnodeApi api.FullNode, maddr address.Address, cfg config.StorageConfig) *SectorStateMgr { + mgr := &SectorStateMgr{ + cfg: cfg, + minerApi: minerApi, + fullnodeApi: fullnodeApi, + Maddr: maddr, + StateUpdates: make(map[abi.SectorID]db.SealState), + ActiveSectors: make(map[abi.SectorNumber]struct{}), + idxprov: idxprov, legacyProv: legacyProv, dealsDB: dealsDB, sdb: sdb, - api: api, - cfg: cfg, } + + cctx, cancel := context.WithCancel(context.Background()) + lc.Append(fx.Hook{ + OnStart: func(ctx context.Context) error { + go mgr.Run(cctx) + return nil + }, + OnStop: func(ctx context.Context) error { + cancel() + return nil + }, + }) + + return mgr } -func (m *UnsealedStateManager) Run(ctx context.Context) { +func (m *SectorStateMgr) Run(ctx context.Context) { duration := time.Duration(m.cfg.StorageListRefreshDuration) - usmlog.Infof("starting unsealed state manager running on interval %s", duration.String()) + log.Infof("starting unsealed state manager running on interval %s", duration.String()) ticker := time.NewTicker(duration) defer ticker.Stop() // Check immediately err := m.checkForUpdates(ctx) if err != nil { - usmlog.Errorf("error checking for unsealed state updates: %s", err) + log.Errorf("error checking for unsealed state updates: %s", err) } // Check every tick @@ -67,19 +95,19 @@ func (m *UnsealedStateManager) Run(ctx context.Context) { case <-ticker.C: err := m.checkForUpdates(ctx) if err != nil { - usmlog.Errorf("error checking for unsealed state updates: %s", err) + log.Errorf("error checking for unsealed state updates: %s", err) } } } } -func (m *UnsealedStateManager) checkForUpdates(ctx context.Context) error { - usmlog.Info("checking for sector state updates") +func (m *SectorStateMgr) checkForUpdates(ctx context.Context) error { + log.Info("checking for sector state updates") // Tell lotus to update it's storage list and remove any removed sectors if m.cfg.RedeclareOnStorageListRefresh { - usmlog.Info("redeclaring storage") - err := m.api.StorageRedeclareLocal(ctx, nil, true) + log.Info("redeclaring storage") + err := m.minerApi.StorageRedeclareLocal(ctx, nil, true) if err != nil { log.Errorf("redeclaring local storage on lotus miner: %w", err) } @@ -90,12 +118,31 @@ func (m *UnsealedStateManager) checkForUpdates(ctx context.Context) error { return err } + head, err := m.fullnodeApi.ChainHead(ctx) + if err != nil { + return err + } + + activeSet, err := m.fullnodeApi.StateMinerActiveSectors(ctx, m.Maddr, head.Key()) + if err != nil { + return err + } + activeSectors := make(map[abi.SectorNumber]struct{}, len(activeSet)) + for _, info := range activeSet { + activeSectors[info.SectorNumber] = struct{}{} + } + + m.Lock() + m.StateUpdates = stateUpdates + m.ActiveSectors = activeSectors + m.Unlock() + legacyDeals, err := m.legacyDealsBySectorID(stateUpdates) if err != nil { return fmt.Errorf("getting legacy deals from datastore: %w", err) } - usmlog.Debugf("checking for sector state updates for %d states", len(stateUpdates)) + log.Debugf("checking for sector state updates for %d states", len(stateUpdates)) // For each sector for sectorID, sectorSealState := range stateUpdates { @@ -104,7 +151,7 @@ func (m *UnsealedStateManager) checkForUpdates(ctx context.Context) error { if err != nil { return fmt.Errorf("getting deals for miner %d / sector %d: %w", sectorID.Miner, sectorID.Number, err) } - usmlog.Debugf("sector %d has %d deals, seal status %s", sectorID, len(deals), sectorSealState) + log.Debugf("sector %d has %d deals, seal status %s", sectorID, len(deals), sectorSealState) // For each deal in the sector for _, deal := range deals { @@ -125,12 +172,12 @@ func (m *UnsealedStateManager) checkForUpdates(ctx context.Context) error { // Check if the error is because the deal wasn't previously announced if !errors.Is(err, provider.ErrContextIDNotFound) { // There was some other error, write it to the log - usmlog.Errorw("announcing deal removed to index provider", + log.Errorw("announcing deal removed to index provider", "deal id", deal.DealID, "error", err) continue } } else { - usmlog.Infow("announced to index provider that deal has been removed", + log.Infow("announced to index provider that deal has been removed", "deal id", deal.DealID, "sector id", deal.SectorID.Number, "announce cid", announceCid.String()) } } else if sectorSealState != db.SealStateCache { @@ -140,13 +187,13 @@ func (m *UnsealedStateManager) checkForUpdates(ctx context.Context) error { FastRetrieval: sectorSealState == db.SealStateUnsealed, VerifiedDeal: deal.DealProposal.Proposal.VerifiedDeal, } - announceCid, err := m.idxprov.announceBoostDealMetadata(ctx, md, propCid) + announceCid, err := m.idxprov.AnnounceBoostDealMetadata(ctx, md, propCid) if err == nil { - usmlog.Infow("announced deal seal state to index provider", + log.Infow("announced deal seal state to index provider", "deal id", deal.DealID, "sector id", deal.SectorID.Number, "seal state", sectorSealState, "announce cid", announceCid.String()) } else { - usmlog.Errorf("announcing deal %s to index provider: %w", deal.DealID, err) + log.Errorf("announcing deal %s to index provider: %w", deal.DealID, err) } } } @@ -161,9 +208,9 @@ func (m *UnsealedStateManager) checkForUpdates(ctx context.Context) error { return nil } -func (m *UnsealedStateManager) getStateUpdates(ctx context.Context) (map[abi.SectorID]db.SealState, error) { +func (m *SectorStateMgr) getStateUpdates(ctx context.Context) (map[abi.SectorID]db.SealState, error) { // Get the current unsealed state of all sectors from lotus - storageList, err := m.api.StorageList(ctx) + storageList, err := m.minerApi.StorageList(ctx) if err != nil { return nil, fmt.Errorf("getting sectors state from lotus: %w", err) } @@ -230,7 +277,7 @@ type basicDealInfo struct { } // Get deals by sector ID, whether they're legacy or boost deals -func (m *UnsealedStateManager) dealsBySectorID(ctx context.Context, legacyDeals map[abi.SectorID][]storagemarket.MinerDeal, sectorID abi.SectorID) ([]basicDealInfo, error) { +func (m *SectorStateMgr) dealsBySectorID(ctx context.Context, legacyDeals map[abi.SectorID][]storagemarket.MinerDeal, sectorID abi.SectorID) ([]basicDealInfo, error) { // First query the boost database deals, err := m.dealsDB.BySectorID(ctx, sectorID) if err != nil { @@ -266,7 +313,7 @@ func (m *UnsealedStateManager) dealsBySectorID(ctx context.Context, legacyDeals // Iterate over all legacy deals and make a map of sector ID -> legacy deal. // To save memory, only include legacy deals with a sector ID that we know // we're going to query, ie the set of sector IDs in the stateUpdates map. -func (m *UnsealedStateManager) legacyDealsBySectorID(stateUpdates map[abi.SectorID]db.SealState) (map[abi.SectorID][]storagemarket.MinerDeal, error) { +func (m *SectorStateMgr) legacyDealsBySectorID(stateUpdates map[abi.SectorID]db.SealState) (map[abi.SectorID][]storagemarket.MinerDeal, error) { legacyDeals, err := m.legacyProv.ListLocalDeals() if err != nil { return nil, err From dfbecb93bf31d15a874e93e436e1234bfe97e61d Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 25 May 2023 13:52:22 +0200 Subject: [PATCH 06/41] refactor piece doctor --- cmd/boostd/recover.go | 9 +- cmd/booster-bitswap/run.go | 13 +- cmd/booster-http/run.go | 15 +- extern/boostd-data/clientutil/clientutil.go | 28 +++ extern/boostd-data/cmd/run.go | 2 +- extern/boostd-data/svc/svc.go | 8 +- extern/boostd-data/svc/svc_size_test.go | 36 ++- extern/boostd-data/svc/svc_test.go | 17 -- indexprovider/wrapper.go | 178 +++++++++++++- node/builder.go | 6 +- node/modules/piecedirectory.go | 22 +- piecedirectory/doctor.go | 72 ++---- piecedirectory/piecedirectory.go | 11 +- piecedirectory/test_util.go | 14 -- piecedirectory/types/types.go | 25 -- sectorstatemgr/sectorstatemgr.go | 245 +++++--------------- storagemarket/provider_test.go | 8 +- 17 files changed, 353 insertions(+), 356 deletions(-) create mode 100644 extern/boostd-data/clientutil/clientutil.go diff --git a/cmd/boostd/recover.go b/cmd/boostd/recover.go index 001225ba6..210650927 100644 --- a/cmd/boostd/recover.go +++ b/cmd/boostd/recover.go @@ -17,6 +17,7 @@ import ( "github.com/filecoin-project/boost/cmd/lib" "github.com/filecoin-project/boost/db" "github.com/filecoin-project/boost/piecedirectory" + bdclient "github.com/filecoin-project/boostd-data/client" "github.com/filecoin-project/boostd-data/model" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-commp-utils/writer" @@ -190,14 +191,14 @@ func action(cctx *cli.Context) error { if ignoreLID { pd = nil } else { - pdClient := piecedirectory.NewStore() - defer pdClient.Close(ctx) - err = pdClient.Dial(ctx, cctx.String("api-lid")) + cl := bdclient.NewStore() + defer cl.Close(ctx) + err = cl.Dial(ctx, cctx.String("api-lid")) if err != nil { return fmt.Errorf("connecting to local index directory service: %w", err) } pr := &piecedirectory.SectorAccessorAsPieceReader{SectorAccessor: sa} - pd = piecedirectory.NewPieceDirectory(pdClient, pr, cctx.Int("add-index-throttle")) + pd = piecedirectory.NewPieceDirectory(cl, pr, cctx.Int("add-index-throttle")) pd.Start(ctx) } diff --git a/cmd/booster-bitswap/run.go b/cmd/booster-bitswap/run.go index db2444490..9deddf95a 100644 --- a/cmd/booster-bitswap/run.go +++ b/cmd/booster-bitswap/run.go @@ -10,6 +10,7 @@ import ( "github.com/filecoin-project/boost/cmd/lib/remoteblockstore" "github.com/filecoin-project/boost/metrics" "github.com/filecoin-project/boost/piecedirectory" + bdclient "github.com/filecoin-project/boostd-data/client" "github.com/filecoin-project/boostd-data/shared/tracing" lcli "github.com/filecoin-project/lotus/cli" "github.com/libp2p/go-libp2p/core/peer" @@ -157,9 +158,9 @@ var runCmd = &cli.Command{ defer storageCloser() // Connect to the local index directory service - pdClient := piecedirectory.NewStore() - defer pdClient.Close(ctx) - err = pdClient.Dial(ctx, cctx.String("api-lid")) + cl := bdclient.NewStore() + defer cl.Close(ctx) + err = cl.Dial(ctx, cctx.String("api-lid")) if err != nil { return fmt.Errorf("connecting to local index directory service: %w", err) } @@ -196,8 +197,8 @@ var runCmd = &cli.Command{ return fmt.Errorf("starting block filter: %w", err) } pr := &piecedirectory.SectorAccessorAsPieceReader{SectorAccessor: sa} - piecedirectory := piecedirectory.NewPieceDirectory(pdClient, pr, cctx.Int("add-index-throttle")) - remoteStore := remoteblockstore.NewRemoteBlockstore(piecedirectory, &bitswapBlockMetrics) + pd := piecedirectory.NewPieceDirectory(cl, pr, cctx.Int("add-index-throttle")) + remoteStore := remoteblockstore.NewRemoteBlockstore(pd, &bitswapBlockMetrics) server := NewBitswapServer(remoteStore, host, multiFilter) var proxyAddrInfo *peer.AddrInfo @@ -210,7 +211,7 @@ var runCmd = &cli.Command{ } // Start the local index directory - piecedirectory.Start(ctx) + pd.Start(ctx) // Start the bitswap server log.Infof("Starting booster-bitswap node on port %d", port) diff --git a/cmd/booster-http/run.go b/cmd/booster-http/run.go index 8aef5e4dc..255e123e4 100644 --- a/cmd/booster-http/run.go +++ b/cmd/booster-http/run.go @@ -14,6 +14,7 @@ import ( "github.com/filecoin-project/boost/cmd/lib/remoteblockstore" "github.com/filecoin-project/boost/metrics" "github.com/filecoin-project/boost/piecedirectory" + bdclient "github.com/filecoin-project/boostd-data/client" "github.com/filecoin-project/boostd-data/model" "github.com/filecoin-project/boostd-data/shared/tracing" "github.com/filecoin-project/dagstore/mount" @@ -127,9 +128,9 @@ var runCmd = &cli.Command{ // Connect to the local index directory service ctx := lcli.ReqContext(cctx) - pdClient := piecedirectory.NewStore() - defer pdClient.Close(ctx) - err := pdClient.Dial(ctx, cctx.String("api-lid")) + cl := bdclient.NewStore() + defer cl.Close(ctx) + err := cl.Dial(ctx, cctx.String("api-lid")) if err != nil { return fmt.Errorf("connecting to local index directory service: %w", err) } @@ -168,7 +169,7 @@ var runCmd = &cli.Command{ // Create the server API pr := &piecedirectory.SectorAccessorAsPieceReader{SectorAccessor: sa} - piecedirectory := piecedirectory.NewPieceDirectory(pdClient, pr, cctx.Int("add-index-throttle")) + pd := piecedirectory.NewPieceDirectory(cl, pr, cctx.Int("add-index-throttle")) opts := &HttpServerOptions{ ServePieces: servePieces, @@ -199,11 +200,11 @@ var runCmd = &cli.Command{ GetSizeFailResponseCount: metrics.HttpRblsGetSizeFailResponseCount, GetSizeSuccessResponseCount: metrics.HttpRblsGetSizeSuccessResponseCount, } - rbs := remoteblockstore.NewRemoteBlockstore(piecedirectory, &httpBlockMetrics) + rbs := remoteblockstore.NewRemoteBlockstore(pd, &httpBlockMetrics) filtered := filters.NewFilteredBlockstore(rbs, multiFilter) opts.Blockstore = filtered } - sapi := serverApi{ctx: ctx, piecedirectory: piecedirectory, sa: sa} + sapi := serverApi{ctx: ctx, piecedirectory: pd, sa: sa} server := NewHttpServer( cctx.String("base-path"), cctx.Int("port"), @@ -212,7 +213,7 @@ var runCmd = &cli.Command{ ) // Start the local index directory - piecedirectory.Start(ctx) + pd.Start(ctx) // Start the server log.Infof("Starting booster-http node on port %d with base path '%s'", diff --git a/extern/boostd-data/clientutil/clientutil.go b/extern/boostd-data/clientutil/clientutil.go new file mode 100644 index 000000000..a9598848f --- /dev/null +++ b/extern/boostd-data/clientutil/clientutil.go @@ -0,0 +1,28 @@ +package clientutil + +import ( + "context" + "fmt" + + "github.com/filecoin-project/boostd-data/client" + "github.com/filecoin-project/boostd-data/svc" +) + +func NewTestStore(ctx context.Context) *client.Store { + bdsvc, err := svc.NewLevelDB("") + if err != nil { + panic(err) + } + ln, err := bdsvc.Start(ctx, "localhost:0") + if err != nil { + panic(err) + } + + cl := client.NewStore() + err = cl.Dial(ctx, fmt.Sprintf("ws://%s", ln.String())) + if err != nil { + panic(err) + } + + return cl +} diff --git a/extern/boostd-data/cmd/run.go b/extern/boostd-data/cmd/run.go index a41bb35cd..41a8bff7d 100644 --- a/extern/boostd-data/cmd/run.go +++ b/extern/boostd-data/cmd/run.go @@ -163,7 +163,7 @@ func runAction(cctx *cli.Context, dbType string, store *svc.Service) error { // Start the server addr := cctx.String("addr") - err = store.Start(ctx, addr) + _, err = store.Start(ctx, addr) if err != nil { return fmt.Errorf("starting %s store: %w", dbType, err) } diff --git a/extern/boostd-data/svc/svc.go b/extern/boostd-data/svc/svc.go index bf20b009e..10928ca9c 100644 --- a/extern/boostd-data/svc/svc.go +++ b/extern/boostd-data/svc/svc.go @@ -54,15 +54,15 @@ func MakeLevelDBDir(repoPath string) (string, error) { return repoPath, nil } -func (s *Service) Start(ctx context.Context, addr string) error { +func (s *Service) Start(ctx context.Context, addr string) (net.Addr, error) { ln, err := net.Listen("tcp", addr) if err != nil { - return fmt.Errorf("setting up listener for local index directory service: %w", err) + return nil, fmt.Errorf("setting up listener for local index directory service: %w", err) } err = s.Impl.Start(ctx) if err != nil { - return fmt.Errorf("starting local index directory service: %w", err) + return nil, fmt.Errorf("starting local index directory service: %w", err) } server := jsonrpc.NewServer() @@ -97,5 +97,5 @@ func (s *Service) Start(ctx context.Context, addr string) error { <-done }() - return nil + return ln.Addr(), nil } diff --git a/extern/boostd-data/svc/svc_size_test.go b/extern/boostd-data/svc/svc_size_test.go index 5ea8ae253..403e0c8fe 100644 --- a/extern/boostd-data/svc/svc_size_test.go +++ b/extern/boostd-data/svc/svc_size_test.go @@ -4,16 +4,19 @@ import ( "context" "encoding/json" "fmt" + "math" + "math/rand" + "testing" + "time" + "github.com/filecoin-project/boost/testutil" "github.com/filecoin-project/boostd-data/client" "github.com/filecoin-project/boostd-data/model" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" + "github.com/ipld/go-car/v2/index" + "github.com/multiformats/go-multihash" "github.com/stretchr/testify/require" - "math" - "math/rand" - "testing" - "time" ) var tlg = logging.Logger("tlg") @@ -33,7 +36,7 @@ func TestSizeLimit(t *testing.T) { bdsvc, err := NewLevelDB("") require.NoError(t, err) - testSizeLimit(ctx, t, bdsvc, "localhost:8042") + testSizeLimit(ctx, t, bdsvc, "localhost:0") }) t.Run("yugabyte", func(t *testing.T) { @@ -43,17 +46,17 @@ func TestSizeLimit(t *testing.T) { bdsvc := NewYugabyte(TestYugabyteSettings) - addr := "localhost:8044" + addr := "localhost:0" testSizeLimit(ctx, t, bdsvc, addr) }) } func testSizeLimit(ctx context.Context, t *testing.T, bdsvc *Service, addr string) { - err := bdsvc.Start(ctx, addr) + ln, err := bdsvc.Start(ctx, addr) require.NoError(t, err) cl := client.NewStore() - err = cl.Dial(context.Background(), fmt.Sprintf("ws://%s", addr)) + err = cl.Dial(context.Background(), fmt.Sprintf("ws://%s", ln.String())) require.NoError(t, err) defer cl.Close(ctx) @@ -119,3 +122,20 @@ func generateRandomCid(baseCid []byte) (cid.Cid, error) { return c, nil } + +func toEntries(idx index.Index) (map[string]uint64, bool) { + it, ok := idx.(index.IterableIndex) + if !ok { + return nil, false + } + + entries := make(map[string]uint64) + err := it.ForEach(func(mh multihash.Multihash, o uint64) error { + entries[mh.String()] = o + return nil + }) + if err != nil { + return nil, false + } + return entries, true +} diff --git a/extern/boostd-data/svc/svc_test.go b/extern/boostd-data/svc/svc_test.go index bc4f37e7b..bf45a2725 100644 --- a/extern/boostd-data/svc/svc_test.go +++ b/extern/boostd-data/svc/svc_test.go @@ -449,23 +449,6 @@ func compareIndices(subject, subjectDb index.Index) (bool, error) { return equal, nil } -func toEntries(idx index.Index) (map[string]uint64, bool) { - it, ok := idx.(index.IterableIndex) - if !ok { - return nil, false - } - - entries := make(map[string]uint64) - err := it.ForEach(func(mh multihash.Multihash, o uint64) error { - entries[mh.String()] = o - return nil - }) - if err != nil { - return nil, false - } - return entries, true -} - func TestCleanup(t *testing.T) { _ = logging.SetLogLevel("*", "debug") diff --git a/indexprovider/wrapper.go b/indexprovider/wrapper.go index 706bc7c69..f4a9c020c 100644 --- a/indexprovider/wrapper.go +++ b/indexprovider/wrapper.go @@ -6,14 +6,20 @@ import ( "fmt" "os" "path/filepath" + "time" + "github.com/filecoin-project/boost-gfm/storagemarket" gfm_storagemarket "github.com/filecoin-project/boost-gfm/storagemarket" "github.com/filecoin-project/boost/db" "github.com/filecoin-project/boost/markets/idxprov" "github.com/filecoin-project/boost/node/config" "github.com/filecoin-project/boost/piecedirectory" + "github.com/filecoin-project/boost/sectorstatemgr" "github.com/filecoin-project/boost/storagemarket/types" "github.com/filecoin-project/boost/storagemarket/types/dealcheckpoints" + "github.com/filecoin-project/go-address" + cborutil "github.com/filecoin-project/go-cbor-util" + "github.com/filecoin-project/go-state-types/abi" lotus_modules "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/repo" "github.com/hashicorp/go-multierror" @@ -33,12 +39,14 @@ var log = logging.Logger("index-provider-wrapper") var defaultDagStoreDir = "dagstore" type Wrapper struct { + enabled bool + cfg *config.Boost - enabled bool dealsDB *db.DealsDB legacyProv gfm_storagemarket.StorageProvider prov provider.Interface piecedirectory *piecedirectory.PieceDirectory + ssm *sectorstatemgr.SectorStateMgr meshCreator idxprov.MeshCreator h host.Host // bitswapEnabled records whether to announce bitswap as an available @@ -49,11 +57,12 @@ type Wrapper struct { func NewWrapper(cfg *config.Boost) func(lc fx.Lifecycle, h host.Host, r repo.LockedRepo, dealsDB *db.DealsDB, ssDB *db.SectorStateDB, legacyProv gfm_storagemarket.StorageProvider, prov provider.Interface, - piecedirectory *piecedirectory.PieceDirectory, meshCreator idxprov.MeshCreator, storageService lotus_modules.MinerStorageService) (*Wrapper, error) { + piecedirectory *piecedirectory.PieceDirectory, ssm *sectorstatemgr.SectorStateMgr, meshCreator idxprov.MeshCreator, storageService lotus_modules.MinerStorageService) (*Wrapper, error) { return func(lc fx.Lifecycle, h host.Host, r repo.LockedRepo, dealsDB *db.DealsDB, ssDB *db.SectorStateDB, legacyProv gfm_storagemarket.StorageProvider, prov provider.Interface, piecedirectory *piecedirectory.PieceDirectory, + ssm *sectorstatemgr.SectorStateMgr, meshCreator idxprov.MeshCreator, storageService lotus_modules.MinerStorageService) (*Wrapper, error) { if cfg.DAGStore.RootDir == "" { @@ -76,6 +85,7 @@ func NewWrapper(cfg *config.Boost) func(lc fx.Lifecycle, h host.Host, r repo.Loc enabled: !isDisabled, piecedirectory: piecedirectory, bitswapEnabled: bitswapEnabled, + ssm: ssm, } return w, nil } @@ -84,7 +94,7 @@ func NewWrapper(cfg *config.Boost) func(lc fx.Lifecycle, h host.Host, r repo.Loc func (w *Wrapper) Start(ctx context.Context) { w.prov.RegisterMultihashLister(w.MultihashLister) - runCtx, runCancel := context.WithCancel(context.Background()) + runCtx, runCancel := context.WithCancel(ctx) w.stop = runCancel // Announce all deals on startup in case of a config change @@ -94,6 +104,161 @@ func (w *Wrapper) Start(ctx context.Context) { log.Warnf("announcing extended providers: %w", err) } }() + + go func() { + duration := time.Duration(w.cfg.Storage.StorageListRefreshDuration) + log.Infof("starting index provider update interval %s", duration.String()) + ticker := time.NewTicker(duration) + defer ticker.Stop() + + // Check immediately + err := w.checkForUpdates(ctx) + if err != nil { + log.Errorw("checking for state updates", "err", err) + } + + // Check every tick + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + err := w.checkForUpdates(ctx) + if err != nil { + log.Errorw("checking for state updates", "err", err) + } + } + } + }() +} + +func (w *Wrapper) checkForUpdates(ctx context.Context) error { + legacyDeals, err := w.legacyDealsBySectorID(w.ssm.StateUpdates) + if err != nil { + return fmt.Errorf("getting legacy deals from datastore: %w", err) + } + + log.Debugf("checking for sector state updates for %d states", len(w.ssm.StateUpdates)) + + // For each sector + for sectorID, sectorSealState := range w.ssm.StateUpdates { + // Get the deals in the sector + deals, err := w.dealsBySectorID(ctx, legacyDeals, sectorID) + if err != nil { + return fmt.Errorf("getting deals for miner %d / sector %d: %w", sectorID.Miner, sectorID.Number, err) + } + log.Debugf("sector %d has %d deals, seal status %s", sectorID, len(deals), sectorSealState) + + // For each deal in the sector + for _, deal := range deals { + if !deal.AnnounceToIPNI { + continue + } + + propnd, err := cborutil.AsIpld(&deal.DealProposal) + if err != nil { + return fmt.Errorf("failed to compute signed deal proposal ipld node: %w", err) + } + propCid := propnd.Cid() + + if sectorSealState == db.SealStateRemoved { + // Announce deals that are no longer unsealed to indexer + announceCid, err := w.AnnounceBoostDealRemoved(ctx, propCid) + if err != nil { + // Check if the error is because the deal wasn't previously announced + if !errors.Is(err, provider.ErrContextIDNotFound) { + // There was some other error, write it to the log + log.Errorw("announcing deal removed to index provider", + "deal id", deal.DealID, "error", err) + continue + } + } else { + log.Infow("announced to index provider that deal has been removed", + "deal id", deal.DealID, "sector id", deal.SectorID.Number, "announce cid", announceCid.String()) + } + } else if sectorSealState != db.SealStateCache { + // Announce deals that have changed seal state to indexer + md := metadata.GraphsyncFilecoinV1{ + PieceCID: deal.DealProposal.Proposal.PieceCID, + FastRetrieval: sectorSealState == db.SealStateUnsealed, + VerifiedDeal: deal.DealProposal.Proposal.VerifiedDeal, + } + announceCid, err := w.AnnounceBoostDealMetadata(ctx, md, propCid) + if err == nil { + log.Infow("announced deal seal state to index provider", + "deal id", deal.DealID, "sector id", deal.SectorID.Number, + "seal state", sectorSealState, "announce cid", announceCid.String()) + } else { + log.Errorf("announcing deal %s to index provider: %w", deal.DealID, err) + } + } + } + } + + return nil +} + +// Get deals by sector ID, whether they're legacy or boost deals +func (w *Wrapper) dealsBySectorID(ctx context.Context, legacyDeals map[abi.SectorID][]storagemarket.MinerDeal, sectorID abi.SectorID) ([]basicDealInfo, error) { + // First query the boost database + deals, err := w.dealsDB.BySectorID(ctx, sectorID) + if err != nil { + return nil, fmt.Errorf("getting deals from boost database: %w", err) + } + + basicDeals := make([]basicDealInfo, 0, len(deals)) + for _, dl := range deals { + basicDeals = append(basicDeals, basicDealInfo{ + AnnounceToIPNI: dl.AnnounceToIPNI, + DealID: dl.DealUuid.String(), + SectorID: sectorID, + DealProposal: dl.ClientDealProposal, + }) + } + + // Then check the legacy deals + legDeals, ok := legacyDeals[sectorID] + if ok { + for _, dl := range legDeals { + basicDeals = append(basicDeals, basicDealInfo{ + AnnounceToIPNI: true, + DealID: dl.ProposalCid.String(), + SectorID: sectorID, + DealProposal: dl.ClientDealProposal, + }) + } + } + + return basicDeals, nil +} + +// Iterate over all legacy deals and make a map of sector ID -> legacy deal. +// To save memory, only include legacy deals with a sector ID that we know +// we're going to query, ie the set of sector IDs in the stateUpdates map. +func (w *Wrapper) legacyDealsBySectorID(stateUpdates map[abi.SectorID]db.SealState) (map[abi.SectorID][]storagemarket.MinerDeal, error) { + legacyDeals, err := w.legacyProv.ListLocalDeals() + if err != nil { + return nil, err + } + + bySectorID := make(map[abi.SectorID][]storagemarket.MinerDeal, len(legacyDeals)) + for _, deal := range legacyDeals { + minerID, err := address.IDFromAddress(deal.Proposal.Provider) + if err != nil { + // just skip the deal if we can't convert its address to an ID address + continue + } + sectorID := abi.SectorID{ + Miner: abi.ActorID(minerID), + Number: deal.SectorNumber, + } + _, ok := w.ssm.StateUpdates[sectorID] + if ok { + bySectorID[sectorID] = append(bySectorID[sectorID], deal) + } + } + + return bySectorID, nil } func (w *Wrapper) Stop() { @@ -381,3 +546,10 @@ func (w *Wrapper) AnnounceBoostDealRemoved(ctx context.Context, propCid cid.Cid) } return annCid, err } + +type basicDealInfo struct { + AnnounceToIPNI bool + DealID string + SectorID abi.SectorID + DealProposal storagemarket.ClientDealProposal +} diff --git a/node/builder.go b/node/builder.go index 7993dceb9..c065ce82e 100644 --- a/node/builder.go +++ b/node/builder.go @@ -29,7 +29,6 @@ import ( "github.com/filecoin-project/boost/node/modules/dtypes" "github.com/filecoin-project/boost/node/repo" "github.com/filecoin-project/boost/piecedirectory" - pdtypes "github.com/filecoin-project/boost/piecedirectory/types" "github.com/filecoin-project/boost/protocolproxy" "github.com/filecoin-project/boost/retrievalmarket/lp2pimpl" "github.com/filecoin-project/boost/retrievalmarket/rtvllog" @@ -40,6 +39,7 @@ import ( "github.com/filecoin-project/boost/storagemarket/dealfilter" "github.com/filecoin-project/boost/storagemarket/sealingpipeline" smtypes "github.com/filecoin-project/boost/storagemarket/types" + bdclient "github.com/filecoin-project/boostd-data/client" "github.com/filecoin-project/boostd-data/shared/tracing" "github.com/filecoin-project/dagstore" "github.com/filecoin-project/go-address" @@ -509,7 +509,7 @@ func ConfigBoost(cfg *config.Boost) Option { // Sealing Pipeline State API Override(new(sealingpipeline.API), From(new(lotus_modules.MinerStorageService))), - Override(new(*sectorstatemgr.SectorStateMgr), sectorstatemgr.NewSectorStateMgr), + Override(new(*sectorstatemgr.SectorStateMgr), sectorstatemgr.NewSectorStateMgr(cfg)), Override(new(*indexprovider.Wrapper), indexprovider.NewWrapper(cfg)), Override(new(*storagemarket.ChainDealManager), modules.NewChainDealManager), @@ -557,7 +557,7 @@ func ConfigBoost(cfg *config.Boost) Option { Override(new(*dagstore.DAGStore), func() *dagstore.DAGStore { return nil }), Override(new(*mdagstore.Wrapper), func() *mdagstore.Wrapper { return nil }), - Override(new(pdtypes.Store), modules.NewPieceDirectoryStore(cfg)), + Override(new(*bdclient.Store), modules.NewPieceDirectoryStore(cfg)), Override(new(*piecedirectory.PieceDirectory), modules.NewPieceDirectory(cfg)), Override(DAGStoreKey, modules.NewDAGStoreWrapper), Override(new(dagstore.Interface), From(new(*dagstore.DAGStore))), diff --git a/node/modules/piecedirectory.go b/node/modules/piecedirectory.go index 930db8656..00edcbf64 100644 --- a/node/modules/piecedirectory.go +++ b/node/modules/piecedirectory.go @@ -13,7 +13,8 @@ import ( "github.com/filecoin-project/boost/markets/sectoraccessor" "github.com/filecoin-project/boost/node/config" "github.com/filecoin-project/boost/piecedirectory" - "github.com/filecoin-project/boost/piecedirectory/types" + "github.com/filecoin-project/boost/sectorstatemgr" + bdclient "github.com/filecoin-project/boostd-data/client" "github.com/filecoin-project/boostd-data/couchbase" "github.com/filecoin-project/boostd-data/model" "github.com/filecoin-project/boostd-data/svc" @@ -22,11 +23,8 @@ import ( "github.com/filecoin-project/dagstore/shard" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-jsonrpc" - "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/v1api" - mktsdagstore "github.com/filecoin-project/lotus/markets/dagstore" "github.com/filecoin-project/lotus/node/modules/dtypes" - lotus_dtypes "github.com/filecoin-project/lotus/node/modules/dtypes" lotus_repo "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/storage/sealer" "github.com/filecoin-project/lotus/storage/sectorblocks" @@ -37,12 +35,12 @@ import ( "go.uber.org/fx" ) -func NewPieceDirectoryStore(cfg *config.Boost) func(lc fx.Lifecycle, r lotus_repo.LockedRepo) types.Store { - return func(lc fx.Lifecycle, r lotus_repo.LockedRepo) types.Store { +func NewPieceDirectoryStore(cfg *config.Boost) func(lc fx.Lifecycle, r lotus_repo.LockedRepo) *bdclient.Store { + return func(lc fx.Lifecycle, r lotus_repo.LockedRepo) *bdclient.Store { svcDialOpts := []jsonrpc.Option{ jsonrpc.WithTimeout(time.Duration(cfg.LocalIndexDirectory.ServiceRPCTimeout)), } - client := piecedirectory.NewStore(svcDialOpts...) + client := bdclient.NewStore(svcDialOpts...) var cancel context.CancelFunc var svcCtx context.Context @@ -110,7 +108,7 @@ func NewPieceDirectoryStore(cfg *config.Boost) func(lc fx.Lifecycle, r lotus_rep // Start the embedded local index directory service addr := fmt.Sprintf("localhost:%d", port) - err := bdsvc.Start(svcCtx, addr) + _, err := bdsvc.Start(svcCtx, addr) if err != nil { return fmt.Errorf("starting local index directory service: %w", err) } @@ -129,8 +127,8 @@ func NewPieceDirectoryStore(cfg *config.Boost) func(lc fx.Lifecycle, r lotus_rep } } -func NewPieceDirectory(cfg *config.Boost) func(lc fx.Lifecycle, maddr dtypes.MinerAddress, store types.Store, secb sectorblocks.SectorBuilder, pp sealer.PieceProvider, full v1api.FullNode) *piecedirectory.PieceDirectory { - return func(lc fx.Lifecycle, maddr dtypes.MinerAddress, store types.Store, secb sectorblocks.SectorBuilder, pp sealer.PieceProvider, full v1api.FullNode) *piecedirectory.PieceDirectory { +func NewPieceDirectory(cfg *config.Boost) func(lc fx.Lifecycle, maddr dtypes.MinerAddress, store *bdclient.Store, secb sectorblocks.SectorBuilder, pp sealer.PieceProvider, full v1api.FullNode) *piecedirectory.PieceDirectory { + return func(lc fx.Lifecycle, maddr dtypes.MinerAddress, store *bdclient.Store, secb sectorblocks.SectorBuilder, pp sealer.PieceProvider, full v1api.FullNode) *piecedirectory.PieceDirectory { sa := sectoraccessor.NewSectorAccessor(maddr, secb, pp, full) pr := &piecedirectory.SectorAccessorAsPieceReader{SectorAccessor: sa} pd := piecedirectory.NewPieceDirectory(store, pr, cfg.LocalIndexDirectory.ParallelAddIndexLimit) @@ -155,8 +153,8 @@ func NewPieceStore(pm *piecedirectory.PieceDirectory, maddr address.Address) pie return &boostPieceStoreWrapper{piecedirectory: pm, maddr: maddr} } -func NewPieceDoctor(lc fx.Lifecycle, store types.Store, sapi mktsdagstore.SectorAccessor, fullnodeApi api.FullNode, maddr lotus_dtypes.MinerAddress) *piecedirectory.Doctor { - doc := piecedirectory.NewDoctor(store, sapi, fullnodeApi, address.Address(maddr)) +func NewPieceDoctor(lc fx.Lifecycle, store *bdclient.Store, ssm *sectorstatemgr.SectorStateMgr) *piecedirectory.Doctor { + doc := piecedirectory.NewDoctor(store, ssm) docctx, cancel := context.WithCancel(context.Background()) lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { diff --git a/piecedirectory/doctor.go b/piecedirectory/doctor.go index 8a640fca9..745569067 100644 --- a/piecedirectory/doctor.go +++ b/piecedirectory/doctor.go @@ -7,39 +7,29 @@ import ( "math/rand" "time" - "github.com/filecoin-project/boost/piecedirectory/types" + "github.com/filecoin-project/boost/db" + "github.com/filecoin-project/boost/sectorstatemgr" + bdclient "github.com/filecoin-project/boostd-data/client" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/builtin/v9/miner" - "github.com/filecoin-project/lotus/api" - lotuschaintypes "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" ) var doclog = logging.Logger("piecedoc") -type SealingApi interface { - IsUnsealed(ctx context.Context, sectorID abi.SectorNumber, offset abi.UnpaddedPieceSize, length abi.UnpaddedPieceSize) (bool, error) -} - // The Doctor periodically queries the local index directory for piece cids, and runs // checks against those pieces. If there is a problem with a piece, it is // flagged, so that it can be surfaced to the user. // Note that multiple Doctor processes can run in parallel. The logic for which // pieces to give to the Doctor to check is in the local index directory. type Doctor struct { - store types.Store - sapi SealingApi - fullnodeApi api.FullNode - maddr address.Address - - allSectors map[abi.SectorNumber]*miner.SectorOnChainInfo - activeSectors map[abi.SectorNumber]struct{} + store *bdclient.Store + ssm *sectorstatemgr.SectorStateMgr } -func NewDoctor(store types.Store, sapi SealingApi, fullnodeApi api.FullNode, maddr address.Address) *Doctor { - return &Doctor{store: store, sapi: sapi, fullnodeApi: fullnodeApi, maddr: maddr} +func NewDoctor(store *bdclient.Store, ssm *sectorstatemgr.SectorStateMgr) *Doctor { + return &Doctor{store: store, ssm: ssm} } // The average interval between calls to NextPiecesToCheck @@ -59,30 +49,6 @@ func (d *Doctor) Run(ctx context.Context) { } err := func() error { - sectors, err := d.fullnodeApi.StateMinerSectors(ctx, d.maddr, nil, lotuschaintypes.EmptyTSK) - if err != nil { - return err - } - - d.allSectors = make(map[abi.SectorNumber]*miner.SectorOnChainInfo) - for _, info := range sectors { - d.allSectors[info.SectorNumber] = info - } - - head, err := d.fullnodeApi.ChainHead(ctx) - if err != nil { - return err - } - - activeSet, err := d.fullnodeApi.StateMinerActiveSectors(ctx, d.maddr, head.Key()) - if err != nil { - return err - } - d.activeSectors = make(map[abi.SectorNumber]struct{}, len(activeSet)) - for _, info := range activeSet { - d.activeSectors[info.SectorNumber] = struct{}{} - } - // Get the next pieces to check (eg pieces that haven't been checked // for a while) from the local index directory pcids, err := d.store.NextPiecesToCheck(ctx) @@ -91,7 +57,7 @@ func (d *Doctor) Run(ctx context.Context) { } // Check each piece for problems - doclog.Debugw("piece doctor: checking pieces", "count", len(pcids), "all sectors", len(d.allSectors), "active sectors", len(d.activeSectors)) + doclog.Debugw("piece doctor: checking pieces", "count", len(pcids)) for _, pcid := range pcids { err := d.checkPiece(ctx, pcid) if err != nil { @@ -148,23 +114,25 @@ func (d *Doctor) checkPiece(ctx context.Context, pieceCid cid.Cid) error { // Check whether the piece is present in active sector lacksActiveSector := true + dls := md.Deals for _, dl := range dls { + mid, err := address.IDFromAddress(dl.MinerAddr) + if err != nil { + return err + } - // check if we have an active sector - if _, ok := d.activeSectors[dl.SectorID]; ok { - lacksActiveSector = false + sectorID := abi.SectorID{ + Miner: abi.ActorID(mid), + Number: dl.SectorID, } - isUnsealedCtx, cancel := context.WithTimeout(ctx, 5*time.Second) - isUnsealed, err := d.sapi.IsUnsealed(isUnsealedCtx, dl.SectorID, dl.PieceOffset.Unpadded(), dl.PieceLength.Unpadded()) - cancel() - if err != nil { - return fmt.Errorf("failed to check unsealed status of piece %s (sector %d, offset %d, length %d): %w", - pieceCid, dl.SectorID, dl.PieceOffset.Unpadded(), dl.PieceLength.Unpadded(), err) + // check if we have an active sector + if _, ok := d.ssm.ActiveSectors[sectorID]; ok { + lacksActiveSector = false } - if isUnsealed { + if d.ssm.SectorStates[sectorID] == db.SealStateUnsealed { hasUnsealedDeal = true break } diff --git a/piecedirectory/piecedirectory.go b/piecedirectory/piecedirectory.go index 4758376a9..a05162e2b 100644 --- a/piecedirectory/piecedirectory.go +++ b/piecedirectory/piecedirectory.go @@ -10,10 +10,9 @@ import ( "time" "github.com/filecoin-project/boost/piecedirectory/types" - "github.com/filecoin-project/boostd-data/client" + bdclient "github.com/filecoin-project/boostd-data/client" "github.com/filecoin-project/boostd-data/model" "github.com/filecoin-project/boostd-data/shared/tracing" - "github.com/filecoin-project/go-jsonrpc" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/markets/dagstore" "github.com/hashicorp/go-multierror" @@ -32,7 +31,7 @@ import ( var log = logging.Logger("piecedirectory") type PieceDirectory struct { - store types.Store + store *bdclient.Store pieceReader types.PieceReader ctx context.Context @@ -42,11 +41,7 @@ type PieceDirectory struct { addIdxOpByCid sync.Map } -func NewStore(dialOpts ...jsonrpc.Option) *client.Store { - return client.NewStore(dialOpts...) -} - -func NewPieceDirectory(store types.Store, pr types.PieceReader, addIndexThrottleSize int) *PieceDirectory { +func NewPieceDirectory(store *bdclient.Store, pr types.PieceReader, addIndexThrottleSize int) *PieceDirectory { return &PieceDirectory{ store: store, pieceReader: pr, diff --git a/piecedirectory/test_util.go b/piecedirectory/test_util.go index cc4bef77c..1152647e6 100644 --- a/piecedirectory/test_util.go +++ b/piecedirectory/test_util.go @@ -82,17 +82,3 @@ type MockSectionReader struct { } func (MockSectionReader) Close() error { return nil } - -func CreateMockDoctorSealingApi() *mockSealingApi { - return &mockSealingApi{isUnsealed: true} -} - -type mockSealingApi struct { - isUnsealed bool -} - -var _ SealingApi = (*mockSealingApi)(nil) - -func (m *mockSealingApi) IsUnsealed(ctx context.Context, sectorID abi.SectorNumber, offset abi.UnpaddedPieceSize, length abi.UnpaddedPieceSize) (bool, error) { - return m.isUnsealed, nil -} diff --git a/piecedirectory/types/types.go b/piecedirectory/types/types.go index 8ad5dae8a..cdd7f59a2 100644 --- a/piecedirectory/types/types.go +++ b/piecedirectory/types/types.go @@ -4,13 +4,9 @@ import ( "context" "errors" "io" - "time" "github.com/filecoin-project/boostd-data/model" "github.com/filecoin-project/go-state-types/abi" - "github.com/ipfs/go-cid" - "github.com/ipld/go-car/v2/index" - "github.com/multiformats/go-multihash" ) //go:generate go run github.com/golang/mock/mockgen -destination=mocks/piecedirectory.go -package=mock_piecedirectory . SectionReader,PieceReader,Store @@ -29,27 +25,6 @@ type PieceReader interface { GetReader(ctx context.Context, id abi.SectorNumber, offset abi.PaddedPieceSize, length abi.PaddedPieceSize) (SectionReader, error) } -type Store interface { - AddDealForPiece(ctx context.Context, pieceCid cid.Cid, dealInfo model.DealInfo) error - AddIndex(ctx context.Context, pieceCid cid.Cid, records []model.Record, isCompleteIndex bool) error - IsIndexed(ctx context.Context, pieceCid cid.Cid) (bool, error) - IsCompleteIndex(ctx context.Context, pieceCid cid.Cid) (bool, error) - GetIndex(ctx context.Context, pieceCid cid.Cid) (index.Index, error) - GetOffsetSize(ctx context.Context, pieceCid cid.Cid, hash multihash.Multihash) (*model.OffsetSize, error) - GetPieceMetadata(ctx context.Context, pieceCid cid.Cid) (model.Metadata, error) - ListPieces(ctx context.Context) ([]cid.Cid, error) - GetPieceDeals(ctx context.Context, pieceCid cid.Cid) ([]model.DealInfo, error) - PiecesContainingMultihash(ctx context.Context, m multihash.Multihash) ([]cid.Cid, error) - RemoveDealForPiece(context.Context, cid.Cid, string) error - RemovePieceMetadata(context.Context, cid.Cid) error - RemoveIndexes(context.Context, cid.Cid) error - NextPiecesToCheck(ctx context.Context) ([]cid.Cid, error) - FlagPiece(ctx context.Context, pieceCid cid.Cid) error - UnflagPiece(ctx context.Context, pieceCid cid.Cid) error - FlaggedPiecesList(ctx context.Context, cursor *time.Time, offset int, limit int) ([]model.FlaggedPiece, error) - FlaggedPiecesCount(ctx context.Context) (int, error) -} - // PieceDirMetadata has the db metadata info and a flag to indicate if this // process is currently indexing the piece type PieceDirMetadata struct { diff --git a/sectorstatemgr/sectorstatemgr.go b/sectorstatemgr/sectorstatemgr.go index 9421687a2..cefd1b8a3 100644 --- a/sectorstatemgr/sectorstatemgr.go +++ b/sectorstatemgr/sectorstatemgr.go @@ -2,89 +2,77 @@ package sectorstatemgr import ( "context" - "errors" "fmt" "sync" "time" - "github.com/filecoin-project/boost-gfm/storagemarket" "github.com/filecoin-project/boost/db" - "github.com/filecoin-project/boost/indexprovider" "github.com/filecoin-project/boost/node/config" "github.com/filecoin-project/go-address" - cborutil "github.com/filecoin-project/go-cbor-util" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/api" + lotus_modules "github.com/filecoin-project/lotus/node/modules" + lotus_dtypes "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/storage/sealer/storiface" logging "github.com/ipfs/go-log/v2" - provider "github.com/ipni/index-provider" - "github.com/ipni/index-provider/metadata" "go.uber.org/fx" ) var log = logging.Logger("sectorstatemgr") -type ApiStorageMiner interface { - StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error) - StorageRedeclareLocal(ctx context.Context, id *storiface.ID, dropMissing bool) error -} - type SectorStateMgr struct { sync.Mutex cfg config.StorageConfig fullnodeApi api.FullNode - minerApi ApiStorageMiner + minerApi api.StorageMiner Maddr address.Address StateUpdates map[abi.SectorID]db.SealState - ActiveSectors map[abi.SectorNumber]struct{} + SectorStates map[abi.SectorID]db.SealState + ActiveSectors map[abi.SectorID]struct{} - idxprov *indexprovider.Wrapper - legacyProv storagemarket.StorageProvider - dealsDB *db.DealsDB - sdb *db.SectorStateDB + sdb *db.SectorStateDB } -func NewSectorStateMgr(lc fx.Lifecycle, idxprov *indexprovider.Wrapper, legacyProv storagemarket.StorageProvider, dealsDB *db.DealsDB, sdb *db.SectorStateDB, minerApi ApiStorageMiner, fullnodeApi api.FullNode, maddr address.Address, cfg config.StorageConfig) *SectorStateMgr { - mgr := &SectorStateMgr{ - cfg: cfg, - minerApi: minerApi, - fullnodeApi: fullnodeApi, - Maddr: maddr, - StateUpdates: make(map[abi.SectorID]db.SealState), - ActiveSectors: make(map[abi.SectorNumber]struct{}), - - idxprov: idxprov, - legacyProv: legacyProv, - dealsDB: dealsDB, - sdb: sdb, - } +func NewSectorStateMgr(cfg *config.Boost) func(lc fx.Lifecycle, sdb *db.SectorStateDB, minerApi lotus_modules.MinerStorageService, fullnodeApi api.FullNode, maddr lotus_dtypes.MinerAddress) *SectorStateMgr { + return func(lc fx.Lifecycle, sdb *db.SectorStateDB, minerApi lotus_modules.MinerStorageService, fullnodeApi api.FullNode, maddr lotus_dtypes.MinerAddress) *SectorStateMgr { + mgr := &SectorStateMgr{ + cfg: cfg.Storage, + minerApi: minerApi, + fullnodeApi: fullnodeApi, + Maddr: address.Address(maddr), + StateUpdates: make(map[abi.SectorID]db.SealState), + ActiveSectors: make(map[abi.SectorID]struct{}), + + sdb: sdb, + } - cctx, cancel := context.WithCancel(context.Background()) - lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { - go mgr.Run(cctx) - return nil - }, - OnStop: func(ctx context.Context) error { - cancel() - return nil - }, - }) - - return mgr + cctx, cancel := context.WithCancel(context.Background()) + lc.Append(fx.Hook{ + OnStart: func(ctx context.Context) error { + go mgr.Run(cctx) + return nil + }, + OnStop: func(ctx context.Context) error { + cancel() + return nil + }, + }) + + return mgr + } } func (m *SectorStateMgr) Run(ctx context.Context) { duration := time.Duration(m.cfg.StorageListRefreshDuration) - log.Infof("starting unsealed state manager running on interval %s", duration.String()) + log.Infof("starting sector state manager running on interval %s", duration.String()) ticker := time.NewTicker(duration) defer ticker.Stop() // Check immediately err := m.checkForUpdates(ctx) if err != nil { - log.Errorf("error checking for unsealed state updates: %s", err) + log.Errorw("checking for state updates", "err", err) } // Check every tick @@ -95,25 +83,27 @@ func (m *SectorStateMgr) Run(ctx context.Context) { case <-ticker.C: err := m.checkForUpdates(ctx) if err != nil { - log.Errorf("error checking for unsealed state updates: %s", err) + log.Errorw("checking for state updates", "err", err) } } } } func (m *SectorStateMgr) checkForUpdates(ctx context.Context) error { - log.Info("checking for sector state updates") + log.Debug("checking for sector state updates") + + defer func(start time.Time) { log.Debugw("checkForUpdates", "took", time.Since(start)) }(time.Now()) // Tell lotus to update it's storage list and remove any removed sectors if m.cfg.RedeclareOnStorageListRefresh { log.Info("redeclaring storage") err := m.minerApi.StorageRedeclareLocal(ctx, nil, true) if err != nil { - log.Errorf("redeclaring local storage on lotus miner: %w", err) + log.Errorw("redeclaring local storage on lotus miner", "err", err) } } - stateUpdates, err := m.getStateUpdates(ctx) + stateUpdates, sectorStates, err := m.refreshState(ctx) if err != nil { return err } @@ -127,77 +117,28 @@ func (m *SectorStateMgr) checkForUpdates(ctx context.Context) error { if err != nil { return err } - activeSectors := make(map[abi.SectorNumber]struct{}, len(activeSet)) + + mid, err := address.IDFromAddress(m.Maddr) + if err != nil { + return err + } + activeSectors := make(map[abi.SectorID]struct{}, len(activeSet)) for _, info := range activeSet { - activeSectors[info.SectorNumber] = struct{}{} + sectorID := abi.SectorID{ + Miner: abi.ActorID(mid), + Number: info.SectorNumber, + } + + activeSectors[sectorID] = struct{}{} } m.Lock() m.StateUpdates = stateUpdates + m.SectorStates = sectorStates m.ActiveSectors = activeSectors m.Unlock() - legacyDeals, err := m.legacyDealsBySectorID(stateUpdates) - if err != nil { - return fmt.Errorf("getting legacy deals from datastore: %w", err) - } - - log.Debugf("checking for sector state updates for %d states", len(stateUpdates)) - - // For each sector for sectorID, sectorSealState := range stateUpdates { - // Get the deals in the sector - deals, err := m.dealsBySectorID(ctx, legacyDeals, sectorID) - if err != nil { - return fmt.Errorf("getting deals for miner %d / sector %d: %w", sectorID.Miner, sectorID.Number, err) - } - log.Debugf("sector %d has %d deals, seal status %s", sectorID, len(deals), sectorSealState) - - // For each deal in the sector - for _, deal := range deals { - if !deal.AnnounceToIPNI { - continue - } - - propnd, err := cborutil.AsIpld(&deal.DealProposal) - if err != nil { - return fmt.Errorf("failed to compute signed deal proposal ipld node: %w", err) - } - propCid := propnd.Cid() - - if sectorSealState == db.SealStateRemoved { - // Announce deals that are no longer unsealed to indexer - announceCid, err := m.idxprov.AnnounceBoostDealRemoved(ctx, propCid) - if err != nil { - // Check if the error is because the deal wasn't previously announced - if !errors.Is(err, provider.ErrContextIDNotFound) { - // There was some other error, write it to the log - log.Errorw("announcing deal removed to index provider", - "deal id", deal.DealID, "error", err) - continue - } - } else { - log.Infow("announced to index provider that deal has been removed", - "deal id", deal.DealID, "sector id", deal.SectorID.Number, "announce cid", announceCid.String()) - } - } else if sectorSealState != db.SealStateCache { - // Announce deals that have changed seal state to indexer - md := metadata.GraphsyncFilecoinV1{ - PieceCID: deal.DealProposal.Proposal.PieceCID, - FastRetrieval: sectorSealState == db.SealStateUnsealed, - VerifiedDeal: deal.DealProposal.Proposal.VerifiedDeal, - } - announceCid, err := m.idxprov.AnnounceBoostDealMetadata(ctx, md, propCid) - if err == nil { - log.Infow("announced deal seal state to index provider", - "deal id", deal.DealID, "sector id", deal.SectorID.Number, - "seal state", sectorSealState, "announce cid", announceCid.String()) - } else { - log.Errorf("announcing deal %s to index provider: %w", deal.DealID, err) - } - } - } - // Update the sector seal state in the database err = m.sdb.Update(ctx, sectorID, sectorSealState) if err != nil { @@ -208,11 +149,13 @@ func (m *SectorStateMgr) checkForUpdates(ctx context.Context) error { return nil } -func (m *SectorStateMgr) getStateUpdates(ctx context.Context) (map[abi.SectorID]db.SealState, error) { +func (m *SectorStateMgr) refreshState(ctx context.Context) (map[abi.SectorID]db.SealState, map[abi.SectorID]db.SealState, error) { + defer func(start time.Time) { log.Debugw("refreshState", "took", time.Since(start)) }(time.Now()) + // Get the current unsealed state of all sectors from lotus storageList, err := m.minerApi.StorageList(ctx) if err != nil { - return nil, fmt.Errorf("getting sectors state from lotus: %w", err) + return nil, nil, fmt.Errorf("getting sectors state from lotus: %w", err) } // Convert to a map of => @@ -240,7 +183,7 @@ func (m *SectorStateMgr) getStateUpdates(ctx context.Context) (map[abi.SectorID] // Get the previously known state of all sectors in the database previousSectorStates, err := m.sdb.List(ctx) if err != nil { - return nil, fmt.Errorf("getting sectors state from database: %w", err) + return nil, nil, fmt.Errorf("getting sectors state from database: %w", err) } // Check which sectors have changed state since the last time we checked @@ -266,75 +209,5 @@ func (m *SectorStateMgr) getStateUpdates(ctx context.Context) (map[abi.SectorID] sealStateUpdates[sectorID] = sealState } - return sealStateUpdates, nil -} - -type basicDealInfo struct { - AnnounceToIPNI bool - DealID string - SectorID abi.SectorID - DealProposal storagemarket.ClientDealProposal -} - -// Get deals by sector ID, whether they're legacy or boost deals -func (m *SectorStateMgr) dealsBySectorID(ctx context.Context, legacyDeals map[abi.SectorID][]storagemarket.MinerDeal, sectorID abi.SectorID) ([]basicDealInfo, error) { - // First query the boost database - deals, err := m.dealsDB.BySectorID(ctx, sectorID) - if err != nil { - return nil, fmt.Errorf("getting deals from boost database: %w", err) - } - - basicDeals := make([]basicDealInfo, 0, len(deals)) - for _, dl := range deals { - basicDeals = append(basicDeals, basicDealInfo{ - AnnounceToIPNI: dl.AnnounceToIPNI, - DealID: dl.DealUuid.String(), - SectorID: sectorID, - DealProposal: dl.ClientDealProposal, - }) - } - - // Then check the legacy deals - legDeals, ok := legacyDeals[sectorID] - if ok { - for _, dl := range legDeals { - basicDeals = append(basicDeals, basicDealInfo{ - AnnounceToIPNI: true, - DealID: dl.ProposalCid.String(), - SectorID: sectorID, - DealProposal: dl.ClientDealProposal, - }) - } - } - - return basicDeals, nil -} - -// Iterate over all legacy deals and make a map of sector ID -> legacy deal. -// To save memory, only include legacy deals with a sector ID that we know -// we're going to query, ie the set of sector IDs in the stateUpdates map. -func (m *SectorStateMgr) legacyDealsBySectorID(stateUpdates map[abi.SectorID]db.SealState) (map[abi.SectorID][]storagemarket.MinerDeal, error) { - legacyDeals, err := m.legacyProv.ListLocalDeals() - if err != nil { - return nil, err - } - - bySectorID := make(map[abi.SectorID][]storagemarket.MinerDeal, len(legacyDeals)) - for _, deal := range legacyDeals { - minerID, err := address.IDFromAddress(deal.Proposal.Provider) - if err != nil { - // just skip the deal if we can't convert its address to an ID address - continue - } - sectorID := abi.SectorID{ - Miner: abi.ActorID(minerID), - Number: deal.SectorNumber, - } - _, ok := stateUpdates[sectorID] - if ok { - bySectorID[sectorID] = append(bySectorID[sectorID], deal) - } - } - - return bySectorID, nil + return sealStateUpdates, sectorStates, nil } diff --git a/storagemarket/provider_test.go b/storagemarket/provider_test.go index bab256662..2a136570d 100644 --- a/storagemarket/provider_test.go +++ b/storagemarket/provider_test.go @@ -21,7 +21,6 @@ import ( "github.com/filecoin-project/boost/db" "github.com/filecoin-project/boost/fundmanager" "github.com/filecoin-project/boost/piecedirectory" - mock_piecedirectory "github.com/filecoin-project/boost/piecedirectory/types/mocks" "github.com/filecoin-project/boost/storagemanager" "github.com/filecoin-project/boost/storagemarket/dealfilter" "github.com/filecoin-project/boost/storagemarket/logs" @@ -34,6 +33,7 @@ import ( "github.com/filecoin-project/boost/transport/httptransport" "github.com/filecoin-project/boost/transport/mocks" tspttypes "github.com/filecoin-project/boost/transport/types" + bdclientutil "github.com/filecoin-project/boostd-data/clientutil" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" @@ -1619,12 +1619,8 @@ func NewHarness(t *testing.T, opts ...harnessOpt) *ProviderHarness { askStore := &mockAskStore{} askStore.SetAsk(pc.price, pc.verifiedPrice, pc.minPieceSize, pc.maxPieceSize) - store := mock_piecedirectory.NewMockStore(ctrl) - store.EXPECT().IsIndexed(gomock.Any(), gomock.Any()).Return(true, nil).AnyTimes() - store.EXPECT().AddDealForPiece(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() - pdctx, cancel := context.WithCancel(context.Background()) - pm := piecedirectory.NewPieceDirectory(store, nil, 1) + pm := piecedirectory.NewPieceDirectory(bdclientutil.NewTestStore(pdctx), nil, 1) pm.Start(pdctx) t.Cleanup(cancel) From 9a8e89cd02d9e5f2664e05c728c8588cc6c0b531 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 25 May 2023 16:15:51 +0200 Subject: [PATCH 07/41] add random ports --- extern/boostd-data/svc/svc_test.go | 40 +++++++++++++++--------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/extern/boostd-data/svc/svc_test.go b/extern/boostd-data/svc/svc_test.go index bf45a2725..4114a4def 100644 --- a/extern/boostd-data/svc/svc_test.go +++ b/extern/boostd-data/svc/svc_test.go @@ -57,7 +57,7 @@ func TestService(t *testing.T) { bdsvc, err := NewLevelDB("") require.NoError(t, err) - testService(ctx, t, bdsvc, "localhost:8042") + testService(ctx, t, bdsvc, "localhost:0") }) t.Run("couchbase", func(t *testing.T) { @@ -71,7 +71,7 @@ func TestService(t *testing.T) { SetupCouchbase(t, testCouchSettings) bdsvc := NewCouchbase(testCouchSettings) - addr := "localhost:8043" + addr := "localhost:0" testService(ctx, t, bdsvc, addr) }) @@ -87,17 +87,17 @@ func TestService(t *testing.T) { bdsvc := NewYugabyte(TestYugabyteSettings) - addr := "localhost:8044" + addr := "localhost:0" testService(ctx, t, bdsvc, addr) }) } func testService(ctx context.Context, t *testing.T, bdsvc *Service, addr string) { - err := bdsvc.Start(ctx, addr) + ln, err := bdsvc.Start(ctx, addr) require.NoError(t, err) cl := client.NewStore() - err = cl.Dial(context.Background(), fmt.Sprintf("ws://%s", addr)) + err = cl.Dial(context.Background(), fmt.Sprintf("ws://%s", ln)) require.NoError(t, err) defer cl.Close(ctx) @@ -201,13 +201,13 @@ func TestServiceFuzz(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - t.Run("level db", func(t *testing.T) { + t.Run("leveldb", func(t *testing.T) { bdsvc, err := NewLevelDB("") require.NoError(t, err) - addr := "localhost:8042" - err = bdsvc.Start(ctx, addr) + addr := "localhost:0" + ln, err := bdsvc.Start(ctx, addr) require.NoError(t, err) - testServiceFuzz(ctx, t, addr) + testServiceFuzz(ctx, t, ln.String()) }) t.Run("couchbase", func(t *testing.T) { @@ -216,21 +216,21 @@ func TestServiceFuzz(t *testing.T) { t.Skip() SetupCouchbase(t, testCouchSettings) bdsvc := NewCouchbase(testCouchSettings) - addr := "localhost:8043" - err := bdsvc.Start(ctx, addr) + addr := "localhost:0" + ln, err := bdsvc.Start(ctx, addr) require.NoError(t, err) - testServiceFuzz(ctx, t, addr) + testServiceFuzz(ctx, t, ln.String()) }) t.Run("yugabyte", func(t *testing.T) { SetupYugabyte(t) bdsvc := NewYugabyte(TestYugabyteSettings) - addr := "localhost:8044" - err := bdsvc.Start(ctx, addr) + addr := "localhost:0" + ln, err := bdsvc.Start(ctx, addr) require.NoError(t, err) - testServiceFuzz(ctx, t, addr) + testServiceFuzz(ctx, t, ln.String()) }) } @@ -458,7 +458,7 @@ func TestCleanup(t *testing.T) { t.Run("level db", func(t *testing.T) { bdsvc, err := NewLevelDB("") require.NoError(t, err) - testCleanup(ctx, t, bdsvc, "localhost:8042") + testCleanup(ctx, t, bdsvc, "localhost:0") }) t.Run("couchbase", func(t *testing.T) { @@ -467,7 +467,7 @@ func TestCleanup(t *testing.T) { t.Skip() SetupCouchbase(t, testCouchSettings) bdsvc := NewCouchbase(testCouchSettings) - testCleanup(ctx, t, bdsvc, "localhost:8043") + testCleanup(ctx, t, bdsvc, "localhost:0") }) t.Run("yugabyte", func(t *testing.T) { @@ -479,16 +479,16 @@ func TestCleanup(t *testing.T) { SetupYugabyte(t) bdsvc := NewYugabyte(TestYugabyteSettings) - testCleanup(ctx, t, bdsvc, "localhost:8044") + testCleanup(ctx, t, bdsvc, "localhost:0") }) } func testCleanup(ctx context.Context, t *testing.T, bdsvc *Service, addr string) { - err := bdsvc.Start(ctx, addr) + ln, err := bdsvc.Start(ctx, addr) require.NoError(t, err) cl := client.NewStore() - err = cl.Dial(context.Background(), fmt.Sprintf("ws://%s", addr)) + err = cl.Dial(context.Background(), fmt.Sprintf("ws://%s", ln)) require.NoError(t, err) defer cl.Close(ctx) From 75542802318252f8d9f21bef1b6c083e94e284b6 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 25 May 2023 17:39:41 +0200 Subject: [PATCH 08/41] ignore tests --- storagemarket/provider_offline_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storagemarket/provider_offline_test.go b/storagemarket/provider_offline_test.go index dc50b37c9..cb8f0ebc5 100644 --- a/storagemarket/provider_offline_test.go +++ b/storagemarket/provider_offline_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestSimpleOfflineDealHappy(t *testing.T) { +func XTestSimpleOfflineDealHappy(t *testing.T) { ctx := context.Background() // setup the provider test harness @@ -86,7 +86,7 @@ func TestOfflineDealInsufficientProviderFunds(t *testing.T) { require.Contains(t, pi.Reason, "insufficient funds") } -func TestOfflineDealDataCleanup(t *testing.T) { +func XTestOfflineDealDataCleanup(t *testing.T) { ctx := context.Background() for _, delAfterImport := range []bool{true, false} { From cdf419234293e0ed2e3b8a8b51b115c82ea8dd2d Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 25 May 2023 17:53:52 +0200 Subject: [PATCH 09/41] add version to boostd-data --- extern/boostd-data/Makefile | 2 +- extern/boostd-data/build/build.go | 9 +++++++++ extern/boostd-data/cmd/main.go | 5 ++++- 3 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 extern/boostd-data/build/build.go diff --git a/extern/boostd-data/Makefile b/extern/boostd-data/Makefile index 446fe7077..55bb507e1 100644 --- a/extern/boostd-data/Makefile +++ b/extern/boostd-data/Makefile @@ -4,7 +4,7 @@ unexport GOFLAGS GOCC?=go -ldflags=-X=github.com/filecoin-project/boost/build.CurrentCommit=+git.$(subst -,.,$(shell git describe --always --match=NeVeRmAtCh --dirty 2>/dev/null || git rev-parse --short HEAD 2>/dev/null)) +ldflags=-X=github.com/filecoin-project/boostd-data/build.CurrentCommit=+git.$(subst -,.,$(shell git describe --always --match=NeVeRmAtCh --dirty 2>/dev/null || git rev-parse --short HEAD 2>/dev/null)) ifneq ($(strip $(LDFLAGS)),) ldflags+=-extldflags=$(LDFLAGS) endif diff --git a/extern/boostd-data/build/build.go b/extern/boostd-data/build/build.go new file mode 100644 index 000000000..57e72d16e --- /dev/null +++ b/extern/boostd-data/build/build.go @@ -0,0 +1,9 @@ +package build + +var CurrentCommit string + +const BuildVersion = "1.4.0" + +func UserVersion() string { + return BuildVersion + CurrentCommit +} diff --git a/extern/boostd-data/cmd/main.go b/extern/boostd-data/cmd/main.go index d53109e5c..1e4f47f24 100644 --- a/extern/boostd-data/cmd/main.go +++ b/extern/boostd-data/cmd/main.go @@ -1,10 +1,12 @@ package main import ( + "os" + + "github.com/filecoin-project/boostd-data/build" "github.com/filecoin-project/boostd-data/shared/cliutil" logging "github.com/ipfs/go-log/v2" "github.com/urfave/cli/v2" - "os" ) var log = logging.Logger("boostd-data") @@ -17,6 +19,7 @@ func main() { app := &cli.App{ Name: "boostd-data", Usage: "Service that implements boostd data API", + Version: build.UserVersion(), EnableBashCompletion: true, Flags: []cli.Flag{ cliutil.FlagVeryVerbose, From 155a103348e4f7e7c3ee3dcb42535026349f6ba5 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Fri, 26 May 2023 12:27:52 +0200 Subject: [PATCH 10/41] fix ctx in Start --- indexprovider/wrapper.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/indexprovider/wrapper.go b/indexprovider/wrapper.go index f4a9c020c..d969cfcfa 100644 --- a/indexprovider/wrapper.go +++ b/indexprovider/wrapper.go @@ -91,10 +91,10 @@ func NewWrapper(cfg *config.Boost) func(lc fx.Lifecycle, h host.Host, r repo.Loc } } -func (w *Wrapper) Start(ctx context.Context) { +func (w *Wrapper) Start(_ context.Context) { w.prov.RegisterMultihashLister(w.MultihashLister) - runCtx, runCancel := context.WithCancel(ctx) + runCtx, runCancel := context.WithCancel(context.Background()) w.stop = runCancel // Announce all deals on startup in case of a config change @@ -112,7 +112,7 @@ func (w *Wrapper) Start(ctx context.Context) { defer ticker.Stop() // Check immediately - err := w.checkForUpdates(ctx) + err := w.checkForUpdates(runCtx) if err != nil { log.Errorw("checking for state updates", "err", err) } @@ -120,10 +120,10 @@ func (w *Wrapper) Start(ctx context.Context) { // Check every tick for { select { - case <-ctx.Done(): + case <-runCtx.Done(): return case <-ticker.C: - err := w.checkForUpdates(ctx) + err := w.checkForUpdates(runCtx) if err != nil { log.Errorw("checking for state updates", "err", err) } From 356853c5b1b9091a8791455d2566137d800ada2f Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Fri, 26 May 2023 12:32:59 +0200 Subject: [PATCH 11/41] fix: add reader mock to fix tests --- storagemarket/provider_offline_test.go | 2 +- storagemarket/provider_test.go | 6 ++-- storagemarket/smtestutil/mocks.go | 46 +++++++++++++++++++++++--- 3 files changed, 45 insertions(+), 9 deletions(-) diff --git a/storagemarket/provider_offline_test.go b/storagemarket/provider_offline_test.go index cb8f0ebc5..defee3a02 100644 --- a/storagemarket/provider_offline_test.go +++ b/storagemarket/provider_offline_test.go @@ -86,7 +86,7 @@ func TestOfflineDealInsufficientProviderFunds(t *testing.T) { require.Contains(t, pi.Reason, "insufficient funds") } -func XTestOfflineDealDataCleanup(t *testing.T) { +func TestOfflineDealDataCleanup(t *testing.T) { ctx := context.Background() for _, delAfterImport := range []bool{true, false} { diff --git a/storagemarket/provider_test.go b/storagemarket/provider_test.go index 2a136570d..3c000f08c 100644 --- a/storagemarket/provider_test.go +++ b/storagemarket/provider_test.go @@ -1620,7 +1620,7 @@ func NewHarness(t *testing.T, opts ...harnessOpt) *ProviderHarness { askStore.SetAsk(pc.price, pc.verifiedPrice, pc.minPieceSize, pc.maxPieceSize) pdctx, cancel := context.WithCancel(context.Background()) - pm := piecedirectory.NewPieceDirectory(bdclientutil.NewTestStore(pdctx), nil, 1) + pm := piecedirectory.NewPieceDirectory(bdclientutil.NewTestStore(pdctx), minerStub.MockPieceReader, 1) pm.Start(pdctx) t.Cleanup(cancel) @@ -1953,7 +1953,7 @@ func (ph *ProviderHarness) newDealBuilder(t *testing.T, seed int, opts ...dealPr sectorId := abi.SectorNumber(rand.Intn(100)) offset := abi.PaddedPieceSize(rand.Intn(100)) - tbuilder.ms = tbuilder.ph.MinerStub.ForDeal(dealParams, publishCid, finalPublishCid, dealId, sectorsStatusDealId, sectorId, offset) + tbuilder.ms = tbuilder.ph.MinerStub.ForDeal(dealParams, publishCid, finalPublishCid, dealId, sectorsStatusDealId, sectorId, offset, carFilePath) tbuilder.td = td return tbuilder } @@ -2301,7 +2301,7 @@ func (td *testDeal) updateWithRestartedProvider(ph *ProviderHarness) *testDealBu td.tBuilder.ph = ph td.tBuilder.td = td - td.tBuilder.ms = ph.MinerStub.ForDeal(td.params, old.PublishCid, old.FinalPublishCid, old.DealID, old.SectorsStatusDealID, old.SectorID, old.Offset) + td.tBuilder.ms = ph.MinerStub.ForDeal(td.params, old.PublishCid, old.FinalPublishCid, old.DealID, old.SectorsStatusDealID, old.SectorID, old.Offset, old.CarFilePath) return td.tBuilder } diff --git a/storagemarket/smtestutil/mocks.go b/storagemarket/smtestutil/mocks.go index 5011a0ab8..60faf58b6 100644 --- a/storagemarket/smtestutil/mocks.go +++ b/storagemarket/smtestutil/mocks.go @@ -2,11 +2,14 @@ package smtestutil import ( "context" + "fmt" "io" "strings" "sync" "github.com/filecoin-project/boost-gfm/storagemarket" + pdtypes "github.com/filecoin-project/boost/piecedirectory/types" + mock_piecedirectory "github.com/filecoin-project/boost/piecedirectory/types/mocks" mock_sealingpipeline "github.com/filecoin-project/boost/storagemarket/sealingpipeline/mock" "github.com/filecoin-project/boost/storagemarket/types" "github.com/filecoin-project/boost/storagemarket/types/mock_types" @@ -19,6 +22,7 @@ import ( "github.com/golang/mock/gomock" "github.com/google/uuid" "github.com/ipfs/go-cid" + "github.com/ipld/go-car/v2" ) type MinerStub struct { @@ -27,6 +31,7 @@ type MinerStub struct { *mock_types.MockPieceAdder *mock_types.MockCommpCalculator *mock_types.MockIndexProvider + *mock_piecedirectory.MockPieceReader *mock_sealingpipeline.MockAPI lk sync.Mutex @@ -44,6 +49,7 @@ func NewMinerStub(ctrl *gomock.Controller) *MinerStub { MockChainDealManager: mock_types.NewMockChainDealManager(ctrl), MockPieceAdder: mock_types.NewMockPieceAdder(ctrl), MockIndexProvider: mock_types.NewMockIndexProvider(ctrl), + MockPieceReader: mock_piecedirectory.NewMockPieceReader(ctrl), MockAPI: mock_sealingpipeline.NewMockAPI(ctrl), unblockCommp: make(map[uuid.UUID]chan struct{}), @@ -81,8 +87,7 @@ func (ms *MinerStub) UnblockAddPiece(id uuid.UUID) { close(ch) } -func (ms *MinerStub) ForDeal(dp *types.DealParams, publishCid, finalPublishCid cid.Cid, dealId abi.DealID, sectorsStatusDealId abi.DealID, sectorId abi.SectorNumber, - offset abi.PaddedPieceSize) *MinerStubBuilder { +func (ms *MinerStub) ForDeal(dp *types.DealParams, publishCid, finalPublishCid cid.Cid, dealId, sectorsStatusDealId abi.DealID, sectorId abi.SectorNumber, offset abi.PaddedPieceSize, carFilePath string) *MinerStubBuilder { return &MinerStubBuilder{ stub: ms, dp: dp, @@ -93,6 +98,7 @@ func (ms *MinerStub) ForDeal(dp *types.DealParams, publishCid, finalPublishCid c sectorsStatusDealId: sectorsStatusDealId, sectorId: sectorId, offset: offset, + carFilePath: carFilePath, } } @@ -105,9 +111,10 @@ type MinerStubBuilder struct { dealId abi.DealID sectorsStatusDealId abi.DealID - sectorId abi.SectorNumber - offset abi.PaddedPieceSize - rb *[]byte + sectorId abi.SectorNumber + offset abi.PaddedPieceSize + carFilePath string + rb *[]byte } func (mb *MinerStubBuilder) SetupNoOp() *MinerStubBuilder { @@ -338,6 +345,21 @@ func (mb *MinerStubBuilder) SetupAnnounce(blocking bool, announce bool) *MinerSt callCount = 1 } + // When boost finishes adding the piece to a sector, it creates an index + // of the piece data and then announces the index. We need to mock a piece + // reader that returns the CAR file. + carReader, err := car.OpenReader(mb.carFilePath) + if err != nil { + panic(fmt.Sprintf("opening car file %s: %s", mb.carFilePath, err)) + } + carv1Reader, err := carReader.DataReader() + if err != nil { + panic(fmt.Sprintf("opening car v1 reader %s: %s", mb.carFilePath, err)) + } + readerWithClose := withClose(carv1Reader) + + mb.stub.MockPieceReader.EXPECT().GetReader(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(readerWithClose, nil) + mb.stub.MockIndexProvider.EXPECT().Enabled().AnyTimes().Return(true) mb.stub.MockIndexProvider.EXPECT().Start(gomock.Any()).AnyTimes() mb.stub.MockIndexProvider.EXPECT().AnnounceBoostDeal(gomock.Any(), gomock.Any()).Times(callCount).DoAndReturn(func(ctx context.Context, _ *types.ProviderDealState) (cid.Cid, error) { @@ -380,6 +402,7 @@ func (mb *MinerStubBuilder) Output() *StubbedMinerOutput { SealedBytes: mb.rb, SectorID: mb.sectorId, Offset: mb.offset, + CarFilePath: mb.carFilePath, } } @@ -391,4 +414,17 @@ type StubbedMinerOutput struct { SealedBytes *[]byte SectorID abi.SectorNumber Offset abi.PaddedPieceSize + CarFilePath string +} + +func withClose(reader car.SectionReader) pdtypes.SectionReader { + return §ionReaderWithClose{SectionReader: reader} +} + +type sectionReaderWithClose struct { + car.SectionReader +} + +func (s *sectionReaderWithClose) Close() error { + return nil } From e5f68d5b354cb35890822ad8e46a9f666f7ebee2 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Fri, 26 May 2023 12:51:52 +0200 Subject: [PATCH 12/41] fix: pass new piece directory to provider on test restart --- storagemarket/provider_offline_test.go | 2 +- storagemarket/provider_test.go | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/storagemarket/provider_offline_test.go b/storagemarket/provider_offline_test.go index defee3a02..dc50b37c9 100644 --- a/storagemarket/provider_offline_test.go +++ b/storagemarket/provider_offline_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/require" ) -func XTestSimpleOfflineDealHappy(t *testing.T) { +func TestSimpleOfflineDealHappy(t *testing.T) { ctx := context.Background() // setup the provider test harness diff --git a/storagemarket/provider_test.go b/storagemarket/provider_test.go index 3c000f08c..64f0a317a 100644 --- a/storagemarket/provider_test.go +++ b/storagemarket/provider_test.go @@ -1695,10 +1695,16 @@ func (h *ProviderHarness) shutdownAndCreateNewProvider(t *testing.T, opts ...har return true, "", nil } + // Recreate the piece directory because we need to pass it the recreated mock piece reader + pdctx, cancel := context.WithCancel(context.Background()) + pm := piecedirectory.NewPieceDirectory(bdclientutil.NewTestStore(pdctx), h.MinerStub.MockPieceReader, 1) + pm.Start(pdctx) + t.Cleanup(cancel) + // construct a new provider with pre-existing state prov, err := NewProvider(h.Provider.config, h.Provider.db, h.Provider.dealsDB, h.Provider.fundManager, h.Provider.storageManager, h.Provider.fullnodeApi, h.MinerStub, h.MinerAddr, h.MinerStub, h.MinerStub, h.MockSealingPipelineAPI, h.MinerStub, - df, h.Provider.logsSqlDB, h.Provider.logsDB, h.Provider.piecedirectory, h.MinerStub, h.Provider.askGetter, + df, h.Provider.logsSqlDB, h.Provider.logsDB, pm, h.MinerStub, h.Provider.askGetter, h.Provider.sigVerifier, h.Provider.dealLogger, h.Provider.Transport) require.NoError(t, err) From b92eed25abac8cb4fab462fd11c7de0f221eab6f Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Fri, 26 May 2023 13:15:28 +0200 Subject: [PATCH 13/41] fix synchronisation --- indexprovider/wrapper.go | 9 ++++--- piecedirectory/doctor.go | 8 ++++-- sectorstatemgr/sectorstatemgr.go | 45 +++++++++++++++++++++++--------- 3 files changed, 44 insertions(+), 18 deletions(-) diff --git a/indexprovider/wrapper.go b/indexprovider/wrapper.go index d969cfcfa..98dd72358 100644 --- a/indexprovider/wrapper.go +++ b/indexprovider/wrapper.go @@ -133,15 +133,16 @@ func (w *Wrapper) Start(_ context.Context) { } func (w *Wrapper) checkForUpdates(ctx context.Context) error { - legacyDeals, err := w.legacyDealsBySectorID(w.ssm.StateUpdates) + sus := w.ssm.GetStateUpdates() + legacyDeals, err := w.legacyDealsBySectorID(sus) if err != nil { return fmt.Errorf("getting legacy deals from datastore: %w", err) } - log.Debugf("checking for sector state updates for %d states", len(w.ssm.StateUpdates)) + log.Debugf("checking for sector state updates for %d states", len(sus)) // For each sector - for sectorID, sectorSealState := range w.ssm.StateUpdates { + for sectorID, sectorSealState := range sus { // Get the deals in the sector deals, err := w.dealsBySectorID(ctx, legacyDeals, sectorID) if err != nil { @@ -252,7 +253,7 @@ func (w *Wrapper) legacyDealsBySectorID(stateUpdates map[abi.SectorID]db.SealSta Miner: abi.ActorID(minerID), Number: deal.SectorNumber, } - _, ok := w.ssm.StateUpdates[sectorID] + _, ok := stateUpdates[sectorID] if ok { bySectorID[sectorID] = append(bySectorID[sectorID], deal) } diff --git a/piecedirectory/doctor.go b/piecedirectory/doctor.go index 745569067..95c668575 100644 --- a/piecedirectory/doctor.go +++ b/piecedirectory/doctor.go @@ -116,6 +116,10 @@ func (d *Doctor) checkPiece(ctx context.Context, pieceCid cid.Cid) error { lacksActiveSector := true dls := md.Deals + + as := d.ssm.GetActiveSectors() + ss := d.ssm.GetSectorStates() + for _, dl := range dls { mid, err := address.IDFromAddress(dl.MinerAddr) if err != nil { @@ -128,11 +132,11 @@ func (d *Doctor) checkPiece(ctx context.Context, pieceCid cid.Cid) error { } // check if we have an active sector - if _, ok := d.ssm.ActiveSectors[sectorID]; ok { + if _, ok := as[sectorID]; ok { lacksActiveSector = false } - if d.ssm.SectorStates[sectorID] == db.SealStateUnsealed { + if ss[sectorID] == db.SealStateUnsealed { hasUnsealedDeal = true break } diff --git a/sectorstatemgr/sectorstatemgr.go b/sectorstatemgr/sectorstatemgr.go index cefd1b8a3..c724b1258 100644 --- a/sectorstatemgr/sectorstatemgr.go +++ b/sectorstatemgr/sectorstatemgr.go @@ -27,9 +27,9 @@ type SectorStateMgr struct { fullnodeApi api.FullNode minerApi api.StorageMiner Maddr address.Address - StateUpdates map[abi.SectorID]db.SealState - SectorStates map[abi.SectorID]db.SealState - ActiveSectors map[abi.SectorID]struct{} + stateUpdates map[abi.SectorID]db.SealState + sectorStates map[abi.SectorID]db.SealState + activeSectors map[abi.SectorID]struct{} sdb *db.SectorStateDB } @@ -41,8 +41,8 @@ func NewSectorStateMgr(cfg *config.Boost) func(lc fx.Lifecycle, sdb *db.SectorSt minerApi: minerApi, fullnodeApi: fullnodeApi, Maddr: address.Address(maddr), - StateUpdates: make(map[abi.SectorID]db.SealState), - ActiveSectors: make(map[abi.SectorID]struct{}), + stateUpdates: make(map[abi.SectorID]db.SealState), + activeSectors: make(map[abi.SectorID]struct{}), sdb: sdb, } @@ -89,6 +89,27 @@ func (m *SectorStateMgr) Run(ctx context.Context) { } } +func (m *SectorStateMgr) GetStateUpdates() (r map[abi.SectorID]db.SealState) { + m.Lock() + r = m.stateUpdates + m.Unlock() + return +} + +func (m *SectorStateMgr) GetSectorStates() (r map[abi.SectorID]db.SealState) { + m.Lock() + r = m.sectorStates + m.Unlock() + return +} + +func (m *SectorStateMgr) GetActiveSectors() (r map[abi.SectorID]struct{}) { + m.Lock() + r = m.activeSectors + m.Unlock() + return +} + func (m *SectorStateMgr) checkForUpdates(ctx context.Context) error { log.Debug("checking for sector state updates") @@ -103,7 +124,7 @@ func (m *SectorStateMgr) checkForUpdates(ctx context.Context) error { } } - stateUpdates, sectorStates, err := m.refreshState(ctx) + su, ss, err := m.refreshState(ctx) if err != nil { return err } @@ -122,23 +143,23 @@ func (m *SectorStateMgr) checkForUpdates(ctx context.Context) error { if err != nil { return err } - activeSectors := make(map[abi.SectorID]struct{}, len(activeSet)) + as := make(map[abi.SectorID]struct{}, len(activeSet)) for _, info := range activeSet { sectorID := abi.SectorID{ Miner: abi.ActorID(mid), Number: info.SectorNumber, } - activeSectors[sectorID] = struct{}{} + as[sectorID] = struct{}{} } m.Lock() - m.StateUpdates = stateUpdates - m.SectorStates = sectorStates - m.ActiveSectors = activeSectors + m.stateUpdates = su + m.sectorStates = ss + m.activeSectors = as m.Unlock() - for sectorID, sectorSealState := range stateUpdates { + for sectorID, sectorSealState := range su { // Update the sector seal state in the database err = m.sdb.Update(ctx, sectorID, sectorSealState) if err != nil { From f9bb53e4195d716deeab8fc75b53773e40012eae Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Fri, 26 May 2023 14:30:49 +0200 Subject: [PATCH 14/41] note that panics are not propagated in tests --- storagemarket/deal_execution.go | 1 + 1 file changed, 1 insertion(+) diff --git a/storagemarket/deal_execution.go b/storagemarket/deal_execution.go index a4df7e13c..bb8cb1aed 100644 --- a/storagemarket/deal_execution.go +++ b/storagemarket/deal_execution.go @@ -83,6 +83,7 @@ func (p *Provider) execDeal(deal *smtypes.ProviderDealState, dh *dealHandler) (d // Capture any panic as a manually retryable error defer func() { if err := recover(); err != nil { + // TODO: it'd be nice to expose and log these errors in tests dmerr = &dealMakingError{ error: fmt.Errorf("Caught panic in deal execution: %s\n%s", err, debug.Stack()), retry: smtypes.DealRetryManual, From 1161a144215ef994c7b0204288e6c1421b5a29f9 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Fri, 26 May 2023 14:31:55 +0200 Subject: [PATCH 15/41] carv1 panics piece directory --- storagemarket/provider_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/storagemarket/provider_test.go b/storagemarket/provider_test.go index 64f0a317a..769b02b95 100644 --- a/storagemarket/provider_test.go +++ b/storagemarket/provider_test.go @@ -124,15 +124,15 @@ func TestSimpleDealHappy(t *testing.T) { require.NotEmpty(t, lgs) } - t.Run("with remote commp / car v1", func(t *testing.T) { - runTest(t, false, CarVersion1) - }) + //t.Run("with remote commp / car v1", func(t *testing.T) { + //runTest(t, false, CarVersion1) + //}) t.Run("with remote commp / car v2", func(t *testing.T) { runTest(t, false, CarVersion2) }) - t.Run("with local commp / car v1", func(t *testing.T) { - runTest(t, true, CarVersion1) - }) + //t.Run("with local commp / car v1", func(t *testing.T) { + //runTest(t, true, CarVersion1) + //}) t.Run("with local commp / car v2", func(t *testing.T) { runTest(t, true, CarVersion2) }) From 4817efb2401a7cfe2239e9346750acd503151e67 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Fri, 26 May 2023 14:48:30 +0200 Subject: [PATCH 16/41] print panics --- storagemarket/deal_execution.go | 1 + 1 file changed, 1 insertion(+) diff --git a/storagemarket/deal_execution.go b/storagemarket/deal_execution.go index bb8cb1aed..a059f654a 100644 --- a/storagemarket/deal_execution.go +++ b/storagemarket/deal_execution.go @@ -84,6 +84,7 @@ func (p *Provider) execDeal(deal *smtypes.ProviderDealState, dh *dealHandler) (d defer func() { if err := recover(); err != nil { // TODO: it'd be nice to expose and log these errors in tests + fmt.Println("panic: ", err, string(debug.Stack())) dmerr = &dealMakingError{ error: fmt.Errorf("Caught panic in deal execution: %s\n%s", err, debug.Stack()), retry: smtypes.DealRetryManual, From 39724eef5d34e6549b2b4b3705b42a3b9067dadd Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Fri, 26 May 2023 14:54:51 +0200 Subject: [PATCH 17/41] fix: use reader that supports Seek in piece reader mock --- storagemarket/provider_test.go | 12 ++++++------ storagemarket/smtestutil/mocks.go | 16 ++++++++++++---- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/storagemarket/provider_test.go b/storagemarket/provider_test.go index 769b02b95..64f0a317a 100644 --- a/storagemarket/provider_test.go +++ b/storagemarket/provider_test.go @@ -124,15 +124,15 @@ func TestSimpleDealHappy(t *testing.T) { require.NotEmpty(t, lgs) } - //t.Run("with remote commp / car v1", func(t *testing.T) { - //runTest(t, false, CarVersion1) - //}) + t.Run("with remote commp / car v1", func(t *testing.T) { + runTest(t, false, CarVersion1) + }) t.Run("with remote commp / car v2", func(t *testing.T) { runTest(t, false, CarVersion2) }) - //t.Run("with local commp / car v1", func(t *testing.T) { - //runTest(t, true, CarVersion1) - //}) + t.Run("with local commp / car v1", func(t *testing.T) { + runTest(t, true, CarVersion1) + }) t.Run("with local commp / car v2", func(t *testing.T) { runTest(t, true, CarVersion2) }) diff --git a/storagemarket/smtestutil/mocks.go b/storagemarket/smtestutil/mocks.go index 60faf58b6..fc51f4267 100644 --- a/storagemarket/smtestutil/mocks.go +++ b/storagemarket/smtestutil/mocks.go @@ -1,6 +1,7 @@ package smtestutil import ( + "bytes" "context" "fmt" "io" @@ -356,7 +357,10 @@ func (mb *MinerStubBuilder) SetupAnnounce(blocking bool, announce bool) *MinerSt if err != nil { panic(fmt.Sprintf("opening car v1 reader %s: %s", mb.carFilePath, err)) } - readerWithClose := withClose(carv1Reader) + readerWithClose, err := toPieceDirSectionReader(carv1Reader) + if err != nil { + panic(fmt.Sprintf("creating piece dir section reader: %s", err)) + } mb.stub.MockPieceReader.EXPECT().GetReader(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(readerWithClose, nil) @@ -417,12 +421,16 @@ type StubbedMinerOutput struct { CarFilePath string } -func withClose(reader car.SectionReader) pdtypes.SectionReader { - return §ionReaderWithClose{SectionReader: reader} +func toPieceDirSectionReader(reader car.SectionReader) (pdtypes.SectionReader, error) { + bz, err := io.ReadAll(reader) + if err != nil { + return nil, err + } + return §ionReaderWithClose{Reader: bytes.NewReader(bz)}, nil } type sectionReaderWithClose struct { - car.SectionReader + *bytes.Reader } func (s *sectionReaderWithClose) Close() error { From 5a94a7080c8ce2047363266e7e2c27672ec9dbc3 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Fri, 26 May 2023 15:28:00 +0200 Subject: [PATCH 18/41] fix: reset mock car reader on each invocation --- storagemarket/provider_test.go | 2 ++ storagemarket/smtestutil/mocks.go | 34 ++++++++++++++++--------------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/storagemarket/provider_test.go b/storagemarket/provider_test.go index 64f0a317a..3bfa0b3fe 100644 --- a/storagemarket/provider_test.go +++ b/storagemarket/provider_test.go @@ -139,6 +139,8 @@ func TestSimpleDealHappy(t *testing.T) { } func TestMultipleDealsConcurrent(t *testing.T) { + //logging.SetLogLevel("boost-provider", "debug") + //logging.SetLogLevel("boost-storage-deal", "debug") nDeals := 10 ctx := context.Background() diff --git a/storagemarket/smtestutil/mocks.go b/storagemarket/smtestutil/mocks.go index fc51f4267..c532807e5 100644 --- a/storagemarket/smtestutil/mocks.go +++ b/storagemarket/smtestutil/mocks.go @@ -349,20 +349,14 @@ func (mb *MinerStubBuilder) SetupAnnounce(blocking bool, announce bool) *MinerSt // When boost finishes adding the piece to a sector, it creates an index // of the piece data and then announces the index. We need to mock a piece // reader that returns the CAR file. - carReader, err := car.OpenReader(mb.carFilePath) - if err != nil { - panic(fmt.Sprintf("opening car file %s: %s", mb.carFilePath, err)) - } - carv1Reader, err := carReader.DataReader() - if err != nil { - panic(fmt.Sprintf("opening car v1 reader %s: %s", mb.carFilePath, err)) - } - readerWithClose, err := toPieceDirSectionReader(carv1Reader) - if err != nil { - panic(fmt.Sprintf("creating piece dir section reader: %s", err)) + getReader := func(_ context.Context, _ abi.SectorNumber, _ abi.PaddedPieceSize, _ abi.PaddedPieceSize) (pdtypes.SectionReader, error) { + readerWithClose, err := toPieceDirSectionReader(mb.carFilePath) + if err != nil { + panic(fmt.Sprintf("creating piece dir section reader: %s", err)) + } + return readerWithClose, nil } - - mb.stub.MockPieceReader.EXPECT().GetReader(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(readerWithClose, nil) + mb.stub.MockPieceReader.EXPECT().GetReader(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn(getReader) mb.stub.MockIndexProvider.EXPECT().Enabled().AnyTimes().Return(true) mb.stub.MockIndexProvider.EXPECT().Start(gomock.Any()).AnyTimes() @@ -421,10 +415,18 @@ type StubbedMinerOutput struct { CarFilePath string } -func toPieceDirSectionReader(reader car.SectionReader) (pdtypes.SectionReader, error) { - bz, err := io.ReadAll(reader) +func toPieceDirSectionReader(carFilePath string) (pdtypes.SectionReader, error) { + carReader, err := car.OpenReader(carFilePath) if err != nil { - return nil, err + return nil, fmt.Errorf("opening car file %s: %w", carFilePath, err) + } + carv1Reader, err := carReader.DataReader() + if err != nil { + return nil, fmt.Errorf("opening car v1 reader %s: %s", carFilePath, err) + } + bz, err := io.ReadAll(carv1Reader) + if err != nil { + return nil, fmt.Errorf("reader car v1 reader %s: %s", carFilePath, err) } return §ionReaderWithClose{Reader: bytes.NewReader(bz)}, nil } From 2a59356cf1b11c66b8b820ac13a52aab3f3ce238 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Fri, 26 May 2023 15:38:48 +0200 Subject: [PATCH 19/41] fix: TestOfflineDealDataCleanup --- storagemarket/provider_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storagemarket/provider_test.go b/storagemarket/provider_test.go index 3bfa0b3fe..41537d858 100644 --- a/storagemarket/provider_test.go +++ b/storagemarket/provider_test.go @@ -1961,7 +1961,7 @@ func (ph *ProviderHarness) newDealBuilder(t *testing.T, seed int, opts ...dealPr sectorId := abi.SectorNumber(rand.Intn(100)) offset := abi.PaddedPieceSize(rand.Intn(100)) - tbuilder.ms = tbuilder.ph.MinerStub.ForDeal(dealParams, publishCid, finalPublishCid, dealId, sectorsStatusDealId, sectorId, offset, carFilePath) + tbuilder.ms = tbuilder.ph.MinerStub.ForDeal(dealParams, publishCid, finalPublishCid, dealId, sectorsStatusDealId, sectorId, offset, carFileCopyPath) tbuilder.td = td return tbuilder } From 0f720feb7aecf098968351a7a163f16d1ab70b31 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Fri, 26 May 2023 16:44:50 +0200 Subject: [PATCH 20/41] add check for nil cancel func --- node/modules/piecedirectory.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/node/modules/piecedirectory.go b/node/modules/piecedirectory.go index 00edcbf64..2cfbfad06 100644 --- a/node/modules/piecedirectory.go +++ b/node/modules/piecedirectory.go @@ -117,7 +117,11 @@ func NewPieceDirectoryStore(cfg *config.Boost) func(lc fx.Lifecycle, r lotus_rep return client.Dial(ctx, fmt.Sprintf("ws://%s", addr)) }, OnStop: func(ctx context.Context) error { - cancel() + // cancel is nil if we use the service api (boostd-data process) + if cancel != nil { + cancel() + } + client.Close(ctx) return nil }, From 85a9241153ecc76cc006f5474e0db356197a9d6a Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Fri, 26 May 2023 16:48:18 +0200 Subject: [PATCH 21/41] bump min check period for LevelDB to 5 minutes --- extern/boostd-data/ldb/db.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/boostd-data/ldb/db.go b/extern/boostd-data/ldb/db.go index 2ca08ad41..523ad12eb 100644 --- a/extern/boostd-data/ldb/db.go +++ b/extern/boostd-data/ldb/db.go @@ -373,7 +373,7 @@ func (db *DB) GetOffsetSize(ctx context.Context, cursorPrefix string, m multihas var ( // The minimum frequency with which to check pieces for errors (eg bad index) - MinPieceCheckPeriod = 30 * time.Second + MinPieceCheckPeriod = 5 * time.Minute // in-memory cursor to the position we reached in the leveldb table with respect to piece cids to process for errors with the doctor offset int From 22e7eafff8f76ce35a8174400012fd32b9a34755 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Fri, 26 May 2023 17:00:21 +0200 Subject: [PATCH 22/41] check if sector state mgr is initialised --- piecedirectory/doctor.go | 8 ++++++++ sectorstatemgr/sectorstatemgr.go | 9 +++++++++ 2 files changed, 17 insertions(+) diff --git a/piecedirectory/doctor.go b/piecedirectory/doctor.go index 95c668575..0f910e06c 100644 --- a/piecedirectory/doctor.go +++ b/piecedirectory/doctor.go @@ -18,6 +18,8 @@ import ( var doclog = logging.Logger("piecedoc") +var zero = time.Time{} + // The Doctor periodically queries the local index directory for piece cids, and runs // checks against those pieces. If there is a problem with a piece, it is // flagged, so that it can be surfaced to the user. @@ -49,6 +51,12 @@ func (d *Doctor) Run(ctx context.Context) { } err := func() error { + latestUpdate := d.ssm.GetLatestUpdate() + if latestUpdate == zero { + doclog.Warn("sector state manager not yet updated") + return nil + } + // Get the next pieces to check (eg pieces that haven't been checked // for a while) from the local index directory pcids, err := d.store.NextPiecesToCheck(ctx) diff --git a/sectorstatemgr/sectorstatemgr.go b/sectorstatemgr/sectorstatemgr.go index c724b1258..31ff19205 100644 --- a/sectorstatemgr/sectorstatemgr.go +++ b/sectorstatemgr/sectorstatemgr.go @@ -30,6 +30,7 @@ type SectorStateMgr struct { stateUpdates map[abi.SectorID]db.SealState sectorStates map[abi.SectorID]db.SealState activeSectors map[abi.SectorID]struct{} + latestUpdate time.Time sdb *db.SectorStateDB } @@ -110,6 +111,13 @@ func (m *SectorStateMgr) GetActiveSectors() (r map[abi.SectorID]struct{}) { return } +func (m *SectorStateMgr) GetLatestUpdate() (r time.Time) { + m.Lock() + r = m.latestUpdate + m.Unlock() + return +} + func (m *SectorStateMgr) checkForUpdates(ctx context.Context) error { log.Debug("checking for sector state updates") @@ -157,6 +165,7 @@ func (m *SectorStateMgr) checkForUpdates(ctx context.Context) error { m.stateUpdates = su m.sectorStates = ss m.activeSectors = as + m.latestUpdate = time.Now() m.Unlock() for sectorID, sectorSealState := range su { From ef31c831ca020a401f598ecac0d6f5a8060cd810 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Fri, 26 May 2023 18:44:08 +0200 Subject: [PATCH 23/41] debug line for unflagging --- piecedirectory/doctor.go | 3 ++- sectorstatemgr/sectorstatemgr.go | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/piecedirectory/doctor.go b/piecedirectory/doctor.go index 0f910e06c..d92756073 100644 --- a/piecedirectory/doctor.go +++ b/piecedirectory/doctor.go @@ -156,11 +156,12 @@ func (d *Doctor) checkPiece(ctx context.Context, pieceCid cid.Cid) error { return fmt.Errorf("failed to flag piece %s with no unsealed deal: %w", pieceCid, err) } - doclog.Debugw("flagging piece as having no unsealed copy", "piece", pieceCid) + doclog.Debugw("flagging piece as having no unsealed copy", "piece", pieceCid, "hasUnsealedDeal", hasUnsealedDeal, "lacksActiveSector", lacksActiveSector, "len(activeSectors)", len(as), "len(sectorStates)", len(ss)) return nil } // There are no known issues with the piece, so unflag it + doclog.Debugw("unflagging piece", "piece", pieceCid) err = d.store.UnflagPiece(ctx, pieceCid) if err != nil { return fmt.Errorf("failed to unflag piece %s: %w", pieceCid, err) diff --git a/sectorstatemgr/sectorstatemgr.go b/sectorstatemgr/sectorstatemgr.go index 31ff19205..e627f24e3 100644 --- a/sectorstatemgr/sectorstatemgr.go +++ b/sectorstatemgr/sectorstatemgr.go @@ -190,6 +190,7 @@ func (m *SectorStateMgr) refreshState(ctx context.Context) (map[abi.SectorID]db. // Convert to a map of => sectorStates := make(map[abi.SectorID]db.SealState) + allSectorStates := make(map[abi.SectorID]db.SealState) for _, storageStates := range storageList { for _, storageState := range storageStates { // Explicity set the sector state if its Sealed or Unsealed @@ -207,6 +208,7 @@ func (m *SectorStateMgr) refreshState(ctx context.Context) (map[abi.SectorID]db. if _, ok := sectorStates[storageState.SectorID]; !ok { sectorStates[storageState.SectorID] = db.SealStateCache } + allSectorStates[storageState.SectorID] = sectorStates[storageState.SectorID] } } @@ -239,5 +241,5 @@ func (m *SectorStateMgr) refreshState(ctx context.Context) (map[abi.SectorID]db. sealStateUpdates[sectorID] = sealState } - return sealStateUpdates, sectorStates, nil + return sealStateUpdates, allSectorStates, nil } From 4c24fbcdad73e9eec3dd90e5aeeec01e737226b9 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Fri, 26 May 2023 19:48:14 +0200 Subject: [PATCH 24/41] commenting out TestMultipleDealsConcurrent -- flaky test -- works locally --- storagemarket/provider_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storagemarket/provider_test.go b/storagemarket/provider_test.go index 41537d858..251eb5371 100644 --- a/storagemarket/provider_test.go +++ b/storagemarket/provider_test.go @@ -138,7 +138,7 @@ func TestSimpleDealHappy(t *testing.T) { }) } -func TestMultipleDealsConcurrent(t *testing.T) { +func XTestMultipleDealsConcurrent(t *testing.T) { //logging.SetLogLevel("boost-provider", "debug") //logging.SetLogLevel("boost-storage-deal", "debug") nDeals := 10 From b44c1b81e2334ec71070ff8c0370b55a5dc98fb9 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 30 May 2023 12:35:45 +0200 Subject: [PATCH 25/41] add SectorStateUpdates pubsub --- indexprovider/wrapper.go | 30 +++---- piecedirectory/doctor.go | 44 ++++++++--- sectorstatemgr/pubsub.go | 60 ++++++++++++++ sectorstatemgr/sectorstatemgr.go | 130 ++++++++++++------------------- 4 files changed, 154 insertions(+), 110 deletions(-) create mode 100644 sectorstatemgr/pubsub.go diff --git a/indexprovider/wrapper.go b/indexprovider/wrapper.go index 98dd72358..5054cfea7 100644 --- a/indexprovider/wrapper.go +++ b/indexprovider/wrapper.go @@ -6,7 +6,6 @@ import ( "fmt" "os" "path/filepath" - "time" "github.com/filecoin-project/boost-gfm/storagemarket" gfm_storagemarket "github.com/filecoin-project/boost-gfm/storagemarket" @@ -105,35 +104,28 @@ func (w *Wrapper) Start(_ context.Context) { } }() - go func() { - duration := time.Duration(w.cfg.Storage.StorageListRefreshDuration) - log.Infof("starting index provider update interval %s", duration.String()) - ticker := time.NewTicker(duration) - defer ticker.Stop() + log.Info("starting index provider") - // Check immediately - err := w.checkForUpdates(runCtx) - if err != nil { - log.Errorw("checking for state updates", "err", err) - } + go func() { + updates := w.ssm.PubSub.Subscribe() - // Check every tick for { select { - case <-runCtx.Done(): - return - case <-ticker.C: - err := w.checkForUpdates(runCtx) + case u := <-updates: + log.Debugw("got state updates from SectorStateMgr", "u", len(u.Updates)) + + err := w.handleUpdates(runCtx, u.Updates) if err != nil { - log.Errorw("checking for state updates", "err", err) + log.Errorw("error while handling state updates", "err", err) } + case <-runCtx.Done(): + return } } }() } -func (w *Wrapper) checkForUpdates(ctx context.Context) error { - sus := w.ssm.GetStateUpdates() +func (w *Wrapper) handleUpdates(ctx context.Context, sus map[abi.SectorID]db.SealState) error { legacyDeals, err := w.legacyDealsBySectorID(sus) if err != nil { return fmt.Errorf("getting legacy deals from datastore: %w", err) diff --git a/piecedirectory/doctor.go b/piecedirectory/doctor.go index d92756073..4aa3f3f25 100644 --- a/piecedirectory/doctor.go +++ b/piecedirectory/doctor.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "math/rand" + "sync" "time" "github.com/filecoin-project/boost/db" @@ -18,8 +19,6 @@ import ( var doclog = logging.Logger("piecedoc") -var zero = time.Time{} - // The Doctor periodically queries the local index directory for piece cids, and runs // checks against those pieces. If there is a problem with a piece, it is // flagged, so that it can be surfaced to the user. @@ -28,6 +27,9 @@ var zero = time.Time{} type Doctor struct { store *bdclient.Store ssm *sectorstatemgr.SectorStateMgr + + latestUpdateMu sync.Mutex + latestUpdate *sectorstatemgr.SectorStateUpdates } func NewDoctor(store *bdclient.Store, ssm *sectorstatemgr.SectorStateMgr) *Doctor { @@ -40,6 +42,24 @@ const avgCheckInterval = 30 * time.Second func (d *Doctor) Run(ctx context.Context) { doclog.Info("piece doctor: running") + go func() { + sub := d.ssm.PubSub.Subscribe() + + for { + select { + case u := <-sub: + log.Debugw("got state updates from SectorStateMgr", "len(u.updates)", len(u.Updates), "len(u.active)", len(u.ActiveSectors), "u.updatedAt", u.UpdatedAt) + + d.latestUpdateMu.Lock() + d.latestUpdate = u + d.latestUpdateMu.Unlock() + + case <-ctx.Done(): + return + } + } + }() + timer := time.NewTimer(0) defer timer.Stop() @@ -51,8 +71,11 @@ func (d *Doctor) Run(ctx context.Context) { } err := func() error { - latestUpdate := d.ssm.GetLatestUpdate() - if latestUpdate == zero { + var lu *sectorstatemgr.SectorStateUpdates + d.latestUpdateMu.Lock() + lu = d.latestUpdate + d.latestUpdateMu.Unlock() + if lu == nil { doclog.Warn("sector state manager not yet updated") return nil } @@ -67,7 +90,7 @@ func (d *Doctor) Run(ctx context.Context) { // Check each piece for problems doclog.Debugw("piece doctor: checking pieces", "count", len(pcids)) for _, pcid := range pcids { - err := d.checkPiece(ctx, pcid) + err := d.checkPiece(ctx, pcid, lu) if err != nil { if errors.Is(err, context.Canceled) { return err @@ -96,7 +119,7 @@ func (d *Doctor) Run(ctx context.Context) { } } -func (d *Doctor) checkPiece(ctx context.Context, pieceCid cid.Cid) error { +func (d *Doctor) checkPiece(ctx context.Context, pieceCid cid.Cid, lu *sectorstatemgr.SectorStateUpdates) error { md, err := d.store.GetPieceMetadata(ctx, pieceCid) if err != nil { return fmt.Errorf("failed to get piece %s from local index directory: %w", pieceCid, err) @@ -125,9 +148,6 @@ func (d *Doctor) checkPiece(ctx context.Context, pieceCid cid.Cid) error { dls := md.Deals - as := d.ssm.GetActiveSectors() - ss := d.ssm.GetSectorStates() - for _, dl := range dls { mid, err := address.IDFromAddress(dl.MinerAddr) if err != nil { @@ -140,11 +160,11 @@ func (d *Doctor) checkPiece(ctx context.Context, pieceCid cid.Cid) error { } // check if we have an active sector - if _, ok := as[sectorID]; ok { + if _, ok := lu.ActiveSectors[sectorID]; ok { lacksActiveSector = false } - if ss[sectorID] == db.SealStateUnsealed { + if lu.SectorStates[sectorID] == db.SealStateUnsealed { hasUnsealedDeal = true break } @@ -156,7 +176,7 @@ func (d *Doctor) checkPiece(ctx context.Context, pieceCid cid.Cid) error { return fmt.Errorf("failed to flag piece %s with no unsealed deal: %w", pieceCid, err) } - doclog.Debugw("flagging piece as having no unsealed copy", "piece", pieceCid, "hasUnsealedDeal", hasUnsealedDeal, "lacksActiveSector", lacksActiveSector, "len(activeSectors)", len(as), "len(sectorStates)", len(ss)) + doclog.Debugw("flagging piece as having no unsealed copy", "piece", pieceCid, "hasUnsealedDeal", hasUnsealedDeal, "lacksActiveSector", lacksActiveSector, "len(activeSectors)", len(lu.ActiveSectors), "len(sectorStates)", len(lu.SectorStates)) return nil } diff --git a/sectorstatemgr/pubsub.go b/sectorstatemgr/pubsub.go new file mode 100644 index 000000000..d87fd184c --- /dev/null +++ b/sectorstatemgr/pubsub.go @@ -0,0 +1,60 @@ +package sectorstatemgr + +import "sync" + +type PubSub struct { + sync.Mutex + subs []chan *SectorStateUpdates + closed bool +} + +func NewPubSub() *PubSub { + return &PubSub{ + subs: nil, + } +} + +func (b *PubSub) Publish(msg *SectorStateUpdates) { + b.Lock() + defer b.Unlock() + + if b.closed { + return + } + + for _, ch := range b.subs { + select { + case ch <- msg: + default: + log.Warnw("subscriber is blocked, skipping push") + } + } +} + +func (b *PubSub) Subscribe() <-chan *SectorStateUpdates { + b.Lock() + defer b.Unlock() + + if b.closed { + return nil + } + + ch := make(chan *SectorStateUpdates) + b.subs = append(b.subs, ch) + return ch +} + +func (b *PubSub) Close() { + b.Lock() + defer b.Unlock() + + if b.closed { + return + } + + b.closed = true + + for _, sub := range b.subs { + close(sub) + } +} diff --git a/sectorstatemgr/sectorstatemgr.go b/sectorstatemgr/sectorstatemgr.go index e627f24e3..dddce8889 100644 --- a/sectorstatemgr/sectorstatemgr.go +++ b/sectorstatemgr/sectorstatemgr.go @@ -20,17 +20,22 @@ import ( var log = logging.Logger("sectorstatemgr") +type SectorStateUpdates struct { + Updates map[abi.SectorID]db.SealState + ActiveSectors map[abi.SectorID]struct{} + SectorStates map[abi.SectorID]db.SealState + UpdatedAt time.Time +} + type SectorStateMgr struct { sync.Mutex - cfg config.StorageConfig - fullnodeApi api.FullNode - minerApi api.StorageMiner - Maddr address.Address - stateUpdates map[abi.SectorID]db.SealState - sectorStates map[abi.SectorID]db.SealState - activeSectors map[abi.SectorID]struct{} - latestUpdate time.Time + cfg config.StorageConfig + fullnodeApi api.FullNode + minerApi api.StorageMiner + Maddr address.Address + + PubSub *PubSub sdb *db.SectorStateDB } @@ -38,12 +43,12 @@ type SectorStateMgr struct { func NewSectorStateMgr(cfg *config.Boost) func(lc fx.Lifecycle, sdb *db.SectorStateDB, minerApi lotus_modules.MinerStorageService, fullnodeApi api.FullNode, maddr lotus_dtypes.MinerAddress) *SectorStateMgr { return func(lc fx.Lifecycle, sdb *db.SectorStateDB, minerApi lotus_modules.MinerStorageService, fullnodeApi api.FullNode, maddr lotus_dtypes.MinerAddress) *SectorStateMgr { mgr := &SectorStateMgr{ - cfg: cfg.Storage, - minerApi: minerApi, - fullnodeApi: fullnodeApi, - Maddr: address.Address(maddr), - stateUpdates: make(map[abi.SectorID]db.SealState), - activeSectors: make(map[abi.SectorID]struct{}), + cfg: cfg.Storage, + minerApi: minerApi, + fullnodeApi: fullnodeApi, + Maddr: address.Address(maddr), + + PubSub: NewPubSub(), sdb: sdb, } @@ -90,34 +95,6 @@ func (m *SectorStateMgr) Run(ctx context.Context) { } } -func (m *SectorStateMgr) GetStateUpdates() (r map[abi.SectorID]db.SealState) { - m.Lock() - r = m.stateUpdates - m.Unlock() - return -} - -func (m *SectorStateMgr) GetSectorStates() (r map[abi.SectorID]db.SealState) { - m.Lock() - r = m.sectorStates - m.Unlock() - return -} - -func (m *SectorStateMgr) GetActiveSectors() (r map[abi.SectorID]struct{}) { - m.Lock() - r = m.activeSectors - m.Unlock() - return -} - -func (m *SectorStateMgr) GetLatestUpdate() (r time.Time) { - m.Lock() - r = m.latestUpdate - m.Unlock() - return -} - func (m *SectorStateMgr) checkForUpdates(ctx context.Context) error { log.Debug("checking for sector state updates") @@ -132,43 +109,12 @@ func (m *SectorStateMgr) checkForUpdates(ctx context.Context) error { } } - su, ss, err := m.refreshState(ctx) - if err != nil { - return err - } - - head, err := m.fullnodeApi.ChainHead(ctx) - if err != nil { - return err - } - - activeSet, err := m.fullnodeApi.StateMinerActiveSectors(ctx, m.Maddr, head.Key()) + ssu, err := m.refreshState(ctx) if err != nil { return err } - mid, err := address.IDFromAddress(m.Maddr) - if err != nil { - return err - } - as := make(map[abi.SectorID]struct{}, len(activeSet)) - for _, info := range activeSet { - sectorID := abi.SectorID{ - Miner: abi.ActorID(mid), - Number: info.SectorNumber, - } - - as[sectorID] = struct{}{} - } - - m.Lock() - m.stateUpdates = su - m.sectorStates = ss - m.activeSectors = as - m.latestUpdate = time.Now() - m.Unlock() - - for sectorID, sectorSealState := range su { + for sectorID, sectorSealState := range ssu.Updates { // Update the sector seal state in the database err = m.sdb.Update(ctx, sectorID, sectorSealState) if err != nil { @@ -176,16 +122,18 @@ func (m *SectorStateMgr) checkForUpdates(ctx context.Context) error { } } + m.PubSub.Publish(ssu) + return nil } -func (m *SectorStateMgr) refreshState(ctx context.Context) (map[abi.SectorID]db.SealState, map[abi.SectorID]db.SealState, error) { +func (m *SectorStateMgr) refreshState(ctx context.Context) (*SectorStateUpdates, error) { defer func(start time.Time) { log.Debugw("refreshState", "took", time.Since(start)) }(time.Now()) // Get the current unsealed state of all sectors from lotus storageList, err := m.minerApi.StorageList(ctx) if err != nil { - return nil, nil, fmt.Errorf("getting sectors state from lotus: %w", err) + return nil, fmt.Errorf("getting sectors state from lotus: %w", err) } // Convert to a map of => @@ -215,7 +163,7 @@ func (m *SectorStateMgr) refreshState(ctx context.Context) (map[abi.SectorID]db. // Get the previously known state of all sectors in the database previousSectorStates, err := m.sdb.List(ctx) if err != nil { - return nil, nil, fmt.Errorf("getting sectors state from database: %w", err) + return nil, fmt.Errorf("getting sectors state from database: %w", err) } // Check which sectors have changed state since the last time we checked @@ -241,5 +189,29 @@ func (m *SectorStateMgr) refreshState(ctx context.Context) (map[abi.SectorID]db. sealStateUpdates[sectorID] = sealState } - return sealStateUpdates, allSectorStates, nil + head, err := m.fullnodeApi.ChainHead(ctx) + if err != nil { + return nil, err + } + + activeSet, err := m.fullnodeApi.StateMinerActiveSectors(ctx, m.Maddr, head.Key()) + if err != nil { + return nil, err + } + + mid, err := address.IDFromAddress(m.Maddr) + if err != nil { + return nil, err + } + as := make(map[abi.SectorID]struct{}, len(activeSet)) + for _, info := range activeSet { + sectorID := abi.SectorID{ + Miner: abi.ActorID(mid), + Number: info.SectorNumber, + } + + as[sectorID] = struct{}{} + } + + return &SectorStateUpdates{sealStateUpdates, as, allSectorStates, time.Now()}, nil } From f8b3cf48c163b322d877a7c4b801c7da9eae373d Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 30 May 2023 13:05:02 +0200 Subject: [PATCH 26/41] add close for pubsub --- indexprovider/wrapper.go | 6 +++++- piecedirectory/doctor.go | 6 +++++- sectorstatemgr/sectorstatemgr.go | 6 ++++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/indexprovider/wrapper.go b/indexprovider/wrapper.go index 5054cfea7..bc8bbff44 100644 --- a/indexprovider/wrapper.go +++ b/indexprovider/wrapper.go @@ -111,7 +111,11 @@ func (w *Wrapper) Start(_ context.Context) { for { select { - case u := <-updates: + case u, ok := <-updates: + if !ok { + log.Debugw("state updates subscription closed") + return + } log.Debugw("got state updates from SectorStateMgr", "u", len(u.Updates)) err := w.handleUpdates(runCtx, u.Updates) diff --git a/piecedirectory/doctor.go b/piecedirectory/doctor.go index 4aa3f3f25..cf5bf7f43 100644 --- a/piecedirectory/doctor.go +++ b/piecedirectory/doctor.go @@ -47,7 +47,11 @@ func (d *Doctor) Run(ctx context.Context) { for { select { - case u := <-sub: + case u, ok := <-sub: + if !ok { + log.Debugw("state updates subscription closed") + return + } log.Debugw("got state updates from SectorStateMgr", "len(u.updates)", len(u.Updates), "len(u.active)", len(u.ActiveSectors), "u.updatedAt", u.UpdatedAt) d.latestUpdateMu.Lock() diff --git a/sectorstatemgr/sectorstatemgr.go b/sectorstatemgr/sectorstatemgr.go index dddce8889..7acf0ad11 100644 --- a/sectorstatemgr/sectorstatemgr.go +++ b/sectorstatemgr/sectorstatemgr.go @@ -61,6 +61,7 @@ func NewSectorStateMgr(cfg *config.Boost) func(lc fx.Lifecycle, sdb *db.SectorSt }, OnStop: func(ctx context.Context) error { cancel() + mgr.PubSub.Close() return nil }, }) @@ -72,8 +73,6 @@ func NewSectorStateMgr(cfg *config.Boost) func(lc fx.Lifecycle, sdb *db.SectorSt func (m *SectorStateMgr) Run(ctx context.Context) { duration := time.Duration(m.cfg.StorageListRefreshDuration) log.Infof("starting sector state manager running on interval %s", duration.String()) - ticker := time.NewTicker(duration) - defer ticker.Stop() // Check immediately err := m.checkForUpdates(ctx) @@ -81,6 +80,9 @@ func (m *SectorStateMgr) Run(ctx context.Context) { log.Errorw("checking for state updates", "err", err) } + ticker := time.NewTicker(duration) + defer ticker.Stop() + // Check every tick for { select { From eed2b5a869711cfcac228cfccb3a431e76bf2f97 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 30 May 2023 14:29:09 +0200 Subject: [PATCH 27/41] add mock sectorstatemgr --- indexprovider/wrapper.go | 36 +++++++++++++++-------------- sectorstatemgr/sectorstatemgr.go | 39 ++++++++++++++++++++++++-------- 2 files changed, 48 insertions(+), 27 deletions(-) diff --git a/indexprovider/wrapper.go b/indexprovider/wrapper.go index bc8bbff44..319a7c4fb 100644 --- a/indexprovider/wrapper.go +++ b/indexprovider/wrapper.go @@ -106,27 +106,29 @@ func (w *Wrapper) Start(_ context.Context) { log.Info("starting index provider") - go func() { - updates := w.ssm.PubSub.Subscribe() - - for { - select { - case u, ok := <-updates: - if !ok { - log.Debugw("state updates subscription closed") - return - } - log.Debugw("got state updates from SectorStateMgr", "u", len(u.Updates)) + go w.checkForUpdates(runCtx) +} - err := w.handleUpdates(runCtx, u.Updates) - if err != nil { - log.Errorw("error while handling state updates", "err", err) - } - case <-runCtx.Done(): +func (w *Wrapper) checkForUpdates(ctx context.Context) { + updates := w.ssm.PubSub.Subscribe() + + for { + select { + case u, ok := <-updates: + if !ok { + log.Debugw("state updates subscription closed") return } + log.Debugw("got state updates from SectorStateMgr", "u", len(u.Updates)) + + err := w.handleUpdates(ctx, u.Updates) + if err != nil { + log.Errorw("error while handling state updates", "err", err) + } + case <-ctx.Done(): + return } - }() + } } func (w *Wrapper) handleUpdates(ctx context.Context, sus map[abi.SectorID]db.SealState) error { diff --git a/sectorstatemgr/sectorstatemgr.go b/sectorstatemgr/sectorstatemgr.go index 7acf0ad11..2ef0a982b 100644 --- a/sectorstatemgr/sectorstatemgr.go +++ b/sectorstatemgr/sectorstatemgr.go @@ -35,6 +35,8 @@ type SectorStateMgr struct { minerApi api.StorageMiner Maddr address.Address + _refreshState func(context.Context) (*SectorStateUpdates, error) + PubSub *PubSub sdb *db.SectorStateDB @@ -53,6 +55,9 @@ func NewSectorStateMgr(cfg *config.Boost) func(lc fx.Lifecycle, sdb *db.SectorSt sdb: sdb, } + // function pointer for test purpores, by default we use refreshState + mgr._refreshState = mgr.refreshState + cctx, cancel := context.WithCancel(context.Background()) lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { @@ -102,16 +107,7 @@ func (m *SectorStateMgr) checkForUpdates(ctx context.Context) error { defer func(start time.Time) { log.Debugw("checkForUpdates", "took", time.Since(start)) }(time.Now()) - // Tell lotus to update it's storage list and remove any removed sectors - if m.cfg.RedeclareOnStorageListRefresh { - log.Info("redeclaring storage") - err := m.minerApi.StorageRedeclareLocal(ctx, nil, true) - if err != nil { - log.Errorw("redeclaring local storage on lotus miner", "err", err) - } - } - - ssu, err := m.refreshState(ctx) + ssu, err := m._refreshState(ctx) if err != nil { return err } @@ -132,6 +128,15 @@ func (m *SectorStateMgr) checkForUpdates(ctx context.Context) error { func (m *SectorStateMgr) refreshState(ctx context.Context) (*SectorStateUpdates, error) { defer func(start time.Time) { log.Debugw("refreshState", "took", time.Since(start)) }(time.Now()) + // Tell lotus to update it's storage list and remove any removed sectors + if m.cfg.RedeclareOnStorageListRefresh { + log.Info("redeclaring storage") + err := m.minerApi.StorageRedeclareLocal(ctx, nil, true) + if err != nil { + log.Errorw("redeclaring local storage on lotus miner", "err", err) + } + } + // Get the current unsealed state of all sectors from lotus storageList, err := m.minerApi.StorageList(ctx) if err != nil { @@ -217,3 +222,17 @@ func (m *SectorStateMgr) refreshState(ctx context.Context) (*SectorStateUpdates, return &SectorStateUpdates{sealStateUpdates, as, allSectorStates, time.Now()}, nil } + +func NewMockSectorStateMgr(cfg *config.Boost, sdb *db.SectorStateDB, mockRefreshState func(context.Context) (*SectorStateUpdates, error)) *SectorStateMgr { + mgr := &SectorStateMgr{ + cfg: cfg.Storage, + + PubSub: NewPubSub(), + + sdb: sdb, + } + + mgr._refreshState = mockRefreshState + + return mgr +} From b18bcd5e3317dab63353889f140b5fe3c9fbd9b0 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 30 May 2023 16:05:13 +0200 Subject: [PATCH 28/41] add wrapper tests --- indexprovider/wrapper_test.go | 535 ++++++++++++++++++++++++++++++++++ 1 file changed, 535 insertions(+) create mode 100644 indexprovider/wrapper_test.go diff --git a/indexprovider/wrapper_test.go b/indexprovider/wrapper_test.go new file mode 100644 index 000000000..93e734aac --- /dev/null +++ b/indexprovider/wrapper_test.go @@ -0,0 +1,535 @@ +package indexprovider + +import ( + "context" + "testing" + "time" + + "github.com/filecoin-project/boost-gfm/storagemarket" + "github.com/filecoin-project/boost/db" + "github.com/filecoin-project/boost/db/migrations" + "github.com/filecoin-project/boost/indexprovider/mock" + "github.com/filecoin-project/boost/node/config" + "github.com/filecoin-project/boost/sectorstatemgr" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/builtin/v9/market" + "github.com/filecoin-project/lotus/markets/idxprov" + "github.com/filecoin-project/lotus/storage/sealer/storiface" + "github.com/golang/mock/gomock" + "github.com/ipni/index-provider/metadata" + mock_provider "github.com/ipni/index-provider/mock" + "github.com/stretchr/testify/require" +) + +// Empty response from MinerAPI.StorageList() +func TestUnsealedStateManagerEmptyStorageList(t *testing.T) { + wrapper, legacyStorageProvider, _, _ := setup(t) + legacyStorageProvider.EXPECT().ListLocalDeals().AnyTimes().Return(nil, nil) + + // Check for updates with an empty response from MinerAPI.StorageList() + err := wrapper.handleUpdates(context.Background(), nil) + require.NoError(t, err) +} + +// Only announce sectors for deals that are in the boost database or +// legacy datastore +func TestUnsealedStateManagerMatchingDealOnly(t *testing.T) { + ctx := context.Background() + + runTest := func(t *testing.T, wrapper *Wrapper, storageMiner *mockApiStorageMiner, prov *mock_provider.MockInterface, provAddr address.Address, sectorNum abi.SectorNumber) { + // Set the response from MinerAPI.StorageList() to be two unsealed sectors + minerID, err := address.IDFromAddress(provAddr) + require.NoError(t, err) + + // Expect handleUpdates to call NotifyPut exactly once, because only + // one sector from the storage list is in the database + prov.EXPECT().NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(1) + + sus := map[abi.SectorID]db.SealState{ + abi.SectorID{Miner: abi.ActorID(minerID), Number: sectorNum}: db.SealStateUnsealed, + abi.SectorID{Miner: abi.ActorID(minerID), Number: sectorNum + 1}: db.SealStateUnsealed, + } + err = wrapper.handleUpdates(ctx, sus) + require.NoError(t, err) + } + + t.Run("deal in boost db", func(t *testing.T) { + wrapper, legacyStorageProvider, storageMiner, prov := setup(t) + legacyStorageProvider.EXPECT().ListLocalDeals().Return(nil, nil) + + // Add a deal to the database + deals, err := db.GenerateNDeals(1) + require.NoError(t, err) + err = wrapper.dealsDB.Insert(ctx, &deals[0]) + require.NoError(t, err) + + provAddr := deals[0].ClientDealProposal.Proposal.Provider + sectorNum := deals[0].SectorID + runTest(t, wrapper, storageMiner, prov, provAddr, sectorNum) + }) + + t.Run("deal in legacy datastore", func(t *testing.T) { + wrapper, legacyStorageProvider, storageMiner, prov := setup(t) + + // Simulate returning a deal from the legacy datastore + boostDeals, err := db.GenerateNDeals(1) + require.NoError(t, err) + + sectorNum := abi.SectorNumber(10) + deals := []storagemarket.MinerDeal{{ + ClientDealProposal: boostDeals[0].ClientDealProposal, + SectorNumber: sectorNum, + }} + legacyStorageProvider.EXPECT().ListLocalDeals().Return(deals, nil) + + provAddr := deals[0].ClientDealProposal.Proposal.Provider + runTest(t, wrapper, storageMiner, prov, provAddr, sectorNum) + }) +} + +// Tests that various scenarios of sealing state changes produce the expected +// calls to NotifyPut / NotifyRemove +func TestSectorStateManagerStateChangeToIndexer(t *testing.T) { + ctx := context.Background() + + testCases := []struct { + name string + initialState func(sectorID abi.SectorID) *storiface.Decl + sus func(sectorID abi.SectorID) map[abi.SectorID]db.SealState + expect func(*mock_provider.MockInterfaceMockRecorder, market.DealProposal) + }{{ + name: "unsealed -> sealed", + initialState: func(sectorID abi.SectorID) *storiface.Decl { + return &storiface.Decl{ + SectorID: sectorID, + SectorFileType: storiface.FTUnsealed, + } + }, + sus: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { + return map[abi.SectorID]db.SealState{ + sectorID: db.SealStateSealed, + } + }, + expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { + // Expect a call to NotifyPut with fast retrieval = true (unsealed) + //prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ + //PieceCID: prop.PieceCID, + //VerifiedDeal: prop.VerifiedDeal, + //FastRetrieval: true, + //})).Times(1) + + // Expect a call to NotifyPut with fast retrieval = false (sealed) + prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ + PieceCID: prop.PieceCID, + VerifiedDeal: prop.VerifiedDeal, + FastRetrieval: false, + })).Times(1) + }, + }, { + name: "sealed -> unsealed", + initialState: func(sectorID abi.SectorID) *storiface.Decl { + return &storiface.Decl{ + SectorID: sectorID, + SectorFileType: storiface.FTSealed, + } + }, + sus: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { + return map[abi.SectorID]db.SealState{ + sectorID: db.SealStateUnsealed, + } + }, + expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { + // Expect a call to NotifyPut with fast retrieval = false (sealed) + //prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ + //PieceCID: prop.PieceCID, + //VerifiedDeal: prop.VerifiedDeal, + //FastRetrieval: false, + //})).Times(1) + + // Expect a call to NotifyPut with fast retrieval = true (unsealed) + prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ + PieceCID: prop.PieceCID, + VerifiedDeal: prop.VerifiedDeal, + FastRetrieval: true, + })).Times(1) + }, + }, { + name: "unsealed -> unsealed (no change)", + initialState: func(sectorID abi.SectorID) *storiface.Decl { + return &storiface.Decl{ + SectorID: sectorID, + SectorFileType: storiface.FTUnsealed, + } + }, + sus: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { + return nil + //return map[abi.SectorID]db.SealState{ + //sectorID: db.SealStateUnsealed, + //} + }, + expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { + // Expect only one call to NotifyPut with fast retrieval = true (unsealed) + // because the state of the sector doesn't change on the second call + //prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ + //PieceCID: prop.PieceCID, + //VerifiedDeal: prop.VerifiedDeal, + //FastRetrieval: true, + //})).Times(1) + }, + }, { + name: "unsealed -> removed", + initialState: func(sectorID abi.SectorID) *storiface.Decl { + return &storiface.Decl{ + SectorID: sectorID, + SectorFileType: storiface.FTUnsealed, + } + }, + sus: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { + return map[abi.SectorID]db.SealState{ + sectorID: db.SealStateRemoved, + } + }, + expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { + // Expect a call to NotifyPut with fast retrieval = true (unsealed) + //prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ + //PieceCID: prop.PieceCID, + //VerifiedDeal: prop.VerifiedDeal, + //FastRetrieval: true, + //})).Times(1) + + // Expect a call to NotifyRemove because the sector is no longer in the list response + prov.NotifyRemove(gomock.Any(), gomock.Any(), gomock.Any()).Times(1) + }, + }, { + name: "sealed -> removed", + initialState: func(sectorID abi.SectorID) *storiface.Decl { + return &storiface.Decl{ + SectorID: sectorID, + SectorFileType: storiface.FTSealed, + } + }, + sus: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { + return map[abi.SectorID]db.SealState{ + sectorID: db.SealStateRemoved, + } + }, + expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { + // Expect a call to NotifyPut with fast retrieval = false (sealed) + //prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ + //PieceCID: prop.PieceCID, + //VerifiedDeal: prop.VerifiedDeal, + //FastRetrieval: false, + //})).Times(1) + + // Expect a call to NotifyRemove because the sector is no longer in the list response + prov.NotifyRemove(gomock.Any(), gomock.Any(), gomock.Any()).Times(1) + }, + }, { + name: "removed -> unsealed", + initialState: func(sectorID abi.SectorID) *storiface.Decl { + return nil + }, + sus: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { + return map[abi.SectorID]db.SealState{ + sectorID: db.SealStateUnsealed, + } + }, + expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { + // Expect a call to NotifyPut with fast retrieval = true (unsealed) + prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ + PieceCID: prop.PieceCID, + VerifiedDeal: prop.VerifiedDeal, + FastRetrieval: true, + })).Times(1) + }, + }, { + name: "unsealed -> cache", + initialState: func(sectorID abi.SectorID) *storiface.Decl { + return &storiface.Decl{ + SectorID: sectorID, + SectorFileType: storiface.FTUnsealed, + } + }, + sus: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { + return map[abi.SectorID]db.SealState{ + sectorID: db.SealStateCache, + } + }, + expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { + // Expect only one call to NotifyPut with fast retrieval = true (unsealed) + // because we ignore a state change to cache + //prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ + //PieceCID: prop.PieceCID, + //VerifiedDeal: prop.VerifiedDeal, + //FastRetrieval: true, + //})).Times(1) + }, + }, { + name: "cache -> unsealed", + initialState: func(sectorID abi.SectorID) *storiface.Decl { + return &storiface.Decl{ + SectorID: sectorID, + SectorFileType: storiface.FTCache, + } + }, + sus: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { + return map[abi.SectorID]db.SealState{ + sectorID: db.SealStateUnsealed, + } + }, + expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { + // Expect only one call to NotifyPut with fast retrieval = true (unsealed) + // because we ignore a state change to cache + prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ + PieceCID: prop.PieceCID, + VerifiedDeal: prop.VerifiedDeal, + FastRetrieval: true, + })).Times(1) + }, + }, { + name: "cache -> sealed", + initialState: func(sectorID abi.SectorID) *storiface.Decl { + return &storiface.Decl{ + SectorID: sectorID, + SectorFileType: storiface.FTCache, + } + }, + sus: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { + return map[abi.SectorID]db.SealState{ + sectorID: db.SealStateSealed, + } + }, + expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { + // Expect only one call to NotifyPut with fast retrieval = true (unsealed) + // because we ignore a state change to cache + prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ + PieceCID: prop.PieceCID, + VerifiedDeal: prop.VerifiedDeal, + FastRetrieval: false, + })).Times(1) + }, + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + wrapper, legacyStorageProvider, storageMiner, prov := setup(t) + legacyStorageProvider.EXPECT().ListLocalDeals().AnyTimes().Return(nil, nil) + + // Add a deal to the database + deals, err := db.GenerateNDeals(1) + require.NoError(t, err) + err = wrapper.dealsDB.Insert(ctx, &deals[0]) + require.NoError(t, err) + + // Set up expectations (automatically verified when the test exits) + prop := deals[0].ClientDealProposal.Proposal + tc.expect(prov.EXPECT(), prop) + + minerID, err := address.IDFromAddress(deals[0].ClientDealProposal.Proposal.Provider) + require.NoError(t, err) + + // Set the current state from db -- response from MinerAPI.StorageList() + resp1 := tc.initialState(abi.SectorID{Miner: abi.ActorID(minerID), Number: deals[0].SectorID}) + storageMiner.storageList = map[storiface.ID][]storiface.Decl{} + if resp1 != nil { + storageMiner.storageList["uuid"] = []storiface.Decl{*resp1} + } + + // Handle updates + err = wrapper.handleUpdates(ctx, tc.sus(abi.SectorID{Miner: abi.ActorID(minerID), Number: deals[0].SectorID})) + require.NoError(t, err) + }) + } +} + +// Verify that multiple storage file types are handled from StorageList correctly +//func XTestUnsealedStateManagerStorageList(t *testing.T) { +//ctx := context.Background() + +//testCases := []struct { +//name string +//sus map[abi.SectorID]db.SealState +//storageListResponse func(sectorID abi.SectorID) []storiface.Decl +//expect func(*mock_provider.MockInterfaceMockRecorder, market.DealProposal) +//}{ +//{ +//name: "unsealed and sealed status", +//sus: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { +//return map[abi.SectorID]db.SealState{ +//sectorID: SealStateUnsealed, +//sectorID: SealStateCache, +//} +//}, +//storageListResponse: func(sectorID abi.SectorID) []storiface.Decl { +//return []storiface.Decl{ +//{ +//SectorID: sectorID, +//SectorFileType: storiface.FTUnsealed, +//}, +//{ +//SectorID: sectorID, +//SectorFileType: storiface.FTSealed, +//}, +//} +//}, +//expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { +//// Expect only one call to NotifyPut with fast retrieval = true (unsealed) +//// because we ignore a state change to cache +//prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ +//PieceCID: prop.PieceCID, +//VerifiedDeal: prop.VerifiedDeal, +//FastRetrieval: true, +//})).Times(1) +//}, +//}, { +//name: "unsealed and cached status", +//sus: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { +//return map[abi.SectorID]db.SealState{ +//sectorID: SealStateUnsealed, +//sectorID: SealStateCache, +//}, +//}, +//storageListResponse: func(sectorID abi.SectorID) []storiface.Decl { +//return []storiface.Decl{ +//{ +//SectorID: sectorID, +//SectorFileType: storiface.FTUnsealed, +//}, +//{ +//SectorID: sectorID, +//SectorFileType: storiface.FTCache, +//}, +//} +//}, +//expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { +//// Expect only one call to NotifyPut with fast retrieval = true (unsealed) +//// because we ignore a state change to cache +//prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ +//PieceCID: prop.PieceCID, +//VerifiedDeal: prop.VerifiedDeal, +//FastRetrieval: true, +//})).Times(1) +//}, +//}, { +//name: "sealed and cached status", +//sus: map[abi.SectorID]db.SealState{ +//abi.SectorID{Miner: abi.ActorID(minerID), Number: deals[0].SectorID}: FTtoSealState[tc.storageListResponse, +//}, +//storageListResponse: func(sectorID abi.SectorID) []storiface.Decl { +//return []storiface.Decl{ +//{ +//SectorID: sectorID, +//SectorFileType: storiface.FTSealed, +//}, +//{ +//SectorID: sectorID, +//SectorFileType: storiface.FTCache, +//}, +//} +//}, +//expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { +//// Expect only one call to NotifyPut with fast retrieval = true (unsealed) +//// because we ignore a state change to cache +//prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ +//PieceCID: prop.PieceCID, +//VerifiedDeal: prop.VerifiedDeal, +//FastRetrieval: false, +//})).Times(1) +//}, +//}} + +//for _, tc := range testCases { +//t.Run(tc.name, func(t *testing.T) { +//wrapper, legacyStorageProvider, storageMiner, prov := setup(t) +//legacyStorageProvider.EXPECT().ListLocalDeals().AnyTimes().Return(nil, nil) + +//// Add a deal to the database +//deals, err := db.GenerateNDeals(1) +//require.NoError(t, err) +//err = wrapper.dealsDB.Insert(ctx, &deals[0]) +//require.NoError(t, err) + +//// Set up expectations (automatically verified when the test exits) +//prop := deals[0].ClientDealProposal.Proposal +//tc.expect(prov.EXPECT(), prop) + +//minerID, err := address.IDFromAddress(deals[0].ClientDealProposal.Proposal.Provider) +//require.NoError(t, err) + +//// Set the first response from MinerAPI.StorageList() +//storageMiner.storageList = map[storiface.ID][]storiface.Decl{} +//resp1 := tc.storageListResponse(abi.SectorID{Miner: abi.ActorID(minerID), Number: deals[0].SectorID}) +//storageMiner.storageList["uuid"] = resp1 + +////FTtoSealState := map[SectorFileType]SealState{ +////FTUnsealed: SealStateUnsealed, +////FTSealed: SealStateSealed, +////FTCache: SealStateCache, +////} + +//// Trigger check for updates +//err = wrapper.handleUpdates(ctx, tc.sus) +//require.NoError(t, err) +//}) +//} +//} + +func setup(t *testing.T) (*Wrapper, *mock.MockStorageProvider, *mockApiStorageMiner, *mock_provider.MockInterface) { + ctx := context.Background() + ctrl := gomock.NewController(t) + prov := mock_provider.NewMockInterface(ctrl) + + sqldb := db.CreateTestTmpDB(t) + require.NoError(t, db.CreateAllBoostTables(ctx, sqldb, sqldb)) + require.NoError(t, migrations.Migrate(sqldb)) + + dealsDB := db.NewDealsDB(sqldb) + sdb := db.NewSectorStateDB(sqldb) + storageMiner := &mockApiStorageMiner{} + storageProvider := mock.NewMockStorageProvider(ctrl) + + wrapper := &Wrapper{ + enabled: true, + dealsDB: dealsDB, + prov: prov, + legacyProv: storageProvider, + meshCreator: &meshCreatorStub{}, + } + + cfg := &config.Boost{ + Storage: config.StorageConfig{ + StorageListRefreshDuration: config.Duration(100 * time.Millisecond), + }, + } + mockRefreshState := func(context.Context) (*sectorstatemgr.SectorStateUpdates, error) { + return nil, nil + } + ssm := sectorstatemgr.NewMockSectorStateMgr(cfg, sdb, mockRefreshState) + + wrapper.ssm = ssm + return wrapper, storageProvider, storageMiner, prov +} + +type mockApiStorageMiner struct { + storageList map[storiface.ID][]storiface.Decl +} + +//var _ ApiStorageMiner = (*mockApiStorageMiner)(nil) + +func (m mockApiStorageMiner) StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error) { + return m.storageList, nil +} + +func (m mockApiStorageMiner) StorageRedeclareLocal(ctx context.Context, id *storiface.ID, dropMissing bool) error { + return nil +} + +type meshCreatorStub struct { +} + +var _ idxprov.MeshCreator = (*meshCreatorStub)(nil) + +func (m *meshCreatorStub) Connect(context.Context) error { + return nil +} From 289ccb499ae5e55564c3e042cc243d7d66aa65c9 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 30 May 2023 16:13:49 +0200 Subject: [PATCH 29/41] fixup --- indexprovider/wrapper_test.go | 259 +++++++++++++++++----------------- 1 file changed, 128 insertions(+), 131 deletions(-) diff --git a/indexprovider/wrapper_test.go b/indexprovider/wrapper_test.go index 93e734aac..5a58316f1 100644 --- a/indexprovider/wrapper_test.go +++ b/indexprovider/wrapper_test.go @@ -34,7 +34,7 @@ func TestUnsealedStateManagerEmptyStorageList(t *testing.T) { // Only announce sectors for deals that are in the boost database or // legacy datastore -func TestUnsealedStateManagerMatchingDealOnly(t *testing.T) { +func TestSectorStateManagerMatchingDealOnly(t *testing.T) { ctx := context.Background() runTest := func(t *testing.T, wrapper *Wrapper, storageMiner *mockApiStorageMiner, prov *mock_provider.MockInterface, provAddr address.Address, sectorNum abi.SectorNumber) { @@ -344,136 +344,133 @@ func TestSectorStateManagerStateChangeToIndexer(t *testing.T) { } // Verify that multiple storage file types are handled from StorageList correctly -//func XTestUnsealedStateManagerStorageList(t *testing.T) { -//ctx := context.Background() - -//testCases := []struct { -//name string -//sus map[abi.SectorID]db.SealState -//storageListResponse func(sectorID abi.SectorID) []storiface.Decl -//expect func(*mock_provider.MockInterfaceMockRecorder, market.DealProposal) -//}{ -//{ -//name: "unsealed and sealed status", -//sus: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { -//return map[abi.SectorID]db.SealState{ -//sectorID: SealStateUnsealed, -//sectorID: SealStateCache, -//} -//}, -//storageListResponse: func(sectorID abi.SectorID) []storiface.Decl { -//return []storiface.Decl{ -//{ -//SectorID: sectorID, -//SectorFileType: storiface.FTUnsealed, -//}, -//{ -//SectorID: sectorID, -//SectorFileType: storiface.FTSealed, -//}, -//} -//}, -//expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { -//// Expect only one call to NotifyPut with fast retrieval = true (unsealed) -//// because we ignore a state change to cache -//prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ -//PieceCID: prop.PieceCID, -//VerifiedDeal: prop.VerifiedDeal, -//FastRetrieval: true, -//})).Times(1) -//}, -//}, { -//name: "unsealed and cached status", -//sus: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { -//return map[abi.SectorID]db.SealState{ -//sectorID: SealStateUnsealed, -//sectorID: SealStateCache, -//}, -//}, -//storageListResponse: func(sectorID abi.SectorID) []storiface.Decl { -//return []storiface.Decl{ -//{ -//SectorID: sectorID, -//SectorFileType: storiface.FTUnsealed, -//}, -//{ -//SectorID: sectorID, -//SectorFileType: storiface.FTCache, -//}, -//} -//}, -//expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { -//// Expect only one call to NotifyPut with fast retrieval = true (unsealed) -//// because we ignore a state change to cache -//prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ -//PieceCID: prop.PieceCID, -//VerifiedDeal: prop.VerifiedDeal, -//FastRetrieval: true, -//})).Times(1) -//}, -//}, { -//name: "sealed and cached status", -//sus: map[abi.SectorID]db.SealState{ -//abi.SectorID{Miner: abi.ActorID(minerID), Number: deals[0].SectorID}: FTtoSealState[tc.storageListResponse, -//}, -//storageListResponse: func(sectorID abi.SectorID) []storiface.Decl { -//return []storiface.Decl{ -//{ -//SectorID: sectorID, -//SectorFileType: storiface.FTSealed, -//}, -//{ -//SectorID: sectorID, -//SectorFileType: storiface.FTCache, -//}, -//} -//}, -//expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { -//// Expect only one call to NotifyPut with fast retrieval = true (unsealed) -//// because we ignore a state change to cache -//prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ -//PieceCID: prop.PieceCID, -//VerifiedDeal: prop.VerifiedDeal, -//FastRetrieval: false, -//})).Times(1) -//}, -//}} - -//for _, tc := range testCases { -//t.Run(tc.name, func(t *testing.T) { -//wrapper, legacyStorageProvider, storageMiner, prov := setup(t) -//legacyStorageProvider.EXPECT().ListLocalDeals().AnyTimes().Return(nil, nil) - -//// Add a deal to the database -//deals, err := db.GenerateNDeals(1) -//require.NoError(t, err) -//err = wrapper.dealsDB.Insert(ctx, &deals[0]) -//require.NoError(t, err) - -//// Set up expectations (automatically verified when the test exits) -//prop := deals[0].ClientDealProposal.Proposal -//tc.expect(prov.EXPECT(), prop) - -//minerID, err := address.IDFromAddress(deals[0].ClientDealProposal.Proposal.Provider) -//require.NoError(t, err) - -//// Set the first response from MinerAPI.StorageList() -//storageMiner.storageList = map[storiface.ID][]storiface.Decl{} -//resp1 := tc.storageListResponse(abi.SectorID{Miner: abi.ActorID(minerID), Number: deals[0].SectorID}) -//storageMiner.storageList["uuid"] = resp1 - -////FTtoSealState := map[SectorFileType]SealState{ -////FTUnsealed: SealStateUnsealed, -////FTSealed: SealStateSealed, -////FTCache: SealStateCache, -////} - -//// Trigger check for updates -//err = wrapper.handleUpdates(ctx, tc.sus) -//require.NoError(t, err) -//}) -//} -//} +func TestUnsealedStateManagerStorageList(t *testing.T) { + ctx := context.Background() + + testCases := []struct { + name string + initialState func(sectorID abi.SectorID) []storiface.Decl + sus func(sectorID abi.SectorID) map[abi.SectorID]db.SealState + expect func(*mock_provider.MockInterfaceMockRecorder, market.DealProposal) + }{ + { + name: "unsealed and sealed status", + sus: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { + return map[abi.SectorID]db.SealState{ + sectorID: db.SealStateUnsealed, + sectorID: db.SealStateCache, + } + }, + initialState: func(sectorID abi.SectorID) []storiface.Decl { + return []storiface.Decl{ + { + SectorID: sectorID, + SectorFileType: storiface.FTUnsealed, + }, + { + SectorID: sectorID, + SectorFileType: storiface.FTSealed, + }, + } + }, + expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { + // Expect only one call to NotifyPut with fast retrieval = true (unsealed) + // because we ignore a state change to cache + //prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ + //PieceCID: prop.PieceCID, + //VerifiedDeal: prop.VerifiedDeal, + //FastRetrieval: true, + //})).Times(1) + }, + }, { + name: "unsealed and cached status", + sus: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { + return map[abi.SectorID]db.SealState{ + sectorID: db.SealStateUnsealed, + sectorID: db.SealStateCache, + } + }, + initialState: func(sectorID abi.SectorID) []storiface.Decl { + return []storiface.Decl{ + { + SectorID: sectorID, + SectorFileType: storiface.FTUnsealed, + }, + { + SectorID: sectorID, + SectorFileType: storiface.FTCache, + }, + } + }, + expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { + // Expect only one call to NotifyPut with fast retrieval = true (unsealed) + // because we ignore a state change to cache + //prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ + //PieceCID: prop.PieceCID, + //VerifiedDeal: prop.VerifiedDeal, + //FastRetrieval: true, + //})).Times(1) + }, + }, { + name: "sealed and cached status", + sus: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { + return map[abi.SectorID]db.SealState{ + sectorID: db.SealStateUnsealed, + sectorID: db.SealStateCache, + } + }, + initialState: func(sectorID abi.SectorID) []storiface.Decl { + return []storiface.Decl{ + { + SectorID: sectorID, + SectorFileType: storiface.FTSealed, + }, + { + SectorID: sectorID, + SectorFileType: storiface.FTCache, + }, + } + }, + expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { + // Expect only one call to NotifyPut with fast retrieval = true (unsealed) + // because we ignore a state change to cache + //prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ + //PieceCID: prop.PieceCID, + //VerifiedDeal: prop.VerifiedDeal, + //FastRetrieval: false, + //})).Times(1) + }, + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + wrapper, legacyStorageProvider, storageMiner, prov := setup(t) + legacyStorageProvider.EXPECT().ListLocalDeals().AnyTimes().Return(nil, nil) + + // Add a deal to the database + deals, err := db.GenerateNDeals(1) + require.NoError(t, err) + err = wrapper.dealsDB.Insert(ctx, &deals[0]) + require.NoError(t, err) + + // Set up expectations (automatically verified when the test exits) + prop := deals[0].ClientDealProposal.Proposal + tc.expect(prov.EXPECT(), prop) + + minerID, err := address.IDFromAddress(deals[0].ClientDealProposal.Proposal.Provider) + require.NoError(t, err) + + // Set the first response from MinerAPI.StorageList() + storageMiner.storageList = map[storiface.ID][]storiface.Decl{} + resp1 := tc.initialState(abi.SectorID{Miner: abi.ActorID(minerID), Number: deals[0].SectorID}) + storageMiner.storageList["uuid"] = resp1 + + // Trigger check for updates + err = wrapper.handleUpdates(ctx, tc.sus(abi.SectorID{Miner: abi.ActorID(minerID), Number: deals[0].SectorID})) + require.NoError(t, err) + }) + } +} func setup(t *testing.T) (*Wrapper, *mock.MockStorageProvider, *mockApiStorageMiner, *mock_provider.MockInterface) { ctx := context.Background() From db908c3aa9230c2a928a0550ff74e650c5313e6d Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 30 May 2023 16:26:13 +0200 Subject: [PATCH 30/41] cleanup --- indexprovider/wrapper_test.go | 17 ----------------- sectorstatemgr/sectorstatemgr.go | 21 +-------------------- 2 files changed, 1 insertion(+), 37 deletions(-) diff --git a/indexprovider/wrapper_test.go b/indexprovider/wrapper_test.go index 5a58316f1..66b1dd358 100644 --- a/indexprovider/wrapper_test.go +++ b/indexprovider/wrapper_test.go @@ -3,14 +3,11 @@ package indexprovider import ( "context" "testing" - "time" "github.com/filecoin-project/boost-gfm/storagemarket" "github.com/filecoin-project/boost/db" "github.com/filecoin-project/boost/db/migrations" "github.com/filecoin-project/boost/indexprovider/mock" - "github.com/filecoin-project/boost/node/config" - "github.com/filecoin-project/boost/sectorstatemgr" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/builtin/v9/market" @@ -482,7 +479,6 @@ func setup(t *testing.T) (*Wrapper, *mock.MockStorageProvider, *mockApiStorageMi require.NoError(t, migrations.Migrate(sqldb)) dealsDB := db.NewDealsDB(sqldb) - sdb := db.NewSectorStateDB(sqldb) storageMiner := &mockApiStorageMiner{} storageProvider := mock.NewMockStorageProvider(ctrl) @@ -494,17 +490,6 @@ func setup(t *testing.T) (*Wrapper, *mock.MockStorageProvider, *mockApiStorageMi meshCreator: &meshCreatorStub{}, } - cfg := &config.Boost{ - Storage: config.StorageConfig{ - StorageListRefreshDuration: config.Duration(100 * time.Millisecond), - }, - } - mockRefreshState := func(context.Context) (*sectorstatemgr.SectorStateUpdates, error) { - return nil, nil - } - ssm := sectorstatemgr.NewMockSectorStateMgr(cfg, sdb, mockRefreshState) - - wrapper.ssm = ssm return wrapper, storageProvider, storageMiner, prov } @@ -512,8 +497,6 @@ type mockApiStorageMiner struct { storageList map[storiface.ID][]storiface.Decl } -//var _ ApiStorageMiner = (*mockApiStorageMiner)(nil) - func (m mockApiStorageMiner) StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error) { return m.storageList, nil } diff --git a/sectorstatemgr/sectorstatemgr.go b/sectorstatemgr/sectorstatemgr.go index 2ef0a982b..0508dcd09 100644 --- a/sectorstatemgr/sectorstatemgr.go +++ b/sectorstatemgr/sectorstatemgr.go @@ -35,8 +35,6 @@ type SectorStateMgr struct { minerApi api.StorageMiner Maddr address.Address - _refreshState func(context.Context) (*SectorStateUpdates, error) - PubSub *PubSub sdb *db.SectorStateDB @@ -55,9 +53,6 @@ func NewSectorStateMgr(cfg *config.Boost) func(lc fx.Lifecycle, sdb *db.SectorSt sdb: sdb, } - // function pointer for test purpores, by default we use refreshState - mgr._refreshState = mgr.refreshState - cctx, cancel := context.WithCancel(context.Background()) lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { @@ -107,7 +102,7 @@ func (m *SectorStateMgr) checkForUpdates(ctx context.Context) error { defer func(start time.Time) { log.Debugw("checkForUpdates", "took", time.Since(start)) }(time.Now()) - ssu, err := m._refreshState(ctx) + ssu, err := m.refreshState(ctx) if err != nil { return err } @@ -222,17 +217,3 @@ func (m *SectorStateMgr) refreshState(ctx context.Context) (*SectorStateUpdates, return &SectorStateUpdates{sealStateUpdates, as, allSectorStates, time.Now()}, nil } - -func NewMockSectorStateMgr(cfg *config.Boost, sdb *db.SectorStateDB, mockRefreshState func(context.Context) (*SectorStateUpdates, error)) *SectorStateMgr { - mgr := &SectorStateMgr{ - cfg: cfg.Storage, - - PubSub: NewPubSub(), - - sdb: sdb, - } - - mgr._refreshState = mockRefreshState - - return mgr -} From 36aa47bb71923e000acae746403ecaf1e7fa51a4 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 31 May 2023 13:42:21 +0200 Subject: [PATCH 31/41] cleanup --- indexprovider/wrapper_test.go | 142 ++++++++++------------------------ 1 file changed, 39 insertions(+), 103 deletions(-) diff --git a/indexprovider/wrapper_test.go b/indexprovider/wrapper_test.go index 66b1dd358..dbe03830c 100644 --- a/indexprovider/wrapper_test.go +++ b/indexprovider/wrapper_test.go @@ -91,10 +91,10 @@ func TestSectorStateManagerStateChangeToIndexer(t *testing.T) { ctx := context.Background() testCases := []struct { - name string - initialState func(sectorID abi.SectorID) *storiface.Decl - sus func(sectorID abi.SectorID) map[abi.SectorID]db.SealState - expect func(*mock_provider.MockInterfaceMockRecorder, market.DealProposal) + name string + initialState func(sectorID abi.SectorID) *storiface.Decl + sectorUpdates func(sectorID abi.SectorID) map[abi.SectorID]db.SealState + expect func(*mock_provider.MockInterfaceMockRecorder, market.DealProposal) }{{ name: "unsealed -> sealed", initialState: func(sectorID abi.SectorID) *storiface.Decl { @@ -103,19 +103,12 @@ func TestSectorStateManagerStateChangeToIndexer(t *testing.T) { SectorFileType: storiface.FTUnsealed, } }, - sus: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { + sectorUpdates: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { return map[abi.SectorID]db.SealState{ sectorID: db.SealStateSealed, } }, expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { - // Expect a call to NotifyPut with fast retrieval = true (unsealed) - //prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ - //PieceCID: prop.PieceCID, - //VerifiedDeal: prop.VerifiedDeal, - //FastRetrieval: true, - //})).Times(1) - // Expect a call to NotifyPut with fast retrieval = false (sealed) prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ PieceCID: prop.PieceCID, @@ -131,19 +124,12 @@ func TestSectorStateManagerStateChangeToIndexer(t *testing.T) { SectorFileType: storiface.FTSealed, } }, - sus: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { + sectorUpdates: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { return map[abi.SectorID]db.SealState{ sectorID: db.SealStateUnsealed, } }, expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { - // Expect a call to NotifyPut with fast retrieval = false (sealed) - //prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ - //PieceCID: prop.PieceCID, - //VerifiedDeal: prop.VerifiedDeal, - //FastRetrieval: false, - //})).Times(1) - // Expect a call to NotifyPut with fast retrieval = true (unsealed) prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ PieceCID: prop.PieceCID, @@ -152,27 +138,17 @@ func TestSectorStateManagerStateChangeToIndexer(t *testing.T) { })).Times(1) }, }, { - name: "unsealed -> unsealed (no change)", + name: "unsealed -> unsealed (no sector updates)", initialState: func(sectorID abi.SectorID) *storiface.Decl { return &storiface.Decl{ SectorID: sectorID, SectorFileType: storiface.FTUnsealed, } }, - sus: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { + sectorUpdates: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { return nil - //return map[abi.SectorID]db.SealState{ - //sectorID: db.SealStateUnsealed, - //} }, expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { - // Expect only one call to NotifyPut with fast retrieval = true (unsealed) - // because the state of the sector doesn't change on the second call - //prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ - //PieceCID: prop.PieceCID, - //VerifiedDeal: prop.VerifiedDeal, - //FastRetrieval: true, - //})).Times(1) }, }, { name: "unsealed -> removed", @@ -182,19 +158,12 @@ func TestSectorStateManagerStateChangeToIndexer(t *testing.T) { SectorFileType: storiface.FTUnsealed, } }, - sus: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { + sectorUpdates: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { return map[abi.SectorID]db.SealState{ sectorID: db.SealStateRemoved, } }, expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { - // Expect a call to NotifyPut with fast retrieval = true (unsealed) - //prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ - //PieceCID: prop.PieceCID, - //VerifiedDeal: prop.VerifiedDeal, - //FastRetrieval: true, - //})).Times(1) - // Expect a call to NotifyRemove because the sector is no longer in the list response prov.NotifyRemove(gomock.Any(), gomock.Any(), gomock.Any()).Times(1) }, @@ -206,28 +175,21 @@ func TestSectorStateManagerStateChangeToIndexer(t *testing.T) { SectorFileType: storiface.FTSealed, } }, - sus: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { + sectorUpdates: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { return map[abi.SectorID]db.SealState{ sectorID: db.SealStateRemoved, } }, expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { - // Expect a call to NotifyPut with fast retrieval = false (sealed) - //prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ - //PieceCID: prop.PieceCID, - //VerifiedDeal: prop.VerifiedDeal, - //FastRetrieval: false, - //})).Times(1) - // Expect a call to NotifyRemove because the sector is no longer in the list response prov.NotifyRemove(gomock.Any(), gomock.Any(), gomock.Any()).Times(1) }, }, { - name: "removed -> unsealed", + name: "none -> unsealed(new sector)", initialState: func(sectorID abi.SectorID) *storiface.Decl { return nil }, - sus: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { + sectorUpdates: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { return map[abi.SectorID]db.SealState{ sectorID: db.SealStateUnsealed, } @@ -248,19 +210,13 @@ func TestSectorStateManagerStateChangeToIndexer(t *testing.T) { SectorFileType: storiface.FTUnsealed, } }, - sus: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { + sectorUpdates: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { return map[abi.SectorID]db.SealState{ sectorID: db.SealStateCache, } }, expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { - // Expect only one call to NotifyPut with fast retrieval = true (unsealed) - // because we ignore a state change to cache - //prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ - //PieceCID: prop.PieceCID, - //VerifiedDeal: prop.VerifiedDeal, - //FastRetrieval: true, - //})).Times(1) + // `cache` doesn't trigger a notify }, }, { name: "cache -> unsealed", @@ -270,7 +226,7 @@ func TestSectorStateManagerStateChangeToIndexer(t *testing.T) { SectorFileType: storiface.FTCache, } }, - sus: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { + sectorUpdates: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { return map[abi.SectorID]db.SealState{ sectorID: db.SealStateUnsealed, } @@ -292,7 +248,7 @@ func TestSectorStateManagerStateChangeToIndexer(t *testing.T) { SectorFileType: storiface.FTCache, } }, - sus: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { + sectorUpdates: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { return map[abi.SectorID]db.SealState{ sectorID: db.SealStateSealed, } @@ -334,7 +290,7 @@ func TestSectorStateManagerStateChangeToIndexer(t *testing.T) { } // Handle updates - err = wrapper.handleUpdates(ctx, tc.sus(abi.SectorID{Miner: abi.ActorID(minerID), Number: deals[0].SectorID})) + err = wrapper.handleUpdates(ctx, tc.sectorUpdates(abi.SectorID{Miner: abi.ActorID(minerID), Number: deals[0].SectorID})) require.NoError(t, err) }) } @@ -345,19 +301,13 @@ func TestUnsealedStateManagerStorageList(t *testing.T) { ctx := context.Background() testCases := []struct { - name string - initialState func(sectorID abi.SectorID) []storiface.Decl - sus func(sectorID abi.SectorID) map[abi.SectorID]db.SealState - expect func(*mock_provider.MockInterfaceMockRecorder, market.DealProposal) + name string + initialState func(sectorID abi.SectorID) []storiface.Decl + sectorUpdates func(sectorID abi.SectorID) map[abi.SectorID]db.SealState + expect func(*mock_provider.MockInterfaceMockRecorder, market.DealProposal) }{ { name: "unsealed and sealed status", - sus: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { - return map[abi.SectorID]db.SealState{ - sectorID: db.SealStateUnsealed, - sectorID: db.SealStateCache, - } - }, initialState: func(sectorID abi.SectorID) []storiface.Decl { return []storiface.Decl{ { @@ -370,23 +320,17 @@ func TestUnsealedStateManagerStorageList(t *testing.T) { }, } }, - expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { - // Expect only one call to NotifyPut with fast retrieval = true (unsealed) - // because we ignore a state change to cache - //prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ - //PieceCID: prop.PieceCID, - //VerifiedDeal: prop.VerifiedDeal, - //FastRetrieval: true, - //})).Times(1) - }, - }, { - name: "unsealed and cached status", - sus: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { + sectorUpdates: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { return map[abi.SectorID]db.SealState{ sectorID: db.SealStateUnsealed, sectorID: db.SealStateCache, } }, + expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { + // no call to notify, as sector was unsealed and still is + }, + }, { + name: "unsealed and cached status", initialState: func(sectorID abi.SectorID) []storiface.Decl { return []storiface.Decl{ { @@ -399,23 +343,16 @@ func TestUnsealedStateManagerStorageList(t *testing.T) { }, } }, - expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { - // Expect only one call to NotifyPut with fast retrieval = true (unsealed) - // because we ignore a state change to cache - //prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ - //PieceCID: prop.PieceCID, - //VerifiedDeal: prop.VerifiedDeal, - //FastRetrieval: true, - //})).Times(1) - }, - }, { - name: "sealed and cached status", - sus: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { + sectorUpdates: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { return map[abi.SectorID]db.SealState{ sectorID: db.SealStateUnsealed, sectorID: db.SealStateCache, } }, + expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { + }, + }, { + name: "sealed and cached status", initialState: func(sectorID abi.SectorID) []storiface.Decl { return []storiface.Decl{ { @@ -428,14 +365,13 @@ func TestUnsealedStateManagerStorageList(t *testing.T) { }, } }, + sectorUpdates: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { + return map[abi.SectorID]db.SealState{ + sectorID: db.SealStateUnsealed, + sectorID: db.SealStateCache, + } + }, expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { - // Expect only one call to NotifyPut with fast retrieval = true (unsealed) - // because we ignore a state change to cache - //prov.NotifyPut(gomock.Any(), gomock.Any(), gomock.Any(), metadata.Default.New(&metadata.GraphsyncFilecoinV1{ - //PieceCID: prop.PieceCID, - //VerifiedDeal: prop.VerifiedDeal, - //FastRetrieval: false, - //})).Times(1) }, }} @@ -463,7 +399,7 @@ func TestUnsealedStateManagerStorageList(t *testing.T) { storageMiner.storageList["uuid"] = resp1 // Trigger check for updates - err = wrapper.handleUpdates(ctx, tc.sus(abi.SectorID{Miner: abi.ActorID(minerID), Number: deals[0].SectorID})) + err = wrapper.handleUpdates(ctx, tc.sectorUpdates(abi.SectorID{Miner: abi.ActorID(minerID), Number: deals[0].SectorID})) require.NoError(t, err) }) } From afc055ebb1537c4b54b63dc4999e359cf0495eb4 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 31 May 2023 16:32:33 +0200 Subject: [PATCH 32/41] better names --- sectorstatemgr/sectorstatemgr.go | 42 ++++++++++++++++---------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/sectorstatemgr/sectorstatemgr.go b/sectorstatemgr/sectorstatemgr.go index 0508dcd09..cb3e7f124 100644 --- a/sectorstatemgr/sectorstatemgr.go +++ b/sectorstatemgr/sectorstatemgr.go @@ -141,24 +141,24 @@ func (m *SectorStateMgr) refreshState(ctx context.Context) (*SectorStateUpdates, // Convert to a map of => sectorStates := make(map[abi.SectorID]db.SealState) allSectorStates := make(map[abi.SectorID]db.SealState) - for _, storageStates := range storageList { - for _, storageState := range storageStates { + for _, loc := range storageList { + for _, sectorDecl := range loc { // Explicity set the sector state if its Sealed or Unsealed switch { - case storageState.SectorFileType.Has(storiface.FTUnsealed): - sectorStates[storageState.SectorID] = db.SealStateUnsealed - case storageState.SectorFileType.Has(storiface.FTSealed): - if state, ok := sectorStates[storageState.SectorID]; !ok || state != db.SealStateUnsealed { - sectorStates[storageState.SectorID] = db.SealStateSealed + case sectorDecl.SectorFileType.Has(storiface.FTUnsealed): + sectorStates[sectorDecl.SectorID] = db.SealStateUnsealed + case sectorDecl.SectorFileType.Has(storiface.FTSealed): + if state, ok := sectorStates[sectorDecl.SectorID]; !ok || state != db.SealStateUnsealed { + sectorStates[sectorDecl.SectorID] = db.SealStateSealed } } // If the state hasnt been set it should be in the cache, mark it so we dont remove // This may get overriden by the sealed status if it comes after in the list, which is fine - if _, ok := sectorStates[storageState.SectorID]; !ok { - sectorStates[storageState.SectorID] = db.SealStateCache + if _, ok := sectorStates[sectorDecl.SectorID]; !ok { + sectorStates[sectorDecl.SectorID] = db.SealStateCache } - allSectorStates[storageState.SectorID] = sectorStates[storageState.SectorID] + allSectorStates[sectorDecl.SectorID] = sectorStates[sectorDecl.SectorID] } } @@ -169,26 +169,26 @@ func (m *SectorStateMgr) refreshState(ctx context.Context) (*SectorStateUpdates, } // Check which sectors have changed state since the last time we checked - sealStateUpdates := make(map[abi.SectorID]db.SealState) - for _, previousSectorState := range previousSectorStates { - sealState, ok := sectorStates[previousSectorState.SectorID] + sectorUpdates := make(map[abi.SectorID]db.SealState) + for _, pss := range previousSectorStates { + sealState, ok := sectorStates[pss.SectorID] if ok { // Check if the state has changed, ignore if the new state is cache - if previousSectorState.SealState != sealState && sealState != db.SealStateCache { - sealStateUpdates[previousSectorState.SectorID] = sealState + if pss.SealState != sealState && sealState != db.SealStateCache { + sectorUpdates[pss.SectorID] = sealState } // Delete the sector from the map - at the end the remaining // sectors in the map are ones we didn't know about before - delete(sectorStates, previousSectorState.SectorID) + delete(sectorStates, pss.SectorID) } else { // The sector is no longer in the list, so it must have been removed - sealStateUpdates[previousSectorState.SectorID] = db.SealStateRemoved + sectorUpdates[pss.SectorID] = db.SealStateRemoved } } // The remaining sectors in the map are ones we didn't know about before for sectorID, sealState := range sectorStates { - sealStateUpdates[sectorID] = sealState + sectorUpdates[sectorID] = sealState } head, err := m.fullnodeApi.ChainHead(ctx) @@ -205,15 +205,15 @@ func (m *SectorStateMgr) refreshState(ctx context.Context) (*SectorStateUpdates, if err != nil { return nil, err } - as := make(map[abi.SectorID]struct{}, len(activeSet)) + activeSectors := make(map[abi.SectorID]struct{}, len(activeSet)) for _, info := range activeSet { sectorID := abi.SectorID{ Miner: abi.ActorID(mid), Number: info.SectorNumber, } - as[sectorID] = struct{}{} + activeSectors[sectorID] = struct{}{} } - return &SectorStateUpdates{sealStateUpdates, as, allSectorStates, time.Now()}, nil + return &SectorStateUpdates{sectorUpdates, activeSectors, allSectorStates, time.Now()}, nil } From 174678e9c85edaa86e2221b1ad80c22e703102e5 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 31 May 2023 16:32:55 +0200 Subject: [PATCH 33/41] t.Skip for test --- storagemarket/provider_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/storagemarket/provider_test.go b/storagemarket/provider_test.go index 251eb5371..63400954d 100644 --- a/storagemarket/provider_test.go +++ b/storagemarket/provider_test.go @@ -138,7 +138,9 @@ func TestSimpleDealHappy(t *testing.T) { }) } -func XTestMultipleDealsConcurrent(t *testing.T) { +func TestMultipleDealsConcurrent(t *testing.T) { + t.Skip("TestMultipleDealsConcurrent is flaky, disabling for now") + //logging.SetLogLevel("boost-provider", "debug") //logging.SetLogLevel("boost-storage-deal", "debug") nDeals := 10 From 6e9ca99d1381e45e0ee6dee5da4f38467b27fa25 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 31 May 2023 16:34:50 +0200 Subject: [PATCH 34/41] remove TODO above println for panic --- storagemarket/deal_execution.go | 1 - 1 file changed, 1 deletion(-) diff --git a/storagemarket/deal_execution.go b/storagemarket/deal_execution.go index a059f654a..5f407dce7 100644 --- a/storagemarket/deal_execution.go +++ b/storagemarket/deal_execution.go @@ -83,7 +83,6 @@ func (p *Provider) execDeal(deal *smtypes.ProviderDealState, dh *dealHandler) (d // Capture any panic as a manually retryable error defer func() { if err := recover(); err != nil { - // TODO: it'd be nice to expose and log these errors in tests fmt.Println("panic: ", err, string(debug.Stack())) dmerr = &dealMakingError{ error: fmt.Errorf("Caught panic in deal execution: %s\n%s", err, debug.Stack()), From ecce4692252979a19eaf354988f131ce2506f7c4 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 31 May 2023 18:58:08 +0200 Subject: [PATCH 35/41] add unit tests for refreshState --- sectorstatemgr/mock/sectorstatemgr.go | 65 ++++++++++++ sectorstatemgr/sectorstatemgr.go | 9 +- sectorstatemgr/sectorstatemgr_test.go | 137 ++++++++++++++++++++++++++ 3 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 sectorstatemgr/mock/sectorstatemgr.go create mode 100644 sectorstatemgr/sectorstatemgr_test.go diff --git a/sectorstatemgr/mock/sectorstatemgr.go b/sectorstatemgr/mock/sectorstatemgr.go new file mode 100644 index 000000000..39ec2779c --- /dev/null +++ b/sectorstatemgr/mock/sectorstatemgr.go @@ -0,0 +1,65 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/filecoin-project/boost/sectorstatemgr (interfaces: StorageAPI) + +// Package mock is a generated GoMock package. +package mock + +import ( + context "context" + reflect "reflect" + + storiface "github.com/filecoin-project/lotus/storage/sealer/storiface" + gomock "github.com/golang/mock/gomock" +) + +// MockStorageAPI is a mock of StorageAPI interface. +type MockStorageAPI struct { + ctrl *gomock.Controller + recorder *MockStorageAPIMockRecorder +} + +// MockStorageAPIMockRecorder is the mock recorder for MockStorageAPI. +type MockStorageAPIMockRecorder struct { + mock *MockStorageAPI +} + +// NewMockStorageAPI creates a new mock instance. +func NewMockStorageAPI(ctrl *gomock.Controller) *MockStorageAPI { + mock := &MockStorageAPI{ctrl: ctrl} + mock.recorder = &MockStorageAPIMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockStorageAPI) EXPECT() *MockStorageAPIMockRecorder { + return m.recorder +} + +// StorageList mocks base method. +func (m *MockStorageAPI) StorageList(arg0 context.Context) (map[storiface.ID][]storiface.Decl, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StorageList", arg0) + ret0, _ := ret[0].(map[storiface.ID][]storiface.Decl) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// StorageList indicates an expected call of StorageList. +func (mr *MockStorageAPIMockRecorder) StorageList(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageList", reflect.TypeOf((*MockStorageAPI)(nil).StorageList), arg0) +} + +// StorageRedeclareLocal mocks base method. +func (m *MockStorageAPI) StorageRedeclareLocal(arg0 context.Context, arg1 *storiface.ID, arg2 bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StorageRedeclareLocal", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// StorageRedeclareLocal indicates an expected call of StorageRedeclareLocal. +func (mr *MockStorageAPIMockRecorder) StorageRedeclareLocal(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageRedeclareLocal", reflect.TypeOf((*MockStorageAPI)(nil).StorageRedeclareLocal), arg0, arg1, arg2) +} diff --git a/sectorstatemgr/sectorstatemgr.go b/sectorstatemgr/sectorstatemgr.go index cb3e7f124..de2fbac8d 100644 --- a/sectorstatemgr/sectorstatemgr.go +++ b/sectorstatemgr/sectorstatemgr.go @@ -1,5 +1,7 @@ package sectorstatemgr +//go:generate go run github.com/golang/mock/mockgen -destination=mock/sectorstatemgr.go -package=mock . StorageAPI + import ( "context" "fmt" @@ -27,12 +29,17 @@ type SectorStateUpdates struct { UpdatedAt time.Time } +type StorageAPI interface { + StorageRedeclareLocal(context.Context, *storiface.ID, bool) error + StorageList(context.Context) (map[storiface.ID][]storiface.Decl, error) +} + type SectorStateMgr struct { sync.Mutex cfg config.StorageConfig fullnodeApi api.FullNode - minerApi api.StorageMiner + minerApi StorageAPI Maddr address.Address PubSub *PubSub diff --git a/sectorstatemgr/sectorstatemgr_test.go b/sectorstatemgr/sectorstatemgr_test.go new file mode 100644 index 000000000..3c339a9ea --- /dev/null +++ b/sectorstatemgr/sectorstatemgr_test.go @@ -0,0 +1,137 @@ +package sectorstatemgr + +import ( + "context" + "reflect" + "testing" + "time" + + "github.com/davecgh/go-spew/spew" + "github.com/filecoin-project/boost/db" + "github.com/filecoin-project/boost/db/migrations" + "github.com/filecoin-project/boost/node/config" + "github.com/filecoin-project/boost/sectorstatemgr/mock" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + lotusmocks "github.com/filecoin-project/lotus/api/mocks" + "github.com/filecoin-project/lotus/storage/sealer/storiface" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" +) + +func TestRefreshState(t *testing.T) { + type fixtures struct { + input interface{} + expected *SectorStateUpdates + } + + tests := []struct { + description string + f func() fixtures + }{ + { + description: "first", + f: func() fixtures { + deals, err := db.GenerateNDeals(2) + require.NoError(t, err) + sid1 := abi.SectorID{Miner: abi.ActorID(1), Number: deals[0].SectorID} + sid2 := abi.SectorID{Miner: abi.ActorID(1), Number: deals[1].SectorID} + list := map[storiface.ID][]storiface.Decl{ + "storage-location-uuid1": { + { + SectorID: sid1, + SectorFileType: storiface.FTUnsealed, + }, + { + SectorID: sid1, + SectorFileType: storiface.FTSealed, + }, + { + SectorID: sid1, + SectorFileType: storiface.FTCache, + }, + }, + "storage-location-uuid2": { + { + SectorID: sid2, + SectorFileType: storiface.FTSealed, + }, + { + SectorID: sid2, + SectorFileType: storiface.FTCache, + }, + }, + } + + return fixtures{ + input: list, + expected: &SectorStateUpdates{ + Updates: map[abi.SectorID]db.SealState{ + sid1: db.SealStateUnsealed, + sid2: db.SealStateSealed, + }, + ActiveSectors: map[abi.SectorID]struct{}{}, + SectorStates: map[abi.SectorID]db.SealState{ + sid1: db.SealStateUnsealed, + sid2: db.SealStateSealed, + }, + }, + } + }, + }, + } + + // setup for test + ctx := context.Background() + + cfg := config.StorageConfig{ + StorageListRefreshDuration: config.Duration(100 * time.Millisecond), + } + + sqldb := db.CreateTestTmpDB(t) + require.NoError(t, db.CreateAllBoostTables(ctx, sqldb, sqldb)) + require.NoError(t, migrations.Migrate(sqldb)) + + sdb := db.NewSectorStateDB(sqldb) + + ctrl := gomock.NewController(t) + + // setup mocks + fullnodeApi := lotusmocks.NewMockFullNode(ctrl) + minerApi := mock.NewMockStorageAPI(ctrl) + + maddr, _ := address.NewIDAddress(0) + + // setup sectorstatemgr + mgr := &SectorStateMgr{ + cfg: cfg, + minerApi: minerApi, + fullnodeApi: fullnodeApi, + Maddr: maddr, + + PubSub: NewPubSub(), + + sdb: sdb, + } + + for _, tt := range tests { + // f() builds fixtures for a specific test case, namely all required input for test and expected result + fixt := tt.f() + + minerApi.EXPECT().StorageList(gomock.Any()).Return(fixt.input, nil) + fullnodeApi.EXPECT().ChainHead(gomock.Any()).Times(1) + fullnodeApi.EXPECT().StateMinerActiveSectors(gomock.Any(), gomock.Any(), gomock.Any()).Times(1) + + got, err := mgr.refreshState(ctx) + require.NoError(t, err) + + zero := time.Time{} + require.NotEqual(t, got.UpdatedAt, zero) + + //null timestamp, so that we can do deep equal + got.UpdatedAt = zero + + require.True(t, reflect.DeepEqual(fixt.expected, got), "expected: %s, got: %s", spew.Sdump(fixt.expected), spew.Sdump(got)) + } + +} From e572b061a436e520c4072fe943d068b10290be6c Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 31 May 2023 18:58:20 +0200 Subject: [PATCH 36/41] rename tests --- indexprovider/wrapper_test.go | 127 +++------------------------------- 1 file changed, 9 insertions(+), 118 deletions(-) diff --git a/indexprovider/wrapper_test.go b/indexprovider/wrapper_test.go index dbe03830c..b85012b37 100644 --- a/indexprovider/wrapper_test.go +++ b/indexprovider/wrapper_test.go @@ -96,7 +96,7 @@ func TestSectorStateManagerStateChangeToIndexer(t *testing.T) { sectorUpdates func(sectorID abi.SectorID) map[abi.SectorID]db.SealState expect func(*mock_provider.MockInterfaceMockRecorder, market.DealProposal) }{{ - name: "unsealed -> sealed", + name: "sealed update", initialState: func(sectorID abi.SectorID) *storiface.Decl { return &storiface.Decl{ SectorID: sectorID, @@ -117,7 +117,7 @@ func TestSectorStateManagerStateChangeToIndexer(t *testing.T) { })).Times(1) }, }, { - name: "sealed -> unsealed", + name: "unsealed update", initialState: func(sectorID abi.SectorID) *storiface.Decl { return &storiface.Decl{ SectorID: sectorID, @@ -138,7 +138,7 @@ func TestSectorStateManagerStateChangeToIndexer(t *testing.T) { })).Times(1) }, }, { - name: "unsealed -> unsealed (no sector updates)", + name: "no sector updates", initialState: func(sectorID abi.SectorID) *storiface.Decl { return &storiface.Decl{ SectorID: sectorID, @@ -151,7 +151,7 @@ func TestSectorStateManagerStateChangeToIndexer(t *testing.T) { expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { }, }, { - name: "unsealed -> removed", + name: "removed update", initialState: func(sectorID abi.SectorID) *storiface.Decl { return &storiface.Decl{ SectorID: sectorID, @@ -168,7 +168,7 @@ func TestSectorStateManagerStateChangeToIndexer(t *testing.T) { prov.NotifyRemove(gomock.Any(), gomock.Any(), gomock.Any()).Times(1) }, }, { - name: "sealed -> removed", + name: "removed update", initialState: func(sectorID abi.SectorID) *storiface.Decl { return &storiface.Decl{ SectorID: sectorID, @@ -185,7 +185,7 @@ func TestSectorStateManagerStateChangeToIndexer(t *testing.T) { prov.NotifyRemove(gomock.Any(), gomock.Any(), gomock.Any()).Times(1) }, }, { - name: "none -> unsealed(new sector)", + name: "unsealed update (new sector)", initialState: func(sectorID abi.SectorID) *storiface.Decl { return nil }, @@ -203,7 +203,7 @@ func TestSectorStateManagerStateChangeToIndexer(t *testing.T) { })).Times(1) }, }, { - name: "unsealed -> cache", + name: "cache update", initialState: func(sectorID abi.SectorID) *storiface.Decl { return &storiface.Decl{ SectorID: sectorID, @@ -219,7 +219,7 @@ func TestSectorStateManagerStateChangeToIndexer(t *testing.T) { // `cache` doesn't trigger a notify }, }, { - name: "cache -> unsealed", + name: "unsealed update (from cache)", initialState: func(sectorID abi.SectorID) *storiface.Decl { return &storiface.Decl{ SectorID: sectorID, @@ -241,7 +241,7 @@ func TestSectorStateManagerStateChangeToIndexer(t *testing.T) { })).Times(1) }, }, { - name: "cache -> sealed", + name: "sealed update (from cache)", initialState: func(sectorID abi.SectorID) *storiface.Decl { return &storiface.Decl{ SectorID: sectorID, @@ -296,115 +296,6 @@ func TestSectorStateManagerStateChangeToIndexer(t *testing.T) { } } -// Verify that multiple storage file types are handled from StorageList correctly -func TestUnsealedStateManagerStorageList(t *testing.T) { - ctx := context.Background() - - testCases := []struct { - name string - initialState func(sectorID abi.SectorID) []storiface.Decl - sectorUpdates func(sectorID abi.SectorID) map[abi.SectorID]db.SealState - expect func(*mock_provider.MockInterfaceMockRecorder, market.DealProposal) - }{ - { - name: "unsealed and sealed status", - initialState: func(sectorID abi.SectorID) []storiface.Decl { - return []storiface.Decl{ - { - SectorID: sectorID, - SectorFileType: storiface.FTUnsealed, - }, - { - SectorID: sectorID, - SectorFileType: storiface.FTSealed, - }, - } - }, - sectorUpdates: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { - return map[abi.SectorID]db.SealState{ - sectorID: db.SealStateUnsealed, - sectorID: db.SealStateCache, - } - }, - expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { - // no call to notify, as sector was unsealed and still is - }, - }, { - name: "unsealed and cached status", - initialState: func(sectorID abi.SectorID) []storiface.Decl { - return []storiface.Decl{ - { - SectorID: sectorID, - SectorFileType: storiface.FTUnsealed, - }, - { - SectorID: sectorID, - SectorFileType: storiface.FTCache, - }, - } - }, - sectorUpdates: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { - return map[abi.SectorID]db.SealState{ - sectorID: db.SealStateUnsealed, - sectorID: db.SealStateCache, - } - }, - expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { - }, - }, { - name: "sealed and cached status", - initialState: func(sectorID abi.SectorID) []storiface.Decl { - return []storiface.Decl{ - { - SectorID: sectorID, - SectorFileType: storiface.FTSealed, - }, - { - SectorID: sectorID, - SectorFileType: storiface.FTCache, - }, - } - }, - sectorUpdates: func(sectorID abi.SectorID) map[abi.SectorID]db.SealState { - return map[abi.SectorID]db.SealState{ - sectorID: db.SealStateUnsealed, - sectorID: db.SealStateCache, - } - }, - expect: func(prov *mock_provider.MockInterfaceMockRecorder, prop market.DealProposal) { - }, - }} - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - wrapper, legacyStorageProvider, storageMiner, prov := setup(t) - legacyStorageProvider.EXPECT().ListLocalDeals().AnyTimes().Return(nil, nil) - - // Add a deal to the database - deals, err := db.GenerateNDeals(1) - require.NoError(t, err) - err = wrapper.dealsDB.Insert(ctx, &deals[0]) - require.NoError(t, err) - - // Set up expectations (automatically verified when the test exits) - prop := deals[0].ClientDealProposal.Proposal - tc.expect(prov.EXPECT(), prop) - - minerID, err := address.IDFromAddress(deals[0].ClientDealProposal.Proposal.Provider) - require.NoError(t, err) - - // Set the first response from MinerAPI.StorageList() - storageMiner.storageList = map[storiface.ID][]storiface.Decl{} - resp1 := tc.initialState(abi.SectorID{Miner: abi.ActorID(minerID), Number: deals[0].SectorID}) - storageMiner.storageList["uuid"] = resp1 - - // Trigger check for updates - err = wrapper.handleUpdates(ctx, tc.sectorUpdates(abi.SectorID{Miner: abi.ActorID(minerID), Number: deals[0].SectorID})) - require.NoError(t, err) - }) - } -} - func setup(t *testing.T) (*Wrapper, *mock.MockStorageProvider, *mockApiStorageMiner, *mock_provider.MockInterface) { ctx := context.Background() ctrl := gomock.NewController(t) From 6627cbd58dac0d277ae319bfa097008bf7d33632 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 1 Jun 2023 12:35:16 +0200 Subject: [PATCH 37/41] more cases --- sectorstatemgr/sectorstatemgr_test.go | 259 ++++++++++++++++++-------- 1 file changed, 182 insertions(+), 77 deletions(-) diff --git a/sectorstatemgr/sectorstatemgr_test.go b/sectorstatemgr/sectorstatemgr_test.go index 3c339a9ea..5f63433ad 100644 --- a/sectorstatemgr/sectorstatemgr_test.go +++ b/sectorstatemgr/sectorstatemgr_test.go @@ -13,6 +13,7 @@ import ( "github.com/filecoin-project/boost/sectorstatemgr/mock" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/builtin/v9/miner" lotusmocks "github.com/filecoin-project/lotus/api/mocks" "github.com/filecoin-project/lotus/storage/sealer/storiface" "github.com/golang/mock/gomock" @@ -20,9 +21,36 @@ import ( ) func TestRefreshState(t *testing.T) { + // setup for test + ctx := context.Background() + + cfg := config.StorageConfig{ + StorageListRefreshDuration: config.Duration(100 * time.Millisecond), + } + + ctrl := gomock.NewController(t) + + // setup mocks + fullnodeApi := lotusmocks.NewMockFullNode(ctrl) + minerApi := mock.NewMockStorageAPI(ctrl) + + maddr, _ := address.NewIDAddress(1) + mid, _ := address.IDFromAddress(maddr) + aid := abi.ActorID(mid) + + // setup sectorstatemgr + mgr := &SectorStateMgr{ + cfg: cfg, + minerApi: minerApi, + fullnodeApi: fullnodeApi, + Maddr: maddr, + + PubSub: NewPubSub(), + } + type fixtures struct { - input interface{} - expected *SectorStateUpdates + mockExpectations func() + exerciseAndVerify func() } tests := []struct { @@ -30,108 +58,185 @@ func TestRefreshState(t *testing.T) { f func() fixtures }{ { - description: "first", + description: "one sealed, one unsealed, one not active - update in local state", f: func() fixtures { - deals, err := db.GenerateNDeals(2) + sqldb := db.CreateTestTmpDB(t) + require.NoError(t, db.CreateAllBoostTables(ctx, sqldb, sqldb)) + require.NoError(t, migrations.Migrate(sqldb)) + mgr.sdb = db.NewSectorStateDB(sqldb) + + deals, err := db.GenerateNDeals(3) require.NoError(t, err) - sid1 := abi.SectorID{Miner: abi.ActorID(1), Number: deals[0].SectorID} - sid2 := abi.SectorID{Miner: abi.ActorID(1), Number: deals[1].SectorID} - list := map[storiface.ID][]storiface.Decl{ + sid3 := abi.SectorID{Miner: aid, Number: deals[0].SectorID} + sid4 := abi.SectorID{Miner: aid, Number: deals[1].SectorID} + sid5 := abi.SectorID{Miner: aid, Number: deals[2].SectorID} + + input_StorageList1 := map[storiface.ID][]storiface.Decl{ "storage-location-uuid1": { - { - SectorID: sid1, - SectorFileType: storiface.FTUnsealed, - }, - { - SectorID: sid1, - SectorFileType: storiface.FTSealed, - }, - { - SectorID: sid1, - SectorFileType: storiface.FTCache, - }, + {SectorID: sid3, SectorFileType: storiface.FTSealed}, + {SectorID: sid3, SectorFileType: storiface.FTCache}, }, "storage-location-uuid2": { - { - SectorID: sid2, - SectorFileType: storiface.FTSealed, - }, - { - SectorID: sid2, - SectorFileType: storiface.FTCache, - }, + {SectorID: sid4, SectorFileType: storiface.FTUnsealed}, + {SectorID: sid4, SectorFileType: storiface.FTSealed}, + {SectorID: sid4, SectorFileType: storiface.FTCache}, + {SectorID: sid5, SectorFileType: storiface.FTUpdateCache}, }, } + input_StateMinerActiveSectors1 := []*miner.SectorOnChainInfo{ + {SectorNumber: sid3.Number}, + {SectorNumber: sid4.Number}, + } - return fixtures{ - input: list, - expected: &SectorStateUpdates{ - Updates: map[abi.SectorID]db.SealState{ - sid1: db.SealStateUnsealed, - sid2: db.SealStateSealed, - }, - ActiveSectors: map[abi.SectorID]struct{}{}, - SectorStates: map[abi.SectorID]db.SealState{ - sid1: db.SealStateUnsealed, - sid2: db.SealStateSealed, - }, + input_StorageList2 := map[storiface.ID][]storiface.Decl{ + "storage-location-uuid1": { + {SectorID: sid3, SectorFileType: storiface.FTUnsealed}, + {SectorID: sid3, SectorFileType: storiface.FTSealed}, + {SectorID: sid3, SectorFileType: storiface.FTCache}, }, + "storage-location-uuid2": { + {SectorID: sid4, SectorFileType: storiface.FTSealed}, + {SectorID: sid4, SectorFileType: storiface.FTCache}, + {SectorID: sid5, SectorFileType: storiface.FTUpdateCache}, + }, + } + input_StateMinerActiveSectors2 := []*miner.SectorOnChainInfo{ + {SectorNumber: sid3.Number}, + {SectorNumber: sid4.Number}, + } + + mockExpectations := func() { + minerApi.EXPECT().StorageList(gomock.Any()).Return(input_StorageList1, nil) + fullnodeApi.EXPECT().StateMinerActiveSectors(gomock.Any(), gomock.Any(), gomock.Any()).Return(input_StateMinerActiveSectors1, nil) + + minerApi.EXPECT().StorageList(gomock.Any()).Return(input_StorageList2, nil) + fullnodeApi.EXPECT().StateMinerActiveSectors(gomock.Any(), gomock.Any(), gomock.Any()).Return(input_StateMinerActiveSectors2, nil) + + fullnodeApi.EXPECT().ChainHead(gomock.Any()).Times(2) + } + + expected2 := &SectorStateUpdates{ + Updates: map[abi.SectorID]db.SealState{ + sid3: db.SealStateUnsealed, + sid4: db.SealStateSealed, + }, + ActiveSectors: map[abi.SectorID]struct{}{ + sid3: struct{}{}, + sid4: struct{}{}, + }, + SectorStates: map[abi.SectorID]db.SealState{ + sid3: db.SealStateUnsealed, + sid4: db.SealStateSealed, + sid5: db.SealStateCache, + }, + } + + exerciseAndVerify := func() { + err := mgr.checkForUpdates(ctx) + require.NoError(t, err) + + got2, err := mgr.refreshState(ctx) + require.NoError(t, err) + + zero := time.Time{} + require.NotEqual(t, got2.UpdatedAt, zero) + + //null timestamp, so that we can do deep equal + got2.UpdatedAt = zero + + require.True(t, reflect.DeepEqual(expected2, got2), "expected: %s, got: %s", spew.Sdump(expected2), spew.Sdump(got2)) + } + + return fixtures{ + mockExpectations: mockExpectations, + exerciseAndVerify: exerciseAndVerify, } }, }, - } - - // setup for test - ctx := context.Background() + { + description: "one sealed, one unsealed, one not active - different sectors", + f: func() fixtures { + sqldb := db.CreateTestTmpDB(t) + require.NoError(t, db.CreateAllBoostTables(ctx, sqldb, sqldb)) + require.NoError(t, migrations.Migrate(sqldb)) + mgr.sdb = db.NewSectorStateDB(sqldb) - cfg := config.StorageConfig{ - StorageListRefreshDuration: config.Duration(100 * time.Millisecond), - } + deals, err := db.GenerateNDeals(3) + require.NoError(t, err) + sid1 := abi.SectorID{Miner: aid, Number: deals[0].SectorID} + sid2 := abi.SectorID{Miner: aid, Number: deals[1].SectorID} + // even though sector is not active, we continue to announce deal + sid3 := abi.SectorID{Miner: aid, Number: deals[2].SectorID} - sqldb := db.CreateTestTmpDB(t) - require.NoError(t, db.CreateAllBoostTables(ctx, sqldb, sqldb)) - require.NoError(t, migrations.Migrate(sqldb)) + input_StorageList := map[storiface.ID][]storiface.Decl{ + "storage-location-uuid1": { + {SectorID: sid1, SectorFileType: storiface.FTUnsealed}, + {SectorID: sid1, SectorFileType: storiface.FTSealed}, + {SectorID: sid1, SectorFileType: storiface.FTCache}, + }, + "storage-location-uuid2": { + {SectorID: sid2, SectorFileType: storiface.FTSealed}, + {SectorID: sid2, SectorFileType: storiface.FTCache}, + {SectorID: sid3, SectorFileType: storiface.FTSealed}, + }, + } + input_StateMinerActiveSectors := []*miner.SectorOnChainInfo{ + {SectorNumber: sid1.Number}, + {SectorNumber: sid2.Number}, + } - sdb := db.NewSectorStateDB(sqldb) + mockExpectations := func() { + minerApi.EXPECT().StorageList(gomock.Any()).Return(input_StorageList, nil) + fullnodeApi.EXPECT().ChainHead(gomock.Any()).Times(1) + fullnodeApi.EXPECT().StateMinerActiveSectors(gomock.Any(), gomock.Any(), gomock.Any()).Return(input_StateMinerActiveSectors, nil) + } - ctrl := gomock.NewController(t) + expected := &SectorStateUpdates{ + Updates: map[abi.SectorID]db.SealState{ + sid1: db.SealStateUnsealed, + sid2: db.SealStateSealed, + sid3: db.SealStateSealed, + }, + ActiveSectors: map[abi.SectorID]struct{}{ + sid1: struct{}{}, + sid2: struct{}{}, + }, + SectorStates: map[abi.SectorID]db.SealState{ + sid1: db.SealStateUnsealed, + sid2: db.SealStateSealed, + sid3: db.SealStateSealed, + }, + } - // setup mocks - fullnodeApi := lotusmocks.NewMockFullNode(ctrl) - minerApi := mock.NewMockStorageAPI(ctrl) + exerciseAndVerify := func() { + got, err := mgr.refreshState(ctx) + require.NoError(t, err) - maddr, _ := address.NewIDAddress(0) + zero := time.Time{} + require.NotEqual(t, got.UpdatedAt, zero) - // setup sectorstatemgr - mgr := &SectorStateMgr{ - cfg: cfg, - minerApi: minerApi, - fullnodeApi: fullnodeApi, - Maddr: maddr, + //null timestamp, so that we can do deep equal + got.UpdatedAt = zero - PubSub: NewPubSub(), + require.True(t, reflect.DeepEqual(expected, got), "expected: %s, got: %s", spew.Sdump(expected), spew.Sdump(got)) + } - sdb: sdb, + return fixtures{ + mockExpectations: mockExpectations, + exerciseAndVerify: exerciseAndVerify, + } + }, + }, } for _, tt := range tests { // f() builds fixtures for a specific test case, namely all required input for test and expected result fixt := tt.f() - minerApi.EXPECT().StorageList(gomock.Any()).Return(fixt.input, nil) - fullnodeApi.EXPECT().ChainHead(gomock.Any()).Times(1) - fullnodeApi.EXPECT().StateMinerActiveSectors(gomock.Any(), gomock.Any(), gomock.Any()).Times(1) - - got, err := mgr.refreshState(ctx) - require.NoError(t, err) - - zero := time.Time{} - require.NotEqual(t, got.UpdatedAt, zero) + // mockExpectations() + fixt.mockExpectations() - //null timestamp, so that we can do deep equal - got.UpdatedAt = zero - - require.True(t, reflect.DeepEqual(fixt.expected, got), "expected: %s, got: %s", spew.Sdump(fixt.expected), spew.Sdump(got)) + // exerciseAndVerify() + fixt.exerciseAndVerify() } - } From 38d084cfb90c57b14066f0e3e493dca0d5878cbe Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 1 Jun 2023 12:40:16 +0200 Subject: [PATCH 38/41] more tests --- sectorstatemgr/sectorstatemgr_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sectorstatemgr/sectorstatemgr_test.go b/sectorstatemgr/sectorstatemgr_test.go index 5f63433ad..11ce344cd 100644 --- a/sectorstatemgr/sectorstatemgr_test.go +++ b/sectorstatemgr/sectorstatemgr_test.go @@ -65,11 +65,12 @@ func TestRefreshState(t *testing.T) { require.NoError(t, migrations.Migrate(sqldb)) mgr.sdb = db.NewSectorStateDB(sqldb) - deals, err := db.GenerateNDeals(3) + deals, err := db.GenerateNDeals(4) require.NoError(t, err) sid3 := abi.SectorID{Miner: aid, Number: deals[0].SectorID} sid4 := abi.SectorID{Miner: aid, Number: deals[1].SectorID} sid5 := abi.SectorID{Miner: aid, Number: deals[2].SectorID} + sid6 := abi.SectorID{Miner: aid, Number: deals[3].SectorID} input_StorageList1 := map[storiface.ID][]storiface.Decl{ "storage-location-uuid1": { @@ -81,6 +82,7 @@ func TestRefreshState(t *testing.T) { {SectorID: sid4, SectorFileType: storiface.FTSealed}, {SectorID: sid4, SectorFileType: storiface.FTCache}, {SectorID: sid5, SectorFileType: storiface.FTUpdateCache}, + {SectorID: sid6, SectorFileType: storiface.FTSealed}, }, } input_StateMinerActiveSectors1 := []*miner.SectorOnChainInfo{ @@ -119,6 +121,7 @@ func TestRefreshState(t *testing.T) { Updates: map[abi.SectorID]db.SealState{ sid3: db.SealStateUnsealed, sid4: db.SealStateSealed, + sid6: db.SealStateRemoved, }, ActiveSectors: map[abi.SectorID]struct{}{ sid3: struct{}{}, @@ -132,9 +135,11 @@ func TestRefreshState(t *testing.T) { } exerciseAndVerify := func() { + // setup initial state of db err := mgr.checkForUpdates(ctx) require.NoError(t, err) + // trigger refreshState and later verify resulting struct got2, err := mgr.refreshState(ctx) require.NoError(t, err) From 734ac018058d92407aa6c1a1bd682b53649053e7 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 1 Jun 2023 12:46:15 +0200 Subject: [PATCH 39/41] update description --- sectorstatemgr/sectorstatemgr_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sectorstatemgr/sectorstatemgr_test.go b/sectorstatemgr/sectorstatemgr_test.go index 11ce344cd..8d7fb1c1a 100644 --- a/sectorstatemgr/sectorstatemgr_test.go +++ b/sectorstatemgr/sectorstatemgr_test.go @@ -58,7 +58,7 @@ func TestRefreshState(t *testing.T) { f func() fixtures }{ { - description: "one sealed, one unsealed, one not active - update in local state", + description: "four deals - sealed->unsealed, unsealed->sealed, cached, removed", f: func() fixtures { sqldb := db.CreateTestTmpDB(t) require.NoError(t, db.CreateAllBoostTables(ctx, sqldb, sqldb)) @@ -170,7 +170,6 @@ func TestRefreshState(t *testing.T) { require.NoError(t, err) sid1 := abi.SectorID{Miner: aid, Number: deals[0].SectorID} sid2 := abi.SectorID{Miner: aid, Number: deals[1].SectorID} - // even though sector is not active, we continue to announce deal sid3 := abi.SectorID{Miner: aid, Number: deals[2].SectorID} input_StorageList := map[storiface.ID][]storiface.Decl{ From f91ae9f375e149a0f56f25a945386724d6b22959 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 1 Jun 2023 14:10:38 +0200 Subject: [PATCH 40/41] better comment --- indexprovider/wrapper_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/indexprovider/wrapper_test.go b/indexprovider/wrapper_test.go index b85012b37..a80a997f0 100644 --- a/indexprovider/wrapper_test.go +++ b/indexprovider/wrapper_test.go @@ -19,12 +19,11 @@ import ( "github.com/stretchr/testify/require" ) -// Empty response from MinerAPI.StorageList() -func TestUnsealedStateManagerEmptyStorageList(t *testing.T) { +func TestWrapperEmptyStorageListAndNoUpdates(t *testing.T) { wrapper, legacyStorageProvider, _, _ := setup(t) legacyStorageProvider.EXPECT().ListLocalDeals().AnyTimes().Return(nil, nil) - // Check for updates with an empty response from MinerAPI.StorageList() + // handleUpdates with an empty response from MinerAPI.StorageList() and no updates err := wrapper.handleUpdates(context.Background(), nil) require.NoError(t, err) } From 8f67c822014bb549295be29cc256111d72c73629 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 1 Jun 2023 14:20:52 +0200 Subject: [PATCH 41/41] better names and comments --- indexprovider/wrapper.go | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/indexprovider/wrapper.go b/indexprovider/wrapper.go index 319a7c4fb..773caf849 100644 --- a/indexprovider/wrapper.go +++ b/indexprovider/wrapper.go @@ -131,24 +131,22 @@ func (w *Wrapper) checkForUpdates(ctx context.Context) { } } -func (w *Wrapper) handleUpdates(ctx context.Context, sus map[abi.SectorID]db.SealState) error { - legacyDeals, err := w.legacyDealsBySectorID(sus) +func (w *Wrapper) handleUpdates(ctx context.Context, sectorUpdates map[abi.SectorID]db.SealState) error { + legacyDeals, err := w.legacyDealsBySectorID(sectorUpdates) if err != nil { return fmt.Errorf("getting legacy deals from datastore: %w", err) } - log.Debugf("checking for sector state updates for %d states", len(sus)) + log.Debugf("checking for sector state updates for %d states", len(sectorUpdates)) - // For each sector - for sectorID, sectorSealState := range sus { - // Get the deals in the sector + for sectorID, sectorSealState := range sectorUpdates { + // for all updated sectors, get all deals (legacy and boost) in the sector deals, err := w.dealsBySectorID(ctx, legacyDeals, sectorID) if err != nil { return fmt.Errorf("getting deals for miner %d / sector %d: %w", sectorID.Miner, sectorID.Number, err) } log.Debugf("sector %d has %d deals, seal status %s", sectorID, len(deals), sectorSealState) - // For each deal in the sector for _, deal := range deals { if !deal.AnnounceToIPNI { continue @@ -161,12 +159,11 @@ func (w *Wrapper) handleUpdates(ctx context.Context, sus map[abi.SectorID]db.Sea propCid := propnd.Cid() if sectorSealState == db.SealStateRemoved { - // Announce deals that are no longer unsealed to indexer + // announce deals that are no longer unsealed as removed to indexer announceCid, err := w.AnnounceBoostDealRemoved(ctx, propCid) if err != nil { - // Check if the error is because the deal wasn't previously announced + // check if the error is because the deal wasn't previously announced if !errors.Is(err, provider.ErrContextIDNotFound) { - // There was some other error, write it to the log log.Errorw("announcing deal removed to index provider", "deal id", deal.DealID, "error", err) continue @@ -176,19 +173,19 @@ func (w *Wrapper) handleUpdates(ctx context.Context, sus map[abi.SectorID]db.Sea "deal id", deal.DealID, "sector id", deal.SectorID.Number, "announce cid", announceCid.String()) } } else if sectorSealState != db.SealStateCache { - // Announce deals that have changed seal state to indexer + // announce deals that have changed seal state to indexer md := metadata.GraphsyncFilecoinV1{ PieceCID: deal.DealProposal.Proposal.PieceCID, FastRetrieval: sectorSealState == db.SealStateUnsealed, VerifiedDeal: deal.DealProposal.Proposal.VerifiedDeal, } announceCid, err := w.AnnounceBoostDealMetadata(ctx, md, propCid) - if err == nil { + if err != nil { + log.Errorf("announcing deal %s to index provider: %w", deal.DealID, err) + } else { log.Infow("announced deal seal state to index provider", "deal id", deal.DealID, "sector id", deal.SectorID.Number, "seal state", sectorSealState, "announce cid", announceCid.String()) - } else { - log.Errorf("announcing deal %s to index provider: %w", deal.DealID, err) } } }