fix: update deal filter to use ExpectedSealDuration and `MaxDealSta…

…rtDelay` (#1609)

* update deal filters

* fix itests

* adjust devnet values

* Update docker/devnet/boost/entrypoint.sh

fix devnet

docker/devnet/boost/entrypoint.sh

@@ -74,5 +74,8 @@ if [ ! -f $BOOST_PATH/.register.boost ]; then
 	echo Super. DONE! Boostd is now configured and will be started soon
 fi
 
+## Override config options
+echo Updating config values
+sed -i 's|ExpectedSealDuration = "24h0m0s"|ExpectedSealDuration = "0h0m10s"|g' $BOOST_PATH/config.toml
 echo Starting boost in dev mode...
 exec boostd -vv run --nosync=true

itests/framework/framework.go

@@ -296,6 +296,8 @@ func (f *TestFramework) Start() error {
 	cfg.Dealmaking.HttpTransferStallCheckPeriod = config.Duration(100 * time.Millisecond)
 	cfg.Storage.ParallelFetchLimit = 10
 
+	cfg.Dealmaking.ExpectedSealDuration = 10
+
 	err = lr.SetConfig(func(raw interface{}) {
 		rcfg := raw.(*config.Boost)
 		*rcfg = *cfg

node/builder.go

@@ -583,9 +583,9 @@ func ConfigBoost(cfg *config.Boost) Option {
 		Override(HandleSetLinkSystem, modules.SetLinkSystem),
 
 		// Boost storage deal filter
-		Override(new(dtypes.StorageDealFilter), modules.BasicDealFilter(cfg.Dealmaking, nil)),
+		Override(new(dtypes.StorageDealFilter), modules.BasicDealFilter(nil)),
 		If(cfg.Dealmaking.Filter != "",
-			Override(new(dtypes.StorageDealFilter), modules.BasicDealFilter(cfg.Dealmaking, dtypes.StorageDealFilter(dealfilter.CliStorageDealFilter(cfg.Dealmaking.Filter)))),
+			Override(new(dtypes.StorageDealFilter), modules.BasicDealFilter(dtypes.StorageDealFilter(dealfilter.CliStorageDealFilter(cfg.Dealmaking.Filter)))),
 		),
 
 		// Lotus markets storage deal filter
@@ -611,7 +611,7 @@ func ConfigBoost(cfg *config.Boost) Option {
 		Override(new(*storageadapter.DealPublisher), storageadapter.NewDealPublisher(&legacyFees, storageadapter.PublishMsgConfig{
 			Period:                  time.Duration(cfg.LotusDealmaking.PublishMsgPeriod),
 			MaxDealsPerMsg:          cfg.LotusDealmaking.MaxDealsPerPublishMsg,
-			StartEpochSealingBuffer: cfg.LotusDealmaking.StartEpochSealingBuffer,
+			StartEpochSealingBuffer: cfg.Dealmaking.StartEpochSealingBuffer,
 		})),
 
 		Override(new(sealer.Unsealer), From(new(lotus_modules.MinerStorageService))),

node/config/types.go

@@ -150,7 +150,9 @@ type DealmakingConfig struct {
 	// This includes the time the deal will need to get transferred and published
 	// before being assigned to a sector
 	ExpectedSealDuration Duration
-	// Maximum amount of time proposed deal StartEpoch can be in future
+	// Maximum amount of time proposed deal StartEpoch can be in the future.
+	// This is applicable only for online deals as offline deals can take long duration
+	// to import the data
 	MaxDealStartDelay Duration
 	// The maximum collateral that the provider will put up against a deal,
 	// as a multiplier of the minimum collateral bound

node/modules/dealfilter.go

@@ -3,20 +3,25 @@ package modules
 import (
 	"context"
 	"fmt"
+	"time"
 
-	"github.com/filecoin-project/boost/node/config"
+	"github.com/filecoin-project/boost-gfm/retrievalmarket"
 	"github.com/filecoin-project/boost/node/modules/dtypes"
 	"github.com/filecoin-project/boost/storagemarket/dealfilter"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/lotus/api/v1api"
+	"github.com/filecoin-project/lotus/build"
 	lotus_repo "github.com/filecoin-project/lotus/node/repo"
 )
 
-func BasicDealFilter(cfg config.DealmakingConfig, userCmd dtypes.StorageDealFilter) func(onlineOk dtypes.ConsiderOnlineStorageDealsConfigFunc,
+func BasicDealFilter(userCmd dtypes.StorageDealFilter) func(onlineOk dtypes.ConsiderOnlineStorageDealsConfigFunc,
 	offlineOk dtypes.ConsiderOfflineStorageDealsConfigFunc,
 	verifiedOk dtypes.ConsiderVerifiedStorageDealsConfigFunc,
 	unverifiedOk dtypes.ConsiderUnverifiedStorageDealsConfigFunc,
 	blocklistFunc dtypes.StorageDealPieceCidBlocklistConfigFunc,
 	expectedSealTimeFunc dtypes.GetExpectedSealDurationFunc,
 	startDelay dtypes.GetMaxDealStartDelayFunc,
+	fullNodeApi v1api.FullNode,
 	r lotus_repo.LockedRepo,
 ) dtypes.StorageDealFilter {
 	return func(onlineOk dtypes.ConsiderOnlineStorageDealsConfigFunc,
@@ -26,6 +31,7 @@ func BasicDealFilter(cfg config.DealmakingConfig, userCmd dtypes.StorageDealFilt
 		blocklistFunc dtypes.StorageDealPieceCidBlocklistConfigFunc,
 		expectedSealTimeFunc dtypes.GetExpectedSealDurationFunc,
 		startDelay dtypes.GetMaxDealStartDelayFunc,
+		fullNodeApi v1api.FullNode,
 		r lotus_repo.LockedRepo,
 	) dtypes.StorageDealFilter {
 		return func(ctx context.Context, params dealfilter.DealFilterParams) (bool, string, error) {
@@ -89,6 +95,38 @@ func BasicDealFilter(cfg config.DealmakingConfig, userCmd dtypes.StorageDealFilt
 				}
 			}
 
+			// Reject if start epoch is less than the minimum time to be taken to seal the deal
+			sealDuration, err := expectedSealTimeFunc()
+			if err != nil {
+				return false, "miner error", err
+			}
+
+			sealEpochs := sealDuration / (time.Duration(build.BlockDelaySecs) * time.Second)
+			ts, err := fullNodeApi.ChainHead(ctx)
+			if err != nil {
+				return false, "failed to get chain head", err
+			}
+			ht := ts.Height()
+			earliest := abi.ChainEpoch(sealEpochs) + ht
+			if deal.ClientDealProposal.Proposal.StartEpoch < earliest {
+				log.Warnw("proposed deal would start before sealing can be completed; rejecting storage deal proposal from client", "piece_cid", deal.ClientDealProposal.Proposal.PieceCID, "client", deal.ClientDealProposal.Proposal.Client.String(), "seal_duration", sealDuration, "earliest", earliest, "curepoch", ht)
+				return false, fmt.Sprintf("cannot seal a sector before %s", deal.ClientDealProposal.Proposal.StartEpoch), nil
+			}
+
+			// Reject if the start epoch is too far in the future and is online
+			// We probably do not want to reject offline deal based on this criteria
+			if !deal.IsOffline {
+				sd, err := startDelay()
+				if err != nil {
+					return false, "miner error", err
+				}
+
+				maxStartEpoch := earliest + abi.ChainEpoch(uint64(sd.Seconds())/build.BlockDelaySecs)
+				if deal.ClientDealProposal.Proposal.StartEpoch > maxStartEpoch {
+					return false, fmt.Sprintf("deal start epoch is too far in the future: %s > %s", deal.ClientDealProposal.Proposal.StartEpoch, maxStartEpoch), nil
+				}
+			}
+
 			if userCmd != nil {
 				return userCmd(ctx, params)
 			}
@@ -97,3 +135,36 @@ func BasicDealFilter(cfg config.DealmakingConfig, userCmd dtypes.StorageDealFilt
 		}
 	}
 }
+
+func RetrievalDealFilter(userFilter dtypes.RetrievalDealFilter) func(onlineOk dtypes.ConsiderOnlineRetrievalDealsConfigFunc,
+	offlineOk dtypes.ConsiderOfflineRetrievalDealsConfigFunc) dtypes.RetrievalDealFilter {
+	return func(onlineOk dtypes.ConsiderOnlineRetrievalDealsConfigFunc,
+		offlineOk dtypes.ConsiderOfflineRetrievalDealsConfigFunc) dtypes.RetrievalDealFilter {
+		return func(ctx context.Context, state retrievalmarket.ProviderDealState) (bool, string, error) {
+			b, err := onlineOk()
+			if err != nil {
+				return false, "miner error", err
+			}
+
+			if !b {
+				log.Warn("online retrieval deal consideration disabled; rejecting retrieval deal proposal from client")
+				return false, "miner is not accepting online retrieval deals", nil
+			}
+
+			b, err = offlineOk()
+			if err != nil {
+				return false, "miner error", err
+			}
+
+			if !b {
+				log.Info("offline retrieval has not been implemented yet")
+			}
+
+			if userFilter != nil {
+				return userFilter(ctx, state)
+			}
+
+			return true, "", nil
+		}
+	}
+}

node/modules/storageminer.go

@@ -84,39 +84,6 @@ var (
 	StorageCounterDSPrefix = "/storage/nextid"
 )
 
-func RetrievalDealFilter(userFilter dtypes.RetrievalDealFilter) func(onlineOk dtypes.ConsiderOnlineRetrievalDealsConfigFunc,
-	offlineOk dtypes.ConsiderOfflineRetrievalDealsConfigFunc) dtypes.RetrievalDealFilter {
-	return func(onlineOk dtypes.ConsiderOnlineRetrievalDealsConfigFunc,
-		offlineOk dtypes.ConsiderOfflineRetrievalDealsConfigFunc) dtypes.RetrievalDealFilter {
-		return func(ctx context.Context, state retrievalmarket.ProviderDealState) (bool, string, error) {
-			b, err := onlineOk()
-			if err != nil {
-				return false, "miner error", err
-			}
-
-			if !b {
-				log.Warn("online retrieval deal consideration disabled; rejecting retrieval deal proposal from client")
-				return false, "miner is not accepting online retrieval deals", nil
-			}
-
-			b, err = offlineOk()
-			if err != nil {
-				return false, "miner error", err
-			}
-
-			if !b {
-				log.Info("offline retrieval has not been implemented yet")
-			}
-
-			if userFilter != nil {
-				return userFilter(ctx, state)
-			}
-
-			return true, "", nil
-		}
-	}
-}
-
 func NewConsiderOnlineStorageDealsConfigFunc(r lotus_repo.LockedRepo) (dtypes.ConsiderOnlineStorageDealsConfigFunc, error) {
 	return func() (out bool, err error) {
 		err = readCfg(r, func(cfg *config.Boost) {