fix: remove boost-gfm and go-data-transfer (#1740)

* remove gfm, pull datatransfer * make gen, docsgen * fix circular depndencies * go mod tidy * fix lint errors * convert RetrievalAskGetter to interface * fix interface pointer * fix dependency injection, move ask db * fix itests * clean up mocks * chore: lots more dt & gs debugging (#1776) * fix lint errs, add back ProviderDataTransfer * regenrate cbor * cleanup go-data-transfer * fix graphsync retrievals * use new client for GS tests * fix selector type * restore selector type * fix: clean up and fix retrieval framework & tests (#1848) * fix: clean up and fix retrieval framework & tests * Use new go-trustless-utils primitives and traversal tools * Simplify Framework#Retrieve * fix: wait for full startup of data transfer when making client * fix lint errors * fix go mod * cleanup itest fixtures, fix gql * go mod tidy * fix directDealProv * handle storage ask * cleanup config * remove graphsync unit tests * add retrieval config headers * update retrieval configs * fix storage ask db command * fix BoostLegacyDealByProposalCid api * make cbor-gen * fix missing api example * migrate config, dump dagstore config --------- Co-authored-by: Rod Vagg <[email protected]>
filecoin-project · Feb 19, 2024 · 3b49cd1 · 3b49cd1
1 parent e552e7c
commit 3b49cd1
Show file tree

Hide file tree

Showing 214 changed files with 32,333 additions and 11,501 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -319,29 +319,14 @@ workflows:
           target: "./itests/dummydeal_test.go"
 
       - test:
-          name: test-itest-markets_v1_deal
-          suite: itest-markets_v1_deal
-          target: "./itests/markets_v1_deal_test.go"
+          name: test-graphsync_identity_cid
+          suite: itest-graphsync_identity_cid
+          target: "./itests/graphsync_identity_cid_test.go"
 
       - test:
-          name: test-itest-markets_v1_identity_cid
-          suite: itest-markets_v1_identity_cid
-          target: "./itests/markets_v1_identity_cid_test.go"
-
-      - test:
-          name: test-itest-markets_v1_offline_deal
-          suite: itest-markets_v1_offline_deal
-          target: "./itests/markets_v1_offline_deal_test.go"
-
-      - test:
-          name: test-itest-markets_v1_retrieval
-          suite: itest-markets_v1_retrieval
-          target: "./itests/markets_v1_retrieval_test.go"
-
-      - test:
-          name: test-itest-disabled_markets_v1_deal
-          suite: itest-disabled_markets_v1_deal
-          target: "./itests/disabled_markets_v1_deal_test.go"
+          name: test-itest-retrieval
+          suite: itest-retrieval
+          target: "./itests/graphsync_retrieval_test.go"
 
       #- test:
           #name: test-itest-direct_deal

diff --git a/api/api.go b/api/api.go
@@ -3,17 +3,10 @@ package api
 import (
 	"context"
 
-	"github.com/filecoin-project/boost-gfm/retrievalmarket"
-	"github.com/filecoin-project/boost-gfm/storagemarket"
 	smtypes "github.com/filecoin-project/boost/storagemarket/types"
-	"github.com/filecoin-project/go-address"
-	datatransfer "github.com/filecoin-project/go-data-transfer"
-	"github.com/filecoin-project/go-state-types/abi"
-	lapi "github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/boost/storagemarket/types/legacytypes"
 	"github.com/google/uuid"
 	"github.com/ipfs/go-cid"
-	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/multiformats/go-multihash"
 )
 
@@ -42,17 +35,9 @@ type Boost interface {
 	BoostDealBySignedProposalCid(ctx context.Context, proposalCid cid.Cid) (*smtypes.ProviderDealState, error)                                  //perm:admin
 	BoostDummyDeal(context.Context, smtypes.DealParams) (*ProviderDealRejectionInfo, error)                                                     //perm:admin
 	BoostIndexerAnnounceDealRemoved(ctx context.Context, propCid cid.Cid) (cid.Cid, error)                                                      //perm:admin
+	BoostLegacyDealByProposalCid(ctx context.Context, propCid cid.Cid) (legacytypes.MinerDeal, error)                                           //perm:admin
 	BoostIndexerAnnounceDeal(ctx context.Context, deal *smtypes.ProviderDealState) (cid.Cid, error)                                             //perm:admin
-	BoostIndexerAnnounceLegacyDeal(ctx context.Context, proposalCid cid.Cid) error                                                              //perm:admin
-	BoostLegacyDealByProposalCid(ctx context.Context, propCid cid.Cid) (storagemarket.MinerDeal, error)                                         //perm:admin
-	BoostDagstoreRegisterShard(ctx context.Context, key string) error                                                                           //perm:admin
-	BoostDagstoreDestroyShard(ctx context.Context, key string) error                                                                            //perm:admin
-	BoostDagstoreInitializeShard(ctx context.Context, key string) error                                                                         //perm:admin
-	BoostDagstoreInitializeAll(ctx context.Context, params DagstoreInitializeAllParams) (<-chan DagstoreInitializeAllEvent, error)              //perm:admin
-	BoostDagstoreRecoverShard(ctx context.Context, key string) error                                                                            //perm:admin
-	BoostDagstoreGC(ctx context.Context) ([]DagstoreShardResult, error)                                                                         //perm:admin
-	BoostDagstorePiecesContainingMultihash(ctx context.Context, mh multihash.Multihash) ([]cid.Cid, error)                                      //perm:read
-	BoostDagstoreListShards(ctx context.Context) ([]DagstoreShardInfo, error)                                                                   //perm:admin
+	BoostIndexerAnnounceLegacyDeal(ctx context.Context, proposalCid cid.Cid) (cid.Cid, error)                                                   //perm:admin
 	BoostMakeDeal(context.Context, smtypes.DealParams) (*ProviderDealRejectionInfo, error)                                                      //perm:write
 	BoostDirectDeal(ctx context.Context, params smtypes.DirectDealParams) (*ProviderDealRejectionInfo, error)                                   //perm:admin
 
@@ -64,75 +49,6 @@ type Boost interface {
 	// MethodGroup: PieceDirectory
 	PdBuildIndexForPieceCid(ctx context.Context, piececid cid.Cid) error //perm:admin
 
-	// RuntimeSubsystems returns the subsystems that are enabled
-	// in this instance.
-	RuntimeSubsystems(ctx context.Context) (lapi.MinerSubsystems, error) //perm:read
-
-	// MethodGroup: LegacyMarket
-	MarketListRetrievalDeals(ctx context.Context) ([]retrievalmarket.ProviderDealState, error)                                                                                           //perm:read
-	MarketSetRetrievalAsk(ctx context.Context, rask *retrievalmarket.Ask) error                                                                                                          //perm:admin
-	MarketGetRetrievalAsk(ctx context.Context) (*retrievalmarket.Ask, error)                                                                                                             //perm:read
-	MarketSetAsk(ctx context.Context, price types.BigInt, verifiedPrice types.BigInt, duration abi.ChainEpoch, minPieceSize abi.PaddedPieceSize, maxPieceSize abi.PaddedPieceSize) error //perm:admin
-	MarketGetAsk(ctx context.Context) (*storagemarket.SignedStorageAsk, error)                                                                                                           //perm:read
-	MarketListDataTransfers(ctx context.Context) ([]DataTransferChannel, error)                                                                                                          //perm:write
-	MarketDataTransferUpdates(ctx context.Context) (<-chan DataTransferChannel, error)                                                                                                   //perm:write
-	MarketRestartDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error                                                        //perm:write
-	MarketCancelDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error                                                         //perm:write
-	MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error                                                                                                        //perm:write
-	MarketListIncompleteDeals(ctx context.Context) ([]storagemarket.MinerDeal, error)                                                                                                    //perm:read
-	MarketPendingDeals(ctx context.Context) (lapi.PendingDealInfo, error)                                                                                                                //perm:write
-	SectorsRefs(context.Context) (map[string][]lapi.SealedRef, error)                                                                                                                    //perm:read
-
-	// MethodGroup: Actor
-	ActorSectorSize(context.Context, address.Address) (abi.SectorSize, error) //perm:read
-
-	// MethodGroup: Deals
-	DealsConsiderOnlineStorageDeals(context.Context) (bool, error)      //perm:admin
-	DealsSetConsiderOnlineStorageDeals(context.Context, bool) error     //perm:admin
-	DealsConsiderOnlineRetrievalDeals(context.Context) (bool, error)    //perm:admin
-	DealsSetConsiderOnlineRetrievalDeals(context.Context, bool) error   //perm:admin
-	DealsPieceCidBlocklist(context.Context) ([]cid.Cid, error)          //perm:admin
-	DealsSetPieceCidBlocklist(context.Context, []cid.Cid) error         //perm:admin
-	DealsConsiderOfflineStorageDeals(context.Context) (bool, error)     //perm:admin
-	DealsSetConsiderOfflineStorageDeals(context.Context, bool) error    //perm:admin
-	DealsConsiderOfflineRetrievalDeals(context.Context) (bool, error)   //perm:admin
-	DealsSetConsiderOfflineRetrievalDeals(context.Context, bool) error  //perm:admin
-	DealsConsiderVerifiedStorageDeals(context.Context) (bool, error)    //perm:admin
-	DealsSetConsiderVerifiedStorageDeals(context.Context, bool) error   //perm:admin
-	DealsConsiderUnverifiedStorageDeals(context.Context) (bool, error)  //perm:admin
-	DealsSetConsiderUnverifiedStorageDeals(context.Context, bool) error //perm:admin
-
 	// MethodGroup: Misc
 	OnlineBackup(context.Context, string) error //perm:admin
 }
-
-// DagstoreShardInfo is the serialized form of dagstore.DagstoreShardInfo that
-// we expose through JSON-RPC to avoid clients having to depend on the
-// dagstore lib.
-type DagstoreShardInfo struct {
-	Key   string
-	State string
-	Error string
-}
-
-// DagstoreShardResult enumerates results per shard.
-type DagstoreShardResult struct {
-	Key     string
-	Success bool
-	Error   string
-}
-
-type DagstoreInitializeAllParams struct {
-	MaxConcurrency int
-	IncludeSealed  bool
-}
-
-// DagstoreInitializeAllEvent represents an initialization event.
-type DagstoreInitializeAllEvent struct {
-	Key     string
-	Event   string // "start", "end"
-	Success bool
-	Error   string
-	Total   int
-	Current int
-}
diff --git a/api/api_full.go b/api/api_full.go
@@ -5,17 +5,11 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/ipfs/go-cid"
-	textselector "github.com/ipld/go-ipld-selector-text-lite"
-
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-bitfield"
-	datatransfer "github.com/filecoin-project/go-data-transfer"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
-
-	"github.com/filecoin-project/boost-gfm/retrievalmarket"
-	"github.com/filecoin-project/boost-gfm/storagemarket"
+	"github.com/ipfs/go-cid"
 
 	"github.com/filecoin-project/go-state-types/builtin/v8/paych"
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
@@ -89,29 +83,6 @@ type Import struct {
 	CARPath string
 }
 
-type DealInfo struct {
-	ProposalCid cid.Cid
-	State       storagemarket.StorageDealStatus
-	Message     string // more information about deal state, particularly errors
-	DealStages  *storagemarket.DealStages
-	Provider    address.Address
-
-	DataRef  *storagemarket.DataRef
-	PieceCID cid.Cid
-	Size     uint64
-
-	PricePerEpoch types.BigInt
-	Duration      uint64
-
-	DealID abi.DealID
-
-	CreationTime time.Time
-	Verified     bool
-
-	TransferChannelID *datatransfer.ChannelID
-	DataTransfer      *DataTransferChannel
-}
-
 type MsgLookup struct {
 	Message   cid.Cid // Can be different than requested, in case it was replaced, but only gas values changed
 	Receipt   types.MessageReceipt
@@ -222,37 +193,6 @@ type MinerPower struct {
 	HasMinPower bool
 }
 
-type QueryOffer struct {
-	Err string
-
-	Root  cid.Cid
-	Piece *cid.Cid
-
-	Size                    uint64
-	MinPrice                types.BigInt
-	UnsealPrice             types.BigInt
-	PaymentInterval         uint64
-	PaymentIntervalIncrease uint64
-	Miner                   address.Address
-	MinerPeer               retrievalmarket.RetrievalPeer
-}
-
-func (o *QueryOffer) Order(client address.Address) RetrievalOrder {
-	return RetrievalOrder{
-		Root:                    o.Root,
-		Piece:                   o.Piece,
-		Size:                    o.Size,
-		Total:                   o.MinPrice,
-		UnsealPrice:             o.UnsealPrice,
-		PaymentInterval:         o.PaymentInterval,
-		PaymentIntervalIncrease: o.PaymentIntervalIncrease,
-		Client:                  client,
-
-		Miner:     o.Miner,
-		MinerPeer: &o.MinerPeer,
-	}
-}
-
 type MarketBalance struct {
 	Escrow big.Int
 	Locked big.Int
@@ -263,24 +203,6 @@ type MarketDeal struct {
 	State    market.DealState
 }
 
-type RetrievalOrder struct {
-	// TODO: make this less unixfs specific
-	Root                  cid.Cid
-	Piece                 *cid.Cid
-	DatamodelPathSelector *textselector.Expression
-	Size                  uint64
-
-	FromLocalCAR string // if specified, get data from a local CARv2 file.
-	// TODO: support offset
-	Total                   types.BigInt
-	UnsealPrice             types.BigInt
-	PaymentInterval         uint64
-	PaymentIntervalIncrease uint64
-	Client                  address.Address
-	Miner                   address.Address
-	MinerPeer               *retrievalmarket.RetrievalPeer
-}
-
 type InvocResult struct {
 	MsgCid         cid.Cid
 	Msg            *types.Message

diff --git a/api/docgen/docgen.go b/api/docgen/docgen.go
@@ -12,11 +12,11 @@ import (
 	"time"
 	"unicode"
 
-	"github.com/filecoin-project/boost-gfm/filestore"
-	"github.com/filecoin-project/boost-gfm/retrievalmarket"
+	"github.com/filecoin-project/boost/datatransfer"
+	"github.com/filecoin-project/boost/retrievalmarket/types/legacyretrievaltypes"
+	"github.com/filecoin-project/boost/storagemarket/types/legacytypes/filestore"
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-bitfield"
-	datatransfer "github.com/filecoin-project/go-data-transfer"
 	"github.com/filecoin-project/go-jsonrpc/auth"
 	"github.com/google/uuid"
 	"github.com/ipfs/go-cid"
@@ -94,7 +94,7 @@ func init() {
 
 	storeIDExample := imports.ID(50)
 	textSelExample := textselector.Expression("Links/21/Hash/Links/42/Hash")
-	clientEvent := retrievalmarket.ClientEventDealAccepted
+	clientEvent := legacyretrievaltypes.ClientEventDealAccepted
 
 	addExample(bitfield.NewFromSet([]uint64{5}))
 	addExample(abi.RegisteredSealProof_StackedDrg32GiBV1_1)
@@ -129,14 +129,15 @@ func init() {
 	addExample(&storeIDExample)
 	addExample(clientEvent)
 	addExample(&clientEvent)
-	addExample(retrievalmarket.ClientEventDealAccepted)
-	addExample(retrievalmarket.DealStatusNew)
+	addExample(legacyretrievaltypes.ClientEventDealAccepted)
+	addExample(legacyretrievaltypes.DealStatusNew)
 	addExample(&textSelExample)
 	addExample(network.ReachabilityPublic)
 	addExample(build.TestNetworkVersion)
 	allocationID := verifreg.AllocationId(0)
 	addExample(allocationID)
 	addExample(&allocationID)
+	addExample(filestore.Path(""))
 	addExample(map[string]int{"name": 42})
 	addExample(map[string]time.Time{"name": time.Unix(1615243938, 0).UTC()})
 	addExample(&types.ExecutionTrace{
@@ -186,10 +187,9 @@ func init() {
 	ExampleValues[reflect.TypeOf(struct{ A multiaddr.Multiaddr }{}).Field(0).Type] = maddr
 
 	// miner specific
-	addExample(filestore.Path(".lotusminer/fstmp123"))
 	si := uint64(12)
 	addExample(&si)
-	addExample(retrievalmarket.DealID(5))
+	addExample(legacyretrievaltypes.DealID(5))
 	addExample(abi.ActorID(1000))
 	addExample(storiface.ID("76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8"))
 	addExample(storiface.FTUnsealed)
@@ -273,15 +273,6 @@ func init() {
 
 	addExample(api.CheckStatusCode(0))
 	addExample(map[string]interface{}{"abc": 123})
-	addExample(api.DagstoreShardResult{
-		Key:   "baga6ea4seaqecmtz7iak33dsfshi627abz4i4665dfuzr3qfs4bmad6dx3iigdq",
-		Error: "<error>",
-	})
-	addExample(api.DagstoreShardInfo{
-		Key:   "baga6ea4seaqecmtz7iak33dsfshi627abz4i4665dfuzr3qfs4bmad6dx3iigdq",
-		State: "ShardStateAvailable",
-		Error: "<error>",
-	})
 	addExample(storiface.ResourceTable)
 	addExample(network.ScopeStat{
 		Memory:             123,