Merge from main to lid branch (#1483)

* fix statx output string (#1451) * fix: flaky TestMultipleDealsConcurrent (#1458) * Add option to serve index provider ads over http (#1452) * feat: option to serve index provider ads over http * fix: config naming, hostname parsing * fix: update docsgen * fix: log announce address * feat: add config for indexer direct announce urls * refactor: always announce over pubsub * fix: docsgen * test: add test case for empty announce address hostname * Add `boostd index announce-latest` command (#1456) * feat: boostd index announce-latest * feat: add announce-latest-http command * fix: default direct announce url * feat: update to index-provider v0.11.2 * Signal to index provider to skip announcements (#1457) * fix: signal to index provider to skip announcements * fix: ensure multihash lister skip error is of type ipld.ErrNotExists --------- Co-authored-by: LexLuthr <[email protected]> * release v1.7.3-rc2 (#1460) * fix: improve stalled retrieval cancellation (#1449) * refactor stalled retrieval cancel * add ctx with timeout * implement suggestions * update err wrapping * fix: set short cancel timeout for unpaid retrievals only --------- Co-authored-by: Dirk McCormick <[email protected]> * feat: enable listen address for booster-http (#1461) * enable listen address * modify tests * fix nil ptr (#1470) * fix: incorrect check when import offline deal data using proposal CID (#1473) * fix incorrect early check * update error msg * fix(server): properly cancel graphsync requests (#1475) * set UI default listen address to localhost (#1476) * feat: display msg params in the mpool UI (#1471) * show msg params * fix: mpool nil pointer * fix width --------- Co-authored-by: Dirk McCormick <[email protected]> * Reset read deadline after reading deal proposal message (#1479) * fix: reset read deadline after reading deal proposal message * fix: increase client request deadline * feat: Show elapsed epoch and PSD wait epochs in UI (#1480) * show epochs * fix devnet UI, use BlockdDelaySecs * fix lint err * Update gql/resolver.go Co-authored-by: dirkmc <[email protected]> --------- Co-authored-by: dirkmc <[email protected]> * release v1.7.3-rc3 (#1481) --------- Co-authored-by: LexLuthr <[email protected]> Co-authored-by: LexLuthr <[email protected]> Co-authored-by: Hannah Howard <[email protected]>
filecoin-project · Jul 20, 2023 · 65e20a6 · 65e20a6
1 parent 319a279
commit 65e20a6
Show file tree

Hide file tree

Showing 26 changed files with 374 additions and 97 deletions.
diff --git a/api/api.go b/api/api.go
@@ -35,6 +35,8 @@ type Boost interface {
 	// MethodGroup: Boost
 	BoostIndexerAnnounceAllDeals(ctx context.Context) error                                                                                     //perm:admin
 	BoostIndexerListMultihashes(ctx context.Context, proposalCid cid.Cid) ([]multihash.Multihash, error)                                        //perm:admin
+	BoostIndexerAnnounceLatest(ctx context.Context) (cid.Cid, error)                                                                            //perm:admin
+	BoostIndexerAnnounceLatestHttp(ctx context.Context, urls []string) (cid.Cid, error)                                                         //perm:admin
 	BoostOfflineDealWithData(ctx context.Context, dealUuid uuid.UUID, filePath string, delAfterImport bool) (*ProviderDealRejectionInfo, error) //perm:admin
 	BoostDeal(ctx context.Context, dealUuid uuid.UUID) (*smtypes.ProviderDealState, error)                                                      //perm:admin
 	BoostDealBySignedProposalCid(ctx context.Context, proposalCid cid.Cid) (*smtypes.ProviderDealState, error)                                  //perm:admin

diff --git a/api/proxy_gen.go b/api/proxy_gen.go
diff --git a/build/openrpc/boost.json.gz b/build/openrpc/boost.json.gz
diff --git a/build/version.go b/build/version.go
@@ -2,7 +2,7 @@ package build
 
 var CurrentCommit string
 
-const BuildVersion = "1.7.0-rc1"
+const BuildVersion = "1.7.3-rc3"
 
 func UserVersion() string {
 	return BuildVersion + CurrentCommit

diff --git a/cmd/boostd/import_data.go b/cmd/boostd/import_data.go
@@ -67,9 +67,6 @@ var importDataCmd = &cli.Command{
 		// If the user has supplied a signed proposal cid
 		deleteAfterImport := cctx.Bool("delete-after-import")
 		if proposalCid != nil {
-			if deleteAfterImport {
-				return fmt.Errorf("legacy deal data cannot be automatically deleted after import (only new deals)")
-			}
 
 			// Look up the deal in the boost database
 			deal, err := napi.BoostDealBySignedProposalCid(cctx.Context, *proposalCid)

diff --git a/cmd/boostd/index.go b/cmd/boostd/index.go
@@ -14,6 +14,8 @@ var indexProvCmd = &cli.Command{
 	Subcommands: []*cli.Command{
 		indexProvAnnounceAllCmd,
 		indexProvListMultihashesCmd,
+		indexProvAnnounceLatest,
+		indexProvAnnounceLatestHttp,
 	},
 }
 
@@ -64,11 +66,63 @@ var indexProvListMultihashesCmd = &cli.Command{
 		if err != nil {
 			return err
 		}
-
+		
 		fmt.Printf("Found %d multihashes for deal with proposal cid %s:\n", len(mhs), propCid)
 		for _, mh := range mhs {
 			fmt.Println("  " + mh.String())
 		}
+
+		return nil
+	},
+}
+
+var indexProvAnnounceLatest = &cli.Command{
+	Name:  "announce-latest",
+	Usage: "Re-publish the latest existing advertisement to pubsub",
+	Action: func(cctx *cli.Context) error {
+		ctx := lcli.ReqContext(cctx)
+
+		napi, closer, err := bcli.GetBoostAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		c, err := napi.BoostIndexerAnnounceLatest(ctx)
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("Announced advertisement with cid %s\n", c)
+		return nil
+	},
+}
+
+var indexProvAnnounceLatestHttp = &cli.Command{
+	Name:  "announce-latest-http",
+	Usage: "Re-publish the latest existing advertisement to specific indexers over http",
+	Flags: []cli.Flag{
+		&cli.StringSliceFlag{
+			Name:     "announce-url",
+			Usage:    "The url(s) to announce to. If not specified, announces to the http urls in config",
+			Required: false,
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		ctx := lcli.ReqContext(cctx)
+
+		napi, closer, err := bcli.GetBoostAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		c, err := napi.BoostIndexerAnnounceLatestHttp(ctx, cctx.StringSlice("announce-url"))
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("Announced advertisement to indexers over http with cid %s\n", c)
 		return nil
 	},
 }
diff --git a/cmd/booster-http/http_test.go b/cmd/booster-http/http_test.go
@@ -22,7 +22,7 @@ const testFile = "test/test_file"
 func TestNewHttpServer(t *testing.T) {
 	// Create a new mock Http server
 	ctrl := gomock.NewController(t)
-	httpServer := NewHttpServer("", 7777, mocks_booster_http.NewMockHttpServerApi(ctrl), nil)
+	httpServer := NewHttpServer("", "0.0.0.0", 7777, mocks_booster_http.NewMockHttpServerApi(ctrl), nil)
 	err := httpServer.Start(context.Background())
 	require.NoError(t, err)
 	waitServerUp(t, 7777)
@@ -41,7 +41,7 @@ func TestHttpGzipResponse(t *testing.T) {
 	// Create a new mock Http server with custom functions
 	ctrl := gomock.NewController(t)
 	mockHttpServer := mocks_booster_http.NewMockHttpServerApi(ctrl)
-	httpServer := NewHttpServer("", 7777, mockHttpServer, nil)
+	httpServer := NewHttpServer("", "0.0.0.0", 7777, mockHttpServer, nil)
 	err := httpServer.Start(context.Background())
 	require.NoError(t, err)
 	waitServerUp(t, 7777)
@@ -99,7 +99,7 @@ func TestHttpInfo(t *testing.T) {
 
 	// Create a new mock Http server
 	ctrl := gomock.NewController(t)
-	httpServer := NewHttpServer("", 7777, mocks_booster_http.NewMockHttpServerApi(ctrl), nil)
+	httpServer := NewHttpServer("", "0.0.0.0", 7777, mocks_booster_http.NewMockHttpServerApi(ctrl), nil)
 	err := httpServer.Start(context.Background())
 	require.NoError(t, err)
 	waitServerUp(t, 7777)

diff --git a/cmd/booster-http/mocks/mock_booster_http.go b/cmd/booster-http/mocks/mock_booster_http.go
diff --git a/cmd/booster-http/run.go b/cmd/booster-http/run.go
@@ -223,8 +223,8 @@ var runCmd = &cli.Command{
 		pd.Start(ctx)
 
 		// Start the server
-		log.Infof("Starting booster-http node on port %d with base path '%s'",
-			cctx.Int("port"), cctx.String("base-path"))
+		log.Infof("Starting booster-http node on listen address %s and port %d with base path '%s'",
+			cctx.String("address"), cctx.Int("port"), cctx.String("base-path"))
 		err = server.Start(ctx)
 		if err != nil {
 			return fmt.Errorf("starting http server: %w", err)

diff --git a/cmd/booster-http/server.go b/cmd/booster-http/server.go
@@ -42,11 +42,12 @@ type apiVersion struct {
 }
 
 type HttpServer struct {
-	path    string
-	port    int
-	api     HttpServerApi
-	opts    HttpServerOptions
-	idxPage string
+	path       string
+	listenAddr string
+	port       int
+	api        HttpServerApi
+	opts       HttpServerOptions
+	idxPage    string
 
 	ctx    context.Context
 	cancel context.CancelFunc
@@ -65,11 +66,11 @@ type HttpServerOptions struct {
 	SupportedResponseFormats []string
 }
 
-func NewHttpServer(path string, port int, api HttpServerApi, opts *HttpServerOptions) *HttpServer {
+func NewHttpServer(path string, listenAddr string, port int, api HttpServerApi, opts *HttpServerOptions) *HttpServer {
 	if opts == nil {
 		opts = &HttpServerOptions{ServePieces: true}
 	}
-	return &HttpServer{path: path, port: port, api: api, opts: *opts, idxPage: parseTemplate(*opts)}
+	return &HttpServer{path: path, listenAddr: listenAddr, port: port, api: api, opts: *opts, idxPage: parseTemplate(*opts)}
 }
 
 func (s *HttpServer) pieceBasePath() string {
@@ -102,7 +103,7 @@ func (s *HttpServer) Start(ctx context.Context) error {
 	handler.HandleFunc("/info", s.handleInfo)
 	handler.Handle("/metrics", metrics.Exporter("booster_http")) // metrics
 	s.server = &http.Server{
-		Addr:    fmt.Sprintf(":%d", s.port),
+		Addr:    fmt.Sprintf("%s:%d", s.listenAddr, s.port),
 		Handler: handler,
 		// This context will be the parent of the context associated with all
 		// incoming requests

diff --git a/cmd/boostx/stats_cmd.go b/cmd/boostx/stats_cmd.go
@@ -2,8 +2,6 @@ package main
 
 import (
 	"fmt"
-	"github.com/filecoin-project/boost/retrievalmarket/lp2pimpl"
-	transports_types "github.com/filecoin-project/boost/retrievalmarket/types"
 	"regexp"
 	"sort"
 	"strings"

diff --git a/documentation/en/api-v1-methods.md b/documentation/en/api-v1-methods.md
@@ -23,6 +23,8 @@
   * [BoostDealBySignedProposalCid](#boostdealbysignedproposalcid)
   * [BoostDummyDeal](#boostdummydeal)
   * [BoostIndexerAnnounceAllDeals](#boostindexerannouncealldeals)
+  * [BoostIndexerAnnounceLatest](#boostindexerannouncelatest)
+  * [BoostIndexerAnnounceLatestHttp](#boostindexerannouncelatesthttp)
   * [BoostIndexerListMultihashes](#boostindexerlistmultihashes)
   * [BoostMakeDeal](#boostmakedeal)
   * [BoostOfflineDealWithData](#boostofflinedealwithdata)
@@ -560,6 +562,41 @@ Inputs: `null`
 
 Response: `{}`
 
+### BoostIndexerAnnounceLatest
+
+
+Perms: admin
+
+Inputs: `null`
+
+Response:
+```json
+{
+  "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
+}
+```
+
+### BoostIndexerAnnounceLatestHttp
+
+
+Perms: admin
+
+Inputs:
+```json
+[
+  [
+    "string value"
+  ]
+]
+```
+
+Response:
+```json
+{
+  "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
+}
+```
+
 ### BoostIndexerListMultihashes
 
 

diff --git a/go.mod b/go.mod
@@ -80,7 +80,7 @@ require (
 	github.com/ipld/go-car/v2 v2.7.0
 	github.com/ipld/go-ipld-prime v0.20.0
 	github.com/ipld/go-ipld-selector-text-lite v0.0.1
-	github.com/ipni/index-provider v0.11.1
+	github.com/ipni/index-provider v0.11.2
 	github.com/ipni/storetheindex v0.5.10
 	github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c
 	github.com/jpillora/backoff v1.0.0

diff --git a/go.sum b/go.sum
@@ -989,8 +989,8 @@ github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd
 github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo=
 github.com/ipld/go-ipld-selector-text-lite v0.0.1 h1:lNqFsQpBHc3p5xHob2KvEg/iM5dIFn6iw4L/Hh+kS1Y=
 github.com/ipld/go-ipld-selector-text-lite v0.0.1/go.mod h1:U2CQmFb+uWzfIEF3I1arrDa5rwtj00PrpiwwCO+k1RM=
-github.com/ipni/index-provider v0.11.1 h1:viNfSBvZA9G+Qe6/FGqfZtavnu4tTSfGUoWEECavqoI=
-github.com/ipni/index-provider v0.11.1/go.mod h1:gB/wN4Mdz4MzikQubjyRRV97iS5BkD4FKB0U/bF/dY4=
+github.com/ipni/index-provider v0.11.2 h1:nvykWK+/ncPTqHiuiJdXp/O0UF0V7iWesjHGKX//NYc=
+github.com/ipni/index-provider v0.11.2/go.mod h1:gB/wN4Mdz4MzikQubjyRRV97iS5BkD4FKB0U/bF/dY4=
 github.com/ipni/storetheindex v0.5.10 h1:r97jIZsXPuwQvePJQuStu2a/kn+Zn8X4MAdA0rU2Pu4=
 github.com/ipni/storetheindex v0.5.10/go.mod h1:SJKFCnSx4X/4ekQuZvq8pVU/7tmxkEv632Qmgu3m2bQ=
 github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c=

diff --git a/gql/resolver.go b/gql/resolver.go
@@ -31,6 +31,7 @@ import (
 	"github.com/filecoin-project/boost/storagemarket/types/dealcheckpoints"
 	"github.com/filecoin-project/boost/transport"
 	"github.com/filecoin-project/lotus/api/v1api"
+	"github.com/filecoin-project/lotus/build"
 	lotus_repo "github.com/filecoin-project/lotus/node/repo"
 	"github.com/google/uuid"
 	"github.com/graph-gophers/graphql-go"