feat: #4910 venus-market,venus : add sub command to mannage piece storage

Makefile

@@ -63,7 +63,7 @@ build: $(BUILD_DEPS)
 	go build -o ./venus-market $(GOFLAGS) ./cmd/venus-market
 
 
-	# docker
+# docker
 .PHONY: docker
 
 docker:

api/impl/venus_market.go

@@ -1,6 +1,7 @@
 package impl
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -26,6 +27,7 @@ import (
 	"github.com/filecoin-project/go-state-types/abi"
 
 	clients2 "github.com/filecoin-project/venus-market/v2/api/clients"
+	"github.com/filecoin-project/venus-market/v2/cli/tablewriter"
 	"github.com/filecoin-project/venus-market/v2/config"
 	"github.com/filecoin-project/venus-market/v2/minermgr"
 	"github.com/filecoin-project/venus-market/v2/models/repo"
@@ -66,6 +68,7 @@ type MarketNodeImpl struct {
 	MinerMgr                                    minermgr.IAddrMgr
 	PaychAPI                                    *paychmgr.PaychAPI
 	Repo                                        repo.Repo
+	Config                                      *config.MarketConfig
 	ConsiderOnlineStorageDealsConfigFunc        config.ConsiderOnlineStorageDealsConfigFunc
 	SetConsiderOnlineStorageDealsConfigFunc     config.SetConsiderOnlineStorageDealsConfigFunc
 	ConsiderOnlineRetrievalDealsConfigFunc      config.ConsiderOnlineRetrievalDealsConfigFunc
@@ -825,3 +828,71 @@ func (m MarketNodeImpl) GetReadUrl(ctx context.Context, s string) (string, error
 func (m MarketNodeImpl) GetWriteUrl(ctx context.Context, s2 string) (string, error) {
 	panic("not support")
 }
+
+func (m MarketNodeImpl) AddFsPieceStorage(ctx context.Context, readonly bool, path string, name string) error {
+	ifs := &config.FsPieceStorage{ReadOnly: readonly, Path: path, Name: name}
+	fsps, err := piecestorage.NewFsPieceStorage(ifs)
+	if err != nil {
+		return err
+	}
+	// add in memory
+	m.PieceStorageMgr.AddPieceStorage(fsps)
+
+	// add to config
+	return m.Config.AddFsPieceStorage(ifs)
+}
+
+func (m MarketNodeImpl) AddS3PieceStorage(ctx context.Context, readonly bool, endpoit string, name string, accessKeyID string, secretAccessKey string, token string) error {
+	ifs := &config.S3PieceStorage{ReadOnly: readonly, EndPoint: endpoit, Name: name, AccessKey: accessKeyID, SecretKey: secretAccessKey, Token: token}
+	s3ps, err := piecestorage.NewS3PieceStorage(ifs)
+	if err != nil {
+		return err
+	}
+	// add in memory
+	m.PieceStorageMgr.AddPieceStorage(s3ps)
+
+	// add to config
+	return m.Config.AddS3PieceStorage(ifs)
+}
+
+func (m MarketNodeImpl) ListPieceStorage(ctx context.Context) (string, error) {
+	storages := m.PieceStorageMgr.GetStorages()
+
+	w := tablewriter.New(tablewriter.Col("index"),
+		tablewriter.Col("name"),
+		tablewriter.Col("readonly"),
+		tablewriter.Col("type"),
+	)
+
+	for idx, storage := range storages {
+
+		w.Write(map[string]interface{}{
+			"index":    idx,
+			"name":     storage.GetName(),
+			"readonly": storage.ReadOnly(),
+			"type":     storage.Type(),
+		})
+	}
+
+	buf := new(bytes.Buffer)
+	err := w.Flush(buf)
+	if err != nil {
+		return "", err
+	}
+	return buf.String(), nil
+}
+
+func (m MarketNodeImpl) GetPieceStorages(ctx context.Context) types.PieceStorageList {
+	return m.PieceStorageMgr.ListStorages()
+}
+
+func (m MarketNodeImpl) RemovePieceStorage(ctx context.Context, name string) error {
+
+	err := m.PieceStorageMgr.RemovePieceStorage(name)
+	if err != nil {
+		return err
+	}
+
+	return m.Config.RemovePieceStorage(name)
+
+}

cli/piece-storage.go

@@ -0,0 +1,218 @@
+package cli
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/filecoin-project/venus-market/v2/cli/tablewriter"
+	"github.com/urfave/cli/v2"
+)
+
+var PieceStorageCmd = &cli.Command{
+	Name:        "piece-storage",
+	Usage:       "Manage piece storage ",
+	Description: "The piece storage will decide where to store pieces and how to store them",
+	Subcommands: []*cli.Command{
+		addFsPieceStorageCmd,
+		addS3PieceStorageCmd,
+		PieceStorageListCmd,
+		PieceStorageRemoveCmd,
+	},
+}
+
+var addFsPieceStorageCmd = &cli.Command{
+	Name:  "add-fs",
+	Usage: "add a local filesystem piece storage",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:    "path",
+			Aliases: []string{"p"},
+			Usage:   "path to the filesystem piece storage",
+		},
+		&cli.BoolFlag{
+			Name:        "read-only",
+			Aliases:     []string{"r"},
+			Usage:       "read-only filesystem piece storage",
+			DefaultText: "false",
+		},
+		// name
+		&cli.StringFlag{
+			Name:    "name",
+			Aliases: []string{"n"},
+			Usage:   "name of the filesystem piece storage",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		nodeApi, closer, err := NewMarketNode(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := ReqContext(cctx)
+		path := cctx.String("path")
+		readOnly := cctx.Bool("read-only")
+		name := cctx.String("name")
+
+		if path == "" {
+			return fmt.Errorf("path is required")
+		}
+		if name == "" {
+			return fmt.Errorf("name is required")
+		}
+
+		err = nodeApi.AddFsPieceStorage(ctx, readOnly, path, name)
+		if err != nil {
+			return err
+		}
+		fmt.Println("Adding filesystem piece storage:", path)
+
+		return nil
+	},
+}
+
+var addS3PieceStorageCmd = &cli.Command{
+	Name:  "add-s3",
+	Usage: "add a object storage for piece storage",
+	Flags: []cli.Flag{
+		// read only
+		&cli.BoolFlag{
+			Name:        "readonly",
+			Aliases:     []string{"r"},
+			Usage:       "set true if you want the piece storage only fro reading",
+			DefaultText: "false",
+		},
+		// Endpoint
+		&cli.StringFlag{
+			Name:    "endpoint",
+			Aliases: []string{"e"},
+			Usage:   "endpoint of the S3 bucket",
+		},
+		// access key
+		&cli.StringFlag{
+			Name:    "access-key",
+			Aliases: []string{"a"},
+			Usage:   "access key of the S3 bucket",
+		},
+		// secret key
+		&cli.StringFlag{
+			Name:    "secret-key",
+			Aliases: []string{"s"},
+			Usage:   "secret key of the S3 bucket",
+		},
+		// token
+		&cli.StringFlag{
+			Name:    "token",
+			Aliases: []string{"t"},
+			Usage:   "token of the S3 bucket",
+		},
+		// name
+		&cli.StringFlag{
+			Name:    "name",
+			Aliases: []string{"n"},
+			Usage:   "name of the S3 bucket",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		nodeApi, closer, err := NewMarketNode(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := ReqContext(cctx)
+
+		readOnly := cctx.Bool("readonly")
+		endpoint := cctx.String("endpoint")
+		accessKey := cctx.String("access-key")
+		secretKey := cctx.String("secret-key")
+		token := cctx.String("token")
+		name := cctx.String("name")
+
+		if endpoint == "" {
+			return fmt.Errorf("endpoint are required")
+		}
+		if accessKey == "" {
+			return fmt.Errorf("access key are required")
+		}
+		if secretKey == "" {
+			return fmt.Errorf("secret key are required")
+		}
+		if name == "" {
+			return fmt.Errorf("name are required")
+		}
+
+		err = nodeApi.AddS3PieceStorage(ctx, readOnly, endpoint, name, accessKey, secretKey, token)
+		if err != nil {
+			return err
+		}
+		fmt.Println("Adding S3 piece storage:", endpoint)
+
+		return nil
+	},
+}
+
+var PieceStorageListCmd = &cli.Command{
+	Name:  "list",
+	Usage: "list piece storages",
+	Action: func(cctx *cli.Context) error {
+		nodeApi, closer, err := NewMarketNode(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+
+		storagelist := nodeApi.GetPieceStorages(ctx)
+
+		w := tablewriter.New(
+			tablewriter.Col("name"),
+			tablewriter.Col("readonly"),
+			tablewriter.Col("path/enter point"),
+			tablewriter.Col("type"),
+		)
+
+		for _, storage := range storagelist.FsStorage {
+
+			w.Write(map[string]interface{}{
+				"Name":                storage.Name,
+				"Readonly":            storage.ReadOnly,
+				"Path or Enter point": storage.Path,
+				"Type":                "file system",
+			})
+		}
+
+		for _, storage := range storagelist.S3Storage {
+			w.Write(map[string]interface{}{
+				"Name":                storage.Name,
+				"Readonly":            storage.ReadOnly,
+				"Path or Enter point": storage.EndPoint,
+				"Type":                "S3",
+			})
+		}
+
+		w.Flush(os.Stdout)
+
+		return nil
+	},
+}
+
+var PieceStorageRemoveCmd = &cli.Command{
+	Name:  "remove",
+	Usage: "remove a piece storage",
+	Action: func(cctx *cli.Context) error {
+		// get idx
+		name := cctx.Args().Get(0)
+		if name == "" {
+			return fmt.Errorf("piece storage name is required")
+		}
+
+		nodeApi, closer, err := NewMarketNode(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+		return nodeApi.RemovePieceStorage(ctx, name)
+	},
+}

cmd/venus-market/main.go

@@ -144,6 +144,7 @@ func main() {
 			cli2.DataTransfersCmd,
 			cli2.DagstoreCmd,
 			cli2.MigrateCmd,
+			cli2.PieceStorageCmd,
 		},
 	}
 

cmd/venus-market/pool-run.go

@@ -87,6 +87,8 @@ func poolDaemon(cctx *cli.Context) error {
 		builder.Override(new(metrics.MetricsCtx), func() context.Context {
 			return metrics2.CtxScope(context.Background(), "venus-market")
 		}),
+		// override marketconfig
+		builder.Override(new(config.MarketConfig), cfg),
 		builder.Override(new(types2.ShutdownChan), shutdownChan),
 		//config
 		config.ConfigServerOpts(cfg),

cmd/venus-market/solo-run.go

@@ -79,6 +79,10 @@ func soloDaemon(cctx *cli.Context) error {
 			return metrics2.CtxScope(context.Background(), "venus-market")
 		}),
 		builder.Override(new(types2.ShutdownChan), shutdownChan),
+
+		// override marketconfig
+		builder.Override(new(config.MarketConfig), cfg),
+
 		//config
 		config.ConfigServerOpts(cfg),