feat: #4910 venus-market,venus : add sub command to mannage piece storage

Makefile

@@ -63,7 +63,7 @@ build: $(BUILD_DEPS)
 	go build -o ./venus-market $(GOFLAGS) ./cmd/venus-market
 
 
-	# docker
+# docker
 .PHONY: docker
 
 docker:

api/impl/venus_market.go

@@ -66,6 +66,7 @@ type MarketNodeImpl struct {
 	MinerMgr                                    minermgr.IAddrMgr
 	PaychAPI                                    *paychmgr.PaychAPI
 	Repo                                        repo.Repo
+	Config                                      *config.MarketConfig
 	ConsiderOnlineStorageDealsConfigFunc        config.ConsiderOnlineStorageDealsConfigFunc
 	SetConsiderOnlineStorageDealsConfigFunc     config.SetConsiderOnlineStorageDealsConfigFunc
 	ConsiderOnlineRetrievalDealsConfigFunc      config.ConsiderOnlineRetrievalDealsConfigFunc
@@ -825,3 +826,50 @@ func (m MarketNodeImpl) GetReadUrl(ctx context.Context, s string) (string, error
 func (m MarketNodeImpl) GetWriteUrl(ctx context.Context, s2 string) (string, error) {
 	panic("not support")
 }
+
+func (m MarketNodeImpl) AddFsPieceStorage(ctx context.Context, readonly bool, path string, name string) error {
+	ifs := &config.FsPieceStorage{ReadOnly: readonly, Path: path, Name: name}
+	fsps, err := piecestorage.NewFsPieceStorage(ifs)
+	if err != nil {
+		return err
+	}
+	// add in memory
+	err = m.PieceStorageMgr.AddPieceStorage(fsps)
+	if err != nil {
+		return err
+	}
+
+	// add to config
+	return m.Config.AddFsPieceStorage(ifs)
+}
+
+func (m MarketNodeImpl) AddS3PieceStorage(ctx context.Context, readonly bool, endpoit string, name string, accessKeyID string, secretAccessKey string, token string) error {
+	ifs := &config.S3PieceStorage{ReadOnly: readonly, EndPoint: endpoit, Name: name, AccessKey: accessKeyID, SecretKey: secretAccessKey, Token: token}
+	s3ps, err := piecestorage.NewS3PieceStorage(ifs)
+	if err != nil {
+		return err
+	}
+	// add in memory
+	err = m.PieceStorageMgr.AddPieceStorage(s3ps)
+	if err != nil {
+		return err
+	}
+
+	// add to config
+	return m.Config.AddS3PieceStorage(ifs)
+}
+
+func (m MarketNodeImpl) GetPieceStorages(ctx context.Context) types.PieceStorageList {
+	return m.PieceStorageMgr.ListStorages()
+}
+
+func (m MarketNodeImpl) RemovePieceStorage(ctx context.Context, name string) error {
+
+	err := m.PieceStorageMgr.RemovePieceStorage(name)
+	if err != nil {
+		return err
+	}
+
+	return m.Config.RemovePieceStorage(name)
+
+}

cli/piece-storage.go

@@ -0,0 +1,218 @@
+package cli
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/filecoin-project/venus-market/v2/cli/tablewriter"
+	"github.com/urfave/cli/v2"
+)
+
+var PieceStorageCmd = &cli.Command{
+	Name:        "piece-storage",
+	Usage:       "Manage piece storage ",
+	Description: "The piece storage will decide where to store pieces and how to store them",
+	Subcommands: []*cli.Command{
+		pieceStorageAddFsCmd,
+		pieceStorageAddS3Cmd,
+		pieceStorageListCmd,
+		pieceStorageRemoveCmd,
+	},
+}
+
+var pieceStorageAddFsCmd = &cli.Command{
+	Name:  "add-fs",
+	Usage: "add a local filesystem piece storage",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:    "path",
+			Aliases: []string{"p"},
+			Usage:   "path to the filesystem piece storage",
+		},
+		&cli.BoolFlag{
+			Name:    "read-only",
+			Aliases: []string{"r"},
+			Usage:   "read-only filesystem piece storage",
+		},
+		// name
+		&cli.StringFlag{
+			Name:    "name",
+			Aliases: []string{"n"},
+			Usage:   "name of the filesystem piece storage",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		nodeApi, closer, err := NewMarketNode(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		if !cctx.IsSet("path") {
+			return fmt.Errorf("path is required")
+		}
+		if !cctx.IsSet("name") {
+			return fmt.Errorf("name is required")
+		}
+
+		ctx := ReqContext(cctx)
+		path := cctx.String("path")
+		readOnly := cctx.Bool("read-only")
+		name := cctx.String("name")
+
+		err = nodeApi.AddFsPieceStorage(ctx, readOnly, path, name)
+		if err != nil {
+			return err
+		}
+		fmt.Println("Adding filesystem piece storage:", path)
+
+		return nil
+	},
+}
+
+var pieceStorageAddS3Cmd = &cli.Command{
+	Name:  "add-s3",
+	Usage: "add a object storage for piece storage",
+	Flags: []cli.Flag{
+		// read only
+		&cli.BoolFlag{
+			Name:        "readonly",
+			Aliases:     []string{"r"},
+			Usage:       "set true if you want the piece storage only fro reading",
+			DefaultText: "false",
+		},
+		// Endpoint
+		&cli.StringFlag{
+			Name:    "endpoint",
+			Aliases: []string{"e"},
+			Usage:   "endpoint of the S3 bucket",
+		},
+		// access key
+		&cli.StringFlag{
+			Name:    "access-key",
+			Aliases: []string{"a"},
+			Usage:   "access key of the S3 bucket",
+		},
+		// secret key
+		&cli.StringFlag{
+			Name:    "secret-key",
+			Aliases: []string{"s"},
+			Usage:   "secret key of the S3 bucket",
+		},
+		// token
+		&cli.StringFlag{
+			Name:    "token",
+			Aliases: []string{"t"},
+			Usage:   "token of the S3 bucket",
+		},
+		// name
+		&cli.StringFlag{
+			Name:    "name",
+			Aliases: []string{"n"},
+			Usage:   "name of the S3 bucket",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		nodeApi, closer, err := NewMarketNode(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := ReqContext(cctx)
+
+		if !cctx.IsSet("endpoint") {
+			return fmt.Errorf("endpoint is required")
+		}
+		if !cctx.IsSet("access-key") {
+			return fmt.Errorf("access-key is required")
+		}
+		if !cctx.IsSet("secret-key") {
+			return fmt.Errorf("secret-key is required")
+		}
+		if !cctx.IsSet("name") {
+			return fmt.Errorf("name is required")
+		}
+
+		readOnly := cctx.Bool("readonly")
+		endpoint := cctx.String("endpoint")
+		accessKey := cctx.String("access-key")
+		secretKey := cctx.String("secret-key")
+		token := cctx.String("token")
+		name := cctx.String("name")
+
+		err = nodeApi.AddS3PieceStorage(ctx, readOnly, endpoint, name, accessKey, secretKey, token)
+		if err != nil {
+			return err
+		}
+		fmt.Println("Adding S3 piece storage:", endpoint)
+
+		return nil
+	},
+}
+
+var pieceStorageListCmd = &cli.Command{
+	Name:  "list",
+	Usage: "list piece storages",
+	Action: func(cctx *cli.Context) error {
+		nodeApi, closer, err := NewMarketNode(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+
+		storagelist := nodeApi.GetPieceStorages(ctx)
+
+		w := tablewriter.New(
+			tablewriter.Col("name"),
+			tablewriter.Col("readonly"),
+			tablewriter.Col("path/enter point"),
+			tablewriter.Col("type"),
+		)
+
+		for _, storage := range storagelist.FsStorage {
+
+			w.Write(map[string]interface{}{
+				"Name":                storage.Name,
+				"Readonly":            storage.ReadOnly,
+				"Path or Enter point": storage.Path,
+				"Type":                "file system",
+			})
+		}
+
+		for _, storage := range storagelist.S3Storage {
+			w.Write(map[string]interface{}{
+				"Name":                storage.Name,
+				"Readonly":            storage.ReadOnly,
+				"Path or Enter point": storage.EndPoint,
+				"Type":                "S3",
+			})
+		}
+
+		w.Flush(os.Stdout)
+
+		return nil
+	},
+}
+
+var pieceStorageRemoveCmd = &cli.Command{
+	Name:      "remove",
+	ArgsUsage: "<name>",
+	Usage:     "remove a piece storage",
+	Action: func(cctx *cli.Context) error {
+		// get idx
+		name := cctx.Args().Get(0)
+		if name == "" {
+			return fmt.Errorf("piece storage name is required")
+		}
+
+		nodeApi, closer, err := NewMarketNode(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+		return nodeApi.RemovePieceStorage(ctx, name)
+	},
+}

cmd/venus-market/main.go

@@ -144,6 +144,7 @@ func main() {
 			cli2.DataTransfersCmd,
 			cli2.DagstoreCmd,
 			cli2.MigrateCmd,
+			cli2.PieceStorageCmd,
 		},
 	}
 

cmd/venus-market/pool-run.go

@@ -87,6 +87,8 @@ func poolDaemon(cctx *cli.Context) error {
 		builder.Override(new(metrics.MetricsCtx), func() context.Context {
 			return metrics2.CtxScope(context.Background(), "venus-market")
 		}),
+		// override marketconfig
+		builder.Override(new(config.MarketConfig), cfg),
 		builder.Override(new(types2.ShutdownChan), shutdownChan),
 		//config
 		config.ConfigServerOpts(cfg),

cmd/venus-market/solo-run.go

@@ -79,6 +79,10 @@ func soloDaemon(cctx *cli.Context) error {
 			return metrics2.CtxScope(context.Background(), "venus-market")
 		}),
 		builder.Override(new(types2.ShutdownChan), shutdownChan),
+
+		// override marketconfig
+		builder.Override(new(config.MarketConfig), cfg),
+
 		//config
 		config.ConfigServerOpts(cfg),
 

config/config.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"encoding"
+	"fmt"
 	"time"
 
 	"github.com/filecoin-project/go-address"
@@ -171,11 +172,13 @@ type PieceStorage struct {
 }
 
 type FsPieceStorage struct {
+	Name     string
 	ReadOnly bool
 	Path     string
 }
 
 type S3PieceStorage struct {
+	Name     string
 	ReadOnly bool
 	EndPoint string
 
@@ -267,6 +270,34 @@ type MarketConfig struct {
 	MaxMarketBalanceAddFee types.FIL
 }
 
+func (m *MarketConfig) RemovePieceStorage(name string) error {
+	for i, s := range m.PieceStorage.Fs {
+		if s.Name == name {
+			m.PieceStorage.Fs = append(m.PieceStorage.Fs[:i], m.PieceStorage.Fs[i+1:]...)
+			SaveConfig(m)
+			return nil
+		}
+	}
+	for i, s := range m.PieceStorage.S3 {
+		if s.Name == name {
+			m.PieceStorage.S3 = append(m.PieceStorage.S3[:i], m.PieceStorage.S3[i+1:]...)
+			SaveConfig(m)
+			return nil
+		}
+	}
+	return fmt.Errorf("piece storage %s not found", name)
+}
+
+func (cfg *MarketConfig) AddFsPieceStorage(fsps *FsPieceStorage) error {
+	cfg.PieceStorage.Fs = append(cfg.PieceStorage.Fs, fsps)
+	return SaveConfig(cfg)
+}
+
+func (cfg *MarketConfig) AddS3PieceStorage(fsps *S3PieceStorage) error {
+	cfg.PieceStorage.S3 = append(cfg.PieceStorage.S3, fsps)
+	return SaveConfig(cfg)
+}
+
 type MarketClientConfig struct {
 	Home `toml:"-"`
 	Common

piecestorage/filestore.go

@@ -116,10 +116,19 @@ func (f *fsPieceStorage) ReadOnly() bool {
 	return f.fsCfg.ReadOnly
 }
 
+func (f *fsPieceStorage) GetName() string {
+	return f.fsCfg.Name
+}
+
 func NewFsPieceStorage(fsCfg *config.FsPieceStorage) (IPieceStorage, error) {
 	fs := &fsPieceStorage{baseUrl: fsCfg.Path, fsCfg: fsCfg}
 	if err := fs.Validate(fsCfg.Path); err != nil {
 		return nil, err
 	}
 	return fs, nil
 }
+
+func NewS3PieceStorage(s3Cfg *config.S3PieceStorage) (IPieceStorage, error) {
+	s3 := &s3PieceStorage{s3Cfg: s3Cfg}
+	return s3, nil
+}