feat: #4910 venus-market,venus : add sub command to mannage piece storage

Makefile

@@ -63,7 +63,7 @@ build: $(BUILD_DEPS)
 	go build -o ./venus-market $(GOFLAGS) ./cmd/venus-market
 
 
-	# docker
+# docker
 .PHONY: docker
 
 docker:

api/impl/venus_market.go

@@ -66,6 +66,7 @@ type MarketNodeImpl struct {
 	MinerMgr                                    minermgr.IAddrMgr
 	PaychAPI                                    *paychmgr.PaychAPI
 	Repo                                        repo.Repo
+	Config                                      *config.MarketConfig
 	ConsiderOnlineStorageDealsConfigFunc        config.ConsiderOnlineStorageDealsConfigFunc
 	SetConsiderOnlineStorageDealsConfigFunc     config.SetConsiderOnlineStorageDealsConfigFunc
 	ConsiderOnlineRetrievalDealsConfigFunc      config.ConsiderOnlineRetrievalDealsConfigFunc
@@ -825,3 +826,48 @@ func (m MarketNodeImpl) GetReadUrl(ctx context.Context, s string) (string, error
 func (m MarketNodeImpl) GetWriteUrl(ctx context.Context, s2 string) (string, error) {
 	panic("not support")
 }
+
+func (m MarketNodeImpl) AddFsPieceStorage(ctx context.Context, readonly bool, path string, name string) error {
+	ifs := &config.FsPieceStorage{ReadOnly: readonly, Path: path, Name: name}
+	fsps, err := piecestorage.NewFsPieceStorage(ifs)
+	if err != nil {
+		return err
+	}
+	// add in memory
+	err = m.PieceStorageMgr.AddPieceStorage(fsps)
+	if err != nil {
+		return err
+	}
+
+	// add to config
+	return m.Config.AddFsPieceStorage(ifs)
+}
+
+func (m MarketNodeImpl) AddS3PieceStorage(ctx context.Context, readonly bool, endpoit string, name string, accessKeyID string, secretAccessKey string, token string) error {
+	ifs := &config.S3PieceStorage{ReadOnly: readonly, EndPoint: endpoit, Name: name, AccessKey: accessKeyID, SecretKey: secretAccessKey, Token: token}
+	s3ps, err := piecestorage.NewS3PieceStorage(ifs)
+	if err != nil {
+		return err
+	}
+	// add in memory
+	err = m.PieceStorageMgr.AddPieceStorage(s3ps)
+	if err != nil {
+		return err
+	}
+
+	// add to config
+	return m.Config.AddS3PieceStorage(ifs)
+}
+
+func (m MarketNodeImpl) GetPieceStorages(ctx context.Context) types.PieceStorageInfos {
+	return m.PieceStorageMgr.ListStorageInfos()
+}
+
+func (m MarketNodeImpl) RemovePieceStorage(ctx context.Context, name string) error {
+	err := m.PieceStorageMgr.RemovePieceStorage(name)
+	if err != nil {
+		return err
+	}
+
+	return m.Config.RemovePieceStorage(name)
+}

cli/piece-storage.go

@@ -0,0 +1,232 @@
+package cli
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+
+	"github.com/chzyer/readline"
+	"github.com/filecoin-project/venus-market/v2/cli/tablewriter"
+	"github.com/urfave/cli/v2"
+)
+
+var PieceStorageCmd = &cli.Command{
+	Name:        "piece-storage",
+	Usage:       "Manage piece storage ",
+	Description: "The piece storage will decide where to store pieces and how to store them",
+	Subcommands: []*cli.Command{
+		pieceStorageAddFsCmd,
+		pieceStorageAddS3Cmd,
+		pieceStorageListCmd,
+		pieceStorageRemoveCmd,
+	},
+}
+
+var pieceStorageAddFsCmd = &cli.Command{
+	Name:  "add-fs",
+	Usage: "add a local filesystem piece storage",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:    "path",
+			Aliases: []string{"p"},
+			Usage:   "path to the filesystem piece storage",
+		},
+		&cli.BoolFlag{
+			Name:    "read-only",
+			Aliases: []string{"r"},
+			Usage:   "read-only filesystem piece storage",
+		},
+		// name
+		&cli.StringFlag{
+			Name:    "name",
+			Aliases: []string{"n"},
+			Usage:   "name of the filesystem piece storage",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		nodeApi, closer, err := NewMarketNode(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		if !cctx.IsSet("path") {
+			return fmt.Errorf("path is required")
+		}
+		if !cctx.IsSet("name") {
+			return fmt.Errorf("name is required")
+		}
+
+		ctx := ReqContext(cctx)
+		path := cctx.String("path")
+		readOnly := cctx.Bool("read-only")
+		name := cctx.String("name")
+
+		err = nodeApi.AddFsPieceStorage(ctx, readOnly, path, name)
+		if err != nil {
+			return err
+		}
+		fmt.Println("Adding filesystem piece storage:", path)
+
+		return nil
+	},
+}
+
+var pieceStorageAddS3Cmd = &cli.Command{
+	Name:  "add-s3",
+	Usage: "add a object storage for piece storage",
+	Flags: []cli.Flag{
+		// read only
+		&cli.BoolFlag{
+			Name:        "readonly",
+			Aliases:     []string{"r"},
+			Usage:       "set true if you want the piece storage only fro reading",
+			DefaultText: "false",
+		},
+		// Endpoint
+		&cli.StringFlag{
+			Name:    "endpoint",
+			Aliases: []string{"e"},
+			Usage:   "endpoint of the S3 bucket",
+		},
+		// name
+		&cli.StringFlag{
+			Name:    "name",
+			Aliases: []string{"n"},
+			Usage:   "name of the S3 bucket",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		nodeApi, closer, err := NewMarketNode(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := ReqContext(cctx)
+
+		if !cctx.IsSet("endpoint") {
+			return fmt.Errorf("endpoint is required")
+		}
+		if !cctx.IsSet("name") {
+			return fmt.Errorf("name is required")
+		}
+
+		// get access key , secret key ,token interactivelly
+		getS3Credentials := func() (string, string, string, error) {
+			// var accessKey, secretKey, token string
+			afmt := NewAppFmt(cctx.App)
+			cs := readline.NewCancelableStdin(afmt.Stdin)
+			go func() {
+				<-ctx.Done()
+				cs.Close() // nolint:errcheck
+			}()
+
+			rl := bufio.NewReader(cs)
+			afmt.Println("Please enter your S3 access key:")
+			_accessKey, _, err := rl.ReadLine()
+			if err != nil {
+				return "", "", "", err
+			}
+			accessKey := string(_accessKey)
+
+			afmt.Println("Please enter your S3 secret key:")
+			_secretKey, _, err := rl.ReadLine()
+			if err != nil {
+				return "", "", "", err
+			}
+			secretKey := string(_secretKey)
+
+			afmt.Println("Please enter your S3 token: (it could be empty)")
+			_token, _, err := rl.ReadLine()
+			if err != nil {
+				return "", "", "", err
+			}
+			token := string(_token)
+
+			return accessKey, secretKey, token, err
+		}
+
+		accessKey, secretKey, token, err := getS3Credentials()
+		if err != nil {
+			return err
+		}
+		readOnly := cctx.Bool("readonly")
+		endpoint := cctx.String("endpoint")
+		name := cctx.String("name")
+
+		err = nodeApi.AddS3PieceStorage(ctx, readOnly, endpoint, name, accessKey, secretKey, token)
+		if err != nil {
+			return err
+		}
+		fmt.Println("Adding S3 piece storage:", endpoint)
+
+		return nil
+	},
+}
+
+var pieceStorageListCmd = &cli.Command{
+	Name:  "list",
+	Usage: "list piece storages",
+	Action: func(cctx *cli.Context) error {
+		nodeApi, closer, err := NewMarketNode(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+
+		storagelist := nodeApi.GetPieceStorages(ctx)
+
+		w := tablewriter.New(
+			tablewriter.Col("name"),
+			tablewriter.Col("readonly"),
+			tablewriter.Col("path/enter point"),
+			tablewriter.Col("type"),
+		)
+
+		for _, storage := range storagelist.FsStorage {
+
+			w.Write(map[string]interface{}{
+				"Name":                storage.Name,
+				"Readonly":            storage.ReadOnly,
+				"Path or Enter point": storage.Path,
+				"Type":                "file system",
+			})
+		}
+
+		for _, storage := range storagelist.S3Storage {
+			w.Write(map[string]interface{}{
+				"Name":                storage.Name,
+				"Readonly":            storage.ReadOnly,
+				"Path or Enter point": storage.EndPoint,
+				"Type":                "S3",
+			})
+		}
+
+		w.Flush(os.Stdout)
+
+		return nil
+	},
+}
+
+var pieceStorageRemoveCmd = &cli.Command{
+	Name:      "remove",
+	ArgsUsage: "<name>",
+	Usage:     "remove a piece storage",
+	Action: func(cctx *cli.Context) error {
+		// get idx
+		name := cctx.Args().Get(0)
+		if name == "" {
+			return fmt.Errorf("piece storage name is required")
+		}
+
+		nodeApi, closer, err := NewMarketNode(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+		return nodeApi.RemovePieceStorage(ctx, name)
+	},
+}

cmd/venus-market/main.go

@@ -146,6 +146,7 @@ func main() {
 			cli2.DataTransfersCmd,
 			cli2.DagstoreCmd,
 			cli2.MigrateCmd,
+			cli2.PieceStorageCmd,
 		},
 	}
 

cmd/venus-market/pool-run.go

@@ -83,6 +83,8 @@ func poolDaemon(cctx *cli.Context) error {
 		builder.Override(new(metrics.MetricsCtx), func() context.Context {
 			return metrics2.CtxScope(context.Background(), "venus-market")
 		}),
+		// override marketconfig
+		builder.Override(new(config.MarketConfig), cfg),
 		builder.Override(new(types2.ShutdownChan), shutdownChan),
 		//config
 		config.ConfigServerOpts(cfg),

cmd/venus-market/solo-run.go

@@ -74,6 +74,10 @@ func soloDaemon(cctx *cli.Context) error {
 			return metrics2.CtxScope(context.Background(), "venus-market")
 		}),
 		builder.Override(new(types2.ShutdownChan), shutdownChan),
+
+		// override marketconfig
+		builder.Override(new(config.MarketConfig), cfg),
+
 		//config
 		config.ConfigServerOpts(cfg),
 

config/config.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"encoding"
+	"fmt"
 	"time"
 
 	"github.com/filecoin-project/go-address"
@@ -178,11 +179,13 @@ type PieceStorage struct {
 }
 
 type FsPieceStorage struct {
+	Name     string
 	ReadOnly bool
 	Path     string
 }
 
 type S3PieceStorage struct {
+	Name     string
 	ReadOnly bool
 	EndPoint string
 
@@ -274,6 +277,32 @@ type MarketConfig struct {
 	MaxMarketBalanceAddFee types.FIL
 }
 
+func (m *MarketConfig) RemovePieceStorage(name string) error {
+	for i, s := range m.PieceStorage.Fs {
+		if s.Name == name {
+			m.PieceStorage.Fs = append(m.PieceStorage.Fs[:i], m.PieceStorage.Fs[i+1:]...)
+			return SaveConfig(m)
+		}
+	}
+	for i, s := range m.PieceStorage.S3 {
+		if s.Name == name {
+			m.PieceStorage.S3 = append(m.PieceStorage.S3[:i], m.PieceStorage.S3[i+1:]...)
+			return SaveConfig(m)
+		}
+	}
+	return fmt.Errorf("piece storage %s not found", name)
+}
+
+func (cfg *MarketConfig) AddFsPieceStorage(fsps *FsPieceStorage) error {
+	cfg.PieceStorage.Fs = append(cfg.PieceStorage.Fs, fsps)
+	return SaveConfig(cfg)
+}
+
+func (cfg *MarketConfig) AddS3PieceStorage(fsps *S3PieceStorage) error {
+	cfg.PieceStorage.S3 = append(cfg.PieceStorage.S3, fsps)
+	return SaveConfig(cfg)
+}
+
 type MarketClientConfig struct {
 	Home `toml:"-"`
 	Common

go.mod

@@ -29,7 +29,7 @@ require (
 	github.com/filecoin-project/specs-actors/v7 v7.0.1
 	github.com/filecoin-project/specs-actors/v8 v8.0.1
 	github.com/filecoin-project/specs-storage v0.4.1
-	github.com/filecoin-project/venus v1.6.0
+	github.com/filecoin-project/venus v1.6.1-0.20220705010019-246520384fea
 	github.com/filecoin-project/venus-auth v1.6.0
 	github.com/filecoin-project/venus-messager v1.6.0
 	github.com/gbrlsnchs/jwt/v3 v3.0.1
@@ -335,5 +335,4 @@ replace (
 	github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi
 	github.com/filecoin-project/go-fil-markets => github.com/hunjixin/go-fil-markets v1.20.1-v16-2-fix.0.20220629015016-77caeda1b4b7
 	github.com/filecoin-project/go-jsonrpc => github.com/ipfs-force-community/go-jsonrpc v0.1.4-0.20210721095535-a67dff16de21
-
 )