feat: #4910 venus-market,venus : add sub command to mannage piece sto…

…rage (#153)

* feat: add subcommand to manage piece storage

Makefile

@@ -63,7 +63,7 @@ build: $(BUILD_DEPS)
 	go build -o ./venus-market $(GOFLAGS) ./cmd/venus-market
 
 
-	# docker
+# docker
 .PHONY: docker
 
 docker:

api/impl/venus_market.go

@@ -66,6 +66,7 @@ type MarketNodeImpl struct {
 	MinerMgr                                    minermgr.IAddrMgr
 	PaychAPI                                    *paychmgr.PaychAPI
 	Repo                                        repo.Repo
+	Config                                      *config.MarketConfig
 	ConsiderOnlineStorageDealsConfigFunc        config.ConsiderOnlineStorageDealsConfigFunc
 	SetConsiderOnlineStorageDealsConfigFunc     config.SetConsiderOnlineStorageDealsConfigFunc
 	ConsiderOnlineRetrievalDealsConfigFunc      config.ConsiderOnlineRetrievalDealsConfigFunc
@@ -825,3 +826,48 @@ func (m MarketNodeImpl) GetReadUrl(ctx context.Context, s string) (string, error
 func (m MarketNodeImpl) GetWriteUrl(ctx context.Context, s2 string) (string, error) {
 	panic("not support")
 }
+
+func (m MarketNodeImpl) AddFsPieceStorage(ctx context.Context, readonly bool, path string, name string) error {
+	ifs := &config.FsPieceStorage{ReadOnly: readonly, Path: path, Name: name}
+	fsps, err := piecestorage.NewFsPieceStorage(ifs)
+	if err != nil {
+		return err
+	}
+	// add in memory
+	err = m.PieceStorageMgr.AddPieceStorage(fsps)
+	if err != nil {
+		return err
+	}
+
+	// add to config
+	return m.Config.AddFsPieceStorage(ifs)
+}
+
+func (m MarketNodeImpl) AddS3PieceStorage(ctx context.Context, readonly bool, endpoit string, name string, accessKeyID string, secretAccessKey string, token string) error {
+	ifs := &config.S3PieceStorage{ReadOnly: readonly, EndPoint: endpoit, Name: name, AccessKey: accessKeyID, SecretKey: secretAccessKey, Token: token}
+	s3ps, err := piecestorage.NewS3PieceStorage(ifs)
+	if err != nil {
+		return err
+	}
+	// add in memory
+	err = m.PieceStorageMgr.AddPieceStorage(s3ps)
+	if err != nil {
+		return err
+	}
+
+	// add to config
+	return m.Config.AddS3PieceStorage(ifs)
+}
+
+func (m MarketNodeImpl) GetPieceStorages(ctx context.Context) types.PieceStorageInfos {
+	return m.PieceStorageMgr.ListStorageInfos()
+}
+
+func (m MarketNodeImpl) RemovePieceStorage(ctx context.Context, name string) error {
+	err := m.PieceStorageMgr.RemovePieceStorage(name)
+	if err != nil {
+		return err
+	}
+
+	return m.Config.RemovePieceStorage(name)
+}

cli/piece-storage.go

@@ -0,0 +1,216 @@
+package cli
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/filecoin-project/venus-market/v2/cli/tablewriter"
+	"github.com/urfave/cli/v2"
+)
+
+var PieceStorageCmd = &cli.Command{
+	Name:        "piece-storage",
+	Usage:       "Manage piece storage ",
+	Description: "The piece storage will decide where to store pieces and how to store them",
+	Subcommands: []*cli.Command{
+		pieceStorageAddFsCmd,
+		pieceStorageAddS3Cmd,
+		pieceStorageListCmd,
+		pieceStorageRemoveCmd,
+	},
+}
+
+var pieceStorageAddFsCmd = &cli.Command{
+	Name:  "add-fs",
+	Usage: "add a local filesystem piece storage",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:    "path",
+			Aliases: []string{"p"},
+			Usage:   "path to the filesystem piece storage",
+		},
+		&cli.BoolFlag{
+			Name:    "read-only",
+			Aliases: []string{"r"},
+			Usage:   "read-only filesystem piece storage",
+		},
+		// name
+		&cli.StringFlag{
+			Name:    "name",
+			Aliases: []string{"n"},
+			Usage:   "name of the filesystem piece storage",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		nodeApi, closer, err := NewMarketNode(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		if !cctx.IsSet("path") {
+			return fmt.Errorf("path is required")
+		}
+		if !cctx.IsSet("name") {
+			return fmt.Errorf("name is required")
+		}
+
+		ctx := ReqContext(cctx)
+		path := cctx.String("path")
+		readOnly := cctx.Bool("read-only")
+		name := cctx.String("name")
+
+		err = nodeApi.AddFsPieceStorage(ctx, readOnly, path, name)
+		if err != nil {
+			return err
+		}
+		fmt.Println("Adding filesystem piece storage:", path)
+
+		return nil
+	},
+}
+
+var pieceStorageAddS3Cmd = &cli.Command{
+	Name:  "add-s3",
+	Usage: "add a object storage for piece storage",
+	Flags: []cli.Flag{
+		// read only
+		&cli.BoolFlag{
+			Name:        "readonly",
+			Aliases:     []string{"r"},
+			Usage:       "set true if you want the piece storage only fro reading",
+			DefaultText: "false",
+		},
+		// Endpoint
+		&cli.StringFlag{
+			Name:    "endpoint",
+			Aliases: []string{"e"},
+			Usage:   "endpoint of the S3 bucket",
+		},
+		// name
+		&cli.StringFlag{
+			Name:    "name",
+			Aliases: []string{"n"},
+			Usage:   "name of the S3 bucket",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		nodeApi, closer, err := NewMarketNode(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := ReqContext(cctx)
+
+		if !cctx.IsSet("endpoint") {
+			return fmt.Errorf("endpoint is required")
+		}
+		if !cctx.IsSet("name") {
+			return fmt.Errorf("name is required")
+		}
+
+		// get access key , secret key ,token interactivelly
+		getS3Credentials := func() (string, string, string, error) {
+			// var accessKey, secretKey, token string
+			afmt := NewAppFmt(cctx.App)
+
+			accessKey, err := afmt.GetScret("access key:", true)
+			if err != nil {
+				return "", "", "", err
+			}
+
+			secretKey, err := afmt.GetScret("secret key:", true)
+			if err != nil {
+				return "", "", "", err
+			}
+
+			token, err := afmt.GetScret("token:", true)
+			if err != nil {
+				return "", "", "", err
+			}
+
+			return accessKey, secretKey, token, err
+		}
+
+		accessKey, secretKey, token, err := getS3Credentials()
+		if err != nil {
+			return err
+		}
+		readOnly := cctx.Bool("readonly")
+		endpoint := cctx.String("endpoint")
+		name := cctx.String("name")
+
+		err = nodeApi.AddS3PieceStorage(ctx, readOnly, endpoint, name, accessKey, secretKey, token)
+		if err != nil {
+			return err
+		}
+		fmt.Println("Adding S3 piece storage:", endpoint)
+
+		return nil
+	},
+}
+
+var pieceStorageListCmd = &cli.Command{
+	Name:  "list",
+	Usage: "list piece storages",
+	Action: func(cctx *cli.Context) error {
+		nodeApi, closer, err := NewMarketNode(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+
+		storagelist := nodeApi.GetPieceStorages(ctx)
+
+		w := tablewriter.New(
+			tablewriter.Col("name"),
+			tablewriter.Col("readonly"),
+			tablewriter.Col("path/enter point"),
+			tablewriter.Col("type"),
+		)
+
+		for _, storage := range storagelist.FsStorage {
+
+			w.Write(map[string]interface{}{
+				"Name":                storage.Name,
+				"Readonly":            storage.ReadOnly,
+				"Path or Enter point": storage.Path,
+				"Type":                "file system",
+			})
+		}
+
+		for _, storage := range storagelist.S3Storage {
+			w.Write(map[string]interface{}{
+				"Name":                storage.Name,
+				"Readonly":            storage.ReadOnly,
+				"Path or Enter point": storage.EndPoint,
+				"Type":                "S3",
+			})
+		}
+
+		return w.Flush(os.Stdout)
+	},
+}
+
+var pieceStorageRemoveCmd = &cli.Command{
+	Name:      "remove",
+	ArgsUsage: "<name>",
+	Usage:     "remove a piece storage",
+	Action: func(cctx *cli.Context) error {
+		// get idx
+		name := cctx.Args().Get(0)
+		if name == "" {
+			return fmt.Errorf("piece storage name is required")
+		}
+
+		nodeApi, closer, err := NewMarketNode(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+		return nodeApi.RemovePieceStorage(ctx, name)
+	},
+}

cli/util.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/docker/go-units"
 	"github.com/fatih/color"
+	"github.com/howeyc/gopass"
 	"github.com/ipfs/go-cidutil/cidenc"
 	"github.com/mitchellh/go-homedir"
 	"github.com/multiformats/go-multibase"
@@ -286,14 +287,14 @@ func channelStatusString(useColor bool, status datatransfer.Status) string {
 
 type AppFmt struct {
 	app   *cli.App
-	Stdin io.Reader
+	Stdin gopass.FdReader
 }
 
 func NewAppFmt(a *cli.App) *AppFmt {
-	var stdin io.Reader
+	var stdin gopass.FdReader
 	istdin, ok := a.Metadata["stdin"]
 	if ok {
-		stdin = istdin.(io.Reader)
+		stdin = istdin.(gopass.FdReader)
 	} else {
 		stdin = os.Stdin
 	}
@@ -315,3 +316,11 @@ func (a *AppFmt) Printf(fmtstr string, args ...interface{}) {
 func (a *AppFmt) Scan(args ...interface{}) (int, error) {
 	return fmt.Fscan(a.Stdin, args...)
 }
+
+func (a *AppFmt) GetScret(prompt string, isMasked bool) (string, error) {
+	pw, err := gopass.GetPasswdPrompt(prompt, isMasked, a.Stdin, a.app.Writer)
+	if err != nil {
+		return "", err
+	}
+	return string(pw), nil
+}

cmd/venus-market/main.go

@@ -146,6 +146,7 @@ func main() {
 			cli2.DataTransfersCmd,
 			cli2.DagstoreCmd,
 			cli2.MigrateCmd,
+			cli2.PieceStorageCmd,
 		},
 	}
 

cmd/venus-market/pool-run.go

@@ -83,6 +83,8 @@ func poolDaemon(cctx *cli.Context) error {
 		builder.Override(new(metrics.MetricsCtx), func() context.Context {
 			return metrics2.CtxScope(context.Background(), "venus-market")
 		}),
+		// override marketconfig
+		builder.Override(new(config.MarketConfig), cfg),
 		builder.Override(new(types2.ShutdownChan), shutdownChan),
 		//config
 		config.ConfigServerOpts(cfg),

cmd/venus-market/solo-run.go

@@ -74,6 +74,10 @@ func soloDaemon(cctx *cli.Context) error {
 			return metrics2.CtxScope(context.Background(), "venus-market")
 		}),
 		builder.Override(new(types2.ShutdownChan), shutdownChan),
+
+		// override marketconfig
+		builder.Override(new(config.MarketConfig), cfg),
+
 		//config
 		config.ConfigServerOpts(cfg),