From 57f96019f7b94f7f8b412b3e9a34dbcdfa19a95e Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Wed, 15 Aug 2018 20:02:18 -0400 Subject: [PATCH 01/10] Provide new "cid" sub-command. License: MIT Signed-off-by: Kevin Atkinson --- cmd/ipfs/ipfs.go | 1 + core/commands/cid.go | 307 +++++++++++++++++++++++++++++++++ core/commands/commands_test.go | 6 + core/commands/root.go | 2 + 4 files changed, 316 insertions(+) create mode 100644 core/commands/cid.go diff --git a/cmd/ipfs/ipfs.go b/cmd/ipfs/ipfs.go index 0342007a12a..8e74139b075 100644 --- a/cmd/ipfs/ipfs.go +++ b/cmd/ipfs/ipfs.go @@ -92,4 +92,5 @@ var cmdDetailsMap = map[string]cmdDetails{ "diag/cmds": {cannotRunOnClient: true}, "repo/fsck": {cannotRunOnDaemon: true}, "config/edit": {cannotRunOnDaemon: true, doesNotUseRepo: true}, + "cid": {doesNotUseRepo: true}, } diff --git a/core/commands/cid.go b/core/commands/cid.go new file mode 100644 index 00000000000..284f79c426e --- /dev/null +++ b/core/commands/cid.go @@ -0,0 +1,307 @@ +package commands + +import ( + "fmt" + "io" + "sort" + "strings" + "text/tabwriter" + "unicode" + + cid "gx/ipfs/QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7/go-cid" + mhash "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash" + cidutil "gx/ipfs/QmQJSeE3CX4zos9qeaG8EhecEK9zvrTEfTG84J8C5NVRwt/go-cidutil" + cmdkit "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit" + verifcid "gx/ipfs/QmVkMRSkXrpjqrroEXWuYBvDBnXCdMMY6gsKicBGVGUqKT/go-verifcid" + cmds "gx/ipfs/QmXTmUCBtDUrzDYVzASogLiNph7EBuYqEgPL7QoHNMzUnz/go-ipfs-cmds" + mbase "gx/ipfs/QmekxXDhCxCJRNuzmHreuaT3BsuJcsjcXWNrtV9C8DRHtd/go-multibase" +) + +var CidCmd = &cmds.Command{ + Helptext: cmdkit.HelpText{ + Tagline: "Convert and discover properties of CIDs", + }, + Subcommands: map[string]*cmds.Command{ + "format": cidFmtCmd, + "base32": base32Cmd, + "bases": basesCmd, + "codecs": codecsCmd, + "hashes": hashesCmd, + }, +} + +var cidFmtCmd = &cmds.Command{ + Helptext: cmdkit.HelpText{ + Tagline: "Format and convert a CID in various useful ways.", + LongDescription: ` +Format and converts 's in various useful ways. + +The optional format string either "prefix" or or a printf style format string: +` + cidutil.FormatRef, + }, + Arguments: []cmdkit.Argument{ + cmdkit.StringArg("cid", true, true, "Cids to format."), + }, + Options: []cmdkit.Option{ + cmdkit.StringOption("f", "Format string."), + cmdkit.StringOption("v", "CID version to convert to."), + cmdkit.StringOption("b", "Multibase to display CID in."), + }, + Run: func(req *cmds.Request, resp cmds.ResponseEmitter, env cmds.Environment) error { + fmtStr, _ := req.Options["f"].(string) + verStr, _ := req.Options["v"].(string) + baseStr, _ := req.Options["b"].(string) + + opts := cidFormatOpts{} + + switch fmtStr { + case "": + opts.fmtStr = "%s" + case "prefix": + opts.fmtStr = "%P" + default: + if strings.IndexByte(fmtStr, '%') == -1 { + return fmt.Errorf("invalid format string: %s", fmtStr) + } + opts.fmtStr = fmtStr + } + + switch verStr { + case "": + // noop + case "0": + opts.verConv = toCidV0 + case "1": + opts.verConv = toCidV1 + default: + return fmt.Errorf("invalid cid version: %s\n", verStr) + } + + if baseStr != "" { + encoder, err := mbase.EncoderByName(baseStr) + if err != nil { + return err + } + opts.newBase = encoder.Encoding() + } else { + opts.newBase = mbase.Encoding(-1) + } + + res, err := formatCids(req.Arguments, opts) + if err != nil { + return err + } + cmds.EmitOnce(resp, res) + return nil + }, + Encoders: cmds.EncoderMap{ + cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, val interface{}) error { + for _, v := range val.([]string) { + fmt.Fprintf(w, "%s\n", v) + } + return nil + }), + }, + Type: []string{}, +} + +var base32Cmd = &cmds.Command{ + Helptext: cmdkit.HelpText{ + Tagline: "Convert CIDs to Base32 CID version 1.", + }, + Arguments: []cmdkit.Argument{ + cmdkit.StringArg("cid", true, true, "Cids to convert."), + }, + Run: func(req *cmds.Request, resp cmds.ResponseEmitter, env cmds.Environment) error { + opts := cidFormatOpts{ + fmtStr: "%s", + newBase: mbase.Encoding(mbase.Base32), + verConv: toCidV1, + } + res, err := formatCids(req.Arguments, opts) + if err != nil { + return err + } + + cmds.EmitOnce(resp, res) + return nil + }, + Encoders: cidFmtCmd.Encoders, + Type: cidFmtCmd.Type, +} + +type cidFormatOpts struct { + fmtStr string + newBase mbase.Encoding + verConv func(cid cid.Cid) (cid.Cid, error) +} + +func formatCids(args []string, opts cidFormatOpts) ([]string, error) { + var res []string + for _, cidStr := range args { + c, err := cid.Decode(cidStr) + if err != nil { + return nil, fmt.Errorf("%s: %v", cidStr, err) + } + base := opts.newBase + if base == -1 { + base, _ = cid.ExtractEncoding(cidStr) + } + if opts.verConv != nil { + c, err = opts.verConv(c) + if err != nil { + return nil, fmt.Errorf("%s: %v", cidStr, err) + } + } + str, err := cidutil.Format(opts.fmtStr, base, c) + if _, ok := err.(cidutil.FormatStringError); ok { + return nil, err + } else if err != nil { + return nil, fmt.Errorf("%s: %v", cidStr, err) + } + res = append(res, str) + } + return res, nil +} + +func toCidV0(c cid.Cid) (cid.Cid, error) { + if c.Type() != cid.DagProtobuf { + return cid.Cid{}, fmt.Errorf("can't convert non-protobuf nodes to cidv0") + } + return cid.NewCidV0(c.Hash()), nil +} + +func toCidV1(c cid.Cid) (cid.Cid, error) { + return cid.NewCidV1(c.Type(), c.Hash()), nil +} + +type CodeAndName struct { + Code int + Name string +} + +var basesCmd = &cmds.Command{ + Helptext: cmdkit.HelpText{ + Tagline: "List available multibase encodings.", + }, + Options: []cmdkit.Option{ + cmdkit.BoolOption("prefix", "also include the single leter prefixes in addition to the code"), + cmdkit.BoolOption("numeric", "also include numeric codes"), + }, + Run: func(req *cmds.Request, resp cmds.ResponseEmitter, env cmds.Environment) error { + var res []CodeAndName + // use EncodingToStr in case at some point there are multiple names for a given code + for code, name := range mbase.EncodingToStr { + res = append(res, CodeAndName{int(code), name}) + } + cmds.EmitOnce(resp, res) + return nil + }, + Encoders: cmds.EncoderMap{ + cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w0 io.Writer, val0 interface{}) error { + w := tabwriter.NewWriter(w0, 0, 0, 2, ' ', 0) + prefixes, _ := req.Options["prefix"].(bool) + numeric, _ := req.Options["numeric"].(bool) + val := val0.([]CodeAndName) + sort.Sort(multibaseSorter{val}) + for _, v := range val { + if prefixes && v.Code >= 32 && v.Code < 127 { + fmt.Fprintf(w, "%c\t", v.Code) + } else if prefixes { + // don't display non-printable prefixes + fmt.Fprintf(w, "\t") + } + if numeric { + fmt.Fprintf(w, "%d\t%s\n", v.Code, v.Name) + } else { + fmt.Fprintf(w, "%s\n", v.Name) + } + } + w.Flush() + return nil + }), + }, + Type: []CodeAndName{}, +} + +var codecsCmd = &cmds.Command{ + Helptext: cmdkit.HelpText{ + Tagline: "List available CID codecs.", + }, + Options: []cmdkit.Option{ + cmdkit.BoolOption("numeric", "also include numeric codes"), + }, + Run: func(req *cmds.Request, resp cmds.ResponseEmitter, env cmds.Environment) error { + var res []CodeAndName + // use CodecToStr as there are multiple names for a given code + for code, name := range cid.CodecToStr { + res = append(res, CodeAndName{int(code), name}) + } + cmds.EmitOnce(resp, res) + return nil + }, + Encoders: cmds.EncoderMap{ + cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w0 io.Writer, val0 interface{}) error { + w := tabwriter.NewWriter(w0, 0, 0, 2, ' ', 0) + numeric, _ := req.Options["numeric"].(bool) + val := val0.([]CodeAndName) + sort.Sort(codeAndNameSorter{val}) + for _, v := range val { + if numeric { + fmt.Fprintf(w, "%d\t%s\n", v.Code, v.Name) + } else { + fmt.Fprintf(w, "%s\n", v.Name) + } + } + w.Flush() + return nil + }), + }, + Type: []CodeAndName{}, +} + +var hashesCmd = &cmds.Command{ + Helptext: cmdkit.HelpText{ + Tagline: "List available multihashes.", + }, + Options: codecsCmd.Options, + Run: func(req *cmds.Request, resp cmds.ResponseEmitter, env cmds.Environment) error { + var res []CodeAndName + // use mhash.Codes in case at some point there are multiple names for a given code + for code, name := range mhash.Codes { + if !verifcid.IsGoodHash(code) { + continue + } + res = append(res, CodeAndName{int(code), name}) + } + cmds.EmitOnce(resp, res) + return nil + }, + Encoders: codecsCmd.Encoders, + Type: codecsCmd.Type, +} + +type multibaseSorter struct { + data []CodeAndName +} + +func (s multibaseSorter) Len() int { return len(s.data) } +func (s multibaseSorter) Swap(i, j int) { s.data[i], s.data[j] = s.data[j], s.data[i] } + +func (s multibaseSorter) Less(i, j int) bool { + a := unicode.ToLower(rune(s.data[i].Code)) + b := unicode.ToLower(rune(s.data[j].Code)) + if a != b { + return a < b + } + // lowecase letters should come before uppercase + return s.data[i].Code > s.data[j].Code +} + +type codeAndNameSorter struct { + data []CodeAndName +} + +func (s codeAndNameSorter) Len() int { return len(s.data) } +func (s codeAndNameSorter) Swap(i, j int) { s.data[i], s.data[j] = s.data[j], s.data[i] } +func (s codeAndNameSorter) Less(i, j int) bool { return s.data[i].Code < s.data[j].Code } diff --git a/core/commands/commands_test.go b/core/commands/commands_test.go index ec4b4ba5f04..74d58903488 100644 --- a/core/commands/commands_test.go +++ b/core/commands/commands_test.go @@ -211,6 +211,12 @@ func TestCommands(t *testing.T) { "/urlstore", "/urlstore/add", "/version", + "/cid", + "/cid/format", + "/cid/base32", + "/cid/codecs", + "/cid/bases", + "/cid/hashes", } cmdSet := make(map[string]struct{}) diff --git a/core/commands/root.go b/core/commands/root.go index 0268136f610..900530a216b 100644 --- a/core/commands/root.go +++ b/core/commands/root.go @@ -71,6 +71,7 @@ TOOL COMMANDS version Show ipfs version information update Download and apply go-ipfs updates commands List all available commands + cid Convert and discover properties of CIDs Use 'ipfs --help' to learn more about each command. @@ -143,6 +144,7 @@ var rootSubcommands = map[string]*cmds.Command{ "urlstore": urlStoreCmd, "version": lgc.NewCommand(VersionCmd), "shutdown": daemonShutdownCmd, + "cid": CidCmd, } // RootRO is the readonly version of Root From 30b861bc5a5f47683d587dd81ae4683672ace9fd Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Fri, 24 Aug 2018 23:19:03 -0400 Subject: [PATCH 02/10] Remove "prefix" format string. Use WithDefault("%s') for `-f- option. License: MIT Signed-off-by: Kevin Atkinson --- core/commands/cid.go | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/core/commands/cid.go b/core/commands/cid.go index 284f79c426e..f1b4a5e9fef 100644 --- a/core/commands/cid.go +++ b/core/commands/cid.go @@ -36,14 +36,14 @@ var cidFmtCmd = &cmds.Command{ LongDescription: ` Format and converts 's in various useful ways. -The optional format string either "prefix" or or a printf style format string: +The optional format string is a printf style format string: ` + cidutil.FormatRef, }, Arguments: []cmdkit.Argument{ cmdkit.StringArg("cid", true, true, "Cids to format."), }, Options: []cmdkit.Option{ - cmdkit.StringOption("f", "Format string."), + cmdkit.StringOption("f", "Printf style format string.").WithDefault("%s"), cmdkit.StringOption("v", "CID version to convert to."), cmdkit.StringOption("b", "Multibase to display CID in."), }, @@ -54,17 +54,10 @@ The optional format string either "prefix" or or a printf style format string: opts := cidFormatOpts{} - switch fmtStr { - case "": - opts.fmtStr = "%s" - case "prefix": - opts.fmtStr = "%P" - default: - if strings.IndexByte(fmtStr, '%') == -1 { - return fmt.Errorf("invalid format string: %s", fmtStr) - } - opts.fmtStr = fmtStr + if strings.IndexByte(fmtStr, '%') == -1 { + return fmt.Errorf("invalid format string: %s", fmtStr) } + opts.fmtStr = fmtStr switch verStr { case "": From 98e5999761bf97f9dba8487d70afcdca23540e24 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Sun, 26 Aug 2018 20:10:19 -0400 Subject: [PATCH 03/10] Factor out boiler plate code for PostRun in "ipfs filestore ls". License: MIT Signed-off-by: Kevin Atkinson --- core/commands/commands.go | 40 ++++++++++++++++++++++++++++++++++++++ core/commands/filestore.go | 37 +++++++---------------------------- 2 files changed, 47 insertions(+), 30 deletions(-) diff --git a/core/commands/commands.go b/core/commands/commands.go index 51c707c6e41..a89b279f773 100644 --- a/core/commands/commands.go +++ b/core/commands/commands.go @@ -8,6 +8,7 @@ package commands import ( "fmt" "io" + "os" "sort" "strings" @@ -149,3 +150,42 @@ func unwrapOutput(i interface{}) (interface{}, error) { return <-ch, nil } + +type nonFatalError string + +// streamRes is a helper function to stream results, that possibly +// contain with non-fatal, the helper function is allowed to panic on +// internal errors +func streamRes(procVal func(interface{}, io.Writer) nonFatalError) func(cmds.Response, cmds.ResponseEmitter) error { + return func(res cmds.Response, re cmds.ResponseEmitter) (err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("internal error: %v", r) + } + re.Close() + }() + + var errors bool + for { + v, err := res.Next() + if err != nil { + if err == io.EOF { + break + } + return err + } + + errorMsg := procVal(v, os.Stdout) + + if errorMsg != "" { + errors = true + fmt.Fprintf(os.Stderr, "%s\n", errorMsg) + } + } + + if errors { + return fmt.Errorf("errors while displaying some entries") + } + return nil + } +} diff --git a/core/commands/filestore.go b/core/commands/filestore.go index a45e01ef93f..deb7520d22f 100644 --- a/core/commands/filestore.go +++ b/core/commands/filestore.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "io" - "os" oldCmds "github.com/ipfs/go-ipfs/commands" lgc "github.com/ipfs/go-ipfs/commands/legacy" @@ -73,36 +72,14 @@ The output is: return res.Emit(out) }, PostRun: cmds.PostRunMap{ - cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error { - var errors bool - for { - v, err := res.Next() - if err != nil { - if err == io.EOF { - break - } - return err - } - - r, ok := v.(*filestore.ListRes) - if !ok { - return e.New(e.TypeErr(r, v)) - } - - if r.ErrorMsg != "" { - errors = true - fmt.Fprintf(os.Stderr, "%s\n", r.ErrorMsg) - } else { - fmt.Fprintf(os.Stdout, "%s\n", r.FormatLong()) - } - } - - if errors { - return fmt.Errorf("errors while displaying some entries") + cmds.CLI: streamRes(func(v interface{}, out io.Writer) nonFatalError { + r := v.(*filestore.ListRes) + if r.ErrorMsg != "" { + return nonFatalError(r.ErrorMsg) } - - return nil - }, + fmt.Fprintf(out, "%s\n", r.FormatLong()) + return "" + }), }, Type: filestore.ListRes{}, } From e5622f628a8d5e92f66f81758e3800a766e9d297 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Sun, 26 Aug 2018 20:30:54 -0400 Subject: [PATCH 04/10] Stream results for "ipfs cid format". Note reading input from Stdin is broken. Only the first result is accepted. License: MIT Signed-off-by: Kevin Atkinson --- core/commands/cid.go | 68 +++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 32 deletions(-) diff --git a/core/commands/cid.go b/core/commands/cid.go index f1b4a5e9fef..fc26b692d96 100644 --- a/core/commands/cid.go +++ b/core/commands/cid.go @@ -80,22 +80,25 @@ The optional format string is a printf style format string: opts.newBase = mbase.Encoding(-1) } - res, err := formatCids(req.Arguments, opts) - if err != nil { - return err - } - cmds.EmitOnce(resp, res) - return nil + return emitCids(req, resp, opts) }, - Encoders: cmds.EncoderMap{ - cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, val interface{}) error { - for _, v := range val.([]string) { - fmt.Fprintf(w, "%s\n", v) + PostRun: cmds.PostRunMap{ + cmds.CLI: streamRes(func(v interface{}, out io.Writer) nonFatalError { + r := v.(*CidFormatRes) + if r.ErrorMsg != "" { + return nonFatalError(fmt.Sprintf("%s: %s", r.CidStr, r.ErrorMsg)) } - return nil + fmt.Fprintf(out, "%s\n", r.Formatted) + return "" }), }, - Type: []string{}, + Type: CidFormatRes{}, +} + +type CidFormatRes struct { + CidStr string // Original Cid String passed in + Formatted string // Formated Result + ErrorMsg string // Error } var base32Cmd = &cmds.Command{ @@ -103,7 +106,7 @@ var base32Cmd = &cmds.Command{ Tagline: "Convert CIDs to Base32 CID version 1.", }, Arguments: []cmdkit.Argument{ - cmdkit.StringArg("cid", true, true, "Cids to convert."), + cmdkit.StringArg("cid", true, true, "Cids to convert.").EnableStdin(), }, Run: func(req *cmds.Request, resp cmds.ResponseEmitter, env cmds.Environment) error { opts := cidFormatOpts{ @@ -111,16 +114,10 @@ var base32Cmd = &cmds.Command{ newBase: mbase.Encoding(mbase.Base32), verConv: toCidV1, } - res, err := formatCids(req.Arguments, opts) - if err != nil { - return err - } - - cmds.EmitOnce(resp, res) - return nil + return emitCids(req, resp, opts) }, - Encoders: cidFmtCmd.Encoders, - Type: cidFmtCmd.Type, + PostRun: cidFmtCmd.PostRun, + Type: cidFmtCmd.Type, } type cidFormatOpts struct { @@ -129,12 +126,19 @@ type cidFormatOpts struct { verConv func(cid cid.Cid) (cid.Cid, error) } -func formatCids(args []string, opts cidFormatOpts) ([]string, error) { - var res []string - for _, cidStr := range args { +func emitCids(req *cmds.Request, resp cmds.ResponseEmitter, opts cidFormatOpts) error { + for _, cidStr := range req.Arguments { + emit := func(fmtd string, err error) { + res := &CidFormatRes{CidStr: cidStr, Formatted: fmtd} + if err != nil { + res.ErrorMsg = err.Error() + } + resp.Emit(res) + } c, err := cid.Decode(cidStr) if err != nil { - return nil, fmt.Errorf("%s: %v", cidStr, err) + emit("", err) + continue } base := opts.newBase if base == -1 { @@ -143,18 +147,18 @@ func formatCids(args []string, opts cidFormatOpts) ([]string, error) { if opts.verConv != nil { c, err = opts.verConv(c) if err != nil { - return nil, fmt.Errorf("%s: %v", cidStr, err) + emit("", err) + continue } } str, err := cidutil.Format(opts.fmtStr, base, c) if _, ok := err.(cidutil.FormatStringError); ok { - return nil, err - } else if err != nil { - return nil, fmt.Errorf("%s: %v", cidStr, err) + // no point in continuing if there is a problem with the format string + return err } - res = append(res, str) + emit(str, err) } - return res, nil + return nil } func toCidV0(c cid.Cid) (cid.Cid, error) { From 599bb7302ea93eeed93b64eaebe00b4eace77f53 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Mon, 27 Aug 2018 18:12:52 -0400 Subject: [PATCH 05/10] Right align numbers in "ipfs cid bases|codecs|hashes" output. This also avoid using TabWriter as it doesn't support right aligning a single column and also because its an overkill for this case. License: MIT Signed-off-by: Kevin Atkinson --- core/commands/cid.go | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/core/commands/cid.go b/core/commands/cid.go index fc26b692d96..ffc9df7d19e 100644 --- a/core/commands/cid.go +++ b/core/commands/cid.go @@ -5,7 +5,6 @@ import ( "io" "sort" "strings" - "text/tabwriter" "unicode" cid "gx/ipfs/QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7/go-cid" @@ -195,26 +194,28 @@ var basesCmd = &cmds.Command{ return nil }, Encoders: cmds.EncoderMap{ - cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w0 io.Writer, val0 interface{}) error { - w := tabwriter.NewWriter(w0, 0, 0, 2, ' ', 0) + cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, val0 interface{}) error { prefixes, _ := req.Options["prefix"].(bool) numeric, _ := req.Options["numeric"].(bool) val := val0.([]CodeAndName) sort.Sort(multibaseSorter{val}) for _, v := range val { - if prefixes && v.Code >= 32 && v.Code < 127 { - fmt.Fprintf(w, "%c\t", v.Code) - } else if prefixes { + code := v.Code + if code < 32 || code >= 127 { // don't display non-printable prefixes - fmt.Fprintf(w, "\t") + code = ' ' } - if numeric { - fmt.Fprintf(w, "%d\t%s\n", v.Code, v.Name) - } else { + switch { + case prefixes && numeric: + fmt.Fprintf(w, "%c %5d %s\n", code, v.Code, v.Name) + case prefixes: + fmt.Fprintf(w, "%c %s\n", code, v.Name) + case numeric: + fmt.Fprintf(w, "%5d %s\n", v.Code, v.Name) + default: fmt.Fprintf(w, "%s\n", v.Name) } } - w.Flush() return nil }), }, @@ -238,19 +239,17 @@ var codecsCmd = &cmds.Command{ return nil }, Encoders: cmds.EncoderMap{ - cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w0 io.Writer, val0 interface{}) error { - w := tabwriter.NewWriter(w0, 0, 0, 2, ' ', 0) + cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, val0 interface{}) error { numeric, _ := req.Options["numeric"].(bool) val := val0.([]CodeAndName) sort.Sort(codeAndNameSorter{val}) for _, v := range val { if numeric { - fmt.Fprintf(w, "%d\t%s\n", v.Code, v.Name) + fmt.Fprintf(w, "%5d %s\n", v.Code, v.Name) } else { fmt.Fprintf(w, "%s\n", v.Name) } } - w.Flush() return nil }), }, From f3ea21c821403c6b07bf594dc316163341adb630 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Mon, 27 Aug 2018 18:52:31 -0400 Subject: [PATCH 06/10] Reenable Stdin argument reading. License: MIT Signed-off-by: Kevin Atkinson --- core/commands/cid.go | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/core/commands/cid.go b/core/commands/cid.go index ffc9df7d19e..1211b027d47 100644 --- a/core/commands/cid.go +++ b/core/commands/cid.go @@ -39,7 +39,7 @@ The optional format string is a printf style format string: ` + cidutil.FormatRef, }, Arguments: []cmdkit.Argument{ - cmdkit.StringArg("cid", true, true, "Cids to format."), + cmdkit.StringArg("cid", true, true, "Cids to format.").EnableStdin(), }, Options: []cmdkit.Option{ cmdkit.StringOption("f", "Printf style format string.").WithDefault("%s"), @@ -125,8 +125,37 @@ type cidFormatOpts struct { verConv func(cid cid.Cid) (cid.Cid, error) } +type argumentIterator struct { + args []string + body cmds.StdinArguments +} + +func (i *argumentIterator) next() (string, bool) { + if len(i.args) > 0 { + arg := i.args[0] + i.args = i.args[1:] + return arg, true + } + if i.body == nil || !i.body.Scan() { + return "", false + } + return strings.TrimSpace(i.body.Argument()), true +} + +func (i *argumentIterator) err() error { + if i.body == nil { + return nil + } + return i.body.Err() +} + func emitCids(req *cmds.Request, resp cmds.ResponseEmitter, opts cidFormatOpts) error { - for _, cidStr := range req.Arguments { + itr := argumentIterator{req.Arguments, req.BodyArgs()} + for { + cidStr, ok := itr.next() + if !ok { + break + } emit := func(fmtd string, err error) { res := &CidFormatRes{CidStr: cidStr, Formatted: fmtd} if err != nil { @@ -157,6 +186,10 @@ func emitCids(req *cmds.Request, resp cmds.ResponseEmitter, opts cidFormatOpts) } emit(str, err) } + err := itr.err() + if err != nil { + return err + } return nil } From 88a7e58b582ed81105d09e595a8635d69c502afc Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Mon, 27 Aug 2018 19:39:14 -0400 Subject: [PATCH 07/10] Add basic sharness tests. License: MIT Signed-off-by: Kevin Atkinson --- test/sharness/t0290-cid.sh | 228 +++++++++++++++++++++++++++++++++++++ 1 file changed, 228 insertions(+) create mode 100755 test/sharness/t0290-cid.sh diff --git a/test/sharness/t0290-cid.sh b/test/sharness/t0290-cid.sh new file mode 100755 index 00000000000..ecc2ba19e4b --- /dev/null +++ b/test/sharness/t0290-cid.sh @@ -0,0 +1,228 @@ +#!/usr/bin/env bash + +test_description="Test cid commands" + +. lib/test-lib.sh + +# note: all "ipfs cid" commands should work without requiring a repo + +CIDv0="QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv" +CIDv1="zdj7WZAAFKPvYPPzyJLso2hhxo8a7ZACFQ4DvvfrNXTHidofr" +CIDb32="bafybeibxm2nsadl3fnxv2sxcxmxaco2jl53wpeorjdzidjwf5aqdg7wa6u" + +test_expect_success "cid base32 works" ' + echo $CIDb32 > expected && + ipfs cid base32 $CIDv0 > actual1 && + test_cmp actual1 expected && + ipfs cid base32 $CIDv1 > actual2 && + test_cmp expected actual2 +' + +test_expect_success "cid format -v 1 -b base58btc" ' + echo $CIDv1 > expected && + ipfs cid format -v 1 -b base58btc $CIDv0 > actual1 && + test_cmp actual1 expected && + ipfs cid format -v 1 -b base58btc $CIDb32 > actual2 && + test_cmp expected actual2 +' + +cat < various_cids +QmZZRTyhDpL5Jgift1cHbAhexeE1m2Hw8x8g7rTcPahDvo + QmPhk6cJkRcFfZCdYam4c9MKYjFG9V29LswUnbrFNhtk2S +bafybeihtwdtifv43rn5cyilnmkwofdcxi2suqimmo62vn3etf45gjoiuwy +bafybeiek4tfxkc4ov6jsmb63fzbirrsalnjw24zd5xawo2fgxisd4jmpyq +zdj7WgYfT2gfsgiUxzPYboaRbP9H9CxZE5jVMK9pDDwCcKDCR +zdj7WbTaiJT1fgatdet9Ei9iDB5hdCxkbVyhyh8YTUnXMiwYi +uAXASIDsp4T3Wnd6kXFOQaljH3GFK_ixkjMtVhB9VOBrPK3bp + uAXASIDdmmyANeytvXUriuy4BO0lfd2eR0UjygabF6CAzfsD1 +EOF + +cat < various_cids_base32 +bafybeifgwyq5gs4l2mru5klgwjfmftjvkmbyyjurbupuz2bst7mhmg2hwa +bafybeiauil46g3lb32jemjbl7yspca3twdcg4wwkbsgdgvgdj5fpfv2f64 +bafybeihtwdtifv43rn5cyilnmkwofdcxi2suqimmo62vn3etf45gjoiuwy +bafybeiek4tfxkc4ov6jsmb63fzbirrsalnjw24zd5xawo2fgxisd4jmpyq +bafybeifffq3aeaymxejo37sn5fyaf7nn7hkfmzwdxyjculx3lw4tyhk7uy +bafybeiczsscdsbs7ffqz55asqdf3smv6klcw3gofszvwlyarci47bgf354 +bafybeib3fhqt3vu532sfyu4qnjmmpxdbjl7cyzemznkyih2vhanm6k3w5e +bafybeibxm2nsadl3fnxv2sxcxmxaco2jl53wpeorjdzidjwf5aqdg7wa6u +EOF + +cat < various_cids_v1 +zdj7WgefqQm5HogBQ2bckZuTYYDarRTUZi51GYCnerHD2G86j +zdj7WWnzU3Nbu5rYGWZHKigUXBtAwShs2SHDCM1TQEvC9TeCN +zdj7WmqAbpsfXgiRBtZP1oAP9QWuuY3mqbc5JhpxJkfT3vYCu +zdj7Wen5gtfr7AivXip3zYd1peuq2QfKrqAn4FGiciVWb96YB +zdj7WgYfT2gfsgiUxzPYboaRbP9H9CxZE5jVMK9pDDwCcKDCR +zdj7WbTaiJT1fgatdet9Ei9iDB5hdCxkbVyhyh8YTUnXMiwYi +zdj7WZQrAvnY5ge3FNg5cmCsNwsvpYjdtu2yEmnWYQ4ES7Nzk +zdj7WZAAFKPvYPPzyJLso2hhxo8a7ZACFQ4DvvfrNXTHidofr +EOF + +test_expect_success "cid base32 works from stdin" ' + cat various_cids | ipfs cid base32 > actual && + test_cmp various_cids_base32 actual +' + +test_expect_success "cid format -v 1 -b base58btc works from stdin" ' + cat various_cids | ipfs cid format -v 1 -b base58btc > actual && + test_cmp various_cids_v1 actual +' + +cat < bases_expect + 0 identity +b 98 base32 +B 66 base32upper +c 99 base32pad +C 67 base32padupper +f 102 base16 +F 70 base16upper +m 109 base64 +M 77 base64pad +t 116 base32hexpad +T 84 base32hexpadupper +u 117 base64url +U 85 base64urlpad +v 118 base32hex +V 86 base32hexupper +z 122 base58btc +Z 90 base58flickr +EOF + +cat < codecs_expect + 85 raw + 112 protobuf + 113 cbor + 120 git-raw + 144 eth-block + 145 eth-block-list + 146 eth-tx-trie + 147 eth-tx + 148 eth-tx-receipt-trie + 149 eth-tx-receipt + 150 eth-state-trie + 151 eth-account-snapshot + 152 eth-storage-trie + 176 bitcoin-block + 177 bitcoin-tx + 192 zcash-block + 193 zcash-tx + 224 decred-block + 225 decred-tx +EOF + +cat < hashes_expect + 0 id + 17 sha1 + 18 sha2-256 + 19 sha2-512 + 20 sha3-512 + 21 sha3-384 + 22 sha3-256 + 23 sha3-224 + 25 shake-256 + 26 keccak-224 + 27 keccak-256 + 28 keccak-384 + 29 keccak-512 + 86 dbl-sha2-256 +45588 blake2b-160 +45589 blake2b-168 +45590 blake2b-176 +45591 blake2b-184 +45592 blake2b-192 +45593 blake2b-200 +45594 blake2b-208 +45595 blake2b-216 +45596 blake2b-224 +45597 blake2b-232 +45598 blake2b-240 +45599 blake2b-248 +45600 blake2b-256 +45601 blake2b-264 +45602 blake2b-272 +45603 blake2b-280 +45604 blake2b-288 +45605 blake2b-296 +45606 blake2b-304 +45607 blake2b-312 +45608 blake2b-320 +45609 blake2b-328 +45610 blake2b-336 +45611 blake2b-344 +45612 blake2b-352 +45613 blake2b-360 +45614 blake2b-368 +45615 blake2b-376 +45616 blake2b-384 +45617 blake2b-392 +45618 blake2b-400 +45619 blake2b-408 +45620 blake2b-416 +45621 blake2b-424 +45622 blake2b-432 +45623 blake2b-440 +45624 blake2b-448 +45625 blake2b-456 +45626 blake2b-464 +45627 blake2b-472 +45628 blake2b-480 +45629 blake2b-488 +45630 blake2b-496 +45631 blake2b-504 +45632 blake2b-512 +45652 blake2s-160 +45653 blake2s-168 +45654 blake2s-176 +45655 blake2s-184 +45656 blake2s-192 +45657 blake2s-200 +45658 blake2s-208 +45659 blake2s-216 +45660 blake2s-224 +45661 blake2s-232 +45662 blake2s-240 +45663 blake2s-248 +45664 blake2s-256 +EOF + +test_expect_success "cid bases" ' + cut -c 10- bases_expect > expect && + ipfs cid bases > actual && + test_cmp expect actual +' + +test_expect_success "cid bases --prefix" ' + cut -c 1-3,10- bases_expect > expect && + ipfs cid bases --prefix > actual && + test_cmp expect actual +' + +test_expect_success "cid bases --prefix --numeric" ' + ipfs cid bases --prefix --numeric > actual && + test_cmp bases_expect actual +' + +test_expect_success "cid codecs" ' + cut -c 8- codecs_expect > expect && + ipfs cid codecs > actual + test_cmp expect actual +' + +test_expect_success "cid codecs --numeric" ' + ipfs cid codecs --numeric > actual && + test_cmp codecs_expect actual +' + +test_expect_success "cid hashes" ' + cut -c 8- hashes_expect > expect && + ipfs cid hashes > actual + test_cmp expect actual +' + +test_expect_success "cid hashes --numeric" ' + ipfs cid hashes --numeric > actual && + test_cmp hashes_expect actual +' + +test_done From c0a873c27fe412a1ad76e5e479aa5d6a314381da Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Wed, 29 Aug 2018 23:39:05 -0400 Subject: [PATCH 08/10] Address code review. License: MIT Signed-off-by: Kevin Atkinson --- core/commands/cid.go | 16 ++++++++++++---- core/commands/commands.go | 8 ++++---- core/commands/filestore.go | 2 +- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/core/commands/cid.go b/core/commands/cid.go index 1211b027d47..16c5d19e7fc 100644 --- a/core/commands/cid.go +++ b/core/commands/cid.go @@ -6,6 +6,8 @@ import ( "sort" "strings" "unicode" + + "github.com/ipfs/go-ipfs/core/commands/e" cid "gx/ipfs/QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7/go-cid" mhash "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash" @@ -66,7 +68,7 @@ The optional format string is a printf style format string: case "1": opts.verConv = toCidV1 default: - return fmt.Errorf("invalid cid version: %s\n", verStr) + return fmt.Errorf("invalid cid version: %s", verStr) } if baseStr != "" { @@ -82,7 +84,7 @@ The optional format string is a printf style format string: return emitCids(req, resp, opts) }, PostRun: cmds.PostRunMap{ - cmds.CLI: streamRes(func(v interface{}, out io.Writer) nonFatalError { + cmds.CLI: streamResults(func(v interface{}, out io.Writer) nonFatalError { r := v.(*CidFormatRes) if r.ErrorMsg != "" { return nonFatalError(fmt.Sprintf("%s: %s", r.CidStr, r.ErrorMsg)) @@ -230,7 +232,10 @@ var basesCmd = &cmds.Command{ cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, val0 interface{}) error { prefixes, _ := req.Options["prefix"].(bool) numeric, _ := req.Options["numeric"].(bool) - val := val0.([]CodeAndName) + val, ok := val0.([]CodeAndName) + if !ok { + return e.TypeErr(val, val0) + } sort.Sort(multibaseSorter{val}) for _, v := range val { code := v.Code @@ -274,7 +279,10 @@ var codecsCmd = &cmds.Command{ Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, val0 interface{}) error { numeric, _ := req.Options["numeric"].(bool) - val := val0.([]CodeAndName) + val, ok := val0.([]CodeAndName) + if !ok { + return e.TypeErr(val, val0) + } sort.Sort(codeAndNameSorter{val}) for _, v := range val { if numeric { diff --git a/core/commands/commands.go b/core/commands/commands.go index a89b279f773..fcfafcee395 100644 --- a/core/commands/commands.go +++ b/core/commands/commands.go @@ -153,10 +153,10 @@ func unwrapOutput(i interface{}) (interface{}, error) { type nonFatalError string -// streamRes is a helper function to stream results, that possibly -// contain with non-fatal, the helper function is allowed to panic on -// internal errors -func streamRes(procVal func(interface{}, io.Writer) nonFatalError) func(cmds.Response, cmds.ResponseEmitter) error { +// streamResults is a helper function to stream results that possibly +// contain non-fatal errors. The helper function is allowed to panic +// on internal errors. +func streamResults(procVal func(interface{}, io.Writer) nonFatalError) func(cmds.Response, cmds.ResponseEmitter) error { return func(res cmds.Response, re cmds.ResponseEmitter) (err error) { defer func() { if r := recover(); r != nil { diff --git a/core/commands/filestore.go b/core/commands/filestore.go index deb7520d22f..9dd94269873 100644 --- a/core/commands/filestore.go +++ b/core/commands/filestore.go @@ -72,7 +72,7 @@ The output is: return res.Emit(out) }, PostRun: cmds.PostRunMap{ - cmds.CLI: streamRes(func(v interface{}, out io.Writer) nonFatalError { + cmds.CLI: streamResults(func(v interface{}, out io.Writer) nonFatalError { r := v.(*filestore.ListRes) if r.ErrorMsg != "" { return nonFatalError(r.ErrorMsg) From 7edc7de28822b6ea0fe237e2d1ad6e230919d3f8 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Thu, 30 Aug 2018 03:19:29 -0400 Subject: [PATCH 09/10] Eliminate `emit` closure, so something with errors on call to emit. License: MIT Signed-off-by: Kevin Atkinson --- core/commands/cid.go | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/core/commands/cid.go b/core/commands/cid.go index 16c5d19e7fc..6defab4c862 100644 --- a/core/commands/cid.go +++ b/core/commands/cid.go @@ -6,7 +6,7 @@ import ( "sort" "strings" "unicode" - + "github.com/ipfs/go-ipfs/core/commands/e" cid "gx/ipfs/QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7/go-cid" @@ -153,21 +153,17 @@ func (i *argumentIterator) err() error { func emitCids(req *cmds.Request, resp cmds.ResponseEmitter, opts cidFormatOpts) error { itr := argumentIterator{req.Arguments, req.BodyArgs()} - for { + var emitErr error + for emitErr == nil { cidStr, ok := itr.next() if !ok { break } - emit := func(fmtd string, err error) { - res := &CidFormatRes{CidStr: cidStr, Formatted: fmtd} - if err != nil { - res.ErrorMsg = err.Error() - } - resp.Emit(res) - } + res := &CidFormatRes{CidStr: cidStr} c, err := cid.Decode(cidStr) if err != nil { - emit("", err) + res.ErrorMsg = err.Error() + emitErr = resp.Emit(res) continue } base := opts.newBase @@ -177,7 +173,8 @@ func emitCids(req *cmds.Request, resp cmds.ResponseEmitter, opts cidFormatOpts) if opts.verConv != nil { c, err = opts.verConv(c) if err != nil { - emit("", err) + res.ErrorMsg = err.Error() + emitErr = resp.Emit(res) continue } } @@ -186,7 +183,15 @@ func emitCids(req *cmds.Request, resp cmds.ResponseEmitter, opts cidFormatOpts) // no point in continuing if there is a problem with the format string return err } - emit(str, err) + if err != nil { + res.ErrorMsg = err.Error() + } else { + res.Formatted = str + } + emitErr = resp.Emit(res) + } + if emitErr != nil { + return emitErr } err := itr.err() if err != nil { From 54904a76914121a4017ae22d0c2090caf91f8858 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Thu, 20 Sep 2018 03:20:17 -0400 Subject: [PATCH 10/10] Rename streamResults to streamResult. License: MIT Signed-off-by: Kevin Atkinson --- core/commands/cid.go | 2 +- core/commands/commands.go | 4 ++-- core/commands/filestore.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/commands/cid.go b/core/commands/cid.go index 6defab4c862..aa44e3d685e 100644 --- a/core/commands/cid.go +++ b/core/commands/cid.go @@ -84,7 +84,7 @@ The optional format string is a printf style format string: return emitCids(req, resp, opts) }, PostRun: cmds.PostRunMap{ - cmds.CLI: streamResults(func(v interface{}, out io.Writer) nonFatalError { + cmds.CLI: streamResult(func(v interface{}, out io.Writer) nonFatalError { r := v.(*CidFormatRes) if r.ErrorMsg != "" { return nonFatalError(fmt.Sprintf("%s: %s", r.CidStr, r.ErrorMsg)) diff --git a/core/commands/commands.go b/core/commands/commands.go index fcfafcee395..65757c1cf75 100644 --- a/core/commands/commands.go +++ b/core/commands/commands.go @@ -153,10 +153,10 @@ func unwrapOutput(i interface{}) (interface{}, error) { type nonFatalError string -// streamResults is a helper function to stream results that possibly +// streamResult is a helper function to stream results that possibly // contain non-fatal errors. The helper function is allowed to panic // on internal errors. -func streamResults(procVal func(interface{}, io.Writer) nonFatalError) func(cmds.Response, cmds.ResponseEmitter) error { +func streamResult(procVal func(interface{}, io.Writer) nonFatalError) func(cmds.Response, cmds.ResponseEmitter) error { return func(res cmds.Response, re cmds.ResponseEmitter) (err error) { defer func() { if r := recover(); r != nil { diff --git a/core/commands/filestore.go b/core/commands/filestore.go index 9dd94269873..1a78dd761f1 100644 --- a/core/commands/filestore.go +++ b/core/commands/filestore.go @@ -72,7 +72,7 @@ The output is: return res.Emit(out) }, PostRun: cmds.PostRunMap{ - cmds.CLI: streamResults(func(v interface{}, out io.Writer) nonFatalError { + cmds.CLI: streamResult(func(v interface{}, out io.Writer) nonFatalError { r := v.(*filestore.ListRes) if r.ErrorMsg != "" { return nonFatalError(r.ErrorMsg)