diff --git a/cmd/ipfs/ipfs.go b/cmd/ipfs/ipfs.go index 0342007a12a..8e74139b075 100644 --- a/cmd/ipfs/ipfs.go +++ b/cmd/ipfs/ipfs.go @@ -92,4 +92,5 @@ var cmdDetailsMap = map[string]cmdDetails{ "diag/cmds": {cannotRunOnClient: true}, "repo/fsck": {cannotRunOnDaemon: true}, "config/edit": {cannotRunOnDaemon: true, doesNotUseRepo: true}, + "cid": {doesNotUseRepo: true}, } diff --git a/core/commands/cid.go b/core/commands/cid.go new file mode 100644 index 00000000000..aa44e3d685e --- /dev/null +++ b/core/commands/cid.go @@ -0,0 +1,349 @@ +package commands + +import ( + "fmt" + "io" + "sort" + "strings" + "unicode" + + "github.com/ipfs/go-ipfs/core/commands/e" + + cid "gx/ipfs/QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7/go-cid" + mhash "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash" + cidutil "gx/ipfs/QmQJSeE3CX4zos9qeaG8EhecEK9zvrTEfTG84J8C5NVRwt/go-cidutil" + cmdkit "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit" + verifcid "gx/ipfs/QmVkMRSkXrpjqrroEXWuYBvDBnXCdMMY6gsKicBGVGUqKT/go-verifcid" + cmds "gx/ipfs/QmXTmUCBtDUrzDYVzASogLiNph7EBuYqEgPL7QoHNMzUnz/go-ipfs-cmds" + mbase "gx/ipfs/QmekxXDhCxCJRNuzmHreuaT3BsuJcsjcXWNrtV9C8DRHtd/go-multibase" +) + +var CidCmd = &cmds.Command{ + Helptext: cmdkit.HelpText{ + Tagline: "Convert and discover properties of CIDs", + }, + Subcommands: map[string]*cmds.Command{ + "format": cidFmtCmd, + "base32": base32Cmd, + "bases": basesCmd, + "codecs": codecsCmd, + "hashes": hashesCmd, + }, +} + +var cidFmtCmd = &cmds.Command{ + Helptext: cmdkit.HelpText{ + Tagline: "Format and convert a CID in various useful ways.", + LongDescription: ` +Format and converts 's in various useful ways. + +The optional format string is a printf style format string: +` + cidutil.FormatRef, + }, + Arguments: []cmdkit.Argument{ + cmdkit.StringArg("cid", true, true, "Cids to format.").EnableStdin(), + }, + Options: []cmdkit.Option{ + cmdkit.StringOption("f", "Printf style format string.").WithDefault("%s"), + cmdkit.StringOption("v", "CID version to convert to."), + cmdkit.StringOption("b", "Multibase to display CID in."), + }, + Run: func(req *cmds.Request, resp cmds.ResponseEmitter, env cmds.Environment) error { + fmtStr, _ := req.Options["f"].(string) + verStr, _ := req.Options["v"].(string) + baseStr, _ := req.Options["b"].(string) + + opts := cidFormatOpts{} + + if strings.IndexByte(fmtStr, '%') == -1 { + return fmt.Errorf("invalid format string: %s", fmtStr) + } + opts.fmtStr = fmtStr + + switch verStr { + case "": + // noop + case "0": + opts.verConv = toCidV0 + case "1": + opts.verConv = toCidV1 + default: + return fmt.Errorf("invalid cid version: %s", verStr) + } + + if baseStr != "" { + encoder, err := mbase.EncoderByName(baseStr) + if err != nil { + return err + } + opts.newBase = encoder.Encoding() + } else { + opts.newBase = mbase.Encoding(-1) + } + + return emitCids(req, resp, opts) + }, + PostRun: cmds.PostRunMap{ + cmds.CLI: streamResult(func(v interface{}, out io.Writer) nonFatalError { + r := v.(*CidFormatRes) + if r.ErrorMsg != "" { + return nonFatalError(fmt.Sprintf("%s: %s", r.CidStr, r.ErrorMsg)) + } + fmt.Fprintf(out, "%s\n", r.Formatted) + return "" + }), + }, + Type: CidFormatRes{}, +} + +type CidFormatRes struct { + CidStr string // Original Cid String passed in + Formatted string // Formated Result + ErrorMsg string // Error +} + +var base32Cmd = &cmds.Command{ + Helptext: cmdkit.HelpText{ + Tagline: "Convert CIDs to Base32 CID version 1.", + }, + Arguments: []cmdkit.Argument{ + cmdkit.StringArg("cid", true, true, "Cids to convert.").EnableStdin(), + }, + Run: func(req *cmds.Request, resp cmds.ResponseEmitter, env cmds.Environment) error { + opts := cidFormatOpts{ + fmtStr: "%s", + newBase: mbase.Encoding(mbase.Base32), + verConv: toCidV1, + } + return emitCids(req, resp, opts) + }, + PostRun: cidFmtCmd.PostRun, + Type: cidFmtCmd.Type, +} + +type cidFormatOpts struct { + fmtStr string + newBase mbase.Encoding + verConv func(cid cid.Cid) (cid.Cid, error) +} + +type argumentIterator struct { + args []string + body cmds.StdinArguments +} + +func (i *argumentIterator) next() (string, bool) { + if len(i.args) > 0 { + arg := i.args[0] + i.args = i.args[1:] + return arg, true + } + if i.body == nil || !i.body.Scan() { + return "", false + } + return strings.TrimSpace(i.body.Argument()), true +} + +func (i *argumentIterator) err() error { + if i.body == nil { + return nil + } + return i.body.Err() +} + +func emitCids(req *cmds.Request, resp cmds.ResponseEmitter, opts cidFormatOpts) error { + itr := argumentIterator{req.Arguments, req.BodyArgs()} + var emitErr error + for emitErr == nil { + cidStr, ok := itr.next() + if !ok { + break + } + res := &CidFormatRes{CidStr: cidStr} + c, err := cid.Decode(cidStr) + if err != nil { + res.ErrorMsg = err.Error() + emitErr = resp.Emit(res) + continue + } + base := opts.newBase + if base == -1 { + base, _ = cid.ExtractEncoding(cidStr) + } + if opts.verConv != nil { + c, err = opts.verConv(c) + if err != nil { + res.ErrorMsg = err.Error() + emitErr = resp.Emit(res) + continue + } + } + str, err := cidutil.Format(opts.fmtStr, base, c) + if _, ok := err.(cidutil.FormatStringError); ok { + // no point in continuing if there is a problem with the format string + return err + } + if err != nil { + res.ErrorMsg = err.Error() + } else { + res.Formatted = str + } + emitErr = resp.Emit(res) + } + if emitErr != nil { + return emitErr + } + err := itr.err() + if err != nil { + return err + } + return nil +} + +func toCidV0(c cid.Cid) (cid.Cid, error) { + if c.Type() != cid.DagProtobuf { + return cid.Cid{}, fmt.Errorf("can't convert non-protobuf nodes to cidv0") + } + return cid.NewCidV0(c.Hash()), nil +} + +func toCidV1(c cid.Cid) (cid.Cid, error) { + return cid.NewCidV1(c.Type(), c.Hash()), nil +} + +type CodeAndName struct { + Code int + Name string +} + +var basesCmd = &cmds.Command{ + Helptext: cmdkit.HelpText{ + Tagline: "List available multibase encodings.", + }, + Options: []cmdkit.Option{ + cmdkit.BoolOption("prefix", "also include the single leter prefixes in addition to the code"), + cmdkit.BoolOption("numeric", "also include numeric codes"), + }, + Run: func(req *cmds.Request, resp cmds.ResponseEmitter, env cmds.Environment) error { + var res []CodeAndName + // use EncodingToStr in case at some point there are multiple names for a given code + for code, name := range mbase.EncodingToStr { + res = append(res, CodeAndName{int(code), name}) + } + cmds.EmitOnce(resp, res) + return nil + }, + Encoders: cmds.EncoderMap{ + cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, val0 interface{}) error { + prefixes, _ := req.Options["prefix"].(bool) + numeric, _ := req.Options["numeric"].(bool) + val, ok := val0.([]CodeAndName) + if !ok { + return e.TypeErr(val, val0) + } + sort.Sort(multibaseSorter{val}) + for _, v := range val { + code := v.Code + if code < 32 || code >= 127 { + // don't display non-printable prefixes + code = ' ' + } + switch { + case prefixes && numeric: + fmt.Fprintf(w, "%c %5d %s\n", code, v.Code, v.Name) + case prefixes: + fmt.Fprintf(w, "%c %s\n", code, v.Name) + case numeric: + fmt.Fprintf(w, "%5d %s\n", v.Code, v.Name) + default: + fmt.Fprintf(w, "%s\n", v.Name) + } + } + return nil + }), + }, + Type: []CodeAndName{}, +} + +var codecsCmd = &cmds.Command{ + Helptext: cmdkit.HelpText{ + Tagline: "List available CID codecs.", + }, + Options: []cmdkit.Option{ + cmdkit.BoolOption("numeric", "also include numeric codes"), + }, + Run: func(req *cmds.Request, resp cmds.ResponseEmitter, env cmds.Environment) error { + var res []CodeAndName + // use CodecToStr as there are multiple names for a given code + for code, name := range cid.CodecToStr { + res = append(res, CodeAndName{int(code), name}) + } + cmds.EmitOnce(resp, res) + return nil + }, + Encoders: cmds.EncoderMap{ + cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, val0 interface{}) error { + numeric, _ := req.Options["numeric"].(bool) + val, ok := val0.([]CodeAndName) + if !ok { + return e.TypeErr(val, val0) + } + sort.Sort(codeAndNameSorter{val}) + for _, v := range val { + if numeric { + fmt.Fprintf(w, "%5d %s\n", v.Code, v.Name) + } else { + fmt.Fprintf(w, "%s\n", v.Name) + } + } + return nil + }), + }, + Type: []CodeAndName{}, +} + +var hashesCmd = &cmds.Command{ + Helptext: cmdkit.HelpText{ + Tagline: "List available multihashes.", + }, + Options: codecsCmd.Options, + Run: func(req *cmds.Request, resp cmds.ResponseEmitter, env cmds.Environment) error { + var res []CodeAndName + // use mhash.Codes in case at some point there are multiple names for a given code + for code, name := range mhash.Codes { + if !verifcid.IsGoodHash(code) { + continue + } + res = append(res, CodeAndName{int(code), name}) + } + cmds.EmitOnce(resp, res) + return nil + }, + Encoders: codecsCmd.Encoders, + Type: codecsCmd.Type, +} + +type multibaseSorter struct { + data []CodeAndName +} + +func (s multibaseSorter) Len() int { return len(s.data) } +func (s multibaseSorter) Swap(i, j int) { s.data[i], s.data[j] = s.data[j], s.data[i] } + +func (s multibaseSorter) Less(i, j int) bool { + a := unicode.ToLower(rune(s.data[i].Code)) + b := unicode.ToLower(rune(s.data[j].Code)) + if a != b { + return a < b + } + // lowecase letters should come before uppercase + return s.data[i].Code > s.data[j].Code +} + +type codeAndNameSorter struct { + data []CodeAndName +} + +func (s codeAndNameSorter) Len() int { return len(s.data) } +func (s codeAndNameSorter) Swap(i, j int) { s.data[i], s.data[j] = s.data[j], s.data[i] } +func (s codeAndNameSorter) Less(i, j int) bool { return s.data[i].Code < s.data[j].Code } diff --git a/core/commands/commands.go b/core/commands/commands.go index 51c707c6e41..65757c1cf75 100644 --- a/core/commands/commands.go +++ b/core/commands/commands.go @@ -8,6 +8,7 @@ package commands import ( "fmt" "io" + "os" "sort" "strings" @@ -149,3 +150,42 @@ func unwrapOutput(i interface{}) (interface{}, error) { return <-ch, nil } + +type nonFatalError string + +// streamResult is a helper function to stream results that possibly +// contain non-fatal errors. The helper function is allowed to panic +// on internal errors. +func streamResult(procVal func(interface{}, io.Writer) nonFatalError) func(cmds.Response, cmds.ResponseEmitter) error { + return func(res cmds.Response, re cmds.ResponseEmitter) (err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("internal error: %v", r) + } + re.Close() + }() + + var errors bool + for { + v, err := res.Next() + if err != nil { + if err == io.EOF { + break + } + return err + } + + errorMsg := procVal(v, os.Stdout) + + if errorMsg != "" { + errors = true + fmt.Fprintf(os.Stderr, "%s\n", errorMsg) + } + } + + if errors { + return fmt.Errorf("errors while displaying some entries") + } + return nil + } +} diff --git a/core/commands/commands_test.go b/core/commands/commands_test.go index ec4b4ba5f04..74d58903488 100644 --- a/core/commands/commands_test.go +++ b/core/commands/commands_test.go @@ -211,6 +211,12 @@ func TestCommands(t *testing.T) { "/urlstore", "/urlstore/add", "/version", + "/cid", + "/cid/format", + "/cid/base32", + "/cid/codecs", + "/cid/bases", + "/cid/hashes", } cmdSet := make(map[string]struct{}) diff --git a/core/commands/filestore.go b/core/commands/filestore.go index a45e01ef93f..1a78dd761f1 100644 --- a/core/commands/filestore.go +++ b/core/commands/filestore.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "io" - "os" oldCmds "github.com/ipfs/go-ipfs/commands" lgc "github.com/ipfs/go-ipfs/commands/legacy" @@ -73,36 +72,14 @@ The output is: return res.Emit(out) }, PostRun: cmds.PostRunMap{ - cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error { - var errors bool - for { - v, err := res.Next() - if err != nil { - if err == io.EOF { - break - } - return err - } - - r, ok := v.(*filestore.ListRes) - if !ok { - return e.New(e.TypeErr(r, v)) - } - - if r.ErrorMsg != "" { - errors = true - fmt.Fprintf(os.Stderr, "%s\n", r.ErrorMsg) - } else { - fmt.Fprintf(os.Stdout, "%s\n", r.FormatLong()) - } - } - - if errors { - return fmt.Errorf("errors while displaying some entries") + cmds.CLI: streamResult(func(v interface{}, out io.Writer) nonFatalError { + r := v.(*filestore.ListRes) + if r.ErrorMsg != "" { + return nonFatalError(r.ErrorMsg) } - - return nil - }, + fmt.Fprintf(out, "%s\n", r.FormatLong()) + return "" + }), }, Type: filestore.ListRes{}, } diff --git a/core/commands/root.go b/core/commands/root.go index 0268136f610..900530a216b 100644 --- a/core/commands/root.go +++ b/core/commands/root.go @@ -71,6 +71,7 @@ TOOL COMMANDS version Show ipfs version information update Download and apply go-ipfs updates commands List all available commands + cid Convert and discover properties of CIDs Use 'ipfs --help' to learn more about each command. @@ -143,6 +144,7 @@ var rootSubcommands = map[string]*cmds.Command{ "urlstore": urlStoreCmd, "version": lgc.NewCommand(VersionCmd), "shutdown": daemonShutdownCmd, + "cid": CidCmd, } // RootRO is the readonly version of Root diff --git a/test/sharness/t0290-cid.sh b/test/sharness/t0290-cid.sh new file mode 100755 index 00000000000..ecc2ba19e4b --- /dev/null +++ b/test/sharness/t0290-cid.sh @@ -0,0 +1,228 @@ +#!/usr/bin/env bash + +test_description="Test cid commands" + +. lib/test-lib.sh + +# note: all "ipfs cid" commands should work without requiring a repo + +CIDv0="QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv" +CIDv1="zdj7WZAAFKPvYPPzyJLso2hhxo8a7ZACFQ4DvvfrNXTHidofr" +CIDb32="bafybeibxm2nsadl3fnxv2sxcxmxaco2jl53wpeorjdzidjwf5aqdg7wa6u" + +test_expect_success "cid base32 works" ' + echo $CIDb32 > expected && + ipfs cid base32 $CIDv0 > actual1 && + test_cmp actual1 expected && + ipfs cid base32 $CIDv1 > actual2 && + test_cmp expected actual2 +' + +test_expect_success "cid format -v 1 -b base58btc" ' + echo $CIDv1 > expected && + ipfs cid format -v 1 -b base58btc $CIDv0 > actual1 && + test_cmp actual1 expected && + ipfs cid format -v 1 -b base58btc $CIDb32 > actual2 && + test_cmp expected actual2 +' + +cat < various_cids +QmZZRTyhDpL5Jgift1cHbAhexeE1m2Hw8x8g7rTcPahDvo + QmPhk6cJkRcFfZCdYam4c9MKYjFG9V29LswUnbrFNhtk2S +bafybeihtwdtifv43rn5cyilnmkwofdcxi2suqimmo62vn3etf45gjoiuwy +bafybeiek4tfxkc4ov6jsmb63fzbirrsalnjw24zd5xawo2fgxisd4jmpyq +zdj7WgYfT2gfsgiUxzPYboaRbP9H9CxZE5jVMK9pDDwCcKDCR +zdj7WbTaiJT1fgatdet9Ei9iDB5hdCxkbVyhyh8YTUnXMiwYi +uAXASIDsp4T3Wnd6kXFOQaljH3GFK_ixkjMtVhB9VOBrPK3bp + uAXASIDdmmyANeytvXUriuy4BO0lfd2eR0UjygabF6CAzfsD1 +EOF + +cat < various_cids_base32 +bafybeifgwyq5gs4l2mru5klgwjfmftjvkmbyyjurbupuz2bst7mhmg2hwa +bafybeiauil46g3lb32jemjbl7yspca3twdcg4wwkbsgdgvgdj5fpfv2f64 +bafybeihtwdtifv43rn5cyilnmkwofdcxi2suqimmo62vn3etf45gjoiuwy +bafybeiek4tfxkc4ov6jsmb63fzbirrsalnjw24zd5xawo2fgxisd4jmpyq +bafybeifffq3aeaymxejo37sn5fyaf7nn7hkfmzwdxyjculx3lw4tyhk7uy +bafybeiczsscdsbs7ffqz55asqdf3smv6klcw3gofszvwlyarci47bgf354 +bafybeib3fhqt3vu532sfyu4qnjmmpxdbjl7cyzemznkyih2vhanm6k3w5e +bafybeibxm2nsadl3fnxv2sxcxmxaco2jl53wpeorjdzidjwf5aqdg7wa6u +EOF + +cat < various_cids_v1 +zdj7WgefqQm5HogBQ2bckZuTYYDarRTUZi51GYCnerHD2G86j +zdj7WWnzU3Nbu5rYGWZHKigUXBtAwShs2SHDCM1TQEvC9TeCN +zdj7WmqAbpsfXgiRBtZP1oAP9QWuuY3mqbc5JhpxJkfT3vYCu +zdj7Wen5gtfr7AivXip3zYd1peuq2QfKrqAn4FGiciVWb96YB +zdj7WgYfT2gfsgiUxzPYboaRbP9H9CxZE5jVMK9pDDwCcKDCR +zdj7WbTaiJT1fgatdet9Ei9iDB5hdCxkbVyhyh8YTUnXMiwYi +zdj7WZQrAvnY5ge3FNg5cmCsNwsvpYjdtu2yEmnWYQ4ES7Nzk +zdj7WZAAFKPvYPPzyJLso2hhxo8a7ZACFQ4DvvfrNXTHidofr +EOF + +test_expect_success "cid base32 works from stdin" ' + cat various_cids | ipfs cid base32 > actual && + test_cmp various_cids_base32 actual +' + +test_expect_success "cid format -v 1 -b base58btc works from stdin" ' + cat various_cids | ipfs cid format -v 1 -b base58btc > actual && + test_cmp various_cids_v1 actual +' + +cat < bases_expect + 0 identity +b 98 base32 +B 66 base32upper +c 99 base32pad +C 67 base32padupper +f 102 base16 +F 70 base16upper +m 109 base64 +M 77 base64pad +t 116 base32hexpad +T 84 base32hexpadupper +u 117 base64url +U 85 base64urlpad +v 118 base32hex +V 86 base32hexupper +z 122 base58btc +Z 90 base58flickr +EOF + +cat < codecs_expect + 85 raw + 112 protobuf + 113 cbor + 120 git-raw + 144 eth-block + 145 eth-block-list + 146 eth-tx-trie + 147 eth-tx + 148 eth-tx-receipt-trie + 149 eth-tx-receipt + 150 eth-state-trie + 151 eth-account-snapshot + 152 eth-storage-trie + 176 bitcoin-block + 177 bitcoin-tx + 192 zcash-block + 193 zcash-tx + 224 decred-block + 225 decred-tx +EOF + +cat < hashes_expect + 0 id + 17 sha1 + 18 sha2-256 + 19 sha2-512 + 20 sha3-512 + 21 sha3-384 + 22 sha3-256 + 23 sha3-224 + 25 shake-256 + 26 keccak-224 + 27 keccak-256 + 28 keccak-384 + 29 keccak-512 + 86 dbl-sha2-256 +45588 blake2b-160 +45589 blake2b-168 +45590 blake2b-176 +45591 blake2b-184 +45592 blake2b-192 +45593 blake2b-200 +45594 blake2b-208 +45595 blake2b-216 +45596 blake2b-224 +45597 blake2b-232 +45598 blake2b-240 +45599 blake2b-248 +45600 blake2b-256 +45601 blake2b-264 +45602 blake2b-272 +45603 blake2b-280 +45604 blake2b-288 +45605 blake2b-296 +45606 blake2b-304 +45607 blake2b-312 +45608 blake2b-320 +45609 blake2b-328 +45610 blake2b-336 +45611 blake2b-344 +45612 blake2b-352 +45613 blake2b-360 +45614 blake2b-368 +45615 blake2b-376 +45616 blake2b-384 +45617 blake2b-392 +45618 blake2b-400 +45619 blake2b-408 +45620 blake2b-416 +45621 blake2b-424 +45622 blake2b-432 +45623 blake2b-440 +45624 blake2b-448 +45625 blake2b-456 +45626 blake2b-464 +45627 blake2b-472 +45628 blake2b-480 +45629 blake2b-488 +45630 blake2b-496 +45631 blake2b-504 +45632 blake2b-512 +45652 blake2s-160 +45653 blake2s-168 +45654 blake2s-176 +45655 blake2s-184 +45656 blake2s-192 +45657 blake2s-200 +45658 blake2s-208 +45659 blake2s-216 +45660 blake2s-224 +45661 blake2s-232 +45662 blake2s-240 +45663 blake2s-248 +45664 blake2s-256 +EOF + +test_expect_success "cid bases" ' + cut -c 10- bases_expect > expect && + ipfs cid bases > actual && + test_cmp expect actual +' + +test_expect_success "cid bases --prefix" ' + cut -c 1-3,10- bases_expect > expect && + ipfs cid bases --prefix > actual && + test_cmp expect actual +' + +test_expect_success "cid bases --prefix --numeric" ' + ipfs cid bases --prefix --numeric > actual && + test_cmp bases_expect actual +' + +test_expect_success "cid codecs" ' + cut -c 8- codecs_expect > expect && + ipfs cid codecs > actual + test_cmp expect actual +' + +test_expect_success "cid codecs --numeric" ' + ipfs cid codecs --numeric > actual && + test_cmp codecs_expect actual +' + +test_expect_success "cid hashes" ' + cut -c 8- hashes_expect > expect && + ipfs cid hashes > actual + test_cmp expect actual +' + +test_expect_success "cid hashes --numeric" ' + ipfs cid hashes --numeric > actual && + test_cmp hashes_expect actual +' + +test_done