Merge pull request #5245 from ipfs/fix/5241

add trickle-dag support to the urlstore

core/commands/root.go

@@ -136,7 +136,7 @@ var rootSubcommands = map[string]*cmds.Command{
 	"tar":       lgc.NewCommand(TarCmd),
 	"file":      lgc.NewCommand(unixfs.UnixFSCmd),
 	"update":    lgc.NewCommand(ExternalBinary()),
-	"urlstore":  lgc.NewCommand(urlStoreCmd),
+	"urlstore":  urlStoreCmd,
 	"version":   lgc.NewCommand(VersionCmd),
 	"shutdown":  lgc.NewCommand(daemonShutdownCmd),
 }

core/commands/urlstore.go

@@ -4,21 +4,20 @@ import (
 	"fmt"
 	"io"
 	"net/http"
-	"strings"
 
-	cmds "github.com/ipfs/go-ipfs/commands"
 	filestore "github.com/ipfs/go-ipfs/filestore"
 	balanced "github.com/ipfs/go-ipfs/importer/balanced"
 	ihelper "github.com/ipfs/go-ipfs/importer/helpers"
+	trickle "github.com/ipfs/go-ipfs/importer/trickle"
 
+	cmds "gx/ipfs/QmNueRyPRQiV7PUEpnP4GgGLuK1rKQLaRW7sfPvUetYig1/go-ipfs-cmds"
 	mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash"
 	chunk "gx/ipfs/QmVDjhUMtkRskBFAVNwyXuLSKbeAya7JKPnzAxMKDaK4x4/go-ipfs-chunker"
 	cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
 	cmdkit "gx/ipfs/QmdE4gMduCKCGAcczM2F5ioYDfdeKuPix138wrES1YSr7f/go-ipfs-cmdkit"
 )
 
 var urlStoreCmd = &cmds.Command{
-
 	Subcommands: map[string]*cmds.Command{
 		"add": urlAdd,
 	},
@@ -44,14 +43,17 @@ found.  It may disappear or the semantics can change at any
 time.
 `,
 	},
+	Options: []cmdkit.Option{
+		cmdkit.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."),
+	},
 	Arguments: []cmdkit.Argument{
 		cmdkit.StringArg("url", true, false, "URL to add to IPFS"),
 	},
 	Type: BlockStat{},
 
-	Run: func(req cmds.Request, res cmds.Response) {
-		url := req.Arguments()[0]
-		n, err := req.InvocContext().GetNode()
+	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) {
+		url := req.Arguments[0]
+		n, err := GetNode(env)
 		if err != nil {
 			res.SetError(err, cmdkit.ErrNormal)
 			return
@@ -73,6 +75,8 @@ time.
 			return
 		}
 
+		useTrickledag, _ := req.Options[trickleOptionName].(bool)
+
 		hreq, err := http.NewRequest("GET", url, nil)
 		if err != nil {
 			res.SetError(err, cmdkit.ErrNormal)
@@ -100,23 +104,25 @@ time.
 			URL:       url,
 		}
 
-		blc, err := balanced.Layout(dbp.New(chk))
+		layout := balanced.Layout
+		if useTrickledag {
+			layout = trickle.Layout
+		}
+		root, err := layout(dbp.New(chk))
 		if err != nil {
 			res.SetError(err, cmdkit.ErrNormal)
 			return
 		}
 
-		res.SetOutput(BlockStat{
-			Key:  blc.Cid().String(),
+		cmds.EmitOnce(res, BlockStat{
+			Key:  root.Cid().String(),
 			Size: int(hres.ContentLength),
 		})
 	},
-	Marshalers: cmds.MarshalerMap{
-		cmds.Text: func(res cmds.Response) (io.Reader, error) {
-			ch := res.Output().(<-chan interface{})
-			bs0 := <-ch
-			bs := bs0.(*BlockStat)
-			return strings.NewReader(bs.Key + "\n"), nil
-		},
+	Encoders: cmds.EncoderMap{
+		cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, bs *BlockStat) error {
+			_, err := fmt.Fprintln(w, bs.Key)
+			return err
+		}),
 	},
 }

test/sharness/t0272-urlstore.sh

@@ -124,6 +124,12 @@ test_expect_success "get large file via urlstore" '
   test_cmp file3 file3.actual
 '
 
+test_expect_success "check that the trickle option works" '
+  HASHat=$(ipfs add -q --cid-version=1 --raw-leaves=true -n --trickle file3) &&
+  HASHut=$(ipfs urlstore add --trickle http://127.0.0.1:$GWAY_PORT/ipfs/$HASH3a) &&
+  test $HASHat = $HASHut
+'
+
 test_kill_ipfs_daemon
 
 test_expect_success "files can not be retrieved via the urlstore" '
@@ -132,4 +138,13 @@ test_expect_success "files can not be retrieved via the urlstore" '
   test_must_fail ipfs cat $HASH3 > /dev/null
 '
 
+test_expect_success "check that the hashes were correct" '
+  HASH1e=$(ipfs add -q -n --cid-version=1 --raw-leaves=true file1) &&
+  HASH2e=$(ipfs add -q -n --cid-version=1 --raw-leaves=true file2) &&
+  HASH3e=$(ipfs add -q -n --cid-version=1 --raw-leaves=true file3) &&
+  test $HASH1e = $HASH1 &&
+  test $HASH2e = $HASH2 &&
+  test $HASH3e = $HASH3
+'
+
 test_done