From ded60a73562a9df9a67ef328d364431a40d1a37c Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sat, 15 Oct 2016 09:06:44 -0700 Subject: [PATCH 1/5] unixfs: allow use of raw merkledag nodes for unixfs files License: MIT Signed-off-by: Jeromy --- blocks/blocks.go | 7 ++- core/commands/add.go | 22 +++++---- core/corehttp/gateway_handler.go | 15 ++++-- core/coreunix/add.go | 34 +++++++------ fuse/readonly/ipfs_test.go | 7 +-- importer/balanced/balanced_test.go | 7 ++- importer/balanced/builder.go | 16 ++++-- importer/helpers/dagbuilder.go | 53 ++++++++++++-------- importer/helpers/helpers.go | 25 ++++++++-- importer/importer.go | 8 +-- importer/importer_test.go | 10 ++-- importer/trickle/trickle_test.go | 10 +++- importer/trickle/trickledag.go | 20 ++++++-- merkledag/merkledag.go | 38 ++++++--------- merkledag/merkledag_test.go | 78 ++++++++++++++++++++++++++++++ merkledag/node.go | 2 +- merkledag/raw.go | 46 ++++++++++++++++++ merkledag/utils/utils.go | 15 +++--- mfs/dir.go | 7 +-- mfs/mfs_test.go | 11 +++-- mfs/ops.go | 5 +- unixfs/io/dagreader.go | 66 ++++++++++++++----------- unixfs/mod/dagmodifier.go | 29 ++++++++--- unixfs/test/utils.go | 9 ++-- 24 files changed, 390 insertions(+), 150 deletions(-) create mode 100644 merkledag/raw.go diff --git a/blocks/blocks.go b/blocks/blocks.go index 4d5b64422fd..ff125367f1d 100644 --- a/blocks/blocks.go +++ b/blocks/blocks.go @@ -37,8 +37,11 @@ func NewBlock(data []byte) *BasicBlock { // we are able to be confident that the data is correct func NewBlockWithCid(data []byte, c *cid.Cid) (*BasicBlock, error) { if u.Debug { - // TODO: fix assumptions - chkc := cid.NewCidV0(u.Hash(data)) + chkc, err := c.Prefix().Sum(data) + if err != nil { + return nil, err + } + if !chkc.Equals(c) { return nil, ErrWrongHash } diff --git a/core/commands/add.go b/core/commands/add.go index e2b35e4cb1c..4215fad30ce 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -23,15 +23,16 @@ import ( var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded") const ( - quietOptionName = "quiet" - silentOptionName = "silent" - progressOptionName = "progress" - trickleOptionName = "trickle" - wrapOptionName = "wrap-with-directory" - hiddenOptionName = "hidden" - onlyHashOptionName = "only-hash" - chunkerOptionName = "chunker" - pinOptionName = "pin" + quietOptionName = "quiet" + silentOptionName = "silent" + progressOptionName = "progress" + trickleOptionName = "trickle" + wrapOptionName = "wrap-with-directory" + hiddenOptionName = "hidden" + onlyHashOptionName = "only-hash" + chunkerOptionName = "chunker" + pinOptionName = "pin" + rawLeavesOptionName = "raw-leaves" ) var AddCmd = &cmds.Command{ @@ -78,6 +79,7 @@ You can now refer to the added file in a gateway, like so: cmds.BoolOption(hiddenOptionName, "H", "Include files that are hidden. Only takes effect on recursive add.").Default(false), cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm to use."), cmds.BoolOption(pinOptionName, "Pin this object when adding.").Default(true), + cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes. (experimental)"), }, PreRun: func(req cmds.Request) error { if quiet, _, _ := req.Option(quietOptionName).Bool(); quiet { @@ -135,6 +137,7 @@ You can now refer to the added file in a gateway, like so: silent, _, _ := req.Option(silentOptionName).Bool() chunker, _, _ := req.Option(chunkerOptionName).String() dopin, _, _ := req.Option(pinOptionName).Bool() + rawblks, _, _ := req.Option(rawLeavesOptionName).Bool() if hash { nilnode, err := core.NewNode(n.Context(), &core.BuildCfg{ @@ -174,6 +177,7 @@ You can now refer to the added file in a gateway, like so: fileAdder.Wrap = wrap fileAdder.Pin = dopin fileAdder.Silent = silent + fileAdder.RawLeaves = rawblks if hash { md := dagtest.Mock() diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index 9b7572d8821..3e71129dcf6 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -1,6 +1,7 @@ package corehttp import ( + "context" "errors" "fmt" "io" @@ -18,10 +19,10 @@ import ( path "github.com/ipfs/go-ipfs/path" uio "github.com/ipfs/go-ipfs/unixfs/io" - "context" routing "gx/ipfs/QmNUgVQTYnXQVrGT2rajZYsuKV8GYdiL91cdZSQDKNPNgE/go-libp2p-routing" humanize "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize" cid "gx/ipfs/QmXUuRadqDq5BuFWzVU6VuKaSjTcNm1gNCtLvvP1TJCW4z/go-cid" + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node" ) const ( @@ -45,7 +46,7 @@ func newGatewayHandler(node *core.IpfsNode, conf GatewayConfig) *gatewayHandler } // TODO(cryptix): find these helpers somewhere else -func (i *gatewayHandler) newDagFromReader(r io.Reader) (*dag.ProtoNode, error) { +func (i *gatewayHandler) newDagFromReader(r io.Reader) (node.Node, error) { // TODO(cryptix): change and remove this helper once PR1136 is merged // return ufs.AddFromReader(i.node, r.Body) return importer.BuildDagFromReader( @@ -353,7 +354,7 @@ func (i *gatewayHandler) putHandler(w http.ResponseWriter, r *http.Request) { return } - var newnode *dag.ProtoNode + var newnode node.Node if rsegs[len(rsegs)-1] == "QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn" { newnode = uio.NewEmptyDirectory() } else { @@ -417,8 +418,14 @@ func (i *gatewayHandler) putHandler(w http.ResponseWriter, r *http.Request) { return } + pbnewnode, ok := newnode.(*dag.ProtoNode) + if !ok { + webError(w, "Cannot read non protobuf nodes through gateway", dag.ErrNotProtobuf, http.StatusBadRequest) + return + } + // object set-data case - pbnd.SetData(newnode.Data()) + pbnd.SetData(pbnewnode.Data()) newcid, err = i.node.DAG.Add(pbnd) if err != nil { diff --git a/core/coreunix/add.go b/core/coreunix/add.go index ff8303ce14d..eaa5ff7e1f8 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -1,6 +1,7 @@ package coreunix import ( + "context" "fmt" "io" "io/ioutil" @@ -13,16 +14,18 @@ import ( "github.com/ipfs/go-ipfs/commands/files" core "github.com/ipfs/go-ipfs/core" "github.com/ipfs/go-ipfs/exchange/offline" - importer "github.com/ipfs/go-ipfs/importer" + balanced "github.com/ipfs/go-ipfs/importer/balanced" "github.com/ipfs/go-ipfs/importer/chunk" + ihelper "github.com/ipfs/go-ipfs/importer/helpers" + trickle "github.com/ipfs/go-ipfs/importer/trickle" dag "github.com/ipfs/go-ipfs/merkledag" mfs "github.com/ipfs/go-ipfs/mfs" "github.com/ipfs/go-ipfs/pin" unixfs "github.com/ipfs/go-ipfs/unixfs" - context "context" logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" cid "gx/ipfs/QmXUuRadqDq5BuFWzVU6VuKaSjTcNm1gNCtLvvP1TJCW4z/go-cid" + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node" ds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore" syncds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore/sync" ) @@ -97,6 +100,7 @@ type Adder struct { Hidden bool Pin bool Trickle bool + RawLeaves bool Silent bool Wrap bool Chunker string @@ -111,22 +115,22 @@ func (adder *Adder) SetMfsRoot(r *mfs.Root) { } // Perform the actual add & pin locally, outputting results to reader -func (adder Adder) add(reader io.Reader) (*dag.ProtoNode, error) { +func (adder Adder) add(reader io.Reader) (node.Node, error) { chnk, err := chunk.FromString(reader, adder.Chunker) if err != nil { return nil, err } + params := ihelper.DagBuilderParams{ + Dagserv: adder.dagService, + RawLeaves: adder.RawLeaves, + Maxlinks: ihelper.DefaultLinksPerBlock, + } if adder.Trickle { - return importer.BuildTrickleDagFromReader( - adder.dagService, - chnk, - ) - } - return importer.BuildDagFromReader( - adder.dagService, - chnk, - ) + return trickle.TrickleLayout(params.New(chnk)) + } + + return balanced.BalancedLayout(params.New(chnk)) } func (adder *Adder) RootNode() (*dag.ProtoNode, error) { @@ -331,7 +335,7 @@ func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, *dag.Pr return gopath.Join(c.String(), filename), dagnode, nil } -func (adder *Adder) addNode(node *dag.ProtoNode, path string) error { +func (adder *Adder) addNode(node node.Node, path string) error { // patch it into the root if path == "" { path = node.Cid().String() @@ -456,7 +460,7 @@ func (adder *Adder) maybePauseForGC() error { } // outputDagnode sends dagnode info over the output channel -func outputDagnode(out chan interface{}, name string, dn *dag.ProtoNode) error { +func outputDagnode(out chan interface{}, name string, dn node.Node) error { if out == nil { return nil } @@ -482,7 +486,7 @@ func NewMemoryDagService() dag.DAGService { } // from core/commands/object.go -func getOutput(dagnode *dag.ProtoNode) (*Object, error) { +func getOutput(dagnode node.Node) (*Object, error) { c := dagnode.Cid() output := &Object{ diff --git a/fuse/readonly/ipfs_test.go b/fuse/readonly/ipfs_test.go index 53a642b7b70..5088c44b126 100644 --- a/fuse/readonly/ipfs_test.go +++ b/fuse/readonly/ipfs_test.go @@ -24,6 +24,7 @@ import ( fstest "github.com/ipfs/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil" cid "gx/ipfs/QmXUuRadqDq5BuFWzVU6VuKaSjTcNm1gNCtLvvP1TJCW4z/go-cid" + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node" u "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util" ) @@ -33,7 +34,7 @@ func maybeSkipFuseTests(t *testing.T) { } } -func randObj(t *testing.T, nd *core.IpfsNode, size int64) (*dag.ProtoNode, []byte) { +func randObj(t *testing.T, nd *core.IpfsNode, size int64) (node.Node, []byte) { buf := make([]byte, size) u.NewTimeSeededRand().Read(buf) read := bytes.NewReader(buf) @@ -74,7 +75,7 @@ func TestIpfsBasicRead(t *testing.T) { defer mnt.Close() fi, data := randObj(t, nd, 10000) - k := fi.Key() + k := fi.Cid() fname := path.Join(mnt.Dir, k.String()) rbuf, err := ioutil.ReadFile(fname) if err != nil { @@ -254,7 +255,7 @@ func TestFileSizeReporting(t *testing.T) { defer mnt.Close() fi, data := randObj(t, nd, 10000) - k := fi.Key() + k := fi.Cid() fname := path.Join(mnt.Dir, k.String()) diff --git a/importer/balanced/balanced_test.go b/importer/balanced/balanced_test.go index 4a896cca780..599fb6a2762 100644 --- a/importer/balanced/balanced_test.go +++ b/importer/balanced/balanced_test.go @@ -28,7 +28,12 @@ func buildTestDag(ds dag.DAGService, spl chunk.Splitter) (*dag.ProtoNode, error) Maxlinks: h.DefaultLinksPerBlock, } - return BalancedLayout(dbp.New(spl)) + nd, err := BalancedLayout(dbp.New(spl)) + if err != nil { + return nil, err + } + + return nd.(*dag.ProtoNode), nil } func getTestDag(t *testing.T, ds dag.DAGService, size int64, blksize int64) (*dag.ProtoNode, []byte) { diff --git a/importer/balanced/builder.go b/importer/balanced/builder.go index 4250e7f81f9..70b3c55b009 100644 --- a/importer/balanced/builder.go +++ b/importer/balanced/builder.go @@ -4,10 +4,11 @@ import ( "errors" h "github.com/ipfs/go-ipfs/importer/helpers" - dag "github.com/ipfs/go-ipfs/merkledag" + + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node" ) -func BalancedLayout(db *h.DagBuilderHelper) (*dag.ProtoNode, error) { +func BalancedLayout(db *h.DagBuilderHelper) (node.Node, error) { var root *h.UnixfsNode for level := 0; !db.Done(); level++ { @@ -56,14 +57,21 @@ func fillNodeRec(db *h.DagBuilderHelper, node *h.UnixfsNode, depth int) error { // Base case if depth <= 0 { // catch accidental -1's in case error above is removed. - return db.FillNodeWithData(node) + child, err := db.GetNextDataNode() + if err != nil { + return err + } + + node.Set(child) + return nil } // while we have room AND we're not done for node.NumChildren() < db.Maxlinks() && !db.Done() { child := h.NewUnixfsNode() - if err := fillNodeRec(db, child, depth-1); err != nil { + err := fillNodeRec(db, child, depth-1) + if err != nil { return err } diff --git a/importer/helpers/dagbuilder.go b/importer/helpers/dagbuilder.go index 497bf036eaf..696b90e4b70 100644 --- a/importer/helpers/dagbuilder.go +++ b/importer/helpers/dagbuilder.go @@ -3,23 +3,30 @@ package helpers import ( "github.com/ipfs/go-ipfs/importer/chunk" dag "github.com/ipfs/go-ipfs/merkledag" + + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node" ) // DagBuilderHelper wraps together a bunch of objects needed to // efficiently create unixfs dag trees type DagBuilderHelper struct { - dserv dag.DAGService - spl chunk.Splitter - recvdErr error - nextData []byte // the next item to return. - maxlinks int - batch *dag.Batch + dserv dag.DAGService + spl chunk.Splitter + recvdErr error + rawLeaves bool + nextData []byte // the next item to return. + maxlinks int + batch *dag.Batch } type DagBuilderParams struct { // Maximum number of links per intermediate node Maxlinks int + // RawLeaves signifies that the importer should use raw ipld nodes as leaves + // instead of using the unixfs TRaw type + RawLeaves bool + // DAGService to write blocks to (required) Dagserv dag.DAGService } @@ -28,10 +35,11 @@ type DagBuilderParams struct { // from chunks object func (dbp *DagBuilderParams) New(spl chunk.Splitter) *DagBuilderHelper { return &DagBuilderHelper{ - dserv: dbp.Dagserv, - spl: spl, - maxlinks: dbp.Maxlinks, - batch: dbp.Dagserv.Batch(), + dserv: dbp.Dagserv, + spl: spl, + rawLeaves: dbp.RawLeaves, + maxlinks: dbp.Maxlinks, + batch: dbp.Dagserv.Batch(), } } @@ -78,9 +86,8 @@ func (db *DagBuilderHelper) FillNodeLayer(node *UnixfsNode) error { // while we have room AND we're not done for node.NumChildren() < db.maxlinks && !db.Done() { - child := NewUnixfsBlock() - - if err := db.FillNodeWithData(child); err != nil { + child, err := db.GetNextDataNode() + if err != nil { return err } @@ -92,21 +99,29 @@ func (db *DagBuilderHelper) FillNodeLayer(node *UnixfsNode) error { return nil } -func (db *DagBuilderHelper) FillNodeWithData(node *UnixfsNode) error { +func (db *DagBuilderHelper) GetNextDataNode() (*UnixfsNode, error) { data := db.Next() if data == nil { // we're done! - return nil + return nil, nil } if len(data) > BlockSizeLimit { - return ErrSizeLimitExceeded + return nil, ErrSizeLimitExceeded } - node.SetData(data) - return nil + if db.rawLeaves { + return &UnixfsNode{ + rawnode: dag.NewRawNode(data), + raw: true, + }, nil + } else { + blk := NewUnixfsBlock() + blk.SetData(data) + return blk, nil + } } -func (db *DagBuilderHelper) Add(node *UnixfsNode) (*dag.ProtoNode, error) { +func (db *DagBuilderHelper) Add(node *UnixfsNode) (node.Node, error) { dn, err := node.GetDagNode() if err != nil { return nil, err diff --git a/importer/helpers/helpers.go b/importer/helpers/helpers.go index 07fe3f99f37..caface2f13a 100644 --- a/importer/helpers/helpers.go +++ b/importer/helpers/helpers.go @@ -1,12 +1,14 @@ package helpers import ( + "context" "fmt" - "context" chunk "github.com/ipfs/go-ipfs/importer/chunk" dag "github.com/ipfs/go-ipfs/merkledag" ft "github.com/ipfs/go-ipfs/unixfs" + + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node" ) // BlockSizeLimit specifies the maximum size an imported block can have. @@ -37,8 +39,10 @@ var ErrSizeLimitExceeded = fmt.Errorf("object size limit exceeded") // UnixfsNode is a struct created to aid in the generation // of unixfs DAG trees type UnixfsNode struct { - node *dag.ProtoNode - ufmt *ft.FSNode + raw bool + rawnode *dag.RawNode + node *dag.ProtoNode + ufmt *ft.FSNode } // NewUnixfsNode creates a new Unixfs node to represent a file @@ -74,6 +78,15 @@ func (n *UnixfsNode) NumChildren() int { return n.ufmt.NumChildren() } +func (n *UnixfsNode) Set(other *UnixfsNode) { + n.node = other.node + n.raw = other.raw + n.rawnode = other.rawnode + if other.ufmt != nil { + n.ufmt.Data = other.ufmt.Data + } +} + func (n *UnixfsNode) GetChild(ctx context.Context, i int, ds dag.DAGService) (*UnixfsNode, error) { nd, err := n.node.Links()[i].GetNode(ctx, ds) if err != nil { @@ -126,7 +139,11 @@ func (n *UnixfsNode) SetData(data []byte) { // getDagNode fills out the proper formatting for the unixfs node // inside of a DAG node and returns the dag node -func (n *UnixfsNode) GetDagNode() (*dag.ProtoNode, error) { +func (n *UnixfsNode) GetDagNode() (node.Node, error) { + if n.raw { + return n.rawnode, nil + } + data, err := n.ufmt.GetBytes() if err != nil { return nil, err diff --git a/importer/importer.go b/importer/importer.go index f5cddf3ff3c..92d32cfc8cb 100644 --- a/importer/importer.go +++ b/importer/importer.go @@ -12,14 +12,16 @@ import ( h "github.com/ipfs/go-ipfs/importer/helpers" trickle "github.com/ipfs/go-ipfs/importer/trickle" dag "github.com/ipfs/go-ipfs/merkledag" + logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node" ) var log = logging.Logger("importer") // Builds a DAG from the given file, writing created blocks to disk as they are // created -func BuildDagFromFile(fpath string, ds dag.DAGService) (*dag.ProtoNode, error) { +func BuildDagFromFile(fpath string, ds dag.DAGService) (node.Node, error) { stat, err := os.Lstat(fpath) if err != nil { return nil, err @@ -38,7 +40,7 @@ func BuildDagFromFile(fpath string, ds dag.DAGService) (*dag.ProtoNode, error) { return BuildDagFromReader(ds, chunk.NewSizeSplitter(f, chunk.DefaultBlockSize)) } -func BuildDagFromReader(ds dag.DAGService, spl chunk.Splitter) (*dag.ProtoNode, error) { +func BuildDagFromReader(ds dag.DAGService, spl chunk.Splitter) (node.Node, error) { dbp := h.DagBuilderParams{ Dagserv: ds, Maxlinks: h.DefaultLinksPerBlock, @@ -47,7 +49,7 @@ func BuildDagFromReader(ds dag.DAGService, spl chunk.Splitter) (*dag.ProtoNode, return bal.BalancedLayout(dbp.New(spl)) } -func BuildTrickleDagFromReader(ds dag.DAGService, spl chunk.Splitter) (*dag.ProtoNode, error) { +func BuildTrickleDagFromReader(ds dag.DAGService, spl chunk.Splitter) (node.Node, error) { dbp := h.DagBuilderParams{ Dagserv: ds, Maxlinks: h.DefaultLinksPerBlock, diff --git a/importer/importer_test.go b/importer/importer_test.go index 611a62d393c..cd04fb8dbf3 100644 --- a/importer/importer_test.go +++ b/importer/importer_test.go @@ -2,19 +2,21 @@ package importer import ( "bytes" + "context" "io" "io/ioutil" "testing" - context "context" chunk "github.com/ipfs/go-ipfs/importer/chunk" dag "github.com/ipfs/go-ipfs/merkledag" mdtest "github.com/ipfs/go-ipfs/merkledag/test" uio "github.com/ipfs/go-ipfs/unixfs/io" + + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node" u "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util" ) -func getBalancedDag(t testing.TB, size int64, blksize int64) (*dag.ProtoNode, dag.DAGService) { +func getBalancedDag(t testing.TB, size int64, blksize int64) (node.Node, dag.DAGService) { ds := mdtest.Mock() r := io.LimitReader(u.NewTimeSeededRand(), size) nd, err := BuildDagFromReader(ds, chunk.NewSizeSplitter(r, blksize)) @@ -24,7 +26,7 @@ func getBalancedDag(t testing.TB, size int64, blksize int64) (*dag.ProtoNode, da return nd, ds } -func getTrickleDag(t testing.TB, size int64, blksize int64) (*dag.ProtoNode, dag.DAGService) { +func getTrickleDag(t testing.TB, size int64, blksize int64) (node.Node, dag.DAGService) { ds := mdtest.Mock() r := io.LimitReader(u.NewTimeSeededRand(), size) nd, err := BuildTrickleDagFromReader(ds, chunk.NewSizeSplitter(r, blksize)) @@ -100,7 +102,7 @@ func BenchmarkTrickleReadFull(b *testing.B) { runReadBench(b, nd, ds) } -func runReadBench(b *testing.B, nd *dag.ProtoNode, ds dag.DAGService) { +func runReadBench(b *testing.B, nd node.Node, ds dag.DAGService) { for i := 0; i < b.N; i++ { ctx, cancel := context.WithCancel(context.Background()) read, err := uio.NewDagReader(ctx, nd, ds) diff --git a/importer/trickle/trickle_test.go b/importer/trickle/trickle_test.go index bad45a7d9c3..192a711968a 100644 --- a/importer/trickle/trickle_test.go +++ b/importer/trickle/trickle_test.go @@ -2,6 +2,7 @@ package trickle import ( "bytes" + "context" "fmt" "io" "io/ioutil" @@ -9,7 +10,6 @@ import ( "os" "testing" - "context" chunk "github.com/ipfs/go-ipfs/importer/chunk" h "github.com/ipfs/go-ipfs/importer/helpers" merkledag "github.com/ipfs/go-ipfs/merkledag" @@ -17,6 +17,7 @@ import ( pin "github.com/ipfs/go-ipfs/pin" ft "github.com/ipfs/go-ipfs/unixfs" uio "github.com/ipfs/go-ipfs/unixfs/io" + u "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util" ) @@ -31,7 +32,12 @@ func buildTestDag(ds merkledag.DAGService, spl chunk.Splitter) (*merkledag.Proto return nil, err } - return nd, VerifyTrickleDagStructure(nd, ds, dbp.Maxlinks, layerRepeat) + pbnd, ok := nd.(*merkledag.ProtoNode) + if !ok { + return nil, merkledag.ErrNotProtobuf + } + + return pbnd, VerifyTrickleDagStructure(pbnd, ds, dbp.Maxlinks, layerRepeat) } //Test where calls to read are smaller than the chunk size diff --git a/importer/trickle/trickledag.go b/importer/trickle/trickledag.go index 9341407e14e..3c6816f6bf8 100644 --- a/importer/trickle/trickledag.go +++ b/importer/trickle/trickledag.go @@ -8,6 +8,8 @@ import ( h "github.com/ipfs/go-ipfs/importer/helpers" dag "github.com/ipfs/go-ipfs/merkledag" ft "github.com/ipfs/go-ipfs/unixfs" + + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node" ) // layerRepeat specifies how many times to append a child tree of a @@ -15,7 +17,7 @@ import ( // improves seek speeds. const layerRepeat = 4 -func TrickleLayout(db *h.DagBuilderHelper) (*dag.ProtoNode, error) { +func TrickleLayout(db *h.DagBuilderHelper) (node.Node, error) { root := h.NewUnixfsNode() if err := db.FillNodeLayer(root); err != nil { return nil, err @@ -66,7 +68,12 @@ func fillTrickleRec(db *h.DagBuilderHelper, node *h.UnixfsNode, depth int) error } // TrickleAppend appends the data in `db` to the dag, using the Trickledag format -func TrickleAppend(ctx context.Context, base *dag.ProtoNode, db *h.DagBuilderHelper) (out *dag.ProtoNode, err_out error) { +func TrickleAppend(ctx context.Context, basen node.Node, db *h.DagBuilderHelper) (out node.Node, err_out error) { + base, ok := basen.(*dag.ProtoNode) + if !ok { + return nil, dag.ErrNotProtobuf + } + defer func() { if err_out == nil { if err := db.Close(); err != nil { @@ -229,8 +236,13 @@ func trickleDepthInfo(node *h.UnixfsNode, maxlinks int) (int, int) { // VerifyTrickleDagStructure checks that the given dag matches exactly the trickle dag datastructure // layout -func VerifyTrickleDagStructure(nd *dag.ProtoNode, ds dag.DAGService, direct int, layerRepeat int) error { - return verifyTDagRec(nd, -1, direct, layerRepeat, ds) +func VerifyTrickleDagStructure(nd node.Node, ds dag.DAGService, direct int, layerRepeat int) error { + pbnd, ok := nd.(*dag.ProtoNode) + if !ok { + return dag.ErrNotProtobuf + } + + return verifyTDagRec(pbnd, -1, direct, layerRepeat, ds) } // Recursive call for verifying the structure of a trickledag diff --git a/merkledag/merkledag.go b/merkledag/merkledag.go index 22392789297..b6a8d8558cd 100644 --- a/merkledag/merkledag.go +++ b/merkledag/merkledag.go @@ -85,23 +85,29 @@ func (n *dagService) Get(ctx context.Context, c *cid.Cid) (node.Node, error) { return nil, fmt.Errorf("Failed to get block for %s: %v", c, err) } - var res node.Node + return decodeBlock(b) +} + +func decodeBlock(b blocks.Block) (node.Node, error) { + c := b.Cid() + switch c.Type() { case cid.Protobuf: - out, err := DecodeProtobuf(b.RawData()) + decnd, err := DecodeProtobuf(b.RawData()) if err != nil { if strings.Contains(err.Error(), "Unmarshal failed") { return nil, fmt.Errorf("The block referred to by '%s' was not a valid merkledag node", c) } return nil, fmt.Errorf("Failed to decode Protocol Buffers: %v", err) } - out.cached = c - res = out + + decnd.cached = b.Cid() + return decnd, nil + case cid.Raw: + return NewRawNode(b.RawData()), nil default: - return nil, fmt.Errorf("unrecognized formatting type") + return nil, fmt.Errorf("unrecognized object type: %s", c.Type()) } - - return res, nil } func (n *dagService) GetLinks(ctx context.Context, c *cid.Cid) ([]*node.Link, error) { @@ -164,24 +170,12 @@ func (ds *dagService) GetMany(ctx context.Context, keys []*cid.Cid) <-chan *Node return } - c := b.Cid() - - var nd node.Node - switch c.Type() { - case cid.Protobuf: - decnd, err := DecodeProtobuf(b.RawData()) - if err != nil { - out <- &NodeOption{Err: err} - return - } - decnd.cached = b.Cid() - nd = decnd - default: - out <- &NodeOption{Err: fmt.Errorf("unrecognized object type: %s", c.Type())} + nd, err := decodeBlock(b) + if err != nil { + out <- &NodeOption{Err: err} return } - // buffered, no need to select out <- &NodeOption{Node: nd} count++ diff --git a/merkledag/merkledag_test.go b/merkledag/merkledag_test.go index 310134fa099..a0e91e8a0b0 100644 --- a/merkledag/merkledag_test.go +++ b/merkledag/merkledag_test.go @@ -373,3 +373,81 @@ func TestBasicAddGet(t *testing.T) { t.Fatal("output didnt match input") } } + +func TestGetRawNodes(t *testing.T) { + rn := NewRawNode([]byte("test")) + + ds := dstest.Mock() + + c, err := ds.Add(rn) + if err != nil { + t.Fatal(err) + } + + if !c.Equals(rn.Cid()) { + t.Fatal("output cids didnt match") + } + + out, err := ds.Get(context.TODO(), c) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(out.RawData(), []byte("test")) { + t.Fatal("raw block should match input data") + } + + if out.Links() != nil { + t.Fatal("raw blocks shouldnt have links") + } + + if out.Tree() != nil { + t.Fatal("tree should return no paths in a raw block") + } + + size, err := out.Size() + if err != nil { + t.Fatal(err) + } + if size != 4 { + t.Fatal("expected size to be 4") + } + + ns, err := out.Stat() + if err != nil { + t.Fatal(err) + } + + if ns.DataSize != 4 { + t.Fatal("expected size to be 4, got: ", ns.DataSize) + } + + _, _, err = out.Resolve([]string{"foo"}) + if err != ErrLinkNotFound { + t.Fatal("shouldnt find links under raw blocks") + } +} + +func TestProtoNodeResolve(t *testing.T) { + + nd := new(ProtoNode) + nd.SetLinks([]*node.Link{{Name: "foo"}}) + + lnk, left, err := nd.Resolve([]string{"foo", "bar"}) + if err != nil { + t.Fatal(err) + } + + if len(left) != 1 || left[0] != "bar" { + t.Fatal("expected the single path element 'bar' to remain") + } + + if lnk.Name != "foo" { + t.Fatal("how did we get anything else?") + } + + tvals := nd.Tree() + if len(tvals) != 1 || tvals[0] != "foo" { + t.Fatal("expected tree to return []{\"foo\"}") + } +} diff --git a/merkledag/node.go b/merkledag/node.go index 4c01c9c9c1b..b0fca652b2c 100644 --- a/merkledag/node.go +++ b/merkledag/node.go @@ -36,7 +36,7 @@ func NodeWithData(d []byte) *ProtoNode { } // AddNodeLink adds a link to another node. -func (n *ProtoNode) AddNodeLink(name string, that *ProtoNode) error { +func (n *ProtoNode) AddNodeLink(name string, that node.Node) error { n.encoded = nil lnk, err := node.MakeLink(that) diff --git a/merkledag/raw.go b/merkledag/raw.go new file mode 100644 index 00000000000..deb2e1a1fe7 --- /dev/null +++ b/merkledag/raw.go @@ -0,0 +1,46 @@ +package merkledag + +import ( + "github.com/ipfs/go-ipfs/blocks" + + cid "gx/ipfs/QmXUuRadqDq5BuFWzVU6VuKaSjTcNm1gNCtLvvP1TJCW4z/go-cid" + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node" + u "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util" +) + +type RawNode struct { + blocks.Block +} + +func NewRawNode(data []byte) *RawNode { + h := u.Hash(data) + c := cid.NewCidV1(cid.Raw, h) + blk, _ := blocks.NewBlockWithCid(data, c) + + return &RawNode{blk} +} + +func (rn *RawNode) Links() []*node.Link { + return nil +} + +func (rn *RawNode) Resolve(path []string) (*node.Link, []string, error) { + return nil, nil, ErrLinkNotFound +} + +func (rn *RawNode) Tree() []string { + return nil +} + +func (rn *RawNode) Size() (uint64, error) { + return uint64(len(rn.RawData())), nil +} + +func (rn *RawNode) Stat() (*node.NodeStat, error) { + return &node.NodeStat{ + CumulativeSize: len(rn.RawData()), + DataSize: len(rn.RawData()), + }, nil +} + +var _ node.Node = (*RawNode)(nil) diff --git a/merkledag/utils/utils.go b/merkledag/utils/utils.go index a44d94621bc..7ef67b93943 100644 --- a/merkledag/utils/utils.go +++ b/merkledag/utils/utils.go @@ -1,17 +1,18 @@ package dagutils import ( + "context" "errors" - context "context" - ds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore" - syncds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore/sync" - bstore "github.com/ipfs/go-ipfs/blocks/blockstore" bserv "github.com/ipfs/go-ipfs/blockservice" offline "github.com/ipfs/go-ipfs/exchange/offline" dag "github.com/ipfs/go-ipfs/merkledag" path "github.com/ipfs/go-ipfs/path" + + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node" + ds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore" + syncds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore/sync" ) type Editor struct { @@ -50,7 +51,7 @@ func (e *Editor) GetDagService() dag.DAGService { return e.tmp } -func addLink(ctx context.Context, ds dag.DAGService, root *dag.ProtoNode, childname string, childnd *dag.ProtoNode) (*dag.ProtoNode, error) { +func addLink(ctx context.Context, ds dag.DAGService, root *dag.ProtoNode, childname string, childnd node.Node) (*dag.ProtoNode, error) { if childname == "" { return nil, errors.New("cannot create link with no name!") } @@ -76,7 +77,7 @@ func addLink(ctx context.Context, ds dag.DAGService, root *dag.ProtoNode, childn return root, nil } -func (e *Editor) InsertNodeAtPath(ctx context.Context, pth string, toinsert *dag.ProtoNode, create func() *dag.ProtoNode) error { +func (e *Editor) InsertNodeAtPath(ctx context.Context, pth string, toinsert node.Node, create func() *dag.ProtoNode) error { splpath := path.SplitList(pth) nd, err := e.insertNodeAtPath(ctx, e.root, splpath, toinsert, create) if err != nil { @@ -86,7 +87,7 @@ func (e *Editor) InsertNodeAtPath(ctx context.Context, pth string, toinsert *dag return nil } -func (e *Editor) insertNodeAtPath(ctx context.Context, root *dag.ProtoNode, path []string, toinsert *dag.ProtoNode, create func() *dag.ProtoNode) (*dag.ProtoNode, error) { +func (e *Editor) insertNodeAtPath(ctx context.Context, root *dag.ProtoNode, path []string, toinsert node.Node, create func() *dag.ProtoNode) (*dag.ProtoNode, error) { if len(path) == 1 { return addLink(ctx, e.tmp, root, path[0], toinsert) } diff --git a/mfs/dir.go b/mfs/dir.go index 3a1c7be8ef3..e8004c80fdc 100644 --- a/mfs/dir.go +++ b/mfs/dir.go @@ -1,6 +1,7 @@ package mfs import ( + "context" "errors" "fmt" "os" @@ -9,11 +10,11 @@ import ( "sync" "time" - context "context" - dag "github.com/ipfs/go-ipfs/merkledag" ft "github.com/ipfs/go-ipfs/unixfs" ufspb "github.com/ipfs/go-ipfs/unixfs/pb" + + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node" ) var ErrNotYetImplemented = errors.New("not yet implemented") @@ -323,7 +324,7 @@ func (d *Directory) Flush() error { } // AddChild adds the node 'nd' under this directory giving it the name 'name' -func (d *Directory) AddChild(name string, nd *dag.ProtoNode) error { +func (d *Directory) AddChild(name string, nd node.Node) error { d.lock.Lock() defer d.lock.Unlock() diff --git a/mfs/mfs_test.go b/mfs/mfs_test.go index dcec37356c1..4ac1b4a744d 100644 --- a/mfs/mfs_test.go +++ b/mfs/mfs_test.go @@ -42,12 +42,12 @@ func getDagserv(t *testing.T) dag.DAGService { return dag.NewDAGService(blockserv) } -func getRandFile(t *testing.T, ds dag.DAGService, size int64) *dag.ProtoNode { +func getRandFile(t *testing.T, ds dag.DAGService, size int64) node.Node { r := io.LimitReader(u.NewTimeSeededRand(), size) return fileNodeFromReader(t, ds, r) } -func fileNodeFromReader(t *testing.T, ds dag.DAGService, r io.Reader) *dag.ProtoNode { +func fileNodeFromReader(t *testing.T, ds dag.DAGService, r io.Reader) node.Node { nd, err := importer.BuildDagFromReader(ds, chunk.DefaultSplitter(r)) if err != nil { t.Fatal(err) @@ -125,7 +125,12 @@ func compStrArrs(a, b []string) bool { return true } -func assertFileAtPath(ds dag.DAGService, root *Directory, exp *dag.ProtoNode, pth string) error { +func assertFileAtPath(ds dag.DAGService, root *Directory, expn node.Node, pth string) error { + exp, ok := expn.(*dag.ProtoNode) + if !ok { + return dag.ErrNotProtobuf + } + parts := path.SplitList(pth) cur := root for i, d := range parts[:len(parts)-1] { diff --git a/mfs/ops.go b/mfs/ops.go index 6464d840411..1c1fef82b77 100644 --- a/mfs/ops.go +++ b/mfs/ops.go @@ -7,8 +7,9 @@ import ( gopath "path" "strings" - dag "github.com/ipfs/go-ipfs/merkledag" path "github.com/ipfs/go-ipfs/path" + + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node" ) // Mv moves the file or directory at 'src' to 'dst' @@ -87,7 +88,7 @@ func lookupDir(r *Root, path string) (*Directory, error) { } // PutNode inserts 'nd' at 'path' in the given mfs -func PutNode(r *Root, path string, nd *dag.ProtoNode) error { +func PutNode(r *Root, path string, nd node.Node) error { dirp, filename := gopath.Split(path) if filename == "" { return fmt.Errorf("cannot create file with empty name") diff --git a/unixfs/io/dagreader.go b/unixfs/io/dagreader.go index 086e5038ce7..4eb3e04c61d 100644 --- a/unixfs/io/dagreader.go +++ b/unixfs/io/dagreader.go @@ -2,17 +2,18 @@ package io import ( "bytes" + "context" "errors" "fmt" "io" "os" - "context" - proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto" - mdag "github.com/ipfs/go-ipfs/merkledag" ft "github.com/ipfs/go-ipfs/unixfs" ftpb "github.com/ipfs/go-ipfs/unixfs/pb" + + proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto" + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node" ) var ErrIsDir = errors.New("this dag node is a directory") @@ -58,36 +59,45 @@ type ReadSeekCloser interface { // NewDagReader creates a new reader object that reads the data represented by // the given node, using the passed in DAGService for data retreival -func NewDagReader(ctx context.Context, n *mdag.ProtoNode, serv mdag.DAGService) (*DagReader, error) { - pb := new(ftpb.Data) - if err := proto.Unmarshal(n.Data(), pb); err != nil { - return nil, err - } - - switch pb.GetType() { - case ftpb.Data_Directory: - // Dont allow reading directories - return nil, ErrIsDir - case ftpb.Data_File, ftpb.Data_Raw: - return NewDataFileReader(ctx, n, pb, serv), nil - case ftpb.Data_Metadata: - if len(n.Links()) == 0 { - return nil, errors.New("incorrectly formatted metadata object") - } - child, err := n.Links()[0].GetNode(ctx, serv) - if err != nil { +func NewDagReader(ctx context.Context, n node.Node, serv mdag.DAGService) (*DagReader, error) { + switch n := n.(type) { + case *mdag.RawNode: + return &DagReader{ + buf: NewRSNCFromBytes(n.RawData()), + }, nil + case *mdag.ProtoNode: + pb := new(ftpb.Data) + if err := proto.Unmarshal(n.Data(), pb); err != nil { return nil, err } - childpb, ok := child.(*mdag.ProtoNode) - if !ok { - return nil, mdag.ErrNotProtobuf + switch pb.GetType() { + case ftpb.Data_Directory: + // Dont allow reading directories + return nil, ErrIsDir + case ftpb.Data_File, ftpb.Data_Raw: + return NewDataFileReader(ctx, n, pb, serv), nil + case ftpb.Data_Metadata: + if len(n.Links()) == 0 { + return nil, errors.New("incorrectly formatted metadata object") + } + child, err := n.Links()[0].GetNode(ctx, serv) + if err != nil { + return nil, err + } + + childpb, ok := child.(*mdag.ProtoNode) + if !ok { + return nil, mdag.ErrNotProtobuf + } + return NewDagReader(ctx, childpb, serv) + case ftpb.Data_Symlink: + return nil, ErrCantReadSymlinks + default: + return nil, ft.ErrUnrecognizedType } - return NewDagReader(ctx, childpb, serv) - case ftpb.Data_Symlink: - return nil, ErrCantReadSymlinks default: - return nil, ft.ErrUnrecognizedType + return nil, fmt.Errorf("unrecognized node type") } } diff --git a/unixfs/mod/dagmodifier.go b/unixfs/mod/dagmodifier.go index 3479ab4a488..fe59436ee47 100644 --- a/unixfs/mod/dagmodifier.go +++ b/unixfs/mod/dagmodifier.go @@ -2,6 +2,7 @@ package mod import ( "bytes" + "context" "errors" "io" "os" @@ -13,10 +14,10 @@ import ( ft "github.com/ipfs/go-ipfs/unixfs" uio "github.com/ipfs/go-ipfs/unixfs/io" - context "context" logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" cid "gx/ipfs/QmXUuRadqDq5BuFWzVU6VuKaSjTcNm1gNCtLvvP1TJCW4z/go-cid" proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto" + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node" ) var ErrSeekFail = errors.New("failed to seek properly") @@ -45,9 +46,14 @@ type DagModifier struct { read *uio.DagReader } -func NewDagModifier(ctx context.Context, from *mdag.ProtoNode, serv mdag.DAGService, spl chunk.SplitterGen) (*DagModifier, error) { +func NewDagModifier(ctx context.Context, from node.Node, serv mdag.DAGService, spl chunk.SplitterGen) (*DagModifier, error) { + pbn, ok := from.(*mdag.ProtoNode) + if !ok { + return nil, mdag.ErrNotProtobuf + } + return &DagModifier{ - curNode: from.Copy(), + curNode: pbn.Copy(), dagserv: serv, splitter: spl, ctx: ctx, @@ -109,7 +115,13 @@ func (dm *DagModifier) expandSparse(size int64) error { if err != nil { return err } - dm.curNode = nnode + + pbnnode, ok := nnode.(*mdag.ProtoNode) + if !ok { + return mdag.ErrNotProtobuf + } + + dm.curNode = pbnnode return nil } @@ -197,7 +209,12 @@ func (dm *DagModifier) Sync() error { return err } - dm.curNode = nd + pbnode, ok := nd.(*mdag.ProtoNode) + if !ok { + return mdag.ErrNotProtobuf + } + + dm.curNode = pbnode } dm.writeStart += uint64(buflen) @@ -288,7 +305,7 @@ func (dm *DagModifier) modifyDag(node *mdag.ProtoNode, offset uint64, data io.Re } // appendData appends the blocks from the given chan to the end of this dag -func (dm *DagModifier) appendData(node *mdag.ProtoNode, spl chunk.Splitter) (*mdag.ProtoNode, error) { +func (dm *DagModifier) appendData(node *mdag.ProtoNode, spl chunk.Splitter) (node.Node, error) { dbp := &help.DagBuilderParams{ Dagserv: dm.dagserv, Maxlinks: help.DefaultLinksPerBlock, diff --git a/unixfs/test/utils.go b/unixfs/test/utils.go index 26755cec57d..abe292300ff 100644 --- a/unixfs/test/utils.go +++ b/unixfs/test/utils.go @@ -2,6 +2,7 @@ package testu import ( "bytes" + "context" "fmt" "io" "io/ioutil" @@ -13,7 +14,7 @@ import ( mdagmock "github.com/ipfs/go-ipfs/merkledag/test" ft "github.com/ipfs/go-ipfs/unixfs" - context "context" + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node" u "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util" ) @@ -27,7 +28,7 @@ func GetDAGServ() mdag.DAGService { return mdagmock.Mock() } -func GetNode(t testing.TB, dserv mdag.DAGService, data []byte) *mdag.ProtoNode { +func GetNode(t testing.TB, dserv mdag.DAGService, data []byte) node.Node { in := bytes.NewReader(data) node, err := imp.BuildTrickleDagFromReader(dserv, SizeSplitterGen(500)(in)) if err != nil { @@ -37,11 +38,11 @@ func GetNode(t testing.TB, dserv mdag.DAGService, data []byte) *mdag.ProtoNode { return node } -func GetEmptyNode(t testing.TB, dserv mdag.DAGService) *mdag.ProtoNode { +func GetEmptyNode(t testing.TB, dserv mdag.DAGService) node.Node { return GetNode(t, dserv, []byte{}) } -func GetRandomNode(t testing.TB, dserv mdag.DAGService, size int64) ([]byte, *mdag.ProtoNode) { +func GetRandomNode(t testing.TB, dserv mdag.DAGService, size int64) ([]byte, node.Node) { in := io.LimitReader(u.NewTimeSeededRand(), size) buf, err := ioutil.ReadAll(in) if err != nil { From 3796e7020e3570c7574f9cb74a7dbb3e03511cbf Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 18 Oct 2016 11:07:32 -0700 Subject: [PATCH 2/5] raw dag: make raw nodes work in cat and get, add tests License: MIT Signed-off-by: Jeromy --- importer/helpers/helpers.go | 9 +++++- test/sharness/t0040-add-and-cat.sh | 16 ++++++---- unixfs/io/dagreader.go | 51 ++++++++++++++++-------------- 3 files changed, 45 insertions(+), 31 deletions(-) diff --git a/importer/helpers/helpers.go b/importer/helpers/helpers.go index caface2f13a..58007daa760 100644 --- a/importer/helpers/helpers.go +++ b/importer/helpers/helpers.go @@ -105,7 +105,7 @@ func (n *UnixfsNode) GetChild(ctx context.Context, i int, ds dag.DAGService) (*U // the passed in DagBuilderHelper is used to store the child node an // pin it locally so it doesnt get lost func (n *UnixfsNode) AddChild(child *UnixfsNode, db *DagBuilderHelper) error { - n.ufmt.AddBlockSize(child.ufmt.FileSize()) + n.ufmt.AddBlockSize(child.DataSize()) childnode, err := child.GetDagNode() if err != nil { @@ -137,6 +137,13 @@ func (n *UnixfsNode) SetData(data []byte) { n.ufmt.Data = data } +func (n *UnixfsNode) DataSize() uint64 { + if n.raw { + return uint64(len(n.rawnode.RawData())) + } + return n.ufmt.FileSize() +} + // getDagNode fills out the proper formatting for the unixfs node // inside of a DAG node and returns the dag node func (n *UnixfsNode) GetDagNode() (node.Node, error) { diff --git a/test/sharness/t0040-add-and-cat.sh b/test/sharness/t0040-add-and-cat.sh index adc930ec301..6ace9a55bbc 100755 --- a/test/sharness/t0040-add-and-cat.sh +++ b/test/sharness/t0040-add-and-cat.sh @@ -87,6 +87,9 @@ test_add_cat_file() { } test_add_cat_5MB() { + ADD_FLAGS="$1" + EXP_HASH="$2" + test_expect_success "generate 5MB file using go-random" ' random 5242880 41 >mountdir/bigfile ' @@ -98,17 +101,16 @@ test_add_cat_5MB() { ' test_expect_success "'ipfs add bigfile' succeeds" ' - ipfs add mountdir/bigfile >actual || + ipfs add $ADD_FLAGS mountdir/bigfile >actual || test_fsh cat daemon_err ' test_expect_success "'ipfs add bigfile' output looks good" ' - HASH="QmSr7FqYkxYWGoSfy8ZiaMWQ5vosb18DQGCzjwEQnVHkTb" && - echo "added $HASH bigfile" >expected && + echo "added $EXP_HASH bigfile" >expected && test_cmp expected actual ' test_expect_success "'ipfs cat' succeeds" ' - ipfs cat "$HASH" >actual + ipfs cat "$EXP_HASH" >actual ' test_expect_success "'ipfs cat' output looks good" ' @@ -116,7 +118,7 @@ test_add_cat_5MB() { ' test_expect_success FUSE "cat ipfs/bigfile succeeds" ' - cat "ipfs/$HASH" >actual + cat "ipfs/$EXP_HASH" >actual ' test_expect_success FUSE "cat ipfs/bigfile looks good" ' @@ -380,7 +382,9 @@ test_expect_success "go-random is installed" ' type random ' -test_add_cat_5MB +test_add_cat_5MB "" "QmSr7FqYkxYWGoSfy8ZiaMWQ5vosb18DQGCzjwEQnVHkTb" + +test_add_cat_5MB --raw-leaves "QmefsDaD3YVphd86mxjJfPLceKv8by98aB6J6sJxK13xS2" test_add_cat_expensive diff --git a/unixfs/io/dagreader.go b/unixfs/io/dagreader.go index 4eb3e04c61d..44945dd3115 100644 --- a/unixfs/io/dagreader.go +++ b/unixfs/io/dagreader.go @@ -129,33 +129,36 @@ func (dr *DagReader) precalcNextBuf(ctx context.Context) error { } dr.linkPosition++ - nxtpb, ok := nxt.(*mdag.ProtoNode) - if !ok { - return mdag.ErrNotProtobuf - } - - pb := new(ftpb.Data) - err = proto.Unmarshal(nxtpb.Data(), pb) - if err != nil { - return fmt.Errorf("incorrectly formatted protobuf: %s", err) - } + switch nxt := nxt.(type) { + case *mdag.ProtoNode: + pb := new(ftpb.Data) + err = proto.Unmarshal(nxt.Data(), pb) + if err != nil { + return fmt.Errorf("incorrectly formatted protobuf: %s", err) + } - switch pb.GetType() { - case ftpb.Data_Directory: - // A directory should not exist within a file - return ft.ErrInvalidDirLocation - case ftpb.Data_File: - dr.buf = NewDataFileReader(dr.ctx, nxtpb, pb, dr.serv) - return nil - case ftpb.Data_Raw: - dr.buf = NewRSNCFromBytes(pb.GetData()) + switch pb.GetType() { + case ftpb.Data_Directory: + // A directory should not exist within a file + return ft.ErrInvalidDirLocation + case ftpb.Data_File: + dr.buf = NewDataFileReader(dr.ctx, nxt, pb, dr.serv) + return nil + case ftpb.Data_Raw: + dr.buf = NewRSNCFromBytes(pb.GetData()) + return nil + case ftpb.Data_Metadata: + return errors.New("shouldnt have had metadata object inside file") + case ftpb.Data_Symlink: + return errors.New("shouldnt have had symlink inside file") + default: + return ft.ErrUnrecognizedType + } + case *mdag.RawNode: + dr.buf = NewRSNCFromBytes(nxt.RawData()) return nil - case ftpb.Data_Metadata: - return errors.New("shouldnt have had metadata object inside file") - case ftpb.Data_Symlink: - return errors.New("shouldnt have had symlink inside file") default: - return ft.ErrUnrecognizedType + return errors.New("unrecognized node type in DagReader") } } From 8ce9963289bcba3f596353f28a942814b5bfff65 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 18 Oct 2016 14:41:47 -0700 Subject: [PATCH 3/5] fix add/cat of small files License: MIT Signed-off-by: Jeromy --- core/commands/files/files.go | 12 +++++-- core/corehttp/gateway_test.go | 16 +++++---- core/coreunix/add.go | 15 +++------ core/coreunix/cat.go | 8 +---- mfs/dir.go | 50 +++++++++++++++++----------- mfs/file.go | 53 +++++++++++++++++++----------- mfs/mfs_test.go | 13 +++++--- mfs/system.go | 5 +-- test/sharness/t0040-add-and-cat.sh | 21 +++++++++++- 9 files changed, 121 insertions(+), 72 deletions(-) diff --git a/core/commands/files/files.go b/core/commands/files/files.go index eff6b96e649..2445dfd388a 100644 --- a/core/commands/files/files.go +++ b/core/commands/files/files.go @@ -2,6 +2,7 @@ package commands import ( "bytes" + "context" "errors" "fmt" "io" @@ -16,8 +17,8 @@ import ( path "github.com/ipfs/go-ipfs/path" ft "github.com/ipfs/go-ipfs/unixfs" - context "context" logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node" ) var log = logging.Logger("cmds/files") @@ -160,7 +161,12 @@ func statNode(ds dag.DAGService, fsn mfs.FSNode) (*Object, error) { c := nd.Cid() - d, err := ft.FromBytes(nd.Data()) + pbnd, ok := nd.(*dag.ProtoNode) + if !ok { + return nil, dag.ErrNotProtobuf + } + + d, err := ft.FromBytes(pbnd.Data()) if err != nil { return nil, err } @@ -245,7 +251,7 @@ var FilesCpCmd = &cmds.Command{ }, } -func getNodeFromPath(ctx context.Context, node *core.IpfsNode, p string) (*dag.ProtoNode, error) { +func getNodeFromPath(ctx context.Context, node *core.IpfsNode, p string) (node.Node, error) { switch { case strings.HasPrefix(p, "/ipfs/"): np, err := path.ParsePath(p) diff --git a/core/corehttp/gateway_test.go b/core/corehttp/gateway_test.go index ba8a936ac11..42959ffaffd 100644 --- a/core/corehttp/gateway_test.go +++ b/core/corehttp/gateway_test.go @@ -1,6 +1,7 @@ package corehttp import ( + "context" "errors" "io/ioutil" "net/http" @@ -9,14 +10,15 @@ import ( "testing" "time" - context "context" core "github.com/ipfs/go-ipfs/core" coreunix "github.com/ipfs/go-ipfs/core/coreunix" + dag "github.com/ipfs/go-ipfs/merkledag" namesys "github.com/ipfs/go-ipfs/namesys" path "github.com/ipfs/go-ipfs/path" repo "github.com/ipfs/go-ipfs/repo" config "github.com/ipfs/go-ipfs/repo/config" testutil "github.com/ipfs/go-ipfs/thirdparty/testutil" + id "gx/ipfs/QmcRa2qn6iCmap9bjp8jAwkvYAq13AUfxdY3rrYiaJbLum/go-libp2p/p2p/protocol/identify" ci "gx/ipfs/QmfWDLQjGjVe4fr5CoztYW2DYYjRysMJrFe1RCsXLPTf46/go-libp2p-crypto" ) @@ -178,11 +180,13 @@ func TestIPNSHostnameRedirect(t *testing.T) { if err != nil { t.Fatal(err) } + _, dagn2, err := coreunix.AddWrapped(n, strings.NewReader("_"), "index.html") if err != nil { t.Fatal(err) } - dagn1.AddNodeLink("foo", dagn2) + + dagn1.(*dag.ProtoNode).AddNodeLink("foo", dagn2) if err != nil { t.Fatal(err) } @@ -197,7 +201,7 @@ func TestIPNSHostnameRedirect(t *testing.T) { t.Fatal(err) } - k := dagn1.Key() + k := dagn1.Cid() t.Logf("k: %s\n", k) ns["/ipns/example.net"] = path.FromString("/ipfs/" + k.String()) @@ -268,8 +272,8 @@ func TestIPNSHostnameBacklinks(t *testing.T) { if err != nil { t.Fatal(err) } - dagn2.AddNodeLink("bar", dagn3) - dagn1.AddNodeLink("foo? #<'", dagn2) + dagn2.(*dag.ProtoNode).AddNodeLink("bar", dagn3) + dagn1.(*dag.ProtoNode).AddNodeLink("foo? #<'", dagn2) if err != nil { t.Fatal(err) } @@ -287,7 +291,7 @@ func TestIPNSHostnameBacklinks(t *testing.T) { t.Fatal(err) } - k := dagn1.Key() + k := dagn1.Cid() t.Logf("k: %s\n", k) ns["/ipns/example.net"] = path.FromString("/ipfs/" + k.String()) diff --git a/core/coreunix/add.go b/core/coreunix/add.go index eaa5ff7e1f8..34702f088a4 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -104,7 +104,7 @@ type Adder struct { Silent bool Wrap bool Chunker string - root *dag.ProtoNode + root node.Node mr *mfs.Root unlocker bs.Unlocker tempRoot *cid.Cid @@ -133,7 +133,7 @@ func (adder Adder) add(reader io.Reader) (node.Node, error) { return balanced.BalancedLayout(params.New(chnk)) } -func (adder *Adder) RootNode() (*dag.ProtoNode, error) { +func (adder *Adder) RootNode() (node.Node, error) { // for memoizing if adder.root != nil { return adder.root, nil @@ -151,12 +151,7 @@ func (adder *Adder) RootNode() (*dag.ProtoNode, error) { return nil, err } - pbnd, ok := nd.(*dag.ProtoNode) - if !ok { - return nil, dag.ErrNotProtobuf - } - - root = pbnd + root = nd } adder.root = root @@ -189,7 +184,7 @@ func (adder *Adder) PinRoot() error { return adder.pinning.Flush() } -func (adder *Adder) Finalize() (*dag.ProtoNode, error) { +func (adder *Adder) Finalize() (node.Node, error) { root := adder.mr.GetValue() // cant just call adder.RootNode() here as we need the name for printing @@ -311,7 +306,7 @@ func AddR(n *core.IpfsNode, root string) (key string, err error) { // to preserve the filename. // Returns the path of the added file ("/filename"), the DAG node of // the directory, and and error if any. -func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, *dag.ProtoNode, error) { +func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, node.Node, error) { file := files.NewReaderFile(filename, filename, ioutil.NopCloser(r), nil) fileAdder, err := NewAdder(n.Context(), n.Pinning, n.Blockstore, n.DAG) if err != nil { diff --git a/core/coreunix/cat.go b/core/coreunix/cat.go index 1d27ca13a43..38fbc157a3b 100644 --- a/core/coreunix/cat.go +++ b/core/coreunix/cat.go @@ -4,7 +4,6 @@ import ( "context" core "github.com/ipfs/go-ipfs/core" - dag "github.com/ipfs/go-ipfs/merkledag" path "github.com/ipfs/go-ipfs/path" uio "github.com/ipfs/go-ipfs/unixfs/io" ) @@ -15,10 +14,5 @@ func Cat(ctx context.Context, n *core.IpfsNode, pstr string) (*uio.DagReader, er return nil, err } - dnpb, ok := dagNode.(*dag.ProtoNode) - if !ok { - return nil, dag.ErrNotProtobuf - } - - return uio.NewDagReader(ctx, dnpb, n.DAG) + return uio.NewDagReader(ctx, dagNode, n.DAG) } diff --git a/mfs/dir.go b/mfs/dir.go index e8004c80fdc..38bee4cccda 100644 --- a/mfs/dir.go +++ b/mfs/dir.go @@ -89,7 +89,7 @@ func (d *Directory) flushCurrentNode() (*dag.ProtoNode, error) { return d.node.Copy(), nil } -func (d *Directory) updateChild(name string, nd *dag.ProtoNode) error { +func (d *Directory) updateChild(name string, nd node.Node) error { err := d.node.RemoveNodeLink(name) if err != nil && err != dag.ErrNotFound { return err @@ -121,28 +121,40 @@ func (d *Directory) childNode(name string) (FSNode, error) { } // cacheNode caches a node into d.childDirs or d.files and returns the FSNode. -func (d *Directory) cacheNode(name string, nd *dag.ProtoNode) (FSNode, error) { - i, err := ft.FromBytes(nd.Data()) - if err != nil { - return nil, err - } +func (d *Directory) cacheNode(name string, nd node.Node) (FSNode, error) { + switch nd := nd.(type) { + case *dag.ProtoNode: + i, err := ft.FromBytes(nd.Data()) + if err != nil { + return nil, err + } - switch i.GetType() { - case ufspb.Data_Directory: - ndir := NewDirectory(d.ctx, name, nd, d, d.dserv) - d.childDirs[name] = ndir - return ndir, nil - case ufspb.Data_File, ufspb.Data_Raw, ufspb.Data_Symlink: + switch i.GetType() { + case ufspb.Data_Directory: + ndir := NewDirectory(d.ctx, name, nd, d, d.dserv) + d.childDirs[name] = ndir + return ndir, nil + case ufspb.Data_File, ufspb.Data_Raw, ufspb.Data_Symlink: + nfi, err := NewFile(name, nd, d, d.dserv) + if err != nil { + return nil, err + } + d.files[name] = nfi + return nfi, nil + case ufspb.Data_Metadata: + return nil, ErrNotYetImplemented + default: + return nil, ErrInvalidChild + } + case *dag.RawNode: nfi, err := NewFile(name, nd, d, d.dserv) if err != nil { return nil, err } d.files[name] = nfi return nfi, nil - case ufspb.Data_Metadata: - return nil, ErrNotYetImplemented default: - return nil, ErrInvalidChild + return nil, fmt.Errorf("unrecognized node type in cache node") } } @@ -162,8 +174,8 @@ func (d *Directory) Uncache(name string) { // childFromDag searches through this directories dag node for a child link // with the given name -func (d *Directory) childFromDag(name string) (*dag.ProtoNode, error) { - pbn, err := d.node.GetLinkedProtoNode(d.ctx, d.dserv, name) +func (d *Directory) childFromDag(name string) (node.Node, error) { + pbn, err := d.node.GetLinkedNode(d.ctx, d.dserv, name) switch err { case nil: return pbn, nil @@ -249,7 +261,7 @@ func (d *Directory) List() ([]NodeListing, error) { return nil, err } - child.Hash = nd.Key().B58String() + child.Hash = nd.Cid().String() out = append(out, child) } @@ -385,7 +397,7 @@ func (d *Directory) Path() string { return out } -func (d *Directory) GetNode() (*dag.ProtoNode, error) { +func (d *Directory) GetNode() (node.Node, error) { d.lock.Lock() defer d.lock.Unlock() diff --git a/mfs/file.go b/mfs/file.go index 373a9dd1dca..931827ebb62 100644 --- a/mfs/file.go +++ b/mfs/file.go @@ -9,6 +9,8 @@ import ( dag "github.com/ipfs/go-ipfs/merkledag" ft "github.com/ipfs/go-ipfs/unixfs" mod "github.com/ipfs/go-ipfs/unixfs/mod" + + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node" ) type File struct { @@ -19,12 +21,12 @@ type File struct { desclock sync.RWMutex dserv dag.DAGService - node *dag.ProtoNode + node node.Node nodelk sync.Mutex } // NewFile returns a NewFile object with the given parameters -func NewFile(name string, node *dag.ProtoNode, parent childCloser, dserv dag.DAGService) (*File, error) { +func NewFile(name string, node node.Node, parent childCloser, dserv dag.DAGService) (*File, error) { return &File{ dserv: dserv, parent: parent, @@ -44,18 +46,23 @@ func (fi *File) Open(flags int, sync bool) (FileDescriptor, error) { node := fi.node fi.nodelk.Unlock() - fsn, err := ft.FSNodeFromBytes(node.Data()) - if err != nil { - return nil, err - } - - switch fsn.Type { - default: - return nil, fmt.Errorf("unsupported fsnode type for 'file'") - case ft.TSymlink: - return nil, fmt.Errorf("symlinks not yet supported") - case ft.TFile, ft.TRaw: - // OK case + switch node := node.(type) { + case *dag.ProtoNode: + fsn, err := ft.FSNodeFromBytes(node.Data()) + if err != nil { + return nil, err + } + + switch fsn.Type { + default: + return nil, fmt.Errorf("unsupported fsnode type for 'file'") + case ft.TSymlink: + return nil, fmt.Errorf("symlinks not yet supported") + case ft.TFile, ft.TRaw: + // OK case + } + case *dag.RawNode: + // Ok as well. } switch flags { @@ -85,16 +92,22 @@ func (fi *File) Open(flags int, sync bool) (FileDescriptor, error) { func (fi *File) Size() (int64, error) { fi.nodelk.Lock() defer fi.nodelk.Unlock() - pbd, err := ft.FromBytes(fi.node.Data()) - if err != nil { - return 0, err + switch nd := fi.node.(type) { + case *dag.ProtoNode: + pbd, err := ft.FromBytes(nd.Data()) + if err != nil { + return 0, err + } + return int64(pbd.GetFilesize()), nil + case *dag.RawNode: + return int64(len(nd.RawData())), nil + default: + return 0, fmt.Errorf("unrecognized node type in mfs/file.Size()") } - - return int64(pbd.GetFilesize()), nil } // GetNode returns the dag node associated with this file -func (fi *File) GetNode() (*dag.ProtoNode, error) { +func (fi *File) GetNode() (node.Node, error) { fi.nodelk.Lock() defer fi.nodelk.Unlock() return fi.node, nil diff --git a/mfs/mfs_test.go b/mfs/mfs_test.go index 4ac1b4a744d..b7e725fbc22 100644 --- a/mfs/mfs_test.go +++ b/mfs/mfs_test.go @@ -794,7 +794,12 @@ func TestFlushing(t *testing.T) { t.Fatal(err) } - fsnode, err := ft.FSNodeFromBytes(rnd.Data()) + pbrnd, ok := rnd.(*dag.ProtoNode) + if !ok { + t.Fatal(dag.ErrNotProtobuf) + } + + fsnode, err := ft.FSNodeFromBytes(pbrnd.Data()) if err != nil { t.Fatal(err) } @@ -803,10 +808,10 @@ func TestFlushing(t *testing.T) { t.Fatal("root wasnt a directory") } - rnk := rnd.Key() + rnk := rnd.Cid() exp := "QmWMVyhTuyxUrXX3ynz171jq76yY3PktfY9Bxiph7b9ikr" - if rnk.B58String() != exp { - t.Fatalf("dag looks wrong, expected %s, but got %s", exp, rnk.B58String()) + if rnk.String() != exp { + t.Fatalf("dag looks wrong, expected %s, but got %s", exp, rnk.String()) } } diff --git a/mfs/system.go b/mfs/system.go index 2a69a187883..4912c0fd332 100644 --- a/mfs/system.go +++ b/mfs/system.go @@ -10,6 +10,7 @@ package mfs import ( + "context" "errors" "sync" "time" @@ -17,9 +18,9 @@ import ( dag "github.com/ipfs/go-ipfs/merkledag" ft "github.com/ipfs/go-ipfs/unixfs" - context "context" logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" cid "gx/ipfs/QmXUuRadqDq5BuFWzVU6VuKaSjTcNm1gNCtLvvP1TJCW4z/go-cid" + node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node" ) var ErrNotExist = errors.New("no such rootfs") @@ -41,7 +42,7 @@ const ( // FSNode represents any node (directory, root, or file) in the mfs filesystem type FSNode interface { - GetNode() (*dag.ProtoNode, error) + GetNode() (node.Node, error) Flush() error Type() NodeType } diff --git a/test/sharness/t0040-add-and-cat.sh b/test/sharness/t0040-add-and-cat.sh index 6ace9a55bbc..49d2db7178f 100755 --- a/test/sharness/t0040-add-and-cat.sh +++ b/test/sharness/t0040-add-and-cat.sh @@ -126,6 +126,21 @@ test_add_cat_5MB() { ' } +test_add_cat_raw() { + test_expect_success "add a small file with raw-leaves" ' + echo "foobar" > afile && + HASH=$(ipfs add -q --raw-leaves afile) + ' + + test_expect_success "cat that small file" ' + ipfs cat $HASH > afile_out + ' + + test_expect_success "make sure it looks good" ' + test_cmp afile afile_out + ' +} + test_add_cat_expensive() { test_expect_success EXPENSIVE "generate 100MB file using go-random" ' random 104857600 42 >mountdir/bigfile @@ -392,17 +407,21 @@ test_add_named_pipe " Post http://$API_ADDR/api/v0/add?encoding=json&progress=tr test_add_pwd_is_symlink +test_add_cat_raw + test_kill_ipfs_daemon # should work offline test_add_cat_file +test_add_cat_raw + test_expect_success "ipfs add --only-hash succeeds" ' echo "unknown content for only-hash" | ipfs add --only-hash -q > oh_hash ' -#TODO: this doesn't work when online hence separated out from test_add_cat_file +#TODO: this doesnt work when online hence separated out from test_add_cat_file test_expect_success "ipfs cat file fails" ' test_must_fail ipfs cat $(cat oh_hash) ' From 9796a03624fa55e7756e95c350de86da8497d6cc Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 18 Oct 2016 16:03:26 -0700 Subject: [PATCH 4/5] update HashOnRead validation to properly support cids License: MIT Signed-off-by: Jeromy --- blocks/blockstore/blockstore.go | 12 ++++++++---- test/sharness/t0084-repo-read-rehash.sh | 5 +++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/blocks/blockstore/blockstore.go b/blocks/blockstore/blockstore.go index dfa35ec413a..fb8a0f067ff 100644 --- a/blocks/blockstore/blockstore.go +++ b/blocks/blockstore/blockstore.go @@ -100,12 +100,16 @@ func (bs *blockstore) Get(k *cid.Cid) (blocks.Block, error) { } if bs.rehash { - rb := blocks.NewBlock(bdata) - if !rb.Cid().Equals(k) { + rbcid, err := k.Prefix().Sum(bdata) + if err != nil { + return nil, err + } + + if !rbcid.Equals(k) { return nil, ErrHashMismatch - } else { - return rb, nil } + + return blocks.NewBlockWithCid(bdata, rbcid) } else { return blocks.NewBlockWithCid(bdata, k) } diff --git a/test/sharness/t0084-repo-read-rehash.sh b/test/sharness/t0084-repo-read-rehash.sh index 9be2e95cf9c..7a1faa4bd86 100755 --- a/test/sharness/t0084-repo-read-rehash.sh +++ b/test/sharness/t0084-repo-read-rehash.sh @@ -43,6 +43,11 @@ test_check_bad_blocks() { test_check_bad_blocks +test_expect_success "can add and cat a raw-leaf file" ' + HASH=$(echo "stuff" | ipfs add -q --raw-leaves) && + ipfs cat $HASH > /dev/null +' + test_launch_ipfs_daemon test_check_bad_blocks test_kill_ipfs_daemon From bb986ebf4c2be6d0906dd82c68811c11f98e25d6 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 19 Oct 2016 11:17:29 -0700 Subject: [PATCH 5/5] rename DataSize to FileSize License: MIT Signed-off-by: Jeromy --- importer/helpers/helpers.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/importer/helpers/helpers.go b/importer/helpers/helpers.go index 58007daa760..e374507b72e 100644 --- a/importer/helpers/helpers.go +++ b/importer/helpers/helpers.go @@ -105,7 +105,7 @@ func (n *UnixfsNode) GetChild(ctx context.Context, i int, ds dag.DAGService) (*U // the passed in DagBuilderHelper is used to store the child node an // pin it locally so it doesnt get lost func (n *UnixfsNode) AddChild(child *UnixfsNode, db *DagBuilderHelper) error { - n.ufmt.AddBlockSize(child.DataSize()) + n.ufmt.AddBlockSize(child.FileSize()) childnode, err := child.GetDagNode() if err != nil { @@ -137,7 +137,7 @@ func (n *UnixfsNode) SetData(data []byte) { n.ufmt.Data = data } -func (n *UnixfsNode) DataSize() uint64 { +func (n *UnixfsNode) FileSize() uint64 { if n.raw { return uint64(len(n.rawnode.RawData())) }