Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Implement 'Raw Node' node type #3307

Merged
merged 5 commits into from
Oct 24, 2016
Merged

Implement 'Raw Node' node type #3307

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ded60a7
unixfs: allow use of raw merkledag nodes for unixfs files
whyrusleeping Oct 15, 2016
3796e70
raw dag: make raw nodes work in cat and get, add tests
whyrusleeping Oct 18, 2016
8ce9963
fix add/cat of small files
whyrusleeping Oct 18, 2016
9796a03
update HashOnRead validation to properly support cids
whyrusleeping Oct 18, 2016
bb986eb
rename DataSize to FileSize
whyrusleeping Oct 19, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions blocks/blocks.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,11 @@ func NewBlock(data []byte) *BasicBlock {
// we are able to be confident that the data is correct
func NewBlockWithCid(data []byte, c *cid.Cid) (*BasicBlock, error) {
if u.Debug {
// TODO: fix assumptions
chkc := cid.NewCidV0(u.Hash(data))
chkc, err := c.Prefix().Sum(data)
if err != nil {
return nil, err
}

if !chkc.Equals(c) {
return nil, ErrWrongHash
}
Expand Down
12 changes: 8 additions & 4 deletions blocks/blockstore/blockstore.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,12 +100,16 @@ func (bs *blockstore) Get(k *cid.Cid) (blocks.Block, error) {
}

if bs.rehash {
rb := blocks.NewBlock(bdata)
if !rb.Cid().Equals(k) {
rbcid, err := k.Prefix().Sum(bdata)
if err != nil {
return nil, err
}

if !rbcid.Equals(k) {
return nil, ErrHashMismatch
} else {
return rb, nil
}

return blocks.NewBlockWithCid(bdata, rbcid)
} else {
return blocks.NewBlockWithCid(bdata, k)
}
Expand Down
22 changes: 13 additions & 9 deletions core/commands/add.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,16 @@ import (
var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded")

const (
quietOptionName = "quiet"
silentOptionName = "silent"
progressOptionName = "progress"
trickleOptionName = "trickle"
wrapOptionName = "wrap-with-directory"
hiddenOptionName = "hidden"
onlyHashOptionName = "only-hash"
chunkerOptionName = "chunker"
pinOptionName = "pin"
quietOptionName = "quiet"
silentOptionName = "silent"
progressOptionName = "progress"
trickleOptionName = "trickle"
wrapOptionName = "wrap-with-directory"
hiddenOptionName = "hidden"
onlyHashOptionName = "only-hash"
chunkerOptionName = "chunker"
pinOptionName = "pin"
rawLeavesOptionName = "raw-leaves"
)

var AddCmd = &cmds.Command{
Expand Down Expand Up @@ -78,6 +79,7 @@ You can now refer to the added file in a gateway, like so:
cmds.BoolOption(hiddenOptionName, "H", "Include files that are hidden. Only takes effect on recursive add.").Default(false),
cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm to use."),
cmds.BoolOption(pinOptionName, "Pin this object when adding.").Default(true),
cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes. (experimental)"),
},
PreRun: func(req cmds.Request) error {
if quiet, _, _ := req.Option(quietOptionName).Bool(); quiet {
Expand Down Expand Up @@ -135,6 +137,7 @@ You can now refer to the added file in a gateway, like so:
silent, _, _ := req.Option(silentOptionName).Bool()
chunker, _, _ := req.Option(chunkerOptionName).String()
dopin, _, _ := req.Option(pinOptionName).Bool()
rawblks, _, _ := req.Option(rawLeavesOptionName).Bool()

if hash {
nilnode, err := core.NewNode(n.Context(), &core.BuildCfg{
Expand Down Expand Up @@ -174,6 +177,7 @@ You can now refer to the added file in a gateway, like so:
fileAdder.Wrap = wrap
fileAdder.Pin = dopin
fileAdder.Silent = silent
fileAdder.RawLeaves = rawblks

if hash {
md := dagtest.Mock()
Expand Down
12 changes: 9 additions & 3 deletions core/commands/files/files.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package commands

import (
"bytes"
"context"
"errors"
"fmt"
"io"
Expand All @@ -16,8 +17,8 @@ import (
path "github.com/ipfs/go-ipfs/path"
ft "github.com/ipfs/go-ipfs/unixfs"

context "context"
logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
)

var log = logging.Logger("cmds/files")
Expand Down Expand Up @@ -160,7 +161,12 @@ func statNode(ds dag.DAGService, fsn mfs.FSNode) (*Object, error) {

c := nd.Cid()

d, err := ft.FromBytes(nd.Data())
pbnd, ok := nd.(*dag.ProtoNode)
if !ok {
return nil, dag.ErrNotProtobuf
}

d, err := ft.FromBytes(pbnd.Data())
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -245,7 +251,7 @@ var FilesCpCmd = &cmds.Command{
},
}

func getNodeFromPath(ctx context.Context, node *core.IpfsNode, p string) (*dag.ProtoNode, error) {
func getNodeFromPath(ctx context.Context, node *core.IpfsNode, p string) (node.Node, error) {
switch {
case strings.HasPrefix(p, "/ipfs/"):
np, err := path.ParsePath(p)
Expand Down
15 changes: 11 additions & 4 deletions core/corehttp/gateway_handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package corehttp

import (
"context"
"errors"
"fmt"
"io"
Expand All @@ -18,10 +19,10 @@ import (
path "github.com/ipfs/go-ipfs/path"
uio "github.com/ipfs/go-ipfs/unixfs/io"

"context"
routing "gx/ipfs/QmNUgVQTYnXQVrGT2rajZYsuKV8GYdiL91cdZSQDKNPNgE/go-libp2p-routing"
humanize "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize"
cid "gx/ipfs/QmXUuRadqDq5BuFWzVU6VuKaSjTcNm1gNCtLvvP1TJCW4z/go-cid"
node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
)

const (
Expand All @@ -45,7 +46,7 @@ func newGatewayHandler(node *core.IpfsNode, conf GatewayConfig) *gatewayHandler
}

// TODO(cryptix): find these helpers somewhere else
func (i *gatewayHandler) newDagFromReader(r io.Reader) (*dag.ProtoNode, error) {
func (i *gatewayHandler) newDagFromReader(r io.Reader) (node.Node, error) {
// TODO(cryptix): change and remove this helper once PR1136 is merged
// return ufs.AddFromReader(i.node, r.Body)
return importer.BuildDagFromReader(
Expand Down Expand Up @@ -353,7 +354,7 @@ func (i *gatewayHandler) putHandler(w http.ResponseWriter, r *http.Request) {
return
}

var newnode *dag.ProtoNode
var newnode node.Node
if rsegs[len(rsegs)-1] == "QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn" {
newnode = uio.NewEmptyDirectory()
} else {
Expand Down Expand Up @@ -417,8 +418,14 @@ func (i *gatewayHandler) putHandler(w http.ResponseWriter, r *http.Request) {
return
}

pbnewnode, ok := newnode.(*dag.ProtoNode)
if !ok {
webError(w, "Cannot read non protobuf nodes through gateway", dag.ErrNotProtobuf, http.StatusBadRequest)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As far as I can see it is put handler?
Does reading rawnodes through gateway work?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, reading raw nodes through the gateway uses the dagreader, which works fine. This nasty hacky area of code is the writeable gateway, which we don't really use...

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you change "Cannot read" to "Cannot write".

return
}

// object set-data case
pbnd.SetData(newnode.Data())
pbnd.SetData(pbnewnode.Data())

newcid, err = i.node.DAG.Add(pbnd)
if err != nil {
Expand Down
16 changes: 10 additions & 6 deletions core/corehttp/gateway_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package corehttp

import (
"context"
"errors"
"io/ioutil"
"net/http"
Expand All @@ -9,14 +10,15 @@ import (
"testing"
"time"

context "context"
core "github.com/ipfs/go-ipfs/core"
coreunix "github.com/ipfs/go-ipfs/core/coreunix"
dag "github.com/ipfs/go-ipfs/merkledag"
namesys "github.com/ipfs/go-ipfs/namesys"
path "github.com/ipfs/go-ipfs/path"
repo "github.com/ipfs/go-ipfs/repo"
config "github.com/ipfs/go-ipfs/repo/config"
testutil "github.com/ipfs/go-ipfs/thirdparty/testutil"

id "gx/ipfs/QmcRa2qn6iCmap9bjp8jAwkvYAq13AUfxdY3rrYiaJbLum/go-libp2p/p2p/protocol/identify"
ci "gx/ipfs/QmfWDLQjGjVe4fr5CoztYW2DYYjRysMJrFe1RCsXLPTf46/go-libp2p-crypto"
)
Expand Down Expand Up @@ -178,11 +180,13 @@ func TestIPNSHostnameRedirect(t *testing.T) {
if err != nil {
t.Fatal(err)
}

_, dagn2, err := coreunix.AddWrapped(n, strings.NewReader("_"), "index.html")
if err != nil {
t.Fatal(err)
}
dagn1.AddNodeLink("foo", dagn2)

dagn1.(*dag.ProtoNode).AddNodeLink("foo", dagn2)
if err != nil {
t.Fatal(err)
}
Expand All @@ -197,7 +201,7 @@ func TestIPNSHostnameRedirect(t *testing.T) {
t.Fatal(err)
}

k := dagn1.Key()
k := dagn1.Cid()
t.Logf("k: %s\n", k)
ns["/ipns/example.net"] = path.FromString("/ipfs/" + k.String())

Expand Down Expand Up @@ -268,8 +272,8 @@ func TestIPNSHostnameBacklinks(t *testing.T) {
if err != nil {
t.Fatal(err)
}
dagn2.AddNodeLink("bar", dagn3)
dagn1.AddNodeLink("foo? #<'", dagn2)
dagn2.(*dag.ProtoNode).AddNodeLink("bar", dagn3)
dagn1.(*dag.ProtoNode).AddNodeLink("foo? #<'", dagn2)
if err != nil {
t.Fatal(err)
}
Expand All @@ -287,7 +291,7 @@ func TestIPNSHostnameBacklinks(t *testing.T) {
t.Fatal(err)
}

k := dagn1.Key()
k := dagn1.Cid()
t.Logf("k: %s\n", k)
ns["/ipns/example.net"] = path.FromString("/ipfs/" + k.String())

Expand Down
49 changes: 24 additions & 25 deletions core/coreunix/add.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package coreunix

import (
"context"
"fmt"
"io"
"io/ioutil"
Expand All @@ -13,16 +14,18 @@ import (
"github.com/ipfs/go-ipfs/commands/files"
core "github.com/ipfs/go-ipfs/core"
"github.com/ipfs/go-ipfs/exchange/offline"
importer "github.com/ipfs/go-ipfs/importer"
balanced "github.com/ipfs/go-ipfs/importer/balanced"
"github.com/ipfs/go-ipfs/importer/chunk"
ihelper "github.com/ipfs/go-ipfs/importer/helpers"
trickle "github.com/ipfs/go-ipfs/importer/trickle"
dag "github.com/ipfs/go-ipfs/merkledag"
mfs "github.com/ipfs/go-ipfs/mfs"
"github.com/ipfs/go-ipfs/pin"
unixfs "github.com/ipfs/go-ipfs/unixfs"

context "context"
logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
cid "gx/ipfs/QmXUuRadqDq5BuFWzVU6VuKaSjTcNm1gNCtLvvP1TJCW4z/go-cid"
node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
ds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore"
syncds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore/sync"
)
Expand Down Expand Up @@ -97,10 +100,11 @@ type Adder struct {
Hidden bool
Pin bool
Trickle bool
RawLeaves bool
Silent bool
Wrap bool
Chunker string
root *dag.ProtoNode
root node.Node
mr *mfs.Root
unlocker bs.Unlocker
tempRoot *cid.Cid
Expand All @@ -111,25 +115,25 @@ func (adder *Adder) SetMfsRoot(r *mfs.Root) {
}

// Perform the actual add & pin locally, outputting results to reader
func (adder Adder) add(reader io.Reader) (*dag.ProtoNode, error) {
func (adder Adder) add(reader io.Reader) (node.Node, error) {
chnk, err := chunk.FromString(reader, adder.Chunker)
if err != nil {
return nil, err
}
params := ihelper.DagBuilderParams{
Dagserv: adder.dagService,
RawLeaves: adder.RawLeaves,
Maxlinks: ihelper.DefaultLinksPerBlock,
}

if adder.Trickle {
return importer.BuildTrickleDagFromReader(
adder.dagService,
chnk,
)
}
return importer.BuildDagFromReader(
adder.dagService,
chnk,
)
return trickle.TrickleLayout(params.New(chnk))
}

return balanced.BalancedLayout(params.New(chnk))
}

func (adder *Adder) RootNode() (*dag.ProtoNode, error) {
func (adder *Adder) RootNode() (node.Node, error) {
// for memoizing
if adder.root != nil {
return adder.root, nil
Expand All @@ -147,12 +151,7 @@ func (adder *Adder) RootNode() (*dag.ProtoNode, error) {
return nil, err
}

pbnd, ok := nd.(*dag.ProtoNode)
if !ok {
return nil, dag.ErrNotProtobuf
}

root = pbnd
root = nd
}

adder.root = root
Expand Down Expand Up @@ -185,7 +184,7 @@ func (adder *Adder) PinRoot() error {
return adder.pinning.Flush()
}

func (adder *Adder) Finalize() (*dag.ProtoNode, error) {
func (adder *Adder) Finalize() (node.Node, error) {
root := adder.mr.GetValue()

// cant just call adder.RootNode() here as we need the name for printing
Expand Down Expand Up @@ -307,7 +306,7 @@ func AddR(n *core.IpfsNode, root string) (key string, err error) {
// to preserve the filename.
// Returns the path of the added file ("<dir hash>/filename"), the DAG node of
// the directory, and and error if any.
func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, *dag.ProtoNode, error) {
func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, node.Node, error) {
file := files.NewReaderFile(filename, filename, ioutil.NopCloser(r), nil)
fileAdder, err := NewAdder(n.Context(), n.Pinning, n.Blockstore, n.DAG)
if err != nil {
Expand All @@ -331,7 +330,7 @@ func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, *dag.Pr
return gopath.Join(c.String(), filename), dagnode, nil
}

func (adder *Adder) addNode(node *dag.ProtoNode, path string) error {
func (adder *Adder) addNode(node node.Node, path string) error {
// patch it into the root
if path == "" {
path = node.Cid().String()
Expand Down Expand Up @@ -456,7 +455,7 @@ func (adder *Adder) maybePauseForGC() error {
}

// outputDagnode sends dagnode info over the output channel
func outputDagnode(out chan interface{}, name string, dn *dag.ProtoNode) error {
func outputDagnode(out chan interface{}, name string, dn node.Node) error {
if out == nil {
return nil
}
Expand All @@ -482,7 +481,7 @@ func NewMemoryDagService() dag.DAGService {
}

// from core/commands/object.go
func getOutput(dagnode *dag.ProtoNode) (*Object, error) {
func getOutput(dagnode node.Node) (*Object, error) {
c := dagnode.Cid()

output := &Object{
Expand Down
Loading