Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

an attempt at making the editor more efficient #1567

Merged
merged 3 commits into from
Sep 5, 2015
Merged

an attempt at making the editor more efficient #1567

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b522107
an attempt at making the editor more efficient
whyrusleeping Aug 12, 2015
30d10ce
add-only-hash no longer stores entirety of everything in memory
whyrusleeping Aug 14, 2015
b3aee28
move mem-dag construction to its own function, and actually call Writ…
whyrusleeping Aug 17, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions blockservice/blockservice.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,9 @@ func (s *BlockService) GetBlock(ctx context.Context, k key.Key) (*blocks.Block,
log.Debug("Blockservice: Searching bitswap.")
blk, err := s.Exchange.GetBlock(ctx, k)
if err != nil {
if err == blockstore.ErrNotFound {
return nil, ErrNotFound
}
return nil, err
}
return blk, nil
Expand Down
42 changes: 33 additions & 9 deletions core/commands/add.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,16 @@ import (
"path"

"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb"
ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
syncds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/sync"
cxt "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"

bstore "github.com/ipfs/go-ipfs/blocks/blockstore"
bserv "github.com/ipfs/go-ipfs/blockservice"
cmds "github.com/ipfs/go-ipfs/commands"
files "github.com/ipfs/go-ipfs/commands/files"
core "github.com/ipfs/go-ipfs/core"
offline "github.com/ipfs/go-ipfs/exchange/offline"
importer "github.com/ipfs/go-ipfs/importer"
"github.com/ipfs/go-ipfs/importer/chunk"
dag "github.com/ipfs/go-ipfs/merkledag"
Expand Down Expand Up @@ -103,11 +108,12 @@ remains to be implemented.
hidden, _, _ := req.Option(hiddenOptionName).Bool()
chunker, _, _ := req.Option(chunkerOptionName).String()

e := dagutils.NewDagEditor(NewMemoryDagService(), newDirNode())
if hash {
nilnode, err := core.NewNode(n.Context(), &core.BuildCfg{
//TODO: need this to be true or all files
// hashed will be stored in memory!
NilRepo: false,
NilRepo: true,
})
if err != nil {
res.SetError(err, cmds.ErrNormal)
Expand All @@ -122,7 +128,7 @@ remains to be implemented.
fileAdder := adder{
ctx: req.Context(),
node: n,
editor: dagutils.NewDagEditor(n.DAG, newDirNode()),
editor: e,
out: outChan,
chunker: chunker,
progress: progress,
Expand Down Expand Up @@ -166,6 +172,15 @@ remains to be implemented.
return err
}

if !hash {
// copy intermediary nodes from editor to our actual dagservice
err := e.WriteOutputTo(n.DAG)
if err != nil {
log.Error("WRITE OUT: ", err)
return err
}
}

rootnd, err := fileAdder.RootNode()
if err != nil {
return err
Expand All @@ -180,6 +195,7 @@ remains to be implemented.
res.SetError(err, cmds.ErrNormal)
return
}

}()
},
PostRun: func(req cmds.Request, res cmds.Response) {
Expand Down Expand Up @@ -268,6 +284,13 @@ remains to be implemented.
Type: AddedObject{},
}

func NewMemoryDagService() dag.DAGService {
// build mem-datastore for editor's intermediary nodes
bs := bstore.NewBlockstore(syncds.MutexWrap(ds.NewMapDatastore()))
bsrv := bserv.New(bs, offline.Exchange(bs))
return dag.NewDAGService(bsrv)
}

// Internal structure for holding the switches passed to the `add` call
type adder struct {
ctx cxt.Context
Expand Down Expand Up @@ -318,7 +341,8 @@ func (params *adder) RootNode() (*dag.Node, error) {
// if not wrapping, AND one root file, use that hash as root.
if !params.wrap && len(r.Links) == 1 {
var err error
r, err = r.Links[0].GetNode(params.ctx, params.node.DAG)
r, err = r.Links[0].GetNode(params.ctx, params.editor.GetDagService())
log.Error("ERR: ", err)
// no need to output, as we've already done so.
return r, err
}
Expand All @@ -330,16 +354,16 @@ func (params *adder) RootNode() (*dag.Node, error) {

func (params *adder) addNode(node *dag.Node, path string) error {
// patch it into the root
key, err := node.Key()
if err != nil {
return err
}

if path == "" {
key, err := node.Key()
if err != nil {
return err
}

path = key.Pretty()
}

if err := params.editor.InsertNodeAtPath(params.ctx, path, key, newDirNode); err != nil {
if err := params.editor.InsertNodeAtPath(params.ctx, path, node, newDirNode); err != nil {
return err
}

Expand Down
7 changes: 6 additions & 1 deletion core/commands/object.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -623,7 +623,12 @@ func addLinkCaller(req cmds.Request, root *dag.Node) (key.Key, error) {

e := dagutils.NewDagEditor(nd.DAG, root)

err = e.InsertNodeAtPath(req.Context(), path, childk, createfunc)
childnd, err := nd.DAG.Get(req.Context(), childk)
if err != nil {
return "", err
}

err = e.InsertNodeAtPath(req.Context(), path, childnd, createfunc)
if err != nil {
return "", err
}
Expand Down
3 changes: 3 additions & 0 deletions merkledag/merkledag.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,9 @@ func (n *dagService) Get(ctx context.Context, k key.Key) (*Node, error) {

b, err := n.Blocks.GetBlock(ctx, k)
if err != nil {
if err == bserv.ErrNotFound {
return nil, ErrNotFound
}
return nil, err
}

Expand Down
12 changes: 10 additions & 2 deletions merkledag/utils/diff.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,11 @@ func ApplyChange(ctx context.Context, ds dag.DAGService, nd *dag.Node, cs []*Cha
for _, c := range cs {
switch c.Type {
case Add:
err := e.InsertNodeAtPath(ctx, c.Path, c.After, nil)
child, err := ds.Get(ctx, c.After)
if err != nil {
return nil, err
}
err = e.InsertNodeAtPath(ctx, c.Path, child, nil)
if err != nil {
return nil, err
}
Expand All @@ -57,7 +61,11 @@ func ApplyChange(ctx context.Context, ds dag.DAGService, nd *dag.Node, cs []*Cha
if err != nil {
return nil, err
}
err = e.InsertNodeAtPath(ctx, c.Path, c.After, nil)
child, err := ds.Get(ctx, c.After)
if err != nil {
return nil, err
}
err = e.InsertNodeAtPath(ctx, c.Path, child, nil)
if err != nil {
return nil, err
}
Expand Down
48 changes: 36 additions & 12 deletions merkledag/utils/utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (

context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"

key "github.com/ipfs/go-ipfs/blocks/key"
dag "github.com/ipfs/go-ipfs/merkledag"
)

Expand All @@ -26,21 +25,17 @@ func (e *Editor) GetNode() *dag.Node {
return e.root.Copy()
}

func (e *Editor) AddLink(ctx context.Context, childname string, childk key.Key) error {
nd, err := addLink(ctx, e.ds, e.root, childname, childk)
if err != nil {
return err
}
e.root = nd
return nil
func (e *Editor) GetDagService() dag.DAGService {
return e.ds
}

func addLink(ctx context.Context, ds dag.DAGService, root *dag.Node, childname string, childk key.Key) (*dag.Node, error) {
func addLink(ctx context.Context, ds dag.DAGService, root *dag.Node, childname string, childnd *dag.Node) (*dag.Node, error) {
if childname == "" {
return nil, errors.New("cannot create link with no name!")
}

childnd, err := ds.Get(ctx, childk)
// ensure that the node we are adding is in the dagservice
_, err := ds.Add(childnd)
if err != nil {
return nil, err
}
Expand All @@ -58,7 +53,7 @@ func addLink(ctx context.Context, ds dag.DAGService, root *dag.Node, childname s
return root, nil
}

func (e *Editor) InsertNodeAtPath(ctx context.Context, path string, toinsert key.Key, create func() *dag.Node) error {
func (e *Editor) InsertNodeAtPath(ctx context.Context, path string, toinsert *dag.Node, create func() *dag.Node) error {
splpath := strings.Split(path, "/")
nd, err := insertNodeAtPath(ctx, e.ds, e.root, splpath, toinsert, create)
if err != nil {
Expand All @@ -68,7 +63,7 @@ func (e *Editor) InsertNodeAtPath(ctx context.Context, path string, toinsert key
return nil
}

func insertNodeAtPath(ctx context.Context, ds dag.DAGService, root *dag.Node, path []string, toinsert key.Key, create func() *dag.Node) (*dag.Node, error) {
func insertNodeAtPath(ctx context.Context, ds dag.DAGService, root *dag.Node, path []string, toinsert *dag.Node, create func() *dag.Node) (*dag.Node, error) {
if len(path) == 1 {
return addLink(ctx, ds, root, path[0], toinsert)
}
Expand Down Expand Up @@ -151,3 +146,32 @@ func rmLink(ctx context.Context, ds dag.DAGService, root *dag.Node, path []strin

return root, nil
}

func (e *Editor) WriteOutputTo(ds dag.DAGService) error {
return copyDag(e.GetNode(), e.ds, ds)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i think this needs what i wrote in 978c9fa#diff-bd0ca9f1e2d5c7ba128af01ec4c1131cR314

}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this doesn't seem to get called? shouldn't it be called by the add command at the end?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it's called now.


func copyDag(nd *dag.Node, from, to dag.DAGService) error {
_, err := to.Add(nd)
if err != nil {
return err
}

for _, lnk := range nd.Links {
child, err := lnk.GetNode(context.Background(), from)
if err != nil {
if err == dag.ErrNotFound {
// not found means we didnt modify it, and it should
// already be in the target datastore
continue
}
return err
}

err = copyDag(child, from, to)
if err != nil {
return err
}
}
return nil
}
4 changes: 2 additions & 2 deletions merkledag/utils/utils_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ func TestAddLink(t *testing.T) {
}

nd := new(dag.Node)
nnode, err := addLink(context.Background(), ds, nd, "fish", fk)
nnode, err := addLink(context.Background(), ds, nd, "fish", fishnode)
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -104,7 +104,7 @@ func testInsert(t *testing.T, e *Editor, path, data string, create bool, experr
}
}

err = e.InsertNodeAtPath(context.Background(), path, ck, c)
err = e.InsertNodeAtPath(context.Background(), path, child, c)
if experr != "" {
var got string
if err != nil {
Expand Down