Skip to content

Commit

Permalink
Add "filestore" commands to list contents and verify filestore.
Browse files Browse the repository at this point in the history
License: MIT
Signed-off-by: Kevin Atkinson <[email protected]>
  • Loading branch information
kevina committed Apr 27, 2016
1 parent 31e2e21 commit 4ef5531
Show file tree
Hide file tree
Showing 5 changed files with 208 additions and 2 deletions.
100 changes: 100 additions & 0 deletions core/commands/filestore.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
package commands

import (
"errors"
"io"

cmds "github.com/ipfs/go-ipfs/commands"
"github.com/ipfs/go-ipfs/filestore"
"github.com/ipfs/go-ipfs/repo/fsrepo"
)

type chanWriter struct {
ch <-chan *filestore.ListRes
buf string
offset int
}

func (w *chanWriter) Read(p []byte) (int, error) {
if w.offset >= len(w.buf) {
w.offset = 0
res, more := <-w.ch
if !more {
return 0, io.EOF
}
w.buf = res.Format()
}
sz := copy(p, w.buf[w.offset:])
w.offset += sz
return sz, nil
}

var FileStoreCmd = &cmds.Command{
Helptext: cmds.HelpText{
Tagline: "Interact with filestore objects",
},
Subcommands: map[string]*cmds.Command{
"ls": lsFileStore,
"verify": verifyFileStore,
},
}

var lsFileStore = &cmds.Command{
Helptext: cmds.HelpText{
Tagline: "List objects on filestore",
},

Run: func(req cmds.Request, res cmds.Response) {
node, err := req.InvocContext().GetNode()
if err != nil {
res.SetError(err, cmds.ErrNormal)
return
}
fsrepo, ok := node.Repo.Self().(*fsrepo.FSRepo)
if !ok {
res.SetError(errors.New("Not a FSRepo"), cmds.ErrNormal)
return
}
ch := make(chan *filestore.ListRes)
go func() {
defer close(ch)
filestore.List(fsrepo.Filestore(), ch)
}()
res.SetOutput(&chanWriter{ch, "", 0})
},
Marshalers: cmds.MarshalerMap{
cmds.Text: func(res cmds.Response) (io.Reader, error) {
return res.(io.Reader), nil
},
},
}

var verifyFileStore = &cmds.Command{
Helptext: cmds.HelpText{
Tagline: "Verify objects in filestore",
},

Run: func(req cmds.Request, res cmds.Response) {
node, err := req.InvocContext().GetNode()
if err != nil {
res.SetError(err, cmds.ErrNormal)
return
}
fsrepo, ok := node.Repo.Self().(*fsrepo.FSRepo)
if !ok {
res.SetError(errors.New("Not a FSRepo"), cmds.ErrNormal)
return
}
ch := make(chan *filestore.ListRes)
go func() {
defer close(ch)
filestore.Verify(fsrepo.Filestore(), ch)
}()
res.SetOutput(&chanWriter{ch, "", 0})
},
Marshalers: cmds.MarshalerMap{
cmds.Text: func(res cmds.Response) (io.Reader, error) {
return res.(io.Reader), nil
},
},
}
1 change: 1 addition & 0 deletions core/commands/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,7 @@ var rootSubcommands = map[string]*cmds.Command{
"update": ExternalBinary(),
"version": VersionCmd,
"bitswap": BitswapCmd,
"filestore": FileStoreCmd,
}

// RootRO is the readonly version of Root
Expand Down
18 changes: 18 additions & 0 deletions filestore/dataobj.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package filestore

import (
"fmt"
pb "github.com/ipfs/go-ipfs/filestore/pb"
)

Expand Down Expand Up @@ -34,6 +35,23 @@ type DataObj struct {
Data []byte
}

func (d *DataObj) StripData() DataObj {
return DataObj{
d.NoBlockData, d.WholeFile, d.FileRoot,
d.FilePath, d.Offset, d.Size, nil,
}
}

func (d *DataObj) Format() string {
if d.NoBlockData {
return fmt.Sprintf("block %s %d %d", d.FilePath, d.Offset, d.Size)
} else if d.FileRoot {
return fmt.Sprintf("root %s %d %d", d.FilePath, d.Offset, d.Size)
} else {
return fmt.Sprintf("other %s %d %d", d.FilePath, d.Offset, d.Size)
}
}

func (d *DataObj) Marshal() ([]byte, error) {
pd := new(pb.DataObj)

Expand Down
12 changes: 10 additions & 2 deletions filestore/datastore.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,9 +91,17 @@ func (d *Datastore) decode(dataObj interface{}) (*DataObj, error) {
return val, nil
}

type InvalidBlock struct{}

func (e InvalidBlock) Error() string {
return "Datastore: Block Verification Failed"
}

// Get the orignal data out of the DataObj
func (d *Datastore) GetData(key ds.Key, val *DataObj, verify bool) ([]byte, error) {
if val.NoBlockData {
if val == nil {
return nil, errors.New("Nil DataObj")
} else if val.NoBlockData {
file, err := os.Open(val.FilePath)
if err != nil {
return nil, err
Expand All @@ -114,7 +122,7 @@ func (d *Datastore) GetData(key ds.Key, val *DataObj, verify bool) ([]byte, erro
if verify {
newKey := k.Key(u.Hash(data)).DsKey()
if newKey != key {
return nil, errors.New("Datastore: Block Verification Failed")
return nil, InvalidBlock{}
}
}
return data, nil
Expand Down
79 changes: 79 additions & 0 deletions filestore/util.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
package filestore

import (
"fmt"
"io"
"os"

ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore"
"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore/query"
b58 "gx/ipfs/QmT8rehPR3F6bmwL6zjUN8XpiDBFFpMP2myPdC6ApsWfJf/go-base58"
)

const (
StatusOk = 1
StatusMissing = 2
StatusInvalid = 3
StatusError = 4
)

func statusStr(status int) string {
switch status {
case 0:
return ""
case 1:
return "ok "
case 2:
return "missing "
case 3:
return "invalid "
case 4:
return "error "
default:
return "?? "
}
}

type ListRes struct {
Key []byte
DataObj
Status int
}

func (r *ListRes) Format() string {
mhash := b58.Encode(r.Key)
return fmt.Sprintf("%s%s %s\n", statusStr(r.Status), mhash, r.DataObj.Format())
}

func list(d *Datastore, out chan<- *ListRes, verify bool) error {
qr, err := d.Query(query.Query{KeysOnly: true})
if err != nil {
return err
}
for r := range qr.Next() {
if r.Error != nil {
return r.Error
}
key := ds.NewKey(r.Key)
val, _ := d.GetDirect(key)
status := 0
if verify {
_, err := d.GetData(key, val, true)
if err == nil {
status = StatusOk
} else if os.IsNotExist(err) {
status = StatusMissing
} else if _, ok := err.(InvalidBlock); ok || err == io.EOF || err == io.ErrUnexpectedEOF {
status = StatusInvalid
} else {
status = StatusError
}
}
out <- &ListRes{key.Bytes()[1:], val.StripData(), status}
}
return nil
}

func List(d *Datastore, out chan<- *ListRes) error { return list(d, out, false) }

func Verify(d *Datastore, out chan<- *ListRes) error { return list(d, out, true) }

0 comments on commit 4ef5531

Please sign in to comment.