Add "filestore" commands to list contents and verify filestore.

License: MIT
Signed-off-by: Kevin Atkinson <[email protected]>

core/commands/filestore.go

@@ -0,0 +1,100 @@
+package commands
+
+import (
+	"errors"
+	"io"
+
+	cmds "github.com/ipfs/go-ipfs/commands"
+	"github.com/ipfs/go-ipfs/filestore"
+	"github.com/ipfs/go-ipfs/repo/fsrepo"
+)
+
+type chanWriter struct {
+	ch     <-chan *filestore.ListRes
+	buf    string
+	offset int
+}
+
+func (w *chanWriter) Read(p []byte) (int, error) {
+	if w.offset >= len(w.buf) {
+		w.offset = 0
+		res, more := <-w.ch
+		if !more {
+			return 0, io.EOF
+		}
+		w.buf = res.Format()
+	}
+	sz := copy(p, w.buf[w.offset:])
+	w.offset += sz
+	return sz, nil
+}
+
+var FileStoreCmd = &cmds.Command{
+	Helptext: cmds.HelpText{
+		Tagline: "Interact with filestore objects",
+	},
+	Subcommands: map[string]*cmds.Command{
+		"ls":     lsFileStore,
+		"verify": verifyFileStore,
+	},
+}
+
+var lsFileStore = &cmds.Command{
+	Helptext: cmds.HelpText{
+		Tagline: "List objects on filestore",
+	},
+
+	Run: func(req cmds.Request, res cmds.Response) {
+		node, err := req.InvocContext().GetNode()
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+		fsrepo, ok := node.Repo.Self().(*fsrepo.FSRepo)
+		if !ok {
+			res.SetError(errors.New("Not a FSRepo"), cmds.ErrNormal)
+			return
+		}
+		ch := make(chan *filestore.ListRes)
+		go func() {
+			defer close(ch)
+			filestore.List(fsrepo.Filestore(), ch)
+		}()
+		res.SetOutput(&chanWriter{ch, "", 0})
+	},
+	Marshalers: cmds.MarshalerMap{
+		cmds.Text: func(res cmds.Response) (io.Reader, error) {
+			return res.(io.Reader), nil
+		},
+	},
+}
+
+var verifyFileStore = &cmds.Command{
+	Helptext: cmds.HelpText{
+		Tagline: "Verify objects in filestore",
+	},
+
+	Run: func(req cmds.Request, res cmds.Response) {
+		node, err := req.InvocContext().GetNode()
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+		fsrepo, ok := node.Repo.Self().(*fsrepo.FSRepo)
+		if !ok {
+			res.SetError(errors.New("Not a FSRepo"), cmds.ErrNormal)
+			return
+		}
+		ch := make(chan *filestore.ListRes)
+		go func() {
+			defer close(ch)
+			filestore.Verify(fsrepo.Filestore(), ch)
+		}()
+		res.SetOutput(&chanWriter{ch, "", 0})
+	},
+	Marshalers: cmds.MarshalerMap{
+		cmds.Text: func(res cmds.Response) (io.Reader, error) {
+			return res.(io.Reader), nil
+		},
+	},
+}

core/commands/root.go

@@ -123,6 +123,7 @@ var rootSubcommands = map[string]*cmds.Command{
 	"update":    ExternalBinary(),
 	"version":   VersionCmd,
 	"bitswap":   BitswapCmd,
+	"filestore": FileStoreCmd,
 }
 
 // RootRO is the readonly version of Root

filestore/dataobj.go

@@ -1,6 +1,7 @@
 package filestore
 
 import (
+	"fmt"
 	pb "github.com/ipfs/go-ipfs/filestore/pb"
 )
 
@@ -34,6 +35,23 @@ type DataObj struct {
 	Data     []byte
 }
 
+func (d *DataObj) StripData() DataObj {
+	return DataObj{
+		d.NoBlockData, d.WholeFile, d.FileRoot,
+		d.FilePath, d.Offset, d.Size, nil,
+	}
+}
+
+func (d *DataObj) Format() string {
+	if d.NoBlockData {
+		return fmt.Sprintf("block %s %d %d", d.FilePath, d.Offset, d.Size)
+	} else if d.FileRoot {
+		return fmt.Sprintf("root  %s %d %d", d.FilePath, d.Offset, d.Size)
+	} else {
+		return fmt.Sprintf("other %s %d %d", d.FilePath, d.Offset, d.Size)
+	}
+}
+
 func (d *DataObj) Marshal() ([]byte, error) {
 	pd := new(pb.DataObj)
 

filestore/datastore.go

@@ -91,9 +91,17 @@ func (d *Datastore) decode(dataObj interface{}) (*DataObj, error) {
 	return val, nil
 }
 
+type InvalidBlock struct{}
+
+func (e InvalidBlock) Error() string {
+	return "Datastore: Block Verification Failed"
+}
+
 // Get the orignal data out of the DataObj
 func (d *Datastore) GetData(key ds.Key, val *DataObj, verify bool) ([]byte, error) {
-	if val.NoBlockData {
+	if val == nil {
+		return nil, errors.New("Nil DataObj")
+	} else if val.NoBlockData {
 		file, err := os.Open(val.FilePath)
 		if err != nil {
 			return nil, err
@@ -114,7 +122,7 @@ func (d *Datastore) GetData(key ds.Key, val *DataObj, verify bool) ([]byte, erro
 		if verify {
 			newKey := k.Key(u.Hash(data)).DsKey()
 			if newKey != key {
-				return nil, errors.New("Datastore: Block Verification Failed")
+				return nil, InvalidBlock{}
 			}
 		}
 		return data, nil

filestore/util.go

@@ -0,0 +1,79 @@
+package filestore
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore"
+	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore/query"
+	b58 "gx/ipfs/QmT8rehPR3F6bmwL6zjUN8XpiDBFFpMP2myPdC6ApsWfJf/go-base58"
+)
+
+const (
+	StatusOk      = 1
+	StatusMissing = 2
+	StatusInvalid = 3
+	StatusError   = 4
+)
+
+func statusStr(status int) string {
+	switch status {
+	case 0:
+		return ""
+	case 1:
+		return "ok      "
+	case 2:
+		return "missing "
+	case 3:
+		return "invalid "
+	case 4:
+		return "error   "
+	default:
+		return "??      "
+	}
+}
+
+type ListRes struct {
+	Key []byte
+	DataObj
+	Status int
+}
+
+func (r *ListRes) Format() string {
+	mhash := b58.Encode(r.Key)
+	return fmt.Sprintf("%s%s %s\n", statusStr(r.Status), mhash, r.DataObj.Format())
+}
+
+func list(d *Datastore, out chan<- *ListRes, verify bool) error {
+	qr, err := d.Query(query.Query{KeysOnly: true})
+	if err != nil {
+		return err
+	}
+	for r := range qr.Next() {
+		if r.Error != nil {
+			return r.Error
+		}
+		key := ds.NewKey(r.Key)
+		val, _ := d.GetDirect(key)
+		status := 0
+		if verify {
+			_, err := d.GetData(key, val, true)
+			if err == nil {
+				status = StatusOk
+			} else if os.IsNotExist(err) {
+				status = StatusMissing
+			} else if _, ok := err.(InvalidBlock); ok || err == io.EOF || err == io.ErrUnexpectedEOF {
+				status = StatusInvalid
+			} else {
+				status = StatusError
+			}
+		}
+		out <- &ListRes{key.Bytes()[1:], val.StripData(), status}
+	}
+	return nil
+}
+
+func List(d *Datastore, out chan<- *ListRes) error { return list(d, out, false) }
+
+func Verify(d *Datastore, out chan<- *ListRes) error { return list(d, out, true) }