Factor out filestore specific code from DagServices.

This also eliminate the need for AddOpts and ExtraInfo interface.

License: MIT
Signed-off-by: Kevin Atkinson <[email protected]>

blocks/blocks.go

@@ -5,7 +5,6 @@ package blocks
 import (
 	"errors"
 	"fmt"
-	"time"
 
 	key "github.com/ipfs/go-ipfs/blocks/key"
 	mh "gx/ipfs/QmYf7ng2hG5XBtJA3tN34DQ2GUN5HNksEw1rLDkmr6vGku/go-multihash"
@@ -26,21 +25,6 @@ type BasicBlock struct {
 	data      []byte
 }
 
-type FilestoreBlock struct {
-	BasicBlock
-	*DataPtr
-	AddOpts interface{}
-}
-
-// This DataPtr has different AltData than the node DataPtr
-type DataPtr struct {
-	AltData  []byte
-	FilePath string
-	Offset   uint64
-	Size     uint64
-	ModTime  time.Time
-}
-
 // NewBlock creates a Block object from opaque data. It will hash the data.
 func NewBlock(data []byte) *BasicBlock {
 	return &BasicBlock{data: data, multihash: u.Hash(data)}

commands/files/adv_reader.go

@@ -1,53 +1,31 @@
 package files
 
 import (
-	"errors"
 	"io"
-	"time"
+	"os"
 )
 
 // An AdvReader is like a Reader but supports getting the current file
 // path and offset into the file when applicable.
 type AdvReader interface {
 	io.Reader
-	ExtraInfo() ExtraInfo
-	SetExtraInfo(inf ExtraInfo) error
-}
-
-type ExtraInfo interface {
-	Offset() int64
-	AbsPath() string
-	// Clone creates a copy with different offset
-	Clone(offset int64) ExtraInfo
+	PosInfo() *PosInfo
 }
 
 type PosInfo struct {
-	offset  int64
-	absPath string
-}
-
-func (i PosInfo) Offset() int64 { return i.offset }
-
-func (i PosInfo) AbsPath() string { return i.absPath }
-
-func (i PosInfo) Clone(offset int64) ExtraInfo { return PosInfo{offset, i.absPath} }
-
-func NewPosInfo(offset int64, absPath string) PosInfo {
-	return PosInfo{offset, absPath}
+	Offset   uint64
+	FullPath string
+	Stat     os.FileInfo // can be nil
 }
 
 type advReaderAdapter struct {
 	io.Reader
 }
 
-func (advReaderAdapter) ExtraInfo() ExtraInfo {
+func (advReaderAdapter) PosInfo() *PosInfo {
 	return nil
 }
 
-func (advReaderAdapter) SetExtraInfo(_ ExtraInfo) error {
-	return errors.New("Reader does not support setting ExtraInfo.")
-}
-
 func AdvReaderAdapter(r io.Reader) AdvReader {
 	switch t := r.(type) {
 	case AdvReader:
@@ -57,12 +35,3 @@ func AdvReaderAdapter(r io.Reader) AdvReader {
 	}
 }
 
-type InfoForFilestore struct {
-	ExtraInfo
-	AddOpts interface{}
-	ModTime time.Time
-}
-
-func (i InfoForFilestore) Clone(offset int64) ExtraInfo {
-	return InfoForFilestore{i.ExtraInfo.Clone(offset), i.AddOpts, i.ModTime}
-}

commands/files/readerfile.go

@@ -13,12 +13,11 @@ type ReaderFile struct {
 	fullpath string
 	reader   io.ReadCloser
 	stat     os.FileInfo
-	offset   int64
-	baseInfo ExtraInfo
+	offset   uint64
 }
 
 func NewReaderFile(filename, path string, reader io.ReadCloser, stat os.FileInfo) *ReaderFile {
-	return &ReaderFile{filename, path, reader, stat, 0, PosInfo{0, path}}
+	return &ReaderFile{filename, path, reader, stat, 0}
 }
 
 func (f *ReaderFile) IsDirectory() bool {
@@ -37,18 +36,13 @@ func (f *ReaderFile) FullPath() string {
 	return f.fullpath
 }
 
-func (f *ReaderFile) ExtraInfo() ExtraInfo {
-	return f.baseInfo.Clone(f.offset)
-}
-
-func (f *ReaderFile) SetExtraInfo(info ExtraInfo) error {
-	f.baseInfo = info
-	return nil
+func (f *ReaderFile) PosInfo() *PosInfo {
+	return &PosInfo{f.offset,f.fullpath,f.stat}
 }
 
 func (f *ReaderFile) Read(p []byte) (int, error) {
 	res, err := f.reader.Read(p)
-	f.offset += int64(res)
+	f.offset += uint64(res)
 	return res, err
 }
 

core/commands/add.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb"
 	"github.com/ipfs/go-ipfs/core/coreunix"
-	"github.com/ipfs/go-ipfs/filestore"
 	"github.com/ipfs/go-ipfs/filestore/support"
 
 	bserv "github.com/ipfs/go-ipfs/blockservice"
@@ -167,7 +166,9 @@ You can now refer to the added file in a gateway, like so:
 		if nocopy || link {
 			ds.Blockstore = filestore_support.NewBlockstore(ds.Blockstore, n.Repo.Datastore())
 			blockService := bserv.New(ds.Blockstore, n.Exchange)
-			ds.DAG = dag.NewDAGService(blockService)
+			dagService := dag.NewDAGService(blockService)
+			dagService.NodeToBlock = filestore_support.NodeToBlock{}
+			ds.DAG = dagService
 		}
 
 		fileAdder, err := coreunix.NewAdder(ds, outChan)
@@ -183,13 +184,6 @@ You can now refer to the added file in a gateway, like so:
 		fileAdder.Pin = dopin
 		fileAdder.Silent = silent
 
-		if nocopy {
-			fileAdder.AddOpts = filestore.AddNoCopy
-		}
-		if link {
-			fileAdder.AddOpts = filestore.AddLink
-		}
-
 		addAllAndPin := func(f files.File) error {
 			// Iterate over each top-level file and add individually. Otherwise the
 			// single files.File f is treated as a directory, affecting hidden file

core/coreunix/add.go

@@ -2,7 +2,7 @@ package coreunix
 
 import (
 	"bytes"
-	"errors"
+	//"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -104,28 +104,10 @@ type Adder struct {
 	mr       *mfs.Root
 	unlocker bs.Unlocker
 	tempRoot key.Key
-	AddOpts  interface{}
 }
 
 // Perform the actual add & pin locally, outputting results to reader
 func (adder Adder) add(reader files.AdvReader) (*dag.Node, error) {
-	if adder.AddOpts != nil {
-		info := reader.ExtraInfo()
-		if info == nil {
-			return nil, errors.New("Reader does not support ExtraInfo.")
-		}
-		// We need to get the ModTime before any part of the
-		// file is read to catch the case when the file is
-		// modified as we are reading it
-		fileInfo, err := os.Stat(info.AbsPath())
-		if err != nil {
-			return nil, err
-		}
-		err = reader.SetExtraInfo(files.InfoForFilestore{info, adder.AddOpts, fileInfo.ModTime()})
-		if err != nil {
-			return nil, err
-		}
-	}
 	chnk, err := chunk.FromString(reader, adder.Chunker)
 	if err != nil {
 		return nil, err
@@ -553,10 +535,6 @@ func (i *progressReader) Read(p []byte) (int, error) {
 	return n, err
 }
 
-func (i *progressReader) ExtraInfo() files.ExtraInfo {
-	return i.reader.ExtraInfo()
-}
-
-func (i *progressReader) SetExtraInfo(info files.ExtraInfo) error {
-	return i.reader.SetExtraInfo(info)
+func (i *progressReader) PosInfo() *files.PosInfo {
+	return i.reader.PosInfo()
 }

filestore/dataobj.go

@@ -7,19 +7,6 @@ import (
 	"time"
 )
 
-// A hack to get around the fact that the Datastore interface does not
-// accept options
-type DataWOpts struct {
-	DataObj interface{}
-	AddOpts interface{}
-}
-
-// Constants to indicate how the data should be added.
-const (
-	AddNoCopy = 1
-	AddLink   = 2
-)
-
 const (
 	// If NoBlockData is true the Data is missing the Block data
 	// as that is provided by the underlying file

filestore/datastore.go

@@ -36,20 +36,7 @@ func New(d ds.Datastore, fileStorePath string, verify int) (*Datastore, error) {
 }
 
 func (d *Datastore) Put(key ds.Key, value interface{}) (err error) {
-	val, ok := value.(*DataWOpts)
-	if !ok {
-		panic(ds.ErrInvalidType)
-	}
-
-	addType, ok := val.AddOpts.(int)
-	if !ok {
-		panic(ds.ErrInvalidType)
-	}
-	if addType != AddNoCopy {
-		return errors.New("Only \"no-copy\" mode supported for now.")
-	}
-
-	dataObj, ok := val.DataObj.(*DataObj)
+	dataObj, ok := value.(*DataObj)
 	if !ok {
 		panic(ds.ErrInvalidType)
 	}
@@ -87,6 +74,7 @@ func (d *Datastore) put(key ds.Key, dataObj *DataObj) (err error) {
 	if err != nil {
 		return err
 	}
+	log.Debugf("adding block %s\n", b58.Encode(key.Bytes()[1:]))
 	return d.ds.Put(key, data)
 }
 
@@ -167,6 +155,9 @@ func (d *Datastore) GetData(key ds.Key, val *DataObj, verify int, update bool) (
 			}
 			modtime = FromTime(fileInfo.ModTime())
 		}
+		if err != nil {
+			log.Debugf("invalid block: %s: %s\n", b58.Encode(key.Bytes()[1:]), err.Error())
+		}
 		invalid := val.Invalid() || err != nil
 		if err == nil && (verify == VerifyAlways || (verify == VerifyIfChanged && modtime != val.ModTime)) {
 			log.Debugf("verifying block %s\n", b58.Encode(key.Bytes()[1:]))
@@ -182,7 +173,11 @@ func (d *Datastore) GetData(key ds.Key, val *DataObj, verify int, update bool) (
 			_ = d.put(key, &newVal)
 		}
 		if invalid {
-			log.Debugf("invalid block %s\n", b58.Encode(key.Bytes()[1:]))
+			if err != nil {
+				log.Debugf("invalid block %s: %s\n", b58.Encode(key.Bytes()[1:]), err.Error())
+			} else {
+				log.Debugf("invalid block %s\n", b58.Encode(key.Bytes()[1:]))
+			}
 			return nil, InvalidBlock{}
 		} else {
 			return data, nil

filestore/support/blockstore.go

@@ -1,5 +1,3 @@
-// package blockstore implements a thin wrapper over a datastore, giving a
-// clean interface for Getting and Putting block objects.
 package filestore_support
 
 import (
@@ -22,7 +20,6 @@ func NewBlockstore(b bs.GCBlockstore, d ds.Batching) bs.GCBlockstore {
 
 func (bs *blockstore) Put(block blocks.Block) error {
 	k := block.Key().DsKey()
-	println("putting...")
 
 	data := bs.prepareBlock(k, block)
 	if data == nil {
@@ -32,7 +29,6 @@ func (bs *blockstore) Put(block blocks.Block) error {
 }
 
 func (bs *blockstore) PutMany(blocks []blocks.Block) error {
-	println("put many...")
 	t, err := bs.datastore.Batch()
 	if err != nil {
 		return err
@@ -52,20 +48,21 @@ func (bs *blockstore) PutMany(blocks []blocks.Block) error {
 }
 
 func (bs *blockstore) prepareBlock(k ds.Key, block blocks.Block) interface{} {
-	if fsBlock, ok := block.(*blocks.FilestoreBlock); !ok {
+	if fsBlock, ok := block.(*FilestoreBlock); !ok {
+		//println("Non DataObj")
 		// Has is cheaper than Put, so see if we already have it
 		exists, err := bs.datastore.Has(k)
 		if err == nil && exists {
 			return nil // already stored.
 		}
 		return block.Data()
 	} else {
-		println("DataObj")
+		//println("DataObj")
 		d := &fs.DataObj{
-			FilePath: fsBlock.FilePath,
+			FilePath: fsBlock.FullPath,
 			Offset:   fsBlock.Offset,
 			Size:     fsBlock.Size,
-			ModTime:  fs.FromTime(fsBlock.ModTime),
+			ModTime:  fs.FromTime(fsBlock.Stat.ModTime()),
 		}
 		if fsBlock.AltData == nil {
 			d.Flags |= fs.WholeFile | fs.FileRoot
@@ -74,7 +71,7 @@ func (bs *blockstore) prepareBlock(k ds.Key, block blocks.Block) interface{} {
 			d.Flags |= fs.NoBlockData
 			d.Data = fsBlock.AltData
 		}
-		return &fs.DataWOpts{d, fsBlock.AddOpts}
+		return d
 	}
 
 }