diff --git a/filestore/datastore.go b/filestore/datastore.go index d7c8f5b1faef..cb87a9554854 100644 --- a/filestore/datastore.go +++ b/filestore/datastore.go @@ -13,10 +13,13 @@ import ( "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore/query" k "github.com/ipfs/go-ipfs/blocks/key" //mh "gx/ipfs/QmYf7ng2hG5XBtJA3tN34DQ2GUN5HNksEw1rLDkmr6vGku/go-multihash" - //b58 "gx/ipfs/QmT8rehPR3F6bmwL6zjUN8XpiDBFFpMP2myPdC6ApsWfJf/go-base58" + b58 "gx/ipfs/QmT8rehPR3F6bmwL6zjUN8XpiDBFFpMP2myPdC6ApsWfJf/go-base58" u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util" + logging "gx/ipfs/QmaDNZ4QMdBdku1YZWBysufYyoQt1negQGNav6PLYarbY8/go-log" ) +var log = logging.Logger("filestore") + const ( VerifyNever = 0 VerifyIfChanged = 1 @@ -28,8 +31,8 @@ type Datastore struct { verify int } -func New(d ds.Datastore, fileStorePath string) (*Datastore, error) { - return &Datastore{d, VerifyIfChanged}, nil +func New(d ds.Datastore, fileStorePath string, verify int) (*Datastore, error) { + return &Datastore{d, verify}, nil } func (d *Datastore) Put(key ds.Key, value interface{}) (err error) { @@ -131,6 +134,9 @@ func (d *Datastore) GetData(key ds.Key, val *DataObj, verify int, update bool) ( if val == nil { return nil, errors.New("Nil DataObj") } else if val.NoBlockData() { + if verify != VerifyIfChanged { + update = false + } file, err := os.Open(val.FilePath) if err != nil { return nil, err @@ -150,7 +156,7 @@ func (d *Datastore) GetData(key ds.Key, val *DataObj, verify int, update bool) ( } data, err = reconstruct(val.Data, buf) } - if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { return nil, err } modtime := val.ModTime @@ -163,12 +169,12 @@ func (d *Datastore) GetData(key ds.Key, val *DataObj, verify int, update bool) ( } invalid := val.Invalid() || err != nil if err == nil && (verify == VerifyAlways || (verify == VerifyIfChanged && modtime != val.ModTime)) { - //println("verifying") + log.Debugf("verifying block %s\n", b58.Encode(key.Bytes()[1:])) newKey := k.Key(u.Hash(data)).DsKey() invalid = newKey != key } if update && (invalid != val.Invalid() || modtime != val.ModTime) { - //println("updating") + log.Debugf("updating block %s\n", b58.Encode(key.Bytes()[1:])) newVal := *val newVal.SetInvalid(invalid) newVal.ModTime = modtime @@ -176,6 +182,7 @@ func (d *Datastore) GetData(key ds.Key, val *DataObj, verify int, update bool) ( _ = d.put(key, &newVal) } if invalid { + log.Debugf("invalid block %s\n", b58.Encode(key.Bytes()[1:])) return nil, InvalidBlock{} } else { return data, nil diff --git a/repo/config/config.go b/repo/config/config.go index d910ccf65cdc..9daa3b3b402d 100644 --- a/repo/config/config.go +++ b/repo/config/config.go @@ -19,6 +19,7 @@ var log = logging.Logger("config") type Config struct { Identity Identity // local node's peer identity Datastore Datastore // local node's storage + Filestore Filestore // local node's filestore Addresses Addresses // local node's addresses Mounts Mounts // local node's mount points Discovery Discovery // local node's discovery mechanisms diff --git a/repo/config/datastore.go b/repo/config/datastore.go index 52582bd5cb5c..edc2fc1ca8b0 100644 --- a/repo/config/datastore.go +++ b/repo/config/datastore.go @@ -38,3 +38,7 @@ type S3Datastore struct { func DataStorePath(configroot string) (string, error) { return Path(configroot, DefaultDataStoreDirectory) } + +type Filestore struct { + Verify string // one of "always", "ifchanged", "never" +} diff --git a/repo/fsrepo/defaultds.go b/repo/fsrepo/defaultds.go index 123dd44678dd..75e3e9399ffd 100644 --- a/repo/fsrepo/defaultds.go +++ b/repo/fsrepo/defaultds.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "path" + "strings" ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore/flatfs" @@ -71,15 +72,10 @@ func openDefaultDatastore(r *FSRepo) (repo.Datastore, *filestore.Datastore, erro var fileStore *filestore.Datastore if useFileStore { - fileStorePath := path.Join(r.path, fileStoreDir) - fileStoreDB, err := levelds.NewDatastore(fileStorePath, &levelds.Options{ - Compression: ldbopts.NoCompression, - }) + fileStore, err = r.newFilestore() if err != nil { - return nil, nil, fmt.Errorf("unable to open filestore: %v", err) + return nil, nil, err } - fileStore, _ = filestore.New(fileStoreDB, "") - //fileStore.(io.Closer).Close() blocksStore = multi.New(fileStore, metricsBlocks, nil, nil) } @@ -111,3 +107,27 @@ func initDefaultDatastore(repoPath string, conf *config.Config) error { } return nil } + +func (r *FSRepo) newFilestore() (*filestore.Datastore, error) { + fileStorePath := path.Join(r.path, fileStoreDir) + fileStoreDB, err := levelds.NewDatastore(fileStorePath, &levelds.Options{ + Compression: ldbopts.NoCompression, + }) + if err != nil { + return nil, fmt.Errorf("unable to open filestore: %v", err) + } + verify := filestore.VerifyIfChanged + switch strings.ToLower(r.config.Filestore.Verify) { + case "never": + verify = filestore.VerifyNever + case "": + case "ifchanged": + case "if changed": + verify = filestore.VerifyIfChanged + case "always": + verify = filestore.VerifyAlways + default: + return nil, fmt.Errorf("invalid value for Filestore.Verify: %s", r.config.Filestore.Verify) + } + return filestore.New(fileStoreDB, "", verify) +} diff --git a/test/sharness/t0262-filestore-config.sh b/test/sharness/t0262-filestore-config.sh new file mode 100755 index 000000000000..75aa12392958 --- /dev/null +++ b/test/sharness/t0262-filestore-config.sh @@ -0,0 +1,47 @@ +#!/bin/sh +# +# Copyright (c) 2014 Christian Couder +# MIT Licensed; see the LICENSE file in this repository. +# + +test_description="Test filestore" + +. lib/test-filestore-lib.sh +. lib/test-lib.sh + +test_init_ipfs + +test_add_cat_file "add --no-copy" "." + +export IPFS_LOGGING=debug +export IPFS_LOGGING_FMT=nocolor + +HASH="QmVr26fY1tKyspEJBniVhqxQeEjhF78XerGiqWAwraVLQH" + +test_expect_success "file always checked" ' + ipfs config Filestore.Verify always 2> log && + ipfs cat "$HASH" 2> log && + grep -q "verifying block $HASH" log && + ! grep -q "updating block $HASH" log +' + +test_expect_success "file checked after change" ' + ipfs config Filestore.Verify ifchanged 2> log && + echo "HELLO WORLDS!" >mountdir/hello.txt && + test_must_fail ipfs cat "$HASH" 2> log && + grep -q "verifying block $HASH" log && + grep -q "updating block $HASH" log +' + +test_expect_success "file never checked" ' + echo "Hello Worlds!" >mountdir/hello.txt && + ipfs add "$dir"/mountdir/hello.txt >actual 2> log && + ipfs config Filestore.Verify never 2> log && + echo "HELLO Worlds!" >mountdir/hello.txt && + ( ipfs cat "$HASH" || true ) 2> log && + grep -q "BlockService GetBlock" log && # Make sure we are still logging + ! grep -q "verifying block $HASH" log && + ! grep -q "updating block $HASH" log +' + +test_done