Skip to content

Commit

Permalink
Add Filestore.Verify config option to control when block are verified.
Browse files Browse the repository at this point in the history
The default value if "ifchanged" and blocks are only verified when
the backing file has changed.

License: MIT
Signed-off-by: Kevin Atkinson <[email protected]>
  • Loading branch information
kevina committed May 18, 2016
1 parent cc52645 commit e8c4a1b
Show file tree
Hide file tree
Showing 5 changed files with 92 additions and 13 deletions.
19 changes: 13 additions & 6 deletions filestore/datastore.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,13 @@ import (
"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore/query"
k "github.com/ipfs/go-ipfs/blocks/key"
//mh "gx/ipfs/QmYf7ng2hG5XBtJA3tN34DQ2GUN5HNksEw1rLDkmr6vGku/go-multihash"
//b58 "gx/ipfs/QmT8rehPR3F6bmwL6zjUN8XpiDBFFpMP2myPdC6ApsWfJf/go-base58"
b58 "gx/ipfs/QmT8rehPR3F6bmwL6zjUN8XpiDBFFpMP2myPdC6ApsWfJf/go-base58"
u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
logging "gx/ipfs/QmaDNZ4QMdBdku1YZWBysufYyoQt1negQGNav6PLYarbY8/go-log"
)

var log = logging.Logger("filestore")

const (
VerifyNever = 0
VerifyIfChanged = 1
Expand All @@ -28,8 +31,8 @@ type Datastore struct {
verify int
}

func New(d ds.Datastore, fileStorePath string) (*Datastore, error) {
return &Datastore{d, VerifyIfChanged}, nil
func New(d ds.Datastore, fileStorePath string, verify int) (*Datastore, error) {
return &Datastore{d, verify}, nil
}

func (d *Datastore) Put(key ds.Key, value interface{}) (err error) {
Expand Down Expand Up @@ -131,6 +134,9 @@ func (d *Datastore) GetData(key ds.Key, val *DataObj, verify int, update bool) (
if val == nil {
return nil, errors.New("Nil DataObj")
} else if val.NoBlockData() {
if verify != VerifyIfChanged {
update = false
}
file, err := os.Open(val.FilePath)
if err != nil {
return nil, err
Expand All @@ -150,7 +156,7 @@ func (d *Datastore) GetData(key ds.Key, val *DataObj, verify int, update bool) (
}
data, err = reconstruct(val.Data, buf)
}
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
return nil, err
}
modtime := val.ModTime
Expand All @@ -163,19 +169,20 @@ func (d *Datastore) GetData(key ds.Key, val *DataObj, verify int, update bool) (
}
invalid := val.Invalid() || err != nil
if err == nil && (verify == VerifyAlways || (verify == VerifyIfChanged && modtime != val.ModTime)) {
//println("verifying")
log.Debugf("verifying block %s\n", b58.Encode(key.Bytes()[1:]))
newKey := k.Key(u.Hash(data)).DsKey()
invalid = newKey != key
}
if update && (invalid != val.Invalid() || modtime != val.ModTime) {
//println("updating")
log.Debugf("updating block %s\n", b58.Encode(key.Bytes()[1:]))
newVal := *val
newVal.SetInvalid(invalid)
newVal.ModTime = modtime
// ignore errors as they are nonfatal
_ = d.put(key, &newVal)
}
if invalid {
log.Debugf("invalid block %s\n", b58.Encode(key.Bytes()[1:]))
return nil, InvalidBlock{}
} else {
return data, nil
Expand Down
1 change: 1 addition & 0 deletions repo/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ var log = logging.Logger("config")
type Config struct {
Identity Identity // local node's peer identity
Datastore Datastore // local node's storage
Filestore Filestore // local node's filestore
Addresses Addresses // local node's addresses
Mounts Mounts // local node's mount points
Discovery Discovery // local node's discovery mechanisms
Expand Down
4 changes: 4 additions & 0 deletions repo/config/datastore.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,3 +38,7 @@ type S3Datastore struct {
func DataStorePath(configroot string) (string, error) {
return Path(configroot, DefaultDataStoreDirectory)
}

type Filestore struct {
Verify string // one of "always", "ifchanged", "never"
}
34 changes: 27 additions & 7 deletions repo/fsrepo/defaultds.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"io"
"path"
"strings"

ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore"
"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore/flatfs"
Expand Down Expand Up @@ -71,15 +72,10 @@ func openDefaultDatastore(r *FSRepo) (repo.Datastore, *filestore.Datastore, erro

var fileStore *filestore.Datastore
if useFileStore {
fileStorePath := path.Join(r.path, fileStoreDir)
fileStoreDB, err := levelds.NewDatastore(fileStorePath, &levelds.Options{
Compression: ldbopts.NoCompression,
})
fileStore, err = r.newFilestore()
if err != nil {
return nil, nil, fmt.Errorf("unable to open filestore: %v", err)
return nil, nil, err
}
fileStore, _ = filestore.New(fileStoreDB, "")
//fileStore.(io.Closer).Close()
blocksStore = multi.New(fileStore, metricsBlocks, nil, nil)
}

Expand Down Expand Up @@ -111,3 +107,27 @@ func initDefaultDatastore(repoPath string, conf *config.Config) error {
}
return nil
}

func (r *FSRepo) newFilestore() (*filestore.Datastore, error) {
fileStorePath := path.Join(r.path, fileStoreDir)
fileStoreDB, err := levelds.NewDatastore(fileStorePath, &levelds.Options{
Compression: ldbopts.NoCompression,
})
if err != nil {
return nil, fmt.Errorf("unable to open filestore: %v", err)
}
verify := filestore.VerifyIfChanged
switch strings.ToLower(r.config.Filestore.Verify) {
case "never":
verify = filestore.VerifyNever
case "":
case "ifchanged":
case "if changed":
verify = filestore.VerifyIfChanged
case "always":
verify = filestore.VerifyAlways
default:
return nil, fmt.Errorf("invalid value for Filestore.Verify: %s", r.config.Filestore.Verify)
}
return filestore.New(fileStoreDB, "", verify)
}
47 changes: 47 additions & 0 deletions test/sharness/t0262-filestore-config.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
#!/bin/sh
#
# Copyright (c) 2014 Christian Couder
# MIT Licensed; see the LICENSE file in this repository.
#

test_description="Test filestore"

. lib/test-filestore-lib.sh
. lib/test-lib.sh

test_init_ipfs

test_add_cat_file "add --no-copy" "."

export IPFS_LOGGING=debug
export IPFS_LOGGING_FMT=nocolor

HASH="QmVr26fY1tKyspEJBniVhqxQeEjhF78XerGiqWAwraVLQH"

test_expect_success "file always checked" '
ipfs config Filestore.Verify always 2> log &&
ipfs cat "$HASH" 2> log &&
grep -q "verifying block $HASH" log &&
! grep -q "updating block $HASH" log
'

test_expect_success "file checked after change" '
ipfs config Filestore.Verify ifchanged 2> log &&
echo "HELLO WORLDS!" >mountdir/hello.txt &&
test_must_fail ipfs cat "$HASH" 2> log &&
grep -q "verifying block $HASH" log &&
grep -q "updating block $HASH" log
'

test_expect_success "file never checked" '
echo "Hello Worlds!" >mountdir/hello.txt &&
ipfs add "$dir"/mountdir/hello.txt >actual 2> log &&
ipfs config Filestore.Verify never 2> log &&
echo "HELLO Worlds!" >mountdir/hello.txt &&
( ipfs cat "$HASH" || true ) 2> log &&
grep -q "BlockService GetBlock" log && # Make sure we are still logging
! grep -q "verifying block $HASH" log &&
! grep -q "updating block $HASH" log
'

test_done

0 comments on commit e8c4a1b

Please sign in to comment.