From 506bc95023443a163339626e1a00fa4c1cc23ac1 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Sat, 28 May 2016 17:40:20 -0400 Subject: [PATCH] "filestore verify": add ability to specify hashes to verify License: MIT Signed-off-by: Kevin Atkinson --- core/commands/filestore.go | 37 +++++++++++++----- filestore/util/verify.go | 78 +++++++++++++++++++++++++++++++++----- 2 files changed, 96 insertions(+), 19 deletions(-) diff --git a/core/commands/filestore.go b/core/commands/filestore.go index cab6bdeda3d9..a032236c4148 100644 --- a/core/commands/filestore.go +++ b/core/commands/filestore.go @@ -243,7 +243,10 @@ var verifyFileStore = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Verify objects in filestore", ShortDescription: ` -Verify nodes in the filestore. The output is: +Verify nodes in the filestore. If no hashes are specified then +verify everything in the filestore. + +The output is: [ []] where , , , and are the same as in the "ls" command and is one of @@ -255,8 +258,8 @@ as in the "ls" command and is one of incomplete: some of the blocks of the tree could not be read changed: the contents of the backing file have changed - no-file: the backing file can not be found - error: the backing file can be found but could not be read + no-file: the backing file could not be found + error: the backing file was found but could not be read ERROR: the block could not be read due to an internal error @@ -271,9 +274,8 @@ as in the "ls" command and is one of the filestore If --basic is specified then just scan leaf nodes to verify that they -are still valid. Otherwise attempt to reconstruct the contents of of -all nodes and also check for orphan nodes (unless --skip-orphans is -also specified). +are still valid. Otherwise attempt to reconstruct the contents of +all nodes and check for orphan nodes if applicable. The --level option specifies how thorough the checks should be. A current meaning of the levels are: @@ -284,11 +286,14 @@ current meaning of the levels are: The --verbose option specifies what to output. The current values are: 7-9: show everything - 5-6: don't show child nodes with a status of: ok, , or complete + 5-6: don't show child nodes unless there is a problem 3-4: don't show child nodes - 0-2: don't child nodes and don't show root nodes with of: ok or complete + 0-2: don't show root nodes unless there is a problem `, }, + Arguments: []cmds.Argument{ + cmds.StringArg("hash", false, true, "Hashs of nodes to verify."), + }, Options: []cmds.Option{ cmds.BoolOption("basic", "Perform a basic scan of leaf nodes only."), cmds.IntOption("level", "l", "0-9, Verification level.").Default(6), @@ -301,6 +306,11 @@ The --verbose option specifies what to output. The current values are: res.SetError(err, cmds.ErrNormal) return } + args := req.Arguments() + keys := make([]k.Key, 0) + for _, key := range args { + keys = append(keys, k.B58KeyDecode(key)) + } basic, _, err := req.Option("basic").Bool() if err != nil { res.SetError(err, cmds.ErrNormal) @@ -325,12 +335,19 @@ The --verbose option specifies what to output. The current values are: res.SetError(err, cmds.ErrNormal) return } - if basic { + + if basic && len(keys) == 0 { ch, _ := fsutil.VerifyBasic(fs, level, verbose) res.SetOutput(&chanWriter{ch: ch}) - } else { + } else if basic { + ch, _ := fsutil.VerifyKeys(keys, node, fs, level) + res.SetOutput(&chanWriter{ch: ch}) + } else if len(keys) == 0 { ch, _ := fsutil.VerifyFull(node, fs, level, verbose, skipOrphans) res.SetOutput(&chanWriter{ch: ch}) + } else { + ch, _ := fsutil.VerifyKeysFull(keys, node, fs, level, verbose) + res.SetOutput(&chanWriter{ch: ch}) } }, Marshalers: cmds.MarshalerMap{ diff --git a/filestore/util/verify.go b/filestore/util/verify.go index 2d311d56a80a..b3013e1e2dc9 100644 --- a/filestore/util/verify.go +++ b/filestore/util/verify.go @@ -3,6 +3,7 @@ package filestore_util import ( "os" + b "github.com/ipfs/go-ipfs/blocks/blockstore" k "github.com/ipfs/go-ipfs/blocks/key" "github.com/ipfs/go-ipfs/core" . "github.com/ipfs/go-ipfs/filestore" @@ -34,6 +35,42 @@ func VerifyBasic(fs *Datastore, level int, verbose int) (<-chan ListRes, error) return out, nil } +func VerifyKeys(keys []k.Key, node *core.IpfsNode, fs *Datastore, level int) (<-chan ListRes, error) { + out := make(chan ListRes, 16) + verifyWhat := VerifyAlways + if level <= 6 { + verifyWhat = VerifyIfChanged + } + go func() { + defer close(out) + for _, key := range keys { + out <- verifyKey(key, fs, node.Blockstore, verifyWhat) + } + }() + return out, nil +} + +func verifyKey(key k.Key, fs *Datastore, bs b.Blockstore, verifyWhat int) ListRes { + dsKey := key.DsKey() + dataObj, err := fs.GetDirect(dsKey) + if err == nil && dataObj.NoBlockData() { + res := ListRes{dsKey, dataObj, 0} + res.Status = verify(fs, dsKey, dataObj, verifyWhat) + return res + } else if err == nil { + return ListRes{dsKey, dataObj, StatusUnchecked} + } + found, _ := bs.Has(key) + if found { + return ListRes{dsKey, nil, StatusFound} + } else if err == ds.ErrNotFound && !found { + return ListRes{dsKey, nil, StatusKeyNotFound} + } else { + Logger.Errorf("%s: %v", key, err) + return ListRes{dsKey, nil, StatusError} + } +} + func VerifyFull(node *core.IpfsNode, fs *Datastore, level int, verbose int, skipOrphans bool) (<-chan ListRes, error) { p := verifyParams{make(chan ListRes, 16), node, fs, level, verbose, skipOrphans, nil} ch, err := ListKeys(p.fs) @@ -47,6 +84,15 @@ func VerifyFull(node *core.IpfsNode, fs *Datastore, level int, verbose int, skip return p.out, nil } +func VerifyKeysFull(keys []k.Key, node *core.IpfsNode, fs *Datastore, level int, verbose int) (<-chan ListRes, error) { + p := verifyParams{make(chan ListRes, 16), node, fs, level, verbose, true, nil} + go func() { + defer close(p.out) + p.verifyKeys(keys) + }() + return p.out, nil +} + type verifyParams struct { out chan ListRes node *core.IpfsNode @@ -77,6 +123,25 @@ func (p *verifyParams) setStatus(dsKey ds.Key, status int) { } } +func (p *verifyParams) verifyKeys(keys []k.Key) { + p.skipOrphans = true + for _, key := range keys { + dsKey := key.DsKey() + dagNode, dataObj, r := p.get(dsKey) + if dataObj == nil || AnError(r) { + /* nothing to do */ + } else if dataObj.Internal() { + r = p.verifyNode(dagNode) + } else { + r = p.verifyLeaf(dsKey, dataObj) + } + res := ListRes{dsKey, dataObj, r} + res.Status = p.checkIfAppended(res) + p.out <- res + p.out <- EmptyListRes + } +} + func (p *verifyParams) verify(ch <-chan ListRes) { p.seen = make(map[string]int) unsafeToCont := false @@ -89,15 +154,7 @@ func (p *verifyParams) verify(ch <-chan ListRes) { if AnError(r) { /* nothing to do */ } else if res.Internal() && res.WholeFile() { - if dagNode == nil { - // we expect a node, so even if the status is - // okay we should set it to an Error - if !AnError(r) { - r = StatusError - } - } else { - r = p.verifyNode(dagNode) - } + r = p.verifyNode(dagNode) } else if res.WholeFile() { r = p.verifyLeaf(res.Key, res.DataObj) } else { @@ -156,6 +213,9 @@ func (p *verifyParams) checkIfAppended(res ListRes) int { } func (p *verifyParams) verifyNode(n *node.Node) int { + if n == nil { + return StatusError + } complete := true for _, link := range n.Links { key := k.Key(link.Hash).DsKey()