diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b63083c341..20d213728e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,33 @@ # go-ipfs changelog +### 0.4.3 - 2016-09-20 + +There have been no changes since the last release candidate 0.4.3-rc4. \o/ + +### 0.4.3-rc4 - 2016-09-09 + +This release candidate fixes issues in Bitswap and the `ipfs add` command, and improves testing. +We plan for this to be the last release candidate before the release of go-ipfs v0.4.3. + +With this release candidate, we're also moving go-ipfs to Go 1.7, which we expect will yield improvements in runtime performance, memory usage, build time and size of the release binaries. + +- Require Go 1.7. (@whyrusleeping, @Kubuxu, @lgierth, [ipfs/go-ipfs#3163](https://github.com/ipfs/go-ipfs/pull/3163)) + - For this purpose, switch Docker image from Alpine 3.4 to Alpine Edge. +- Fix cancellation of Bitswap `wantlist` entries. (@whyrusleeping, [ipfs/go-ipfs#3182](https://github.com/ipfs/go-ipfs/pull/3182)) +- Fix clearing of `active` state of Bitswap provider queries. (@whyrusleeping, [ipfs/go-ipfs#3169](https://github.com/ipfs/go-ipfs/pull/3169)) +- Fix a panic in the DHT code. (@Kubuxu, [ipfs/go-ipfs#3200](https://github.com/ipfs/go-ipfs/pull/3200)) +- Improve handling of `Identity` field in `ipfs config` command. (@Kubuxu, @whyrusleeping, [ipfs/go-ipfs#3141](https://github.com/ipfs/go-ipfs/pull/3141)) +- Fix explicit adding of symlinked files and directories. (@kevina, [ipfs/go-ipfs#3135](https://github.com/ipfs/go-ipfs/pull/3135)) +- Fix bash auto-completion of `ipfs daemon --unrestricted-api` option. (@lgierth, [ipfs/go-ipfs#3159](https://github.com/ipfs/go-ipfs/pull/3159)) +- Introduce a new timeout tool for tests to avoid licensing issues. (@Kubuxu, [ipfs/go-ipfs#3152](https://github.com/ipfs/go-ipfs/pull/3152)) +- Improve output for migrations of fs-repo. (@lgierth, [ipfs/go-ipfs#3158](https://github.com/ipfs/go-ipfs/pull/3158)) +- Fix info notice of commands taking input from stdin. (@Kubuxu, [ipfs/go-ipfs#3134](https://github.com/ipfs/go-ipfs/pull/3134)) +- Bring back a few tests for stdin handling of `ipfs cat` and `ipfs add`. (@Kubuxu, [ipfs/go-ipfs#3144](https://github.com/ipfs/go-ipfs/pull/3144)) +- Improve sharness tests for `ipfs repo verify` command. (@whyrusleeping, [ipfs/go-ipfs#3148](https://github.com/ipfs/go-ipfs/pull/3148)) +- Improve sharness tests for CORS headers on the gateway. (@Kubuxu, [ipfs/go-ipfs#3142](https://github.com/ipfs/go-ipfs/pull/3142)) +- Improve tests for pinning within `ipfs files`. (@kevina, [ipfs/go-ipfs#3151](https://github.com/ipfs/go-ipfs/pull/3151)) +- Improve tests for the automatic raising of file descriptor limits. (@whyrusleeping, [ipfs/go-ipfs#3149](https://github.com/ipfs/go-ipfs/pull/3149)) + ### 0.4.3-rc3 - 2016-08-09 This release candidate fixes a panic that occurs when input from stdin was diff --git a/blocks/blockstore/caching.go b/blocks/blockstore/caching.go index d482e045958..d1da0f72109 100644 --- a/blocks/blockstore/caching.go +++ b/blocks/blockstore/caching.go @@ -9,16 +9,16 @@ import ( // Next to each option is it aproximate memory usage per unit type CacheOpts struct { - HasBloomFilterSize int // 1 bit + HasBloomFilterSize int // 1 byte HasBloomFilterHashes int // No size, 7 is usually best, consult bloom papers HasARCCacheSize int // 32 bytes } func DefaultCacheOpts() CacheOpts { return CacheOpts{ - HasBloomFilterSize: 512 * 8 * 1024, + HasBloomFilterSize: 512 << 10, HasBloomFilterHashes: 7, - HasARCCacheSize: 64 * 1024, + HasARCCacheSize: 64 << 10, } } @@ -41,7 +41,8 @@ func CachedBlockstore(bs GCBlockstore, cbs, err = newARCCachedBS(ctx, cbs, opts.HasARCCacheSize) } if opts.HasBloomFilterSize != 0 { - cbs, err = bloomCached(cbs, ctx, opts.HasBloomFilterSize, opts.HasBloomFilterHashes) + // *8 because of bytes to bits conversion + cbs, err = bloomCached(cbs, ctx, opts.HasBloomFilterSize*8, opts.HasBloomFilterHashes) } return cbs, err diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index 0850432f3f2..8fb07a333c0 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -228,19 +228,25 @@ func daemonFunc(req cmds.Request, res cmds.Response) { return case fsrepo.ErrNeedMigration: domigrate, found, _ := req.Option(migrateKwd).Bool() - fmt.Println("Found old repo version, migrations need to be run.") + fmt.Println("Found outdated fs-repo, migrations need to be run.") if !found { - domigrate = YesNoPrompt("Run migrations automatically? [y/N]") + domigrate = YesNoPrompt("Run migrations now? [y/N]") } if !domigrate { - res.SetError(fmt.Errorf("please run the migrations manually"), cmds.ErrNormal) + fmt.Println("Not running migrations of fs-repo now.") + fmt.Println("Please get fs-repo-migrations from https://dist.ipfs.io") + res.SetError(fmt.Errorf("fs-repo requires migration"), cmds.ErrNormal) return } err = migrate.RunMigration(fsrepo.RepoVersion) if err != nil { + fmt.Println("The migrations of fs-repo failed:") + fmt.Printf(" %s\n", err) + fmt.Println("If you think this is a bug, please file an issue and include this whole log output.") + fmt.Println(" https://github.com/ipfs/fs-repo-migrations") res.SetError(err, cmds.ErrNormal) return } diff --git a/cmd/ipfs/ulimit_unix.go b/cmd/ipfs/ulimit_unix.go index 1fec41472e8..943c8318869 100644 --- a/cmd/ipfs/ulimit_unix.go +++ b/cmd/ipfs/ulimit_unix.go @@ -30,12 +30,15 @@ func checkAndSetUlimit() error { return fmt.Errorf("error getting rlimit: %s", err) } + var setting bool if rLimit.Cur < ipfsFileDescNum { if rLimit.Max < ipfsFileDescNum { + log.Error("adjusting max") rLimit.Max = ipfsFileDescNum } - fmt.Printf("Adjusting current ulimit to %d.\n", ipfsFileDescNum) + fmt.Printf("Adjusting current ulimit to %d...\n", ipfsFileDescNum) rLimit.Cur = ipfsFileDescNum + setting = true } err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit) @@ -43,5 +46,9 @@ func checkAndSetUlimit() error { return fmt.Errorf("error setting ulimit: %s", err) } + if setting { + fmt.Printf("Successfully raised file descriptor limit to %d.\n", ipfsFileDescNum) + } + return nil } diff --git a/commands/cli/parse.go b/commands/cli/parse.go index 3ac7ea40ea4..a70ab2eb8f1 100644 --- a/commands/cli/parse.go +++ b/commands/cli/parse.go @@ -251,7 +251,7 @@ func parseOpts(args []string, root *cmds.Command) ( return } -const msgStdinInfo = "ipfs: Reading from %s; send Ctrl-d to stop.\n" +const msgStdinInfo = "ipfs: Reading from %s; send Ctrl-d to stop." func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursive, hidden bool, root *cmds.Command) ([]string, []files.File, error) { // ignore stdin on Windows @@ -401,16 +401,14 @@ func appendFile(fpath string, argDef *cmds.Argument, recursive, hidden bool) (fi if err != nil { return nil, err } + cwd, err = filepath.EvalSymlinks(cwd) + if err != nil { + return nil, err + } fpath = cwd } - fpath = filepath.Clean(fpath) - fpath, err := filepath.EvalSymlinks(fpath) - if err != nil { - return nil, err - } - // Repeat ToSlash after EvalSymlinks as it turns path to platform specific - fpath = filepath.ToSlash(fpath) + fpath = filepath.ToSlash(filepath.Clean(fpath)) stat, err := os.Lstat(fpath) if err != nil { @@ -469,6 +467,7 @@ func newMessageReader(r io.ReadCloser, msg string) io.ReadCloser { func (r *messageReader) Read(b []byte) (int, error) { if !r.done { fmt.Fprintln(os.Stderr, r.message) + r.done = true } return r.r.Read(b) diff --git a/core/commands/config.go b/core/commands/config.go index fedae5292f0..cbdbf3e817b 100644 --- a/core/commands/config.go +++ b/core/commands/config.go @@ -15,6 +15,7 @@ import ( repo "github.com/ipfs/go-ipfs/repo" config "github.com/ipfs/go-ipfs/repo/config" fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo" + u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util" ) @@ -162,14 +163,12 @@ included in the output of this command. return } - idmap, ok := cfg["Identity"].(map[string]interface{}) - if !ok { - res.SetError(fmt.Errorf("config has no identity"), cmds.ErrNormal) + err = scrubValue(cfg, []string{config.IdentityTag, config.PrivKeyTag}) + if err != nil { + res.SetError(err, cmds.ErrNormal) return } - delete(idmap, "PrivKey") - output, err := config.HumanOutput(cfg) if err != nil { res.SetError(err, cmds.ErrNormal) @@ -180,6 +179,47 @@ included in the output of this command. }, } +func scrubValue(m map[string]interface{}, key []string) error { + find := func(m map[string]interface{}, k string) (string, interface{}, bool) { + lckey := strings.ToLower(k) + for mkey, val := range m { + lcmkey := strings.ToLower(mkey) + if lckey == lcmkey { + return mkey, val, true + } + } + return "", nil, false + } + + cur := m + for _, k := range key[:len(key)-1] { + foundk, val, ok := find(cur, k) + if !ok { + return fmt.Errorf("failed to find specified key") + } + + if foundk != k { + // case mismatch, calling this an error + return fmt.Errorf("case mismatch in config, expected %q but got %q", k, foundk) + } + + mval, mok := val.(map[string]interface{}) + if !mok { + return fmt.Errorf("%s was not a map", foundk) + } + + cur = mval + } + + todel, _, ok := find(cur, key[len(key)-1]) + if !ok { + return fmt.Errorf("%s, not found", strings.Join(key, ".")) + } + + delete(cur, todel) + return nil +} + var configEditCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Open the config file for editing in $EDITOR.", @@ -250,19 +290,10 @@ func getConfig(r repo.Repo, key string) (*ConfigField, error) { } func setConfig(r repo.Repo, key string, value interface{}) (*ConfigField, error) { - keyF, err := getConfig(r, "Identity.PrivKey") - if err != nil { - return nil, errors.New("failed to get PrivKey") - } - privkey := keyF.Value - err = r.SetConfigKey(key, value) + err := r.SetConfigKey(key, value) if err != nil { return nil, fmt.Errorf("failed to set config value: %s (maybe use --json?)", err) } - err = r.SetConfigKey("Identity.PrivKey", privkey) - if err != nil { - return nil, errors.New("failed to set PrivKey") - } return getConfig(r, key) } @@ -286,7 +317,7 @@ func replaceConfig(r repo.Repo, file io.Reader) error { return errors.New("setting private key with API is not supported") } - keyF, err := getConfig(r, "Identity.PrivKey") + keyF, err := getConfig(r, config.PrivKeySelector) if err != nil { return fmt.Errorf("Failed to get PrivKey") } diff --git a/docs/config.md b/docs/config.md index 0cc845e2905..87f100c7f9a 100644 --- a/docs/config.md +++ b/docs/config.md @@ -101,7 +101,7 @@ Default: `false` A boolean value. If set to true, all block reads from disk will be hashed and verified. This will cause increased CPU utilization. - `BloomFilterSize` -A number representing the size in bits of the blockstore's bloom filter. A value of zero represents the feature being disabled. +A number representing the size in bytes of the blockstore's bloom filter. A value of zero represents the feature being disabled. Default: `0` diff --git a/docs/releases.md b/docs/releases.md index 063b7d20e05..a25fe3274ef 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -12,7 +12,7 @@ will be allowed up to release day. - [ ] all tests pass (no exceptions) - [ ] webui works (for most definitions of 'works') - [ ] CHANGELOG.md has been updated - - use `LAST=v0.4.2 for n in $(git log --oneline --merges --reverse -n-1 $LAST...master | cut -d'#' -f2 | cut -d' ' -f1); do echo https://github.com/ipfs/go-ipfs/pull/$n; done` + - use `LAST=v0.4.2 ; for n in $(git log --oneline --merges --reverse -n-1 $LAST...master | cut -d'#' -f2 | cut -d' ' -f1); do echo https://github.com/ipfs/go-ipfs/pull/$n; done` - [ ] version string in `repo/config/version.go` has been updated - [ ] tag commit with vX.Y.Z - [ ] bump version string in `repo/config/version.go` to `vX.Y.Z-dev` diff --git a/exchange/bitswap/bitswap.go b/exchange/bitswap/bitswap.go index ed914b979e0..de2dce25d80 100644 --- a/exchange/bitswap/bitswap.go +++ b/exchange/bitswap/bitswap.go @@ -17,7 +17,6 @@ import ( bsmsg "github.com/ipfs/go-ipfs/exchange/bitswap/message" bsnet "github.com/ipfs/go-ipfs/exchange/bitswap/network" notifications "github.com/ipfs/go-ipfs/exchange/bitswap/notifications" - wantlist "github.com/ipfs/go-ipfs/exchange/bitswap/wantlist" flags "github.com/ipfs/go-ipfs/flags" "github.com/ipfs/go-ipfs/thirdparty/delay" loggables "gx/ipfs/QmYrv4LgCC8FhG2Ab4bwuq5DqBdwMtx3hMb3KKJDZcr2d7/go-libp2p-loggables" @@ -89,7 +88,7 @@ func New(parent context.Context, p peer.ID, network bsnet.BitSwapNetwork, notifications: notif, engine: decision.NewEngine(ctx, bstore), // TODO close the engine with Close() method network: network, - findKeys: make(chan *wantlist.Entry, sizeBatchRequestChan), + findKeys: make(chan *blockRequest, sizeBatchRequestChan), process: px, newBlocks: make(chan blocks.Block, HasBlockBufferSize), provideKeys: make(chan key.Key, provideKeysBufferSize), @@ -132,7 +131,7 @@ type Bitswap struct { notifications notifications.PubSub // send keys to a worker to find and connect to providers for them - findKeys chan *wantlist.Entry + findKeys chan *blockRequest engine *decision.Engine @@ -149,8 +148,8 @@ type Bitswap struct { } type blockRequest struct { - key key.Key - ctx context.Context + Key key.Key + Ctx context.Context } // GetBlock attempts to retrieve a particular block from peers within the @@ -240,13 +239,50 @@ func (bs *Bitswap) GetBlocks(ctx context.Context, keys []key.Key) (<-chan blocks // NB: Optimization. Assumes that providers of key[0] are likely to // be able to provide for all keys. This currently holds true in most // every situation. Later, this assumption may not hold as true. - req := &wantlist.Entry{ + req := &blockRequest{ Key: keys[0], Ctx: ctx, } + + remaining := make(map[key.Key]struct{}) + for _, k := range keys { + remaining[k] = struct{}{} + } + + out := make(chan blocks.Block) + go func() { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + defer close(out) + defer func() { + var toCancel []key.Key + for k, _ := range remaining { + toCancel = append(toCancel, k) + } + bs.CancelWants(toCancel) + }() + for { + select { + case blk, ok := <-promise: + if !ok { + return + } + + delete(remaining, blk.Key()) + select { + case out <- blk: + case <-ctx.Done(): + return + } + case <-ctx.Done(): + return + } + } + }() + select { case bs.findKeys <- req: - return promise, nil + return out, nil case <-ctx.Done(): return nil, ctx.Err() } diff --git a/exchange/bitswap/bitswap_test.go b/exchange/bitswap/bitswap_test.go index 5428d221c56..49785c6ce73 100644 --- a/exchange/bitswap/bitswap_test.go +++ b/exchange/bitswap/bitswap_test.go @@ -341,7 +341,6 @@ func TestDoubleGet(t *testing.T) { blocks := bg.Blocks(1) ctx1, cancel1 := context.WithCancel(context.Background()) - blkch1, err := instances[1].Exchange.GetBlocks(ctx1, []key.Key{blocks[0].Key()}) if err != nil { t.Fatal(err) @@ -369,11 +368,15 @@ func TestDoubleGet(t *testing.T) { t.Fatal(err) } - blk, ok := <-blkch2 - if !ok { - t.Fatal("expected to get the block here") + select { + case blk, ok := <-blkch2: + if !ok { + t.Fatal("expected to get the block here") + } + t.Log(blk) + case <-time.After(time.Second * 5): + t.Fatal("timed out waiting on block") } - t.Log(blk) for _, inst := range instances { err := inst.Exchange.Close() @@ -382,3 +385,68 @@ func TestDoubleGet(t *testing.T) { } } } + +func TestWantlistCleanup(t *testing.T) { + net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(kNetworkDelay)) + sg := NewTestSessionGenerator(net) + defer sg.Close() + bg := blocksutil.NewBlockGenerator() + + instances := sg.Instances(1)[0] + bswap := instances.Exchange + blocks := bg.Blocks(20) + + var keys []key.Key + for _, b := range blocks { + keys = append(keys, b.Key()) + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*50) + defer cancel() + _, err := bswap.GetBlock(ctx, keys[0]) + if err != context.DeadlineExceeded { + t.Fatal("shouldnt have fetched any blocks") + } + + time.Sleep(time.Millisecond * 50) + + if len(bswap.GetWantlist()) > 0 { + t.Fatal("should not have anyting in wantlist") + } + + ctx, cancel = context.WithTimeout(context.Background(), time.Millisecond*50) + defer cancel() + _, err = bswap.GetBlocks(ctx, keys[:10]) + if err != nil { + t.Fatal(err) + } + + <-ctx.Done() + time.Sleep(time.Millisecond * 50) + + if len(bswap.GetWantlist()) > 0 { + t.Fatal("should not have anyting in wantlist") + } + + _, err = bswap.GetBlocks(context.Background(), keys[:1]) + if err != nil { + t.Fatal(err) + } + + ctx, cancel = context.WithCancel(context.Background()) + _, err = bswap.GetBlocks(ctx, keys[10:]) + if err != nil { + t.Fatal(err) + } + + time.Sleep(time.Millisecond * 50) + if len(bswap.GetWantlist()) != 11 { + t.Fatal("should have 11 keys in wantlist") + } + + cancel() + time.Sleep(time.Millisecond * 50) + if !(len(bswap.GetWantlist()) == 1 && bswap.GetWantlist()[0] == keys[0]) { + t.Fatal("should only have keys[0] in wantlist") + } +} diff --git a/exchange/bitswap/decision/bench_test.go b/exchange/bitswap/decision/bench_test.go index 5a5a34587ad..eabc7cbeb95 100644 --- a/exchange/bitswap/decision/bench_test.go +++ b/exchange/bitswap/decision/bench_test.go @@ -21,6 +21,6 @@ func BenchmarkTaskQueuePush(b *testing.B) { } b.ResetTimer() for i := 0; i < b.N; i++ { - q.Push(wantlist.Entry{Key: key.Key(i), Priority: math.MaxInt32}, peers[i%len(peers)]) + q.Push(&wantlist.Entry{Key: key.Key(i), Priority: math.MaxInt32}, peers[i%len(peers)]) } } diff --git a/exchange/bitswap/decision/engine.go b/exchange/bitswap/decision/engine.go index 8f888851ff6..5a0b99c1948 100644 --- a/exchange/bitswap/decision/engine.go +++ b/exchange/bitswap/decision/engine.go @@ -104,7 +104,7 @@ func NewEngine(ctx context.Context, bs bstore.Blockstore) *Engine { return e } -func (e *Engine) WantlistForPeer(p peer.ID) (out []wl.Entry) { +func (e *Engine) WantlistForPeer(p peer.ID) (out []*wl.Entry) { e.lock.Lock() partner, ok := e.ledgerMap[p] if ok { @@ -233,7 +233,7 @@ func (e *Engine) MessageReceived(p peer.ID, m bsmsg.BitSwapMessage) error { for _, entry := range m.Wantlist() { if entry.Cancel { - log.Debugf("cancel %s", entry.Key) + log.Debugf("%s cancel %s", p, entry.Key) l.CancelWant(entry.Key) e.peerRequestQueue.Remove(entry.Key, p) } else { diff --git a/exchange/bitswap/decision/ledger.go b/exchange/bitswap/decision/ledger.go index dedbbb8e38d..887451dd831 100644 --- a/exchange/bitswap/decision/ledger.go +++ b/exchange/bitswap/decision/ledger.go @@ -87,7 +87,7 @@ func (l *ledger) CancelWant(k key.Key) { l.wantList.Remove(k) } -func (l *ledger) WantListContains(k key.Key) (wl.Entry, bool) { +func (l *ledger) WantListContains(k key.Key) (*wl.Entry, bool) { return l.wantList.Contains(k) } diff --git a/exchange/bitswap/decision/peer_request_queue.go b/exchange/bitswap/decision/peer_request_queue.go index c6eb045c145..6e301869fab 100644 --- a/exchange/bitswap/decision/peer_request_queue.go +++ b/exchange/bitswap/decision/peer_request_queue.go @@ -13,7 +13,7 @@ import ( type peerRequestQueue interface { // Pop returns the next peerRequestTask. Returns nil if the peerRequestQueue is empty. Pop() *peerRequestTask - Push(entry wantlist.Entry, to peer.ID) + Push(entry *wantlist.Entry, to peer.ID) Remove(k key.Key, p peer.ID) // NB: cannot expose simply expose taskQueue.Len because trashed elements @@ -45,7 +45,7 @@ type prq struct { } // Push currently adds a new peerRequestTask to the end of the list -func (tl *prq) Push(entry wantlist.Entry, to peer.ID) { +func (tl *prq) Push(entry *wantlist.Entry, to peer.ID) { tl.lock.Lock() defer tl.lock.Unlock() partner, ok := tl.partners[to] @@ -166,7 +166,7 @@ func (tl *prq) thawRound() { } type peerRequestTask struct { - Entry wantlist.Entry + Entry *wantlist.Entry Target peer.ID // A callback to signal that this task has been completed diff --git a/exchange/bitswap/decision/peer_request_queue_test.go b/exchange/bitswap/decision/peer_request_queue_test.go index 01e07baeed9..59fd9f27350 100644 --- a/exchange/bitswap/decision/peer_request_queue_test.go +++ b/exchange/bitswap/decision/peer_request_queue_test.go @@ -41,7 +41,7 @@ func TestPushPop(t *testing.T) { for _, index := range rand.Perm(len(alphabet)) { // add blocks for all letters letter := alphabet[index] t.Log(partner.String()) - prq.Push(wantlist.Entry{Key: key.Key(letter), Priority: math.MaxInt32 - index}, partner) + prq.Push(&wantlist.Entry{Key: key.Key(letter), Priority: math.MaxInt32 - index}, partner) } for _, consonant := range consonants { prq.Remove(key.Key(consonant), partner) @@ -78,10 +78,10 @@ func TestPeerRepeats(t *testing.T) { // Have each push some blocks for i := 0; i < 5; i++ { - prq.Push(wantlist.Entry{Key: key.Key(i)}, a) - prq.Push(wantlist.Entry{Key: key.Key(i)}, b) - prq.Push(wantlist.Entry{Key: key.Key(i)}, c) - prq.Push(wantlist.Entry{Key: key.Key(i)}, d) + prq.Push(&wantlist.Entry{Key: key.Key(i)}, a) + prq.Push(&wantlist.Entry{Key: key.Key(i)}, b) + prq.Push(&wantlist.Entry{Key: key.Key(i)}, c) + prq.Push(&wantlist.Entry{Key: key.Key(i)}, d) } // now, pop off four entries, there should be one from each diff --git a/exchange/bitswap/message/message.go b/exchange/bitswap/message/message.go index 23a9f14ed6d..8e65b369b17 100644 --- a/exchange/bitswap/message/message.go +++ b/exchange/bitswap/message/message.go @@ -64,7 +64,7 @@ func newMsg(full bool) *impl { } type Entry struct { - wantlist.Entry + *wantlist.Entry Cancel bool } @@ -120,7 +120,7 @@ func (m *impl) addEntry(k key.Key, priority int, cancel bool) { e.Cancel = cancel } else { m.wantlist[k] = Entry{ - Entry: wantlist.Entry{ + Entry: &wantlist.Entry{ Key: k, Priority: priority, }, diff --git a/exchange/bitswap/wantlist/wantlist.go b/exchange/bitswap/wantlist/wantlist.go index 2fcaf0c2909..9c31b4f38c8 100644 --- a/exchange/bitswap/wantlist/wantlist.go +++ b/exchange/bitswap/wantlist/wantlist.go @@ -7,8 +7,6 @@ import ( "sync" key "gx/ipfs/Qmce4Y4zg3sYr7xKM5UueS67vhNni6EeWgCRnb7MbLJMew/go-key" - - "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" ) type ThreadSafe struct { @@ -18,19 +16,17 @@ type ThreadSafe struct { // not threadsafe type Wantlist struct { - set map[key.Key]Entry + set map[key.Key]*Entry } type Entry struct { Key key.Key Priority int - Ctx context.Context - cancel func() RefCnt int } -type entrySlice []Entry +type entrySlice []*Entry func (es entrySlice) Len() int { return len(es) } func (es entrySlice) Swap(i, j int) { es[i], es[j] = es[j], es[i] } @@ -44,41 +40,41 @@ func NewThreadSafe() *ThreadSafe { func New() *Wantlist { return &Wantlist{ - set: make(map[key.Key]Entry), + set: make(map[key.Key]*Entry), } } -func (w *ThreadSafe) Add(k key.Key, priority int) { +func (w *ThreadSafe) Add(k key.Key, priority int) bool { w.lk.Lock() defer w.lk.Unlock() - w.Wantlist.Add(k, priority) + return w.Wantlist.Add(k, priority) } -func (w *ThreadSafe) AddEntry(e Entry) { +func (w *ThreadSafe) AddEntry(e *Entry) bool { w.lk.Lock() defer w.lk.Unlock() - w.Wantlist.AddEntry(e) + return w.Wantlist.AddEntry(e) } -func (w *ThreadSafe) Remove(k key.Key) { +func (w *ThreadSafe) Remove(k key.Key) bool { w.lk.Lock() defer w.lk.Unlock() - w.Wantlist.Remove(k) + return w.Wantlist.Remove(k) } -func (w *ThreadSafe) Contains(k key.Key) (Entry, bool) { +func (w *ThreadSafe) Contains(k key.Key) (*Entry, bool) { w.lk.RLock() defer w.lk.RUnlock() return w.Wantlist.Contains(k) } -func (w *ThreadSafe) Entries() []Entry { +func (w *ThreadSafe) Entries() []*Entry { w.lk.RLock() defer w.lk.RUnlock() return w.Wantlist.Entries() } -func (w *ThreadSafe) SortedEntries() []Entry { +func (w *ThreadSafe) SortedEntries() []*Entry { w.lk.RLock() defer w.lk.RUnlock() return w.Wantlist.SortedEntries() @@ -94,50 +90,50 @@ func (w *Wantlist) Len() int { return len(w.set) } -func (w *Wantlist) Add(k key.Key, priority int) { +func (w *Wantlist) Add(k key.Key, priority int) bool { if e, ok := w.set[k]; ok { e.RefCnt++ - return + return false } - ctx, cancel := context.WithCancel(context.Background()) - w.set[k] = Entry{ + w.set[k] = &Entry{ Key: k, Priority: priority, - Ctx: ctx, - cancel: cancel, RefCnt: 1, } + + return true } -func (w *Wantlist) AddEntry(e Entry) { - if _, ok := w.set[e.Key]; ok { - return +func (w *Wantlist) AddEntry(e *Entry) bool { + if ex, ok := w.set[e.Key]; ok { + ex.RefCnt++ + return false } w.set[e.Key] = e + return true } -func (w *Wantlist) Remove(k key.Key) { +func (w *Wantlist) Remove(k key.Key) bool { e, ok := w.set[k] if !ok { - return + return false } e.RefCnt-- if e.RefCnt <= 0 { delete(w.set, k) - if e.cancel != nil { - e.cancel() - } + return true } + return false } -func (w *Wantlist) Contains(k key.Key) (Entry, bool) { +func (w *Wantlist) Contains(k key.Key) (*Entry, bool) { e, ok := w.set[k] return e, ok } -func (w *Wantlist) Entries() []Entry { +func (w *Wantlist) Entries() []*Entry { var es entrySlice for _, e := range w.set { es = append(es, e) @@ -145,7 +141,7 @@ func (w *Wantlist) Entries() []Entry { return es } -func (w *Wantlist) SortedEntries() []Entry { +func (w *Wantlist) SortedEntries() []*Entry { var es entrySlice for _, e := range w.set { es = append(es, e) diff --git a/exchange/bitswap/wantmanager.go b/exchange/bitswap/wantmanager.go index 189c2e38e32..b39a3a3cca8 100644 --- a/exchange/bitswap/wantmanager.go +++ b/exchange/bitswap/wantmanager.go @@ -75,6 +75,7 @@ func (pm *WantManager) WantBlocks(ctx context.Context, ks []key.Key) { } func (pm *WantManager) CancelWants(ks []key.Key) { + log.Infof("cancel wants: %s", ks) pm.addEntries(context.TODO(), ks, true) } @@ -83,16 +84,17 @@ func (pm *WantManager) addEntries(ctx context.Context, ks []key.Key, cancel bool for i, k := range ks { entries = append(entries, &bsmsg.Entry{ Cancel: cancel, - Entry: wantlist.Entry{ + Entry: &wantlist.Entry{ Key: k, Priority: kMaxPriority - i, - Ctx: ctx, + RefCnt: 1, }, }) } select { case pm.incoming <- entries: case <-pm.ctx.Done(): + case <-ctx.Done(): } } @@ -241,33 +243,31 @@ func (pm *WantManager) Run() { case entries := <-pm.incoming: // add changes to our wantlist + var filtered []*bsmsg.Entry for _, e := range entries { if e.Cancel { - pm.wl.Remove(e.Key) + if pm.wl.Remove(e.Key) { + filtered = append(filtered, e) + } } else { - pm.wl.AddEntry(e.Entry) + if pm.wl.AddEntry(e.Entry) { + filtered = append(filtered, e) + } } } // broadcast those wantlist changes for _, p := range pm.peers { - p.addMessage(entries) + p.addMessage(filtered) } case <-tock.C: // resend entire wantlist every so often (REALLY SHOULDNT BE NECESSARY) var es []*bsmsg.Entry for _, e := range pm.wl.Entries() { - select { - case <-e.Ctx.Done(): - // entry has been cancelled - // simply continue, the entry will be removed from the - // wantlist soon enough - continue - default: - } es = append(es, &bsmsg.Entry{Entry: e}) } + for _, p := range pm.peers { p.outlk.Lock() p.out = bsmsg.New(true) diff --git a/exchange/bitswap/workers.go b/exchange/bitswap/workers.go index bc8ae1c391a..5332a201398 100644 --- a/exchange/bitswap/workers.go +++ b/exchange/bitswap/workers.go @@ -1,16 +1,15 @@ package bitswap import ( + "math/rand" "sync" "time" process "gx/ipfs/QmSF8fPo3jgVBAy8fpdjjYqgG87dkJgUprRBHRd2tmfgpP/goprocess" procctx "gx/ipfs/QmSF8fPo3jgVBAy8fpdjjYqgG87dkJgUprRBHRd2tmfgpP/goprocess/context" - context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" - - wantlist "github.com/ipfs/go-ipfs/exchange/bitswap/wantlist" logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" peer "gx/ipfs/QmWXjJo15p4pzT7cayEwZi2sWgJqLnGDof6ZGMh9xBgU1p/go-libp2p-peer" + context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" key "gx/ipfs/Qmce4Y4zg3sYr7xKM5UueS67vhNni6EeWgCRnb7MbLJMew/go-key" ) @@ -173,9 +172,17 @@ func (bs *Bitswap) rebroadcastWorker(parent context.Context) { } case <-broadcastSignal.C: // resend unfulfilled wantlist keys log.Event(ctx, "Bitswap.Rebroadcast.active") - for _, e := range bs.wm.wl.Entries() { - e := e - bs.findKeys <- &e + entries := bs.wm.wl.Entries() + if len(entries) == 0 { + continue + } + + // TODO: come up with a better strategy for determining when to search + // for new providers for blocks. + i := rand.Intn(len(entries)) + bs.findKeys <- &blockRequest{ + Key: entries[i].Key, + Ctx: ctx, } case <-parent.Done(): return @@ -185,33 +192,37 @@ func (bs *Bitswap) rebroadcastWorker(parent context.Context) { func (bs *Bitswap) providerQueryManager(ctx context.Context) { var activeLk sync.Mutex - active := make(map[key.Key]*wantlist.Entry) + kset := key.NewKeySet() for { select { case e := <-bs.findKeys: activeLk.Lock() - if _, ok := active[e.Key]; ok { + if kset.Has(e.Key) { activeLk.Unlock() continue } - active[e.Key] = e + kset.Add(e.Key) activeLk.Unlock() - go func(e *wantlist.Entry) { + go func(e *blockRequest) { child, cancel := context.WithTimeout(e.Ctx, providerRequestTimeout) defer cancel() providers := bs.network.FindProvidersAsync(child, e.Key, maxProvidersPerRequest) + wg := &sync.WaitGroup{} for p := range providers { + wg.Add(1) go func(p peer.ID) { + defer wg.Done() err := bs.network.ConnectTo(child, p) if err != nil { log.Debug("failed to connect to provider %s: %s", p, err) } }(p) } + wg.Wait() activeLk.Lock() - delete(active, e.Key) + kset.Remove(e.Key) activeLk.Unlock() }(e) diff --git a/misc/completion/ipfs-completion.bash b/misc/completion/ipfs-completion.bash index 0b88ea4c66e..1dc1838c722 100644 --- a/misc/completion/ipfs-completion.bash +++ b/misc/completion/ipfs-completion.bash @@ -104,7 +104,7 @@ _ipfs_config_show() _ipfs_daemon() { _ipfs_comp "--init --routing= --mount --writable --mount-ipfs= \ - --mount-ipns= --disable-transport-encryption \ + --mount-ipns= --unrestricted-api --disable-transport-encryption \ --help" } diff --git a/package.json b/package.json index bcdafa31273..487321baba8 100644 --- a/package.json +++ b/package.json @@ -256,6 +256,12 @@ "hash": "QmYvLYkYiVEi5LBHP2uFqiUaHqH7zWnEuRqoNEuGLNG6JB", "name": "go-libp2p-kad-dht", "version": "1.1.0" + }, + { + "author": "whyrusleeping", + "hash": "QmdCL8M8DXJdSRnwhpDhukX5r8ydjxnzPJpaKrFudDA8yn", + "name": "hang-fds", + "version": "0.0.0" } ], "gxVersion": "0.4.0", diff --git a/repo/config/identity.go b/repo/config/identity.go index edac9298338..61e11cf5b29 100644 --- a/repo/config/identity.go +++ b/repo/config/identity.go @@ -5,6 +5,10 @@ import ( ic "gx/ipfs/QmVoi5es8D5fNHZDqoW6DgDAEPEV5hQp8GBz161vZXiwpQ/go-libp2p-crypto" ) +const IdentityTag = "Identity" +const PrivKeyTag = "PrivKey" +const PrivKeySelector = IdentityTag + "." + PrivKeyTag + // Identity tracks the configuration of the local node's identity. type Identity struct { PeerID string diff --git a/repo/fsrepo/fsrepo.go b/repo/fsrepo/fsrepo.go index 0046f35dd83..b6224d04bdd 100644 --- a/repo/fsrepo/fsrepo.go +++ b/repo/fsrepo/fsrepo.go @@ -482,6 +482,14 @@ func (r *FSRepo) SetConfigKey(key string, value interface{}) error { return err } + // Load private key to guard against it being overwritten. + // NOTE: this is a temporary measure to secure this field until we move + // keys out of the config file. + pkval, err := common.MapGetKV(mapconf, config.PrivKeySelector) + if err != nil { + return err + } + // Get the type of the value associated with the key oldValue, err := common.MapGetKV(mapconf, key) ok := true @@ -523,6 +531,11 @@ func (r *FSRepo) SetConfigKey(key string, value interface{}) error { return err } + // replace private key, in case it was overwritten. + if err := common.MapSetKV(mapconf, config.PrivKeySelector, pkval); err != nil { + return err + } + // This step doubles as to validate the map against the struct // before serialization conf, err := config.FromMap(mapconf) diff --git a/repo/fsrepo/migrations/migrations.go b/repo/fsrepo/migrations/migrations.go index 47c96926dbf..4cd091d1ee7 100644 --- a/repo/fsrepo/migrations/migrations.go +++ b/repo/fsrepo/migrations/migrations.go @@ -37,7 +37,7 @@ func migrationsBinName() string { func RunMigration(newv int) error { migrateBin := migrationsBinName() - fmt.Println(" => checking for migrations binary...") + fmt.Println(" => Looking for suitable fs-repo-migrations binary.") var err error migrateBin, err = exec.LookPath(migrateBin) @@ -47,15 +47,17 @@ func RunMigration(newv int) error { } if err != nil { - fmt.Println(" => usable migrations not found on system, fetching...") + fmt.Println(" => None found, downloading.") + loc, err := GetMigrations() if err != nil { + fmt.Println(" => Failed to download fs-repo-migrations.") return err } err = verifyMigrationSupportsVersion(loc, newv) if err != nil { - return fmt.Errorf("no migration binary found that supports version %d - %s", newv, err) + return fmt.Errorf("no fs-repo-migration binary found for version %d: %s", newv, err) } migrateBin = loc @@ -65,14 +67,15 @@ func RunMigration(newv int) error { cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr - fmt.Printf(" => running migration: '%s -to %d -y'\n\n", migrateBin, newv) + fmt.Printf(" => Running: %s -to %d -y\n", migrateBin, newv) err = cmd.Run() if err != nil { + fmt.Printf(" => Failed: %s -to %d -y\n", migrateBin, newv) return fmt.Errorf("migration failed: %s", err) } - fmt.Println(" => migrations binary completed successfully") + fmt.Printf(" => Success: fs-repo has been migrated to version %d.\n", newv) return nil } @@ -80,21 +83,19 @@ func RunMigration(newv int) error { func GetMigrations() (string, error) { latest, err := GetLatestVersion(DistPath, migrations) if err != nil { - return "", fmt.Errorf("getting latest version of fs-repo-migrations: %s", err) + return "", fmt.Errorf("failed to find latest fs-repo-migrations: %s", err) } dir, err := ioutil.TempDir("", "go-ipfs-migrate") if err != nil { - return "", fmt.Errorf("tempdir: %s", err) + return "", fmt.Errorf("failed to create fs-repo-migrations tempdir: %s", err) } out := filepath.Join(dir, migrationsBinName()) err = GetBinaryForVersion(migrations, migrations, DistPath, latest, out) if err != nil { - fmt.Printf(" => error getting migrations binary: %s\n", err) - fmt.Println(" => could not find or install fs-repo-migrations, please manually install it") - return "", fmt.Errorf("failed to find migrations binary") + return "", fmt.Errorf("failed to download latest fs-repo-migrations: %s", err) } err = os.Chmod(out, 0755) @@ -184,7 +185,6 @@ func httpGet(url string) (*http.Response, error) { } func httpFetch(url string) (io.ReadCloser, error) { - fmt.Printf("fetching url: %s\n", url) resp, err := httpGet(url) if err != nil { return nil, err @@ -196,7 +196,7 @@ func httpFetch(url string) (io.ReadCloser, error) { return nil, fmt.Errorf("error reading error body: %s", err) } - return nil, fmt.Errorf("%s: %s", resp.Status, string(mes)) + return nil, fmt.Errorf("GET %s error: %s: %s", url, resp.Status, string(mes)) } return resp.Body, nil diff --git a/test/Makefile b/test/Makefile index 9ebee7afbdc..d7a320e58a6 100644 --- a/test/Makefile +++ b/test/Makefile @@ -1,11 +1,13 @@ BINS = bin/random bin/multihash bin/ipfs bin/pollEndpoint bin/iptb bin/go-sleep +BINS += bin/go-timeout IPFS_ROOT = ../ IPFS_CMD = ../cmd/ipfs RANDOM_SRC = ../Godeps/_workspace/src/github.com/jbenet/go-random RANDOM_FILES_SRC = ../Godeps/_workspace/src/github.com/jbenet/go-random-files POLLENDPOINT_SRC= ../thirdparty/pollEndpoint GOSLEEP_SRC = ./dependencies/go-sleep +GOTIMEOUT_SRC = ./dependencies/go-timeout export PATH := ../bin:${PATH} @@ -49,16 +51,28 @@ bin/go-sleep: $(call find_go_files, $(GOSLEEP_SRC)) IPFS-BUILD-OPTIONS @echo "*** installing $@ ***" go build $(GOFLAGS) -o bin/go-sleep $(GOSLEEP_SRC) +bin/go-timeout: $(call find_go_files, $(GOTIMEOUT_SRC)) IPFS-BUILD-OPTIONS + @echo "*** installing $@ ***" + go build $(GOFLAGS) -o bin/go-timeout $(GOTIMEOUT_SRC) + # gx dependencies multihash_src: $(eval MULTIHASH_HASH := $(shell cd .. && bin/gx deps find go-multihash)) $(eval MULTIHASH_SRC := gx/ipfs/$(MULTIHASH_HASH)/go-multihash) +hang-fds_src: + $(eval HANG_FDS_HASH := $(shell cd .. && bin/gx deps find hang-fds)) + $(eval HANG_FDS_SRC := gx/ipfs/$(HANG_FDS_HASH)/hang-fds) + bin/multihash: multihash_src $(call find_go_files, $(MULTIHASH_SRC)) IPFS-BUILD-OPTIONS @echo "*** installing $@ ***" go build $(GOFLAGS) -o bin/multihash $(MULTIHASH_SRC)/multihash +bin/hang-fds: hang-fds_src $(call find_go_files, $(HANG_FDS_SRC)) IPFS-BUILD-OPTIONS + @echo "*** installing $@ ***" + go build $(GOFLAGS) -o bin/hang-fds $(HANG_FDS_SRC) + iptb_src: $(eval IPTB_HASH := $(shell cd .. && bin/gx deps find iptb)) $(eval IPTB_SRC := gx/ipfs/$(IPTB_HASH)/iptb) diff --git a/test/bin/.gitignore b/test/bin/.gitignore index 8eec62514ef..c032badeef4 100644 --- a/test/bin/.gitignore +++ b/test/bin/.gitignore @@ -8,4 +8,3 @@ !checkflags !continueyn !verify-go-fmt.sh -!time-out diff --git a/test/bin/time-out b/test/bin/time-out deleted file mode 100755 index 47d863bd0f5..00000000000 --- a/test/bin/time-out +++ /dev/null @@ -1,83 +0,0 @@ -#!/bin/bash -# -# The Bash shell script executes a command with a time-out. -# Upon time-out expiration SIGTERM (15) is sent to the process. If the signal -# is blocked, then the subsequent SIGKILL (9) terminates it. -# -# Based on the Bash documentation example. - -scriptName="${0##*/}" - -declare -i DEFAULT_TIMEOUT=9 -declare -i DEFAULT_INTERVAL=1 -declare -i DEFAULT_DELAY=1 - -# Timeout. -declare -i timeout=DEFAULT_TIMEOUT -# Interval between checks if the process is still alive. -declare -i interval=DEFAULT_INTERVAL -# Delay between posting the SIGTERM signal and destroying the process by SIGKILL. -declare -i delay=DEFAULT_DELAY - -function printUsage() { - cat < 0)); do - sleep $interval - kill -0 $$ || exit 0 - ((t -= interval)) - done - - # Be nice, post SIGTERM first. - # The 'exit 0' below will be executed if any preceeding command fails. - kill -s SIGTERM $$ && kill -0 $$ || exit 0 - sleep $delay - kill -s SIGKILL $$ -) 2> /dev/null & - -exec "$@" diff --git a/test/dependencies/go-timeout/.gitignore b/test/dependencies/go-timeout/.gitignore new file mode 100644 index 00000000000..46c758b1daa --- /dev/null +++ b/test/dependencies/go-timeout/.gitignore @@ -0,0 +1 @@ +go-timeout diff --git a/test/dependencies/go-timeout/LICENSE b/test/dependencies/go-timeout/LICENSE new file mode 100644 index 00000000000..5c30e8cce46 --- /dev/null +++ b/test/dependencies/go-timeout/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2016 Jakub "Kubuxu" Sztandera + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + diff --git a/test/dependencies/go-timeout/main.go b/test/dependencies/go-timeout/main.go new file mode 100644 index 00000000000..326267c8890 --- /dev/null +++ b/test/dependencies/go-timeout/main.go @@ -0,0 +1,55 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/exec" + "strconv" + "syscall" + "time" +) + +func main() { + if len(os.Args) < 3 { + fmt.Fprintf(os.Stderr, + "Usage: %s \n", os.Args[0]) + os.Exit(1) + } + timeout, err := strconv.ParseUint(os.Args[1], 10, 32) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } + ctx, _ := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second) + + cmd := exec.CommandContext(ctx, os.Args[2], os.Args[3:]...) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err = cmd.Start() + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + } + err = cmd.Wait() + + if err != nil { + if ctx.Err() != nil { + os.Exit(124) + } else { + exitErr, ok := err.(*exec.ExitError) + if !ok { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(255) + } + waits, ok := exitErr.Sys().(syscall.WaitStatus) + if !ok { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(255) + } + os.Exit(waits.ExitStatus()) + } + } else { + os.Exit(0) + } +} diff --git a/test/sharness/Makefile b/test/sharness/Makefile index 7a691c6d421..922e1ecbb67 100644 --- a/test/sharness/Makefile +++ b/test/sharness/Makefile @@ -8,7 +8,7 @@ T = $(sort $(wildcard t[0-9][0-9][0-9][0-9]-*.sh)) BINS = bin/random bin/multihash bin/ipfs bin/pollEndpoint \ - bin/iptb bin/go-sleep bin/random-files + bin/iptb bin/go-sleep bin/random-files bin/go-timeout bin/hang-fds SHARNESS = lib/sharness/sharness.sh IPFS_ROOT = ../.. diff --git a/test/sharness/t0021-config.sh b/test/sharness/t0021-config.sh index 1fc35e176b2..3366df9e6e2 100755 --- a/test/sharness/t0021-config.sh +++ b/test/sharness/t0021-config.sh @@ -111,10 +111,10 @@ test_config_cmd() { test_expect_success "'ipfs config replace' injects privkey back" ' ipfs config replace show_config && - grep "\"PrivKey\":" "$IPFS_PATH/config" | grep -e ": \".\+\"" >/dev/null + grep "\"PrivKey\":" "$IPFS_PATH/config" | grep -e ": \".\+\"" >/dev/null ' - test_expect_success "'ipfs config replace' with privkey erors out" ' + test_expect_success "'ipfs config replace' with privkey errors out" ' cp "$IPFS_PATH/config" real_config && test_expect_code 1 ipfs config replace - < real_config 2> replace_out ' @@ -124,6 +124,16 @@ test_config_cmd() { test_cmp replace_out replace_expected ' + test_expect_success "'ipfs config replace' with lower case privkey errors out" ' + cp "$IPFS_PATH/config" real_config && + sed -i -e '\''s/PrivKey/privkey/'\'' real_config && + test_expect_code 1 ipfs config replace - < real_config 2> replace_out + ' + + test_expect_success "output looks good" ' + echo "Error: setting private key with API is not supported" > replace_expected + test_cmp replace_out replace_expected + ' } test_init_ipfs diff --git a/test/sharness/t0040-add-and-cat.sh b/test/sharness/t0040-add-and-cat.sh index 92f001762eb..04f535cd7ba 100755 --- a/test/sharness/t0040-add-and-cat.sh +++ b/test/sharness/t0040-add-and-cat.sh @@ -189,6 +189,18 @@ test_add_named_pipe() { ' } +test_add_pwd_is_symlink() { + test_expect_success "ipfs add -r adds directory content when ./ is symlink" ' + mkdir hellodir && + echo "World" > hellodir/world && + ln -s hellodir hellolink && + ( cd hellolink && + ipfs add -r . > ../actual ) && + grep "added Qma9CyFdG5ffrZCcYSin2uAETygB25cswVwEYYzwfQuhTe" actual && + rm -r hellodir + ' +} + test_launch_ipfs_daemon_and_mount test_expect_success "'ipfs add --help' succeeds" ' @@ -344,6 +356,25 @@ test_expect_success "ipfs cat output looks good" ' test_cmp expected actual ' +test_expect_success "ipfs cat with both arg and stdin" ' + echo "$MARS" | ipfs cat "$VENUS" >actual +' + +test_expect_success "ipfs cat output looks good" ' + cat mountdir/planets/venus.txt >expected && + test_cmp expected actual +' + +test_expect_success "ipfs cat with two args and stdin" ' + echo "$MARS" | ipfs cat "$VENUS" "$VENUS" >actual +' + +test_expect_success "ipfs cat output looks good" ' + cat mountdir/planets/venus.txt mountdir/planets/venus.txt >expected && + test_cmp expected actual +' + + test_expect_success "go-random is installed" ' type random ' @@ -354,6 +385,8 @@ test_add_cat_expensive test_add_named_pipe " Post http://$API_ADDR/api/v0/add?encoding=json&progress=true&r=true&stream-channels=true:" +test_add_pwd_is_symlink + test_kill_ipfs_daemon # should work offline @@ -371,6 +404,8 @@ test_expect_success "ipfs cat file fails" ' test_add_named_pipe "" +test_add_pwd_is_symlink + # Test daemon in offline mode test_launch_ipfs_daemon --offline diff --git a/test/sharness/t0044-add-symlink.sh b/test/sharness/t0044-add-symlink.sh index b7111bbfd39..e2f5f416481 100755 --- a/test/sharness/t0044-add-symlink.sh +++ b/test/sharness/t0044-add-symlink.sh @@ -11,13 +11,9 @@ test_description="Test add -w" test_expect_success "creating files succeeds" ' mkdir -p files/foo && mkdir -p files/bar && - mkdir -p files/badin echo "some text" > files/foo/baz && - ln -s ../foo/baz files/bar/baz && - ln -s files/does/not/exist files/badin/bad && - mkdir -p files2/a/b/c && - echo "some other text" > files2/a/b/c/foo && - ln -s b files2/a/d + ln -s files/foo/baz files/bar/baz && + ln -s files/does/not/exist files/bad ' test_add_symlinks() { @@ -27,21 +23,21 @@ test_add_symlinks() { ' test_expect_success "output looks good" ' - echo QmQRgZT6xVFKJLVVpJDu3WcPkw2iqQ1jqK1F9jmdeq9zAv > filehash_exp && + echo QmWdiHKoeSW8G1u7ATCgpx4yMoUhYaJBQGkyPLkS9goYZ8 > filehash_exp && test_cmp filehash_exp filehash_out ' - test_expect_success "adding a symlink adds the file itself" ' + test_expect_success "adding a symlink adds the link itself" ' ipfs add -q files/bar/baz > goodlink_out ' test_expect_success "output looks good" ' - echo QmcPNXE5zjkWkM24xQ7Bi3VAm8fRxiaNp88jFsij7kSQF1 > goodlink_exp && + echo "QmdocmZeF7qwPT9Z8SiVhMSyKA2KKoA2J7jToW6z6WBmxR" > goodlink_exp && test_cmp goodlink_exp goodlink_out ' test_expect_success "adding a broken symlink works" ' - ipfs add -qr files/badin | head -1 > badlink_out + ipfs add -q files/bad > badlink_out ' test_expect_success "output looks good" ' @@ -51,6 +47,9 @@ test_add_symlinks() { test_expect_success "adding with symlink in middle of path is same as\ adding with no symlink" ' + mkdir -p files2/a/b/c && + echo "some other text" > files2/a/b/c/foo && + ln -s b files2/a/d ipfs add -rq files2/a/b/c > no_sym && ipfs add -rq files2/a/d/c > sym && test_cmp no_sym sym diff --git a/test/sharness/t0060-daemon.sh b/test/sharness/t0060-daemon.sh index 677fbe5aa12..beae67eb53d 100755 --- a/test/sharness/t0060-daemon.sh +++ b/test/sharness/t0060-daemon.sh @@ -121,12 +121,20 @@ test_expect_success "daemon with pipe eventually becomes live" ' test_fsh cat stdin_daemon_out || test_fsh cat stdin_daemon_err || test_fsh cat stdin_poll_apiout || test_fsh cat stdin_poll_apierr ' -ulimit -n 512 +ulimit -S -n 512 TEST_ULIMIT_PRESET=1 test_launch_ipfs_daemon test_expect_success "daemon raised its fd limit" ' - grep "ulimit" actual_daemon > /dev/null + grep "raised file descriptor limit to 1024." actual_daemon > /dev/null +' + +test_expect_success "daemon actually can handle 1024 file descriptors" ' + hang-fds -hold=2s 1000 '$API_MADDR' +' + +test_expect_success "daemon didnt throw any errors" ' + test_expect_code 1 grep "too many open files" daemon_err ' test_kill_ipfs_daemon diff --git a/test/sharness/t0066-migration.sh b/test/sharness/t0066-migration.sh index 78689f32367..facf0d0ae2c 100755 --- a/test/sharness/t0066-migration.sh +++ b/test/sharness/t0066-migration.sh @@ -23,11 +23,11 @@ test_expect_success "manually reset repo version to 3" ' ' test_expect_success "ipfs daemon --migrate=false fails" ' - test_expect_code 1 ipfs daemon --migrate=false 2> false_out + test_expect_code 1 ipfs daemon --migrate=false > false_out ' test_expect_success "output looks good" ' - grep "please run the migrations manually" false_out + grep "Please get fs-repo-migrations from https://dist.ipfs.io" false_out ' test_expect_success "ipfs daemon --migrate=true runs migration" ' @@ -35,8 +35,8 @@ test_expect_success "ipfs daemon --migrate=true runs migration" ' ' test_expect_success "output looks good" ' - grep "running migration" true_out > /dev/null && - grep "binary completed successfully" true_out > /dev/null + grep "Running: " true_out > /dev/null && + grep "Success: fs-repo has been migrated to version 4." true_out > /dev/null ' test_expect_success "'ipfs daemon' prompts to auto migrate" ' @@ -44,9 +44,9 @@ test_expect_success "'ipfs daemon' prompts to auto migrate" ' ' test_expect_success "output looks good" ' - grep "Found old repo version" daemon_out > /dev/null && - grep "Run migrations automatically?" daemon_out > /dev/null && - grep "please run the migrations manually" daemon_err > /dev/null + grep "Found outdated fs-repo" daemon_out > /dev/null && + grep "Run migrations now?" daemon_out > /dev/null && + grep "Please get fs-repo-migrations from https://dist.ipfs.io" daemon_out > /dev/null ' test_done diff --git a/test/sharness/t0086-repo-verify.sh b/test/sharness/t0086-repo-verify.sh new file mode 100755 index 00000000000..f30efb1ebcd --- /dev/null +++ b/test/sharness/t0086-repo-verify.sh @@ -0,0 +1,60 @@ +#!/bin/sh +# +# Copyright (c) 2016 Jeromy Johnson +# MIT Licensed; see the LICENSE file in this repository. +# + +test_description="Test ipfs repo fsck" + +. lib/test-lib.sh + +test_init_ipfs + +sort_rand() { + case `uname` in + Linux) + sort -R + ;; + Darwin) + ruby -e 'puts STDIN.readlines.shuffle' + ;; + *) + echo "unsupported system: $(uname)" + esac +} + +check_random_corruption() { + to_break=$(find "$IPFS_PATH/blocks" -type f | sort_rand | head -n 1) + + test_expect_success "back up file and overwrite it" ' + cp "$to_break" backup_file && + echo "this is super broken" > "$to_break" + ' + + test_expect_success "repo verify detects failure" ' + test_expect_code 1 ipfs repo verify + ' + + test_expect_success "replace the object" ' + cp backup_file "$to_break" + ' + + test_expect_success "ipfs repo verify passes just fine now" ' + ipfs repo verify + ' +} + +test_expect_success "create some files" ' + random-files -depth=3 -dirs=4 -files=10 foobar > /dev/null +' + +test_expect_success "add them all" ' + ipfs add -r -q foobar > /dev/null +' + +for i in `seq 20` +do + check_random_corruption +done + +test_done diff --git a/test/sharness/t0112-gateway-cors.sh b/test/sharness/t0112-gateway-cors.sh index dbb51021ae8..d118abe1656 100755 --- a/test/sharness/t0112-gateway-cors.sh +++ b/test/sharness/t0112-gateway-cors.sh @@ -28,10 +28,12 @@ thash='QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn' test_expect_success "GET to Gateway succeeds" ' curl -svX GET "http://127.0.0.1:$gwport/ipfs/$thash" 2>curl_output ' + +cat curl_output # GET Response from Gateway should contain CORS headers test_expect_success "GET response for Gateway resource looks good" ' - grep "Access-Control-Allow-Origin:" curl_output && - grep "Access-Control-Allow-Methods:" curl_output && + grep "Access-Control-Allow-Origin:" curl_output | grep "\*" && + grep "Access-Control-Allow-Methods:" curl_output | grep " GET\b" && grep "Access-Control-Allow-Headers:" curl_output ' @@ -41,8 +43,8 @@ test_expect_success "OPTIONS to Gateway succeeds" ' ' # OPTION Response from Gateway should contain CORS headers test_expect_success "OPTIONS response for Gateway resource looks good" ' - grep "Access-Control-Allow-Origin:" curl_output && - grep "Access-Control-Allow-Methods:" curl_output && + grep "Access-Control-Allow-Origin:" curl_output | grep "\*" && + grep "Access-Control-Allow-Methods:" curl_output | grep " GET\b" && grep "Access-Control-Allow-Headers:" curl_output ' diff --git a/test/sharness/t0220-bitswap.sh b/test/sharness/t0220-bitswap.sh index 67ba2f33773..6278bf2f3ee 100755 --- a/test/sharness/t0220-bitswap.sh +++ b/test/sharness/t0220-bitswap.sh @@ -11,13 +11,6 @@ test_description="test bitswap commands" test_init_ipfs test_launch_ipfs_daemon -test_expect_success "'ipfs block get' adds hash to wantlist" ' - export NONEXIST=QmeXxaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa && - test_expect_code 1 ipfs block get $NONEXIST --timeout=10ms && - ipfs bitswap wantlist >wantlist_out && - grep $NONEXIST wantlist_out -' - test_expect_success "'ipfs bitswap stat' succeeds" ' ipfs bitswap stat >stat_out ' @@ -29,8 +22,7 @@ bitswap status blocks received: 0 dup blocks received: 0 dup data received: 0 B - wantlist [1 keys] - $NONEXIST + wantlist [0 keys] partners [0] EOF test_cmp expected stat_out @@ -45,12 +37,8 @@ test_expect_success "'ipfs bitswap wantlist -p' works" ' ipfs bitswap wantlist -p "$PEERID" >wantlist_p_out ' -test_expect_failure "'ipfs bitswap wantlist -p' output looks good" ' - test_cmp wantlist_out wantlist_p_out -' - -test_expect_success "'ipfs bitswap unwant' succeeds" ' - ipfs bitswap unwant $NONEXIST +test_expect_success "'ipfs bitswap wantlist -p' output looks good" ' + test_must_be_empty wantlist_p_out ' test_expect_success "hash was removed from wantlist" ' diff --git a/test/sharness/t0252-files-gc.sh b/test/sharness/t0252-files-gc.sh index 68c720e1a80..10928c4247a 100755 --- a/test/sharness/t0252-files-gc.sh +++ b/test/sharness/t0252-files-gc.sh @@ -11,13 +11,15 @@ test_description="test how the unix files api interacts with the gc" test_init_ipfs test_expect_success "object not removed after gc" ' - echo "hello world" | ipfs files write --create /hello.txt && + echo "hello world" > hello.txt && + cat hello.txt | ipfs files write --create /hello.txt && ipfs repo gc && ipfs cat QmVib14uvPnCP73XaCDpwugRuwfTsVbGyWbatHAmLSdZUS ' test_expect_success "/hello.txt still accessible after gc" ' - ipfs files read /hello.txt + ipfs files read /hello.txt > hello-actual && + test_cmp hello.txt hello-actual ' ADIR_HASH=QmbCgoMYVuZq8m1vK31JQx9DorwQdLMF1M3sJ7kygLLqnW @@ -27,7 +29,6 @@ test_expect_success "gc okay after adding incomplete node -- prep" ' ipfs files mkdir /adir && echo "file1" | ipfs files write --create /adir/file1 && echo "file2" | ipfs files write --create /adir/file2 && - ipfs cat $FILE1_HASH && ipfs pin add --recursive=false $ADIR_HASH && ipfs files rm -r /adir && ipfs repo gc && # will remove /adir/file1 and /adir/file2 but not /adir @@ -42,4 +43,34 @@ test_expect_success "gc okay after adding incomplete node" ' ipfs refs $ADIR_HASH ' +test_expect_success "add directory with direct pin" ' + mkdir mydir/ && + echo "hello world!" > mydir/hello.txt && + FILE_UNPINNED=$(ipfs add --pin=false -q -r mydir/hello.txt) && + DIR_PINNED=$(ipfs add --pin=false -q -r mydir | tail -n1) && + ipfs add --pin=false -r mydir && + ipfs pin add --recursive=false $DIR_PINNED && + ipfs cat $FILE_UNPINNED +' + +test_expect_success "run gc and make sure directory contents are removed" ' + ipfs repo gc && + test_must_fail ipfs cat $FILE_UNPINNED +' + +test_expect_success "add incomplete directory and make sure gc is okay" ' + ipfs files cp /ipfs/$DIR_PINNED /mydir && + ipfs repo gc && + test_must_fail ipfs cat $FILE_UNPINNED +' + +test_expect_success "add back directory contents and run gc" ' + ipfs add --pin=false mydir/hello.txt && + ipfs repo gc +' + +test_expect_success "make sure directory contents are not removed" ' + ipfs cat $FILE_UNPINNED +' + test_done diff --git a/test/sharness/t0500-issues-and-regressions-offline.sh b/test/sharness/t0500-issues-and-regressions-offline.sh index 802adea6a8a..bfab467e53e 100755 --- a/test/sharness/t0500-issues-and-regressions-offline.sh +++ b/test/sharness/t0500-issues-and-regressions-offline.sh @@ -8,17 +8,17 @@ test_description="Tests for various fixed issues and regressions." test_expect_success "ipfs init with occupied input works - #2748" ' export IPFS_PATH="ipfs_path" - echo "" | time-out ipfs init && + echo "" | go-timeout 10 ipfs init && rm -rf ipfs_path ' test_init_ipfs -test_expect_success "ipfs cat --help succeeds with no input" ' - time-out ipfs cat --help +test_expect_success "ipfs cat --help succeeds when input remains open" ' + yes | go-timeout 1 ipfs cat --help ' -test_expect_success "ipfs pin ls --help succeeds with no input" ' - time-out ipfs pin ls --help +test_expect_success "ipfs pin ls --help succeeds when input remains open" ' + yes | go-timeout 1 ipfs pin ls --help ' test_expect_success "ipfs add on 1MB from stdin woks" '