Merge pull request #3248 from ipfs/version/0.4.3-rc4

Merge 0.4.3 release candidate changes back into master

CHANGELOG.md

@@ -1,5 +1,33 @@
 # go-ipfs changelog
 
+### 0.4.3 - 2016-09-20
+
+There have been no changes since the last release candidate 0.4.3-rc4. \o/
+
+### 0.4.3-rc4 - 2016-09-09
+
+This release candidate fixes issues in Bitswap and the `ipfs add` command, and improves testing.
+We plan for this to be the last release candidate before the release of go-ipfs v0.4.3.
+
+With this release candidate, we're also moving go-ipfs to Go 1.7, which we expect will yield improvements in runtime performance, memory usage, build time and size of the release binaries.
+
+- Require Go 1.7. (@whyrusleeping, @Kubuxu, @lgierth, [ipfs/go-ipfs#3163](https://github.com/ipfs/go-ipfs/pull/3163))
+  - For this purpose, switch Docker image from Alpine 3.4 to Alpine Edge.
+- Fix cancellation of Bitswap `wantlist` entries. (@whyrusleeping, [ipfs/go-ipfs#3182](https://github.com/ipfs/go-ipfs/pull/3182))
+- Fix clearing of `active` state of Bitswap provider queries. (@whyrusleeping, [ipfs/go-ipfs#3169](https://github.com/ipfs/go-ipfs/pull/3169))
+- Fix a panic in the DHT code. (@Kubuxu, [ipfs/go-ipfs#3200](https://github.com/ipfs/go-ipfs/pull/3200))
+- Improve handling of `Identity` field in `ipfs config` command. (@Kubuxu, @whyrusleeping, [ipfs/go-ipfs#3141](https://github.com/ipfs/go-ipfs/pull/3141))
+- Fix explicit adding of symlinked files and directories. (@kevina, [ipfs/go-ipfs#3135](https://github.com/ipfs/go-ipfs/pull/3135))
+- Fix bash auto-completion of `ipfs daemon --unrestricted-api` option. (@lgierth, [ipfs/go-ipfs#3159](https://github.com/ipfs/go-ipfs/pull/3159))
+- Introduce a new timeout tool for tests to avoid licensing issues. (@Kubuxu, [ipfs/go-ipfs#3152](https://github.com/ipfs/go-ipfs/pull/3152))
+- Improve output for migrations of fs-repo. (@lgierth, [ipfs/go-ipfs#3158](https://github.com/ipfs/go-ipfs/pull/3158))
+- Fix info notice of commands taking input from stdin. (@Kubuxu, [ipfs/go-ipfs#3134](https://github.com/ipfs/go-ipfs/pull/3134))
+- Bring back a few tests for stdin handling of `ipfs cat` and `ipfs add`. (@Kubuxu, [ipfs/go-ipfs#3144](https://github.com/ipfs/go-ipfs/pull/3144))
+- Improve sharness tests for `ipfs repo verify` command. (@whyrusleeping, [ipfs/go-ipfs#3148](https://github.com/ipfs/go-ipfs/pull/3148))
+- Improve sharness tests for CORS headers on the gateway. (@Kubuxu, [ipfs/go-ipfs#3142](https://github.com/ipfs/go-ipfs/pull/3142))
+- Improve tests for pinning within `ipfs files`. (@kevina, [ipfs/go-ipfs#3151](https://github.com/ipfs/go-ipfs/pull/3151))
+- Improve tests for the automatic raising of file descriptor limits. (@whyrusleeping, [ipfs/go-ipfs#3149](https://github.com/ipfs/go-ipfs/pull/3149))
+
 ### 0.4.3-rc3 - 2016-08-09
 
 This release candidate fixes a panic that occurs when input from stdin was

blocks/blockstore/caching.go

@@ -9,16 +9,16 @@ import (
 
 // Next to each option is it aproximate memory usage per unit
 type CacheOpts struct {
-	HasBloomFilterSize   int // 1 bit
+	HasBloomFilterSize   int // 1 byte
 	HasBloomFilterHashes int // No size, 7 is usually best, consult bloom papers
 	HasARCCacheSize      int // 32 bytes
 }
 
 func DefaultCacheOpts() CacheOpts {
 	return CacheOpts{
-		HasBloomFilterSize:   512 * 8 * 1024,
+		HasBloomFilterSize:   512 << 10,
 		HasBloomFilterHashes: 7,
-		HasARCCacheSize:      64 * 1024,
+		HasARCCacheSize:      64 << 10,
 	}
 }
 
@@ -41,7 +41,8 @@ func CachedBlockstore(bs GCBlockstore,
 		cbs, err = newARCCachedBS(ctx, cbs, opts.HasARCCacheSize)
 	}
 	if opts.HasBloomFilterSize != 0 {
-		cbs, err = bloomCached(cbs, ctx, opts.HasBloomFilterSize, opts.HasBloomFilterHashes)
+		// *8 because of bytes to bits conversion
+		cbs, err = bloomCached(cbs, ctx, opts.HasBloomFilterSize*8, opts.HasBloomFilterHashes)
 	}
 
 	return cbs, err

cmd/ipfs/daemon.go

@@ -228,19 +228,25 @@ func daemonFunc(req cmds.Request, res cmds.Response) {
 		return
 	case fsrepo.ErrNeedMigration:
 		domigrate, found, _ := req.Option(migrateKwd).Bool()
-		fmt.Println("Found old repo version, migrations need to be run.")
+		fmt.Println("Found outdated fs-repo, migrations need to be run.")
 
 		if !found {
-			domigrate = YesNoPrompt("Run migrations automatically? [y/N]")
+			domigrate = YesNoPrompt("Run migrations now? [y/N]")
 		}
 
 		if !domigrate {
-			res.SetError(fmt.Errorf("please run the migrations manually"), cmds.ErrNormal)
+			fmt.Println("Not running migrations of fs-repo now.")
+			fmt.Println("Please get fs-repo-migrations from https://dist.ipfs.io")
+			res.SetError(fmt.Errorf("fs-repo requires migration"), cmds.ErrNormal)
 			return
 		}
 
 		err = migrate.RunMigration(fsrepo.RepoVersion)
 		if err != nil {
+			fmt.Println("The migrations of fs-repo failed:")
+			fmt.Printf("  %s\n", err)
+			fmt.Println("If you think this is a bug, please file an issue and include this whole log output.")
+			fmt.Println("  https://github.com/ipfs/fs-repo-migrations")
 			res.SetError(err, cmds.ErrNormal)
 			return
 		}

cmd/ipfs/ulimit_unix.go

@@ -30,18 +30,25 @@ func checkAndSetUlimit() error {
 		return fmt.Errorf("error getting rlimit: %s", err)
 	}
 
+	var setting bool
 	if rLimit.Cur < ipfsFileDescNum {
 		if rLimit.Max < ipfsFileDescNum {
+			log.Error("adjusting max")
 			rLimit.Max = ipfsFileDescNum
 		}
-		fmt.Printf("Adjusting current ulimit to %d.\n", ipfsFileDescNum)
+		fmt.Printf("Adjusting current ulimit to %d...\n", ipfsFileDescNum)
 		rLimit.Cur = ipfsFileDescNum
+		setting = true
 	}
 
 	err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
 	if err != nil {
 		return fmt.Errorf("error setting ulimit: %s", err)
 	}
 
+	if setting {
+		fmt.Printf("Successfully raised file descriptor limit to %d.\n", ipfsFileDescNum)
+	}
+
 	return nil
 }

commands/cli/parse.go

@@ -251,7 +251,7 @@ func parseOpts(args []string, root *cmds.Command) (
 	return
 }
 
-const msgStdinInfo = "ipfs: Reading from %s; send Ctrl-d to stop.\n"
+const msgStdinInfo = "ipfs: Reading from %s; send Ctrl-d to stop."
 
 func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursive, hidden bool, root *cmds.Command) ([]string, []files.File, error) {
 	// ignore stdin on Windows
@@ -401,16 +401,14 @@ func appendFile(fpath string, argDef *cmds.Argument, recursive, hidden bool) (fi
 		if err != nil {
 			return nil, err
 		}
+		cwd, err = filepath.EvalSymlinks(cwd)
+		if err != nil {
+			return nil, err
+		}
 		fpath = cwd
 	}
 
-	fpath = filepath.Clean(fpath)
-	fpath, err := filepath.EvalSymlinks(fpath)
-	if err != nil {
-		return nil, err
-	}
-	// Repeat ToSlash after EvalSymlinks as it turns path to platform specific
-	fpath = filepath.ToSlash(fpath)
+	fpath = filepath.ToSlash(filepath.Clean(fpath))
 
 	stat, err := os.Lstat(fpath)
 	if err != nil {
@@ -469,6 +467,7 @@ func newMessageReader(r io.ReadCloser, msg string) io.ReadCloser {
 func (r *messageReader) Read(b []byte) (int, error) {
 	if !r.done {
 		fmt.Fprintln(os.Stderr, r.message)
+		r.done = true
 	}
 
 	return r.r.Read(b)

core/commands/config.go

@@ -15,6 +15,7 @@ import (
 	repo "github.com/ipfs/go-ipfs/repo"
 	config "github.com/ipfs/go-ipfs/repo/config"
 	fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo"
+
 	u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
 )
 
@@ -162,14 +163,12 @@ included in the output of this command.
 			return
 		}
 
-		idmap, ok := cfg["Identity"].(map[string]interface{})
-		if !ok {
-			res.SetError(fmt.Errorf("config has no identity"), cmds.ErrNormal)
+		err = scrubValue(cfg, []string{config.IdentityTag, config.PrivKeyTag})
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
 			return
 		}
 
-		delete(idmap, "PrivKey")
-
 		output, err := config.HumanOutput(cfg)
 		if err != nil {
 			res.SetError(err, cmds.ErrNormal)
@@ -180,6 +179,47 @@ included in the output of this command.
 	},
 }
 
+func scrubValue(m map[string]interface{}, key []string) error {
+	find := func(m map[string]interface{}, k string) (string, interface{}, bool) {
+		lckey := strings.ToLower(k)
+		for mkey, val := range m {
+			lcmkey := strings.ToLower(mkey)
+			if lckey == lcmkey {
+				return mkey, val, true
+			}
+		}
+		return "", nil, false
+	}
+
+	cur := m
+	for _, k := range key[:len(key)-1] {
+		foundk, val, ok := find(cur, k)
+		if !ok {
+			return fmt.Errorf("failed to find specified key")
+		}
+
+		if foundk != k {
+			// case mismatch, calling this an error
+			return fmt.Errorf("case mismatch in config, expected %q but got %q", k, foundk)
+		}
+
+		mval, mok := val.(map[string]interface{})
+		if !mok {
+			return fmt.Errorf("%s was not a map", foundk)
+		}
+
+		cur = mval
+	}
+
+	todel, _, ok := find(cur, key[len(key)-1])
+	if !ok {
+		return fmt.Errorf("%s, not found", strings.Join(key, "."))
+	}
+
+	delete(cur, todel)
+	return nil
+}
+
 var configEditCmd = &cmds.Command{
 	Helptext: cmds.HelpText{
 		Tagline: "Open the config file for editing in $EDITOR.",
@@ -250,19 +290,10 @@ func getConfig(r repo.Repo, key string) (*ConfigField, error) {
 }
 
 func setConfig(r repo.Repo, key string, value interface{}) (*ConfigField, error) {
-	keyF, err := getConfig(r, "Identity.PrivKey")
-	if err != nil {
-		return nil, errors.New("failed to get PrivKey")
-	}
-	privkey := keyF.Value
-	err = r.SetConfigKey(key, value)
+	err := r.SetConfigKey(key, value)
 	if err != nil {
 		return nil, fmt.Errorf("failed to set config value: %s (maybe use --json?)", err)
 	}
-	err = r.SetConfigKey("Identity.PrivKey", privkey)
-	if err != nil {
-		return nil, errors.New("failed to set PrivKey")
-	}
 	return getConfig(r, key)
 }
 
@@ -286,7 +317,7 @@ func replaceConfig(r repo.Repo, file io.Reader) error {
 		return errors.New("setting private key with API is not supported")
 	}
 
-	keyF, err := getConfig(r, "Identity.PrivKey")
+	keyF, err := getConfig(r, config.PrivKeySelector)
 	if err != nil {
 		return fmt.Errorf("Failed to get PrivKey")
 	}

docs/config.md

@@ -101,7 +101,7 @@ Default: `false`
 A boolean value. If set to true, all block reads from disk will be hashed and verified. This will cause increased CPU utilization.
 
 - `BloomFilterSize`
-A number representing the size in bits of the blockstore's bloom filter. A value of zero represents the feature being disabled.
+A number representing the size in bytes of the blockstore's bloom filter. A value of zero represents the feature being disabled.
 
 Default: `0` 
 

docs/releases.md

@@ -12,7 +12,7 @@ will be allowed up to release day.
 - [ ] all tests pass (no exceptions)
 - [ ] webui works (for most definitions of 'works')
 - [ ] CHANGELOG.md has been updated
-  - use `LAST=v0.4.2 for n in $(git log --oneline --merges --reverse -n-1 $LAST...master | cut -d'#' -f2 | cut -d' ' -f1); do echo https://github.com/ipfs/go-ipfs/pull/$n; done`
+  - use `LAST=v0.4.2 ; for n in $(git log --oneline --merges --reverse -n-1 $LAST...master | cut -d'#' -f2 | cut -d' ' -f1); do echo https://github.com/ipfs/go-ipfs/pull/$n; done`
 - [ ] version string in `repo/config/version.go` has been updated
 - [ ] tag commit with vX.Y.Z
 - [ ] bump version string in `repo/config/version.go` to `vX.Y.Z-dev`

exchange/bitswap/bitswap.go

@@ -17,7 +17,6 @@ import (
 	bsmsg "github.com/ipfs/go-ipfs/exchange/bitswap/message"
 	bsnet "github.com/ipfs/go-ipfs/exchange/bitswap/network"
 	notifications "github.com/ipfs/go-ipfs/exchange/bitswap/notifications"
-	wantlist "github.com/ipfs/go-ipfs/exchange/bitswap/wantlist"
 	flags "github.com/ipfs/go-ipfs/flags"
 	"github.com/ipfs/go-ipfs/thirdparty/delay"
 	loggables "gx/ipfs/QmYrv4LgCC8FhG2Ab4bwuq5DqBdwMtx3hMb3KKJDZcr2d7/go-libp2p-loggables"
@@ -89,7 +88,7 @@ func New(parent context.Context, p peer.ID, network bsnet.BitSwapNetwork,
 		notifications: notif,
 		engine:        decision.NewEngine(ctx, bstore), // TODO close the engine with Close() method
 		network:       network,
-		findKeys:      make(chan *wantlist.Entry, sizeBatchRequestChan),
+		findKeys:      make(chan *blockRequest, sizeBatchRequestChan),
 		process:       px,
 		newBlocks:     make(chan blocks.Block, HasBlockBufferSize),
 		provideKeys:   make(chan key.Key, provideKeysBufferSize),
@@ -132,7 +131,7 @@ type Bitswap struct {
 	notifications notifications.PubSub
 
 	// send keys to a worker to find and connect to providers for them
-	findKeys chan *wantlist.Entry
+	findKeys chan *blockRequest
 
 	engine *decision.Engine
 
@@ -149,8 +148,8 @@ type Bitswap struct {
 }
 
 type blockRequest struct {
-	key key.Key
-	ctx context.Context
+	Key key.Key
+	Ctx context.Context
 }
 
 // GetBlock attempts to retrieve a particular block from peers within the
@@ -240,13 +239,50 @@ func (bs *Bitswap) GetBlocks(ctx context.Context, keys []key.Key) (<-chan blocks
 	// NB: Optimization. Assumes that providers of key[0] are likely to
 	// be able to provide for all keys. This currently holds true in most
 	// every situation. Later, this assumption may not hold as true.
-	req := &wantlist.Entry{
+	req := &blockRequest{
 		Key: keys[0],
 		Ctx: ctx,
 	}
+
+	remaining := make(map[key.Key]struct{})
+	for _, k := range keys {
+		remaining[k] = struct{}{}
+	}
+
+	out := make(chan blocks.Block)
+	go func() {
+		ctx, cancel := context.WithCancel(ctx)
+		defer cancel()
+		defer close(out)
+		defer func() {
+			var toCancel []key.Key
+			for k, _ := range remaining {
+				toCancel = append(toCancel, k)
+			}
+			bs.CancelWants(toCancel)
+		}()
+		for {
+			select {
+			case blk, ok := <-promise:
+				if !ok {
+					return
+				}
+
+				delete(remaining, blk.Key())
+				select {
+				case out <- blk:
+				case <-ctx.Done():
+					return
+				}
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
 	select {
 	case bs.findKeys <- req:
-		return promise, nil
+		return out, nil
 	case <-ctx.Done():
 		return nil, ctx.Err()
 	}