Merge branch 'master' into feat/datastore-pinner

README.md

@@ -378,6 +378,8 @@ When starting a container running ipfs for the first time with an empty data dir
 
     docker run -d --name ipfs_host -e IPFS_PROFILE=server -v $ipfs_staging:/export -v $ipfs_data:/data/ipfs -p 4001:4001 -p 4001:4001/udp -p 127.0.0.1:8080:8080 -p 127.0.0.1:5001:5001 ipfs/go-ipfs:latest
 
+#### Private swarms inside Docker
+
 It is possible to initialize the container with a swarm key file (`/data/ipfs/swarm.key`) using the variables `IPFS_SWARM_KEY` and `IPFS_SWARM_KEY_FILE`. The `IPFS_SWARM_KEY` creates `swarm.key` with the contents of the variable itself, whilst `IPFS_SWARM_KEY_FILE` copies the key from a path stored in the variable. The `IPFS_SWARM_KEY_FILE` **overwrites** the key generated by `IPFS_SWARM_KEY`.
 
     docker run -d --name ipfs_host -e IPFS_SWARM_KEY=<your swarm key> -v $ipfs_staging:/export -v $ipfs_data:/data/ipfs -p 4001:4001 -p 4001:4001/udp -p 127.0.0.1:8080:8080 -p 127.0.0.1:5001:5001 ipfs/go-ipfs:latest
@@ -387,6 +389,20 @@ The swarm key initialization can also be done using docker secrets **(requires d
     cat your_swarm.key | docker secret create swarm_key_secret -
     docker run -d --name ipfs_host --secret swarm_key_secret -e IPFS_SWARM_KEY_FILE=/run/secrets/swarm_key_secret -v $ipfs_staging:/export -v $ipfs_data:/data/ipfs -p 4001:4001 -p 4001:4001/udp -p 127.0.0.1:8080:8080 -p 127.0.0.1:5001:5001 ipfs/go-ipfs:latest
 
+#### Key rotation inside Docker
+
+If needed, it is possible to do key rotation in an ephemeral container that is temporarily executing against a volume that is mounted under `/data/ipfs`:
+
+```sh
+# given container named 'ipfs-test' that persists repo at /path/to/persisted/.ipfs
+$ docker run -d --name ipfs-test -v /path/to/persisted/.ipfs:/data/ipfs ipfs/go-ipfs:v0.7.0 
+$ docker stop ipfs-test  
+
+# key rotation works like this (old key saved under 'old-self')
+$ docker run --rm -it -v /path/to/persisted/.ipfs:/data/ipfs ipfs/go-ipfs:v0.7.0 key rotate -o old-self -t ed25519
+$ docker start ipfs-test # will start with the new key
+```
+
 ### Troubleshooting
 
 If you have previously installed IPFS before and you are running into problems getting a newer version to work, try deleting (or backing up somewhere else) your IPFS config directory (~/.ipfs by default) and rerunning `ipfs init`. This will reinitialize the config file to its defaults and clear out the local datastore of any bad entries.

assets/bindata_version_hash.go

@@ -2,5 +2,5 @@
 package assets
 
 const (
-	BindataVersionHash = "514e5ae28d8adb84955801b56ef47aca44bf9cc8"
+	BindataVersionHash = "605b5945438e1fe2eaf8a6571cca7ecda12d5599"
 )

core/commands/dag/dag.go

@@ -685,7 +685,7 @@ var DagStatCmd = &cmds.Command{
 	Helptext: cmds.HelpText{
 		Tagline: "Gets stats for a DAG",
 		ShortDescription: `
-'ipfs dag size' fetches a dag and returns various statistics about the DAG.
+'ipfs dag stat' fetches a dag and returns various statistics about the DAG.
 Statistics include size and number of blocks.
 
 Note: This command skips duplicate blocks in reporting both size and the number of blocks

core/coreapi/pubsub.go

@@ -118,5 +118,9 @@ func (msg *pubSubMessage) Seq() []byte {
 }
 
 func (msg *pubSubMessage) Topics() []string {
-	return msg.msg.TopicIDs
+	// TODO: handle breaking downstream changes by returning a single string.
+	if msg.msg.Topic == nil {
+		return nil
+	}
+	return []string{*msg.msg.Topic}
 }

core/corehttp/gateway_handler.go

@@ -290,7 +290,12 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
 		goget := r.URL.Query().Get("go-get") == "1"
 		if dirwithoutslash && !goget {
 			// See comment above where originalUrlPath is declared.
-			http.Redirect(w, r, originalUrlPath+"/", 302)
+			suffix := "/"
+			if r.URL.RawQuery != "" {
+				// preserve query parameters
+				suffix = suffix + "?" + r.URL.RawQuery
+			}
+			http.Redirect(w, r, originalUrlPath+suffix, 302)
 			return
 		}
 
@@ -387,8 +392,9 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
 
 	hash := resolvedPath.Cid().String()
 
-	// Storage for gateway URL to be used when linking to other rootIDs. This
-	// will be blank unless subdomain resolution is being used for this request.
+	// Gateway root URL to be used when linking to other rootIDs.
+	// This will be blank unless subdomain or DNSLink resolution is being used
+	// for this request.
 	var gwURL string
 
 	// Get gateway hostname and build gateway URL.
@@ -398,13 +404,16 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
 		gwURL = ""
 	}
 
+	dnslink := hasDNSLinkOrigin(gwURL, urlPath)
+
 	// See comment above where originalUrlPath is declared.
 	tplData := listingTemplateData{
 		GatewayURL:  gwURL,
+		DNSLink:     dnslink,
 		Listing:     dirListing,
 		Size:        size,
 		Path:        urlPath,
-		Breadcrumbs: breadcrumbs(urlPath),
+		Breadcrumbs: breadcrumbs(urlPath, dnslink),
 		BackLink:    backLink,
 		Hash:        hash,
 	}

core/corehttp/gateway_indexPage.go

@@ -13,6 +13,7 @@ import (
 // structs for directory listing
 type listingTemplateData struct {
 	GatewayURL  string
+	DNSLink     bool
 	Listing     []directoryItem
 	Size        string
 	Path        string
@@ -34,16 +35,16 @@ type breadcrumb struct {
 	Path string
 }
 
-func breadcrumbs(urlPath string) []breadcrumb {
+func breadcrumbs(urlPath string, dnslinkOrigin bool) []breadcrumb {
 	var ret []breadcrumb
 
 	p, err := ipfspath.ParsePath(urlPath)
 	if err != nil {
 		// No breadcrumbs, fallback to bare Path in template
 		return ret
 	}
-
 	segs := p.Segments()
+	contentRoot := segs[1]
 	for i, seg := range segs {
 		if i == 0 {
 			ret = append(ret, breadcrumb{Name: seg})
@@ -55,13 +56,38 @@ func breadcrumbs(urlPath string) []breadcrumb {
 		}
 	}
 
+	// Drop the /ipns/<fqdn> prefix from breadcrumb Paths when directory
+	// listing on a DNSLink website (loaded due to Host header in HTTP
+	// request).  Necessary because the hostname most likely won't have a
+	// public gateway mounted.
+	if dnslinkOrigin {
+		prefix := "/ipns/" + contentRoot
+		for i, crumb := range ret {
+			if strings.HasPrefix(crumb.Path, prefix) {
+				ret[i].Path = strings.Replace(crumb.Path, prefix, "", 1)
+			}
+		}
+		// Make contentRoot breadcrumb link to the website root
+		ret[1].Path = "/"
+	}
+
 	return ret
 }
 
 func shortHash(hash string) string {
 	return (hash[0:4] + "\u2026" + hash[len(hash)-4:])
 }
 
+// helper to detect DNSLink website context
+// (when hostname from gwURL is matching /ipns/<fqdn> in path)
+func hasDNSLinkOrigin(gwURL string, path string) bool {
+	if gwURL != "" {
+		fqdn := stripPort(strings.TrimPrefix(gwURL, "//"))
+		return strings.HasPrefix(path, "/ipns/"+fqdn)
+	}
+	return false
+}
+
 var listingTemplate *template.Template
 
 func init() {

core/corehttp/gateway_test.go

@@ -391,6 +391,10 @@ func TestIPNSHostnameRedirect(t *testing.T) {
 	}
 }
 
+// Test directory listing on DNSLink website
+// (scenario when Host header is the same as URL hostname)
+// This is basic regression test: additional end-to-end tests
+// can be found in test/sharness/t0115-gateway-dir-listing.sh
 func TestIPNSHostnameBacklinks(t *testing.T) {
 	ns := mockNamesys{}
 	ts, api, ctx := newTestServerAndNode(t, ns)
@@ -437,15 +441,15 @@ func TestIPNSHostnameBacklinks(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	// expect correct backlinks
+	// expect correct links
 	body, err := ioutil.ReadAll(res.Body)
 	if err != nil {
 		t.Fatalf("error reading response: %s", err)
 	}
 	s := string(body)
 	t.Logf("body: %s\n", string(body))
 
-	if !matchPathOrBreadcrumbs(s, "/ipns/<a href=\"/ipns/example.net\">example.net</a>/<a href=\"/ipns/example.net/foo%3F%20%23%3C%27\">foo? #&lt;&#39;</a>") {
+	if !matchPathOrBreadcrumbs(s, "/ipns/<a href=\"//example.net/\">example.net</a>/<a href=\"//example.net/foo%3F%20%23%3C%27\">foo? #&lt;&#39;</a>") {
 		t.Fatalf("expected a path in directory listing")
 	}
 	if !strings.Contains(s, "<a href=\"/foo%3F%20%23%3C%27/./..\">") {
@@ -454,6 +458,10 @@ func TestIPNSHostnameBacklinks(t *testing.T) {
 	if !strings.Contains(s, "<a href=\"/foo%3F%20%23%3C%27/file.txt\">") {
 		t.Fatalf("expected file in directory listing")
 	}
+	if !strings.Contains(s, "<a class=\"ipfs-hash\" href=\"https://cid.ipfs.io/#") {
+		// https://github.com/ipfs/dir-index-html/issues/42
+		t.Fatalf("expected links to cid.ipfs.io in CID column when on DNSLink website")
+	}
 	if !strings.Contains(s, k2.Cid().String()) {
 		t.Fatalf("expected hash in directory listing")
 	}
@@ -487,6 +495,10 @@ func TestIPNSHostnameBacklinks(t *testing.T) {
 	if !strings.Contains(s, "<a href=\"/file.txt\">") {
 		t.Fatalf("expected file in directory listing")
 	}
+	if !strings.Contains(s, "<a class=\"ipfs-hash\" href=\"https://cid.ipfs.io/#") {
+		// https://github.com/ipfs/dir-index-html/issues/42
+		t.Fatalf("expected links to cid.ipfs.io in CID column when on DNSLink website")
+	}
 	if !strings.Contains(s, k.Cid().String()) {
 		t.Fatalf("expected hash in directory listing")
 	}
@@ -511,7 +523,7 @@ func TestIPNSHostnameBacklinks(t *testing.T) {
 	s = string(body)
 	t.Logf("body: %s\n", string(body))
 
-	if !matchPathOrBreadcrumbs(s, "/ipns/<a href=\"/ipns/example.net\">example.net</a>/<a href=\"/ipns/example.net/foo%3F%20%23%3C%27\">foo? #&lt;&#39;</a>/<a href=\"/ipns/example.net/foo%3F%20%23%3C%27/bar\">bar</a>") {
+	if !matchPathOrBreadcrumbs(s, "/ipns/<a href=\"//example.net/\">example.net</a>/<a href=\"//example.net/foo%3F%20%23%3C%27\">foo? #&lt;&#39;</a>/<a href=\"//example.net/foo%3F%20%23%3C%27/bar\">bar</a>") {
 		t.Fatalf("expected a path in directory listing")
 	}
 	if !strings.Contains(s, "<a href=\"/foo%3F%20%23%3C%27/bar/./..\">") {
@@ -545,7 +557,7 @@ func TestIPNSHostnameBacklinks(t *testing.T) {
 	s = string(body)
 	t.Logf("body: %s\n", string(body))
 
-	if !matchPathOrBreadcrumbs(s, "/ipns/<a href=\"/ipns/example.net\">example.net</a>") {
+	if !matchPathOrBreadcrumbs(s, "/ipns/<a href=\"//example.net/\">example.net</a>") {
 		t.Fatalf("expected a path in directory listing")
 	}
 	if !strings.Contains(s, "<a href=\"/good-prefix/\">") {

core/corehttp/hostname.go

@@ -129,7 +129,7 @@ func HostnameOption() ServeOption {
 				if !gw.NoDNSLink && isDNSLinkRequest(r.Context(), coreAPI, host) {
 					// rewrite path and handle as DNSLink
 					r.URL.Path = "/ipns/" + stripPort(host) + r.URL.Path
-					childMux.ServeHTTP(w, r)
+					childMux.ServeHTTP(w, withHostnameContext(r, host))
 					return
 				}
 
@@ -143,10 +143,6 @@ func HostnameOption() ServeOption {
 			if gw, hostname, ns, rootID, ok := knownSubdomainDetails(host, knownGateways); ok {
 				// Looks like we're using a known gateway in subdomain mode.
 
-				// Add gateway hostname context for linking to other root ids.
-				// Example: localhost/ipfs/{cid}
-				ctx := context.WithValue(r.Context(), "gw-hostname", hostname)
-
 				// Assemble original path prefix.
 				pathPrefix := "/" + ns + "/" + rootID
 
@@ -201,7 +197,7 @@ func HostnameOption() ServeOption {
 				r.URL.Path = pathPrefix + r.URL.Path
 
 				// Serve path request
-				childMux.ServeHTTP(w, r.WithContext(ctx))
+				childMux.ServeHTTP(w, withHostnameContext(r, hostname))
 				return
 			}
 			// We don't have a known gateway. Fallback on DNSLink lookup
@@ -213,7 +209,7 @@ func HostnameOption() ServeOption {
 			if !cfg.Gateway.NoDNSLink && isDNSLinkRequest(r.Context(), coreAPI, host) {
 				// rewrite path and handle as DNSLink
 				r.URL.Path = "/ipns/" + stripPort(host) + r.URL.Path
-				childMux.ServeHTTP(w, r)
+				childMux.ServeHTTP(w, withHostnameContext(r, host))
 				return
 			}
 
@@ -234,6 +230,17 @@ type wildcardHost struct {
 	spec *config.GatewaySpec
 }
 
+// Extends request context to include hostname of a canonical gateway root
+// (subdomain root or dnslink fqdn)
+func withHostnameContext(r *http.Request, hostname string) *http.Request {
+	// This is required for links on directory listing pages to work correctly
+	// on subdomain and dnslink gateways. While DNSlink could read value from
+	// Host header, subdomain gateways have more comples rules (knownSubdomainDetails)
+	// More: https://github.com/ipfs/dir-index-html/issues/42
+	ctx := context.WithValue(r.Context(), "gw-hostname", hostname)
+	return r.WithContext(ctx)
+}
+
 func prepareKnownGateways(publicGateways map[string]*config.GatewaySpec) gatewayHosts {
 	var hosts gatewayHosts
 

core/corehttp/webui.go

@@ -1,7 +1,7 @@
 package corehttp
 
 // TODO: move to IPNS
-const WebUIPath = "/ipfs/bafybeifekmcbbi4nwyj4aasti6x3nuhyli464wfjjfdjg4xnz53lhyiedq" // v2.11.2
+const WebUIPath = "/ipfs/bafybeif4zkmu7qdhkpf3pnhwxipylqleof7rl6ojbe7mq3fzogz6m4xk3i" // v2.11.4
 
 // this is a list of all past webUI paths.
 var WebUIPaths = []string{

core/node/core.go

@@ -3,6 +3,7 @@ package node
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/ipfs/go-bitswap"
 	"github.com/ipfs/go-bitswap/network"
@@ -51,7 +52,10 @@ func Pinning(bstore blockstore.Blockstore, ds format.DAGService, repo repo.Repo)
 	}
 	syncDs := &syncDagService{ds, syncFn}
 
-	pinning, err := dspinner.LoadPinner(rootDS, syncDs)
+	ctx, cancel := context.WithTimeout(context.TODO(), 2*time.Minute)
+	defer cancel()
+
+	pinning, err := dspinner.New(ctx, rootDS, syncDs)
 	if err != nil {
 		return nil, err
 	}