Merge pull request #6678 from ipfs/feat/api-over-unix

daemon: support unix domain sockets for the API/gateway

cmd/ipfs/daemon.go

@@ -416,7 +416,7 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
 	// Setting to 1 lets us multiply it with other stats to add the version labels
 	ipfsInfoMetric.With(prometheus.Labels{
 		"version": version.CurrentVersionNumber,
-		"commit": version.CurrentCommit,
+		"commit":  version.CurrentCommit,
 	}).Set(1)
 
 	// initialize metrics collector
@@ -492,7 +492,11 @@ func serveHTTPApi(req *cmds.Request, cctx *oldcmds.Context) (<-chan error, error
 	for _, listener := range listeners {
 		// we might have listened to /tcp/0 - lets see what we are listing on
 		fmt.Printf("API server listening on %s\n", listener.Multiaddr())
-		fmt.Printf("WebUI: http://%s/webui\n", listener.Addr())
+		// Browsers require TCP.
+		switch listener.Addr().Network() {
+		case "tcp", "tcp4", "tcp6":
+			fmt.Printf("WebUI: http://%s/webui\n", listener.Addr())
+		}
 	}
 
 	// by default, we don't let you load arbitrary ipfs objects through the api,

cmd/ipfs/main.go

@@ -6,6 +6,8 @@ import (
 	"errors"
 	"fmt"
 	"math/rand"
+	"net"
+	"net/http"
 	"os"
 	"runtime/pprof"
 	"strings"
@@ -22,7 +24,7 @@ import (
 
 	"github.com/ipfs/go-ipfs-cmds"
 	"github.com/ipfs/go-ipfs-cmds/cli"
-	"github.com/ipfs/go-ipfs-cmds/http"
+	cmdhttp "github.com/ipfs/go-ipfs-cmds/http"
 	"github.com/ipfs/go-ipfs-config"
 	u "github.com/ipfs/go-ipfs-util"
 	logging "github.com/ipfs/go-log"
@@ -249,23 +251,39 @@ func makeExecutor(req *cmds.Request, env interface{}) (cmds.Executor, error) {
 	if err != nil {
 		return nil, err
 	}
-	_, host, err := manet.DialArgs(apiAddr)
+	network, host, err := manet.DialArgs(apiAddr)
 	if err != nil {
 		return nil, err
 	}
 
 	// Construct the executor.
-	opts := []http.ClientOpt{
-		http.ClientWithAPIPrefix(corehttp.APIPath),
+	opts := []cmdhttp.ClientOpt{
+		cmdhttp.ClientWithAPIPrefix(corehttp.APIPath),
 	}
 
 	// Fallback on a local executor if we (a) have a repo and (b) aren't
 	// forcing a daemon.
 	if !daemonRequested && fsrepo.IsInitialized(cctx.ConfigRoot) {
-		opts = append(opts, http.ClientWithFallback(exe))
+		opts = append(opts, cmdhttp.ClientWithFallback(exe))
+	}
+
+	switch network {
+	case "tcp", "tcp4", "tcp6":
+	case "unix":
+		path := host
+		host = "unix"
+		opts = append(opts, cmdhttp.ClientWithHTTPClient(&http.Client{
+			Transport: &http.Transport{
+				DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+					return net.Dial("unix", path)
+				},
+			},
+		}))
+	default:
+		return nil, fmt.Errorf("unsupported API address: %s", apiAddr)
 	}
 
-	return http.NewClient(host, opts...), nil
+	return cmdhttp.NewClient(host, opts...), nil
 }
 
 // commandDetails returns a command's details for the command given by |path|.

docs/config.md

@@ -77,16 +77,32 @@ Contains information about various listener addresses to be used by this node.
 - `API`
 Multiaddr or array of multiaddrs describing the address to serve the local HTTP API on.
 
+Supported Transports:
+
+* tcp/ip{4,6} - `/ipN/.../tcp/...`
+* unix - `/unix/path/to/socket`
+
 Default: `/ip4/127.0.0.1/tcp/5001`
 
 - `Gateway`
 Multiaddr or array of multiaddrs describing the address to serve the local gateway on.
 
+Supported Transports:
+
+* tcp/ip{4,6} - `/ipN/.../tcp/...`
+* unix - `/unix/path/to/socket`
+
 Default: `/ip4/127.0.0.1/tcp/8080`
 
 - `Swarm`
 Array of multiaddrs describing which addresses to listen on for p2p swarm connections.
 
+Supported Transports:
+
+* tcp/ip{4,6} - `/ipN/.../tcp/...`
+* websocket - `/ipN/.../tcp/.../ws`
+* quic - `/ipN/.../udp/.../quic`
+
 Default:
 ```json
 [

go.mod

@@ -29,7 +29,7 @@ require (
 	github.com/ipfs/go-fs-lock v0.0.1
 	github.com/ipfs/go-ipfs-blockstore v0.1.0
 	github.com/ipfs/go-ipfs-chunker v0.0.1
-	github.com/ipfs/go-ipfs-cmds v0.1.0
+	github.com/ipfs/go-ipfs-cmds v0.1.1
 	github.com/ipfs/go-ipfs-config v0.0.11
 	github.com/ipfs/go-ipfs-ds-help v0.0.1
 	github.com/ipfs/go-ipfs-exchange-interface v0.0.1

go.sum

@@ -183,8 +183,8 @@ github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IW
 github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk=
 github.com/ipfs/go-ipfs-chunker v0.0.1 h1:cHUUxKFQ99pozdahi+uSC/3Y6HeRpi9oTeUHbE27SEw=
 github.com/ipfs/go-ipfs-chunker v0.0.1/go.mod h1:tWewYK0we3+rMbOh7pPFGDyypCtvGcBFymgY4rSDLAw=
-github.com/ipfs/go-ipfs-cmds v0.1.0 h1:0CEde9EcxByej8+L6d1PST57J4ambRPyCTjLG5Ymou8=
-github.com/ipfs/go-ipfs-cmds v0.1.0/go.mod h1:TiK4e7/V31tuEb8YWDF8lN3qrnDH+BS7ZqWIeYJlAs8=
+github.com/ipfs/go-ipfs-cmds v0.1.1 h1:H9/BLf5rcsULHMj/x8gC0e5o+raYhqk1OQsfzbGMNM4=
+github.com/ipfs/go-ipfs-cmds v0.1.1/go.mod h1:k1zMXcOLtljA9iAnZHddbH69yVm5+weRL0snmMD/rK0=
 github.com/ipfs/go-ipfs-config v0.0.11 h1:5/4nas2CQXiKr2/MLxU24GDGTBvtstQIQezuk7ltOQQ=
 github.com/ipfs/go-ipfs-config v0.0.11/go.mod h1:wveA8UT5ywN26oKStByzmz1CO6cXwLKKM6Jn/Hfw08I=
 github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
@@ -596,8 +596,8 @@ github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE=
 github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
-github.com/rs/cors v1.6.0 h1:G9tHG9lebljV9mfp9SNPDL36nCDxmo3zTlAf1YgvzmI=
-github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
+github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
+github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=

test/dependencies/pollEndpoint/main.go

@@ -3,10 +3,6 @@ package main
 
 import (
 	"flag"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"net/url"
 	"os"
 	"time"
 
@@ -16,11 +12,10 @@ import (
 )
 
 var (
-	host     = flag.String("host", "/ip4/127.0.0.1/tcp/5001", "the multiaddr host to dial on")
-	endpoint = flag.String("ep", "/version", "which http endpoint path to hit")
-	tries    = flag.Int("tries", 10, "how many tries to make before failing")
-	timeout  = flag.Duration("tout", time.Second, "how long to wait between attempts")
-	verbose  = flag.Bool("v", false, "verbose logging")
+	host    = flag.String("host", "/ip4/127.0.0.1/tcp/5001", "the multiaddr host to dial on")
+	tries   = flag.Int("tries", 10, "how many tries to make before failing")
+	timeout = flag.Duration("tout", time.Second, "how long to wait between attempts")
+	verbose = flag.Bool("v", false, "verbose logging")
 )
 
 var log = logging.Logger("pollEndpoint")
@@ -33,56 +28,27 @@ func main() {
 	if err != nil {
 		log.Fatal("NewMultiaddr() failed: ", err)
 	}
-	p := addr.Protocols()
-	if len(p) < 2 {
-		log.Fatal("need two protocols in host flag (/ip/tcp): ", addr)
-	}
-	_, host, err := manet.DialArgs(addr)
-	if err != nil {
-		log.Fatal("manet.DialArgs() failed: ", err)
-	}
 
 	if *verbose { // lower log level
 		logging.SetDebugLogging()
 	}
 
-	// construct url to dial
-	var u url.URL
-	u.Scheme = "http"
-	u.Host = host
-	u.Path = *endpoint
-
 	// show what we got
 	start := time.Now()
-	log.Debugf("starting at %s, tries: %d, timeout: %s, url: %s", start, *tries, *timeout, u)
+	log.Debugf("starting at %s, tries: %d, timeout: %s, addr: %s", start, *tries, *timeout, addr)
 
 	for *tries > 0 {
-
-		err := checkOK(http.Get(u.String()))
+		c, err := manet.Dial(addr)
 		if err == nil {
 			log.Debugf("ok -  endpoint reachable with %d tries remaining, took %s", *tries, time.Since(start))
+			c.Close()
 			os.Exit(0)
 		}
-		log.Debug("get failed: ", err)
+		log.Debug("connect failed: ", err)
 		time.Sleep(*timeout)
 		*tries--
 	}
 
 	log.Error("failed.")
 	os.Exit(1)
 }
-
-func checkOK(resp *http.Response, err error) error {
-	if err == nil { // request worked
-		defer resp.Body.Close()
-		if resp.StatusCode == http.StatusOK {
-			return nil
-		}
-		body, err := ioutil.ReadAll(resp.Body)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "pollEndpoint: ioutil.ReadAll() Error: %s", err)
-		}
-		return fmt.Errorf("response not OK. %d %s %q", resp.StatusCode, resp.Status, string(body))
-	}
-	return err
-}

test/sharness/lib/test-lib.sh

@@ -230,7 +230,7 @@ test_launch_ipfs_daemon() {
 
   # we say the daemon is ready when the API server is ready.
   test_expect_success "'ipfs daemon' is ready" '
-    pollEndpoint -ep=/version -host=$API_MADDR -v -tout=1s -tries=60 2>poll_apierr > poll_apiout ||
+    pollEndpoint -host=$API_MADDR -v -tout=1s -tries=60 2>poll_apierr > poll_apiout ||
     test_fsh cat actual_daemon || test_fsh cat daemon_err || test_fsh cat poll_apierr || test_fsh cat poll_apiout
   '
 }

test/sharness/t0040-add-and-cat.sh

@@ -494,7 +494,6 @@ test_add_cat_expensive() {
 }
 
 test_add_named_pipe() {
-  err_prefix=$1
   test_expect_success "useful error message when adding a named pipe" '
     mkfifo named-pipe &&
     test_expect_code 1 ipfs add named-pipe 2>actual &&
@@ -510,7 +509,7 @@ test_add_named_pipe() {
     mkfifo named-pipe-dir/named-pipe &&
     STAT=$(generic_stat named-pipe-dir/named-pipe) &&
     test_expect_code 1 ipfs add -r named-pipe-dir 2>actual &&
-    printf "Error:$err_prefix unrecognized file type for named-pipe-dir/named-pipe: $STAT\n" >expected &&
+    printf "Error: unrecognized file type for named-pipe-dir/named-pipe: $STAT\n" >expected &&
     rm named-pipe-dir/named-pipe &&
     rmdir named-pipe-dir &&
     test_cmp expected actual
@@ -796,7 +795,7 @@ test_add_cat_expensive "--cid-version=1" "bafybeidkj5ecbhrqmzrcee2rw7qwsx24z3364
 # encoded with the blake2b-256 hash funtion
 test_add_cat_expensive '--hash=blake2b-256' "bafykbzaceb26fnq5hz5iopzamcb4yqykya5x6a4nvzdmcyuu4rj2akzs3z7r6"
 
-test_add_named_pipe " Post http://$API_ADDR/api/v0/add?chunker=size-262144&encoding=json&hash=sha2-256&inline-limit=32&pin=true&progress=true&recursive=true&stream-channels=true:"
+test_add_named_pipe
 
 test_add_pwd_is_symlink
 
@@ -827,7 +826,7 @@ test_expect_success "ipfs cat file fails" '
   test_must_fail ipfs cat $(cat oh_hash)
 '
 
-test_add_named_pipe ""
+test_add_named_pipe
 
 test_add_pwd_is_symlink
 

test/sharness/t0060-daemon.sh

@@ -170,7 +170,7 @@ test_expect_success "'ipfs daemon' should be able to run with a pipe attached to
 '
 
 test_expect_success "daemon with pipe eventually becomes live" '
-  pollEndpoint -host='$API_MADDR' -ep=/version -v -tout=1s -tries=10 >stdin_poll_apiout 2>stdin_poll_apierr &&
+  pollEndpoint -host='$API_MADDR' -v -tout=1s -tries=10 >stdin_poll_apiout 2>stdin_poll_apierr &&
   test_kill_repeat_10_sec $DAEMON_PID ||
   test_fsh cat stdin_daemon_out || test_fsh cat stdin_daemon_err || test_fsh cat stdin_poll_apiout || test_fsh cat stdin_poll_apierr
 '

test/sharness/t0067-unix-api.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+#
+# MIT Licensed; see the LICENSE file in this repository.
+#
+
+test_description="Test unix API transport"
+
+. lib/test-lib.sh
+
+test_init_ipfs
+
+# We can't use the trash dir as the full name must be longer less than 108 bytes
+# long (because that's the max unix domain socket path length).
+SOCKDIR="$(mktemp -d "${TMPDIR:-/tmp}/unix-api-sharness.XXXXXX")"
+
+test_expect_success "configure" '
+  peerid=$(ipfs config Identity.PeerID) &&
+  ipfs config Addresses.API "/unix/$SOCKDIR/sock"
+'
+
+test_launch_ipfs_daemon
+
+test_expect_success "client works" '
+  printf "$peerid" >expected &&
+  ipfs --api="/unix/$SOCKDIR/sock" id -f="<id>" >actual &&
+  test_cmp expected actual
+'
+
+test_kill_ipfs_daemon
+test_done

test/sharness/t0111-gateway-writeable.sh

@@ -30,7 +30,7 @@ test_launch_ipfs_daemon
 port=$GWAY_PORT
 
 test_expect_success "ipfs daemon up" '
-  pollEndpoint -host $GWAY_MADDR -ep=/version -v -tout=1s -tries=60 2>poll_apierr > poll_apiout ||
+  pollEndpoint -host $GWAY_MADDR -v -tout=1s -tries=60 2>poll_apierr > poll_apiout ||
   test_fsh cat poll_apierr || test_fsh cat poll_apiout
 '