daemon: support unix domain sockets for the API/gateway

All the work was client-side. Unix domain socket multiaddrs already worked
server-side.

fixes #4218

cmd/ipfs/daemon.go

@@ -469,7 +469,12 @@ func serveHTTPApi(req *cmds.Request, cctx *oldcmds.Context) (<-chan error, error
 		// we might have listened to /tcp/0 - lets see what we are listing on
 		apiMaddr = apiLis.Multiaddr()
 		fmt.Printf("API server listening on %s\n", apiMaddr)
-		fmt.Printf("WebUI: http://%s/webui\n", apiLis.Addr())
+
+		// Browsers require TCP.
+		switch apiLis.Addr().Network() {
+		case "tcp", "tcp4", "tcp6":
+			fmt.Printf("WebUI: http://%s/webui\n", apiLis.Addr())
+		}
 		listeners = append(listeners, apiLis)
 	}
 

cmd/ipfs/main.go

@@ -6,6 +6,8 @@ import (
 	"errors"
 	"fmt"
 	"math/rand"
+	"net"
+	"net/http"
 	"os"
 	"runtime/pprof"
 	"strings"
@@ -22,7 +24,7 @@ import (
 
 	"github.com/ipfs/go-ipfs-cmds"
 	"github.com/ipfs/go-ipfs-cmds/cli"
-	"github.com/ipfs/go-ipfs-cmds/http"
+	cmdhttp "github.com/ipfs/go-ipfs-cmds/http"
 	"github.com/ipfs/go-ipfs-config"
 	u "github.com/ipfs/go-ipfs-util"
 	logging "github.com/ipfs/go-log"
@@ -249,23 +251,39 @@ func makeExecutor(req *cmds.Request, env interface{}) (cmds.Executor, error) {
 	if err != nil {
 		return nil, err
 	}
-	_, host, err := manet.DialArgs(apiAddr)
+	network, host, err := manet.DialArgs(apiAddr)
 	if err != nil {
 		return nil, err
 	}
 
 	// Construct the executor.
-	opts := []http.ClientOpt{
-		http.ClientWithAPIPrefix(corehttp.APIPath),
+	opts := []cmdhttp.ClientOpt{
+		cmdhttp.ClientWithAPIPrefix(corehttp.APIPath),
 	}
 
 	// Fallback on a local executor if we (a) have a repo and (b) aren't
 	// forcing a daemon.
 	if !daemonRequested && fsrepo.IsInitialized(cctx.ConfigRoot) {
-		opts = append(opts, http.ClientWithFallback(exe))
+		opts = append(opts, cmdhttp.ClientWithFallback(exe))
+	}
+
+	switch network {
+	case "tcp", "tcp4", "tcp6":
+	case "unix":
+		path := host
+		host = "unix"
+		opts = append(opts, cmdhttp.ClientWithHTTPClient(&http.Client{
+			Transport: &http.Transport{
+				DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+					return net.Dial("unix", path)
+				},
+			},
+		}))
+	default:
+		return nil, fmt.Errorf("unsupported API address: %s", apiAddr)
 	}
 
-	return http.NewClient(host, opts...), nil
+	return cmdhttp.NewClient(host, opts...), nil
 }
 
 // commandDetails returns a command's details for the command given by |path|.

docs/config.md

@@ -77,16 +77,32 @@ Contains information about various listener addresses to be used by this node.
 - `API`
 Multiaddr or array of multiaddrs describing the address to serve the local HTTP API on.
 
+Supported Transports:
+
+* tcp/ip{4,6} - `/ipN/.../tcp/...`
+* unix - `/unix/path/to/socket`
+
 Default: `/ip4/127.0.0.1/tcp/5001`
 
 - `Gateway`
 Multiaddr or array of multiaddrs describing the address to serve the local gateway on.
 
+Supported Transports:
+
+* tcp/ip{4,6} - `/ipN/.../tcp/...`
+* unix - `/unix/path/to/socket`
+
 Default: `/ip4/127.0.0.1/tcp/8080`
 
 - `Swarm`
 Array of multiaddrs describing which addresses to listen on for p2p swarm connections.
 
+Supported Transports:
+
+* tcp/ip{4,6} - `/ipN/.../tcp/...`
+* websocket - `/ipN/.../tcp/.../ws`
+* quic - `/ipN/.../udp/.../quic`
+
 Default:
 ```json
 [

go.mod

@@ -28,12 +28,12 @@ require (
 	github.com/ipfs/go-fs-lock v0.0.1
 	github.com/ipfs/go-ipfs-blockstore v0.1.0
 	github.com/ipfs/go-ipfs-chunker v0.0.1
-	github.com/ipfs/go-ipfs-cmds v0.1.0
+	github.com/ipfs/go-ipfs-cmds v0.1.1
 	github.com/ipfs/go-ipfs-config v0.0.11
 	github.com/ipfs/go-ipfs-ds-help v0.0.1
 	github.com/ipfs/go-ipfs-exchange-interface v0.0.1
 	github.com/ipfs/go-ipfs-exchange-offline v0.0.1
-	github.com/ipfs/go-ipfs-files v0.0.3
+	github.com/ipfs/go-ipfs-files v0.0.4
 	github.com/ipfs/go-ipfs-posinfo v0.0.1
 	github.com/ipfs/go-ipfs-provider v0.2.1
 	github.com/ipfs/go-ipfs-routing v0.1.0

go.sum

@@ -176,8 +176,8 @@ github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IW
 github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk=
 github.com/ipfs/go-ipfs-chunker v0.0.1 h1:cHUUxKFQ99pozdahi+uSC/3Y6HeRpi9oTeUHbE27SEw=
 github.com/ipfs/go-ipfs-chunker v0.0.1/go.mod h1:tWewYK0we3+rMbOh7pPFGDyypCtvGcBFymgY4rSDLAw=
-github.com/ipfs/go-ipfs-cmds v0.1.0 h1:0CEde9EcxByej8+L6d1PST57J4ambRPyCTjLG5Ymou8=
-github.com/ipfs/go-ipfs-cmds v0.1.0/go.mod h1:TiK4e7/V31tuEb8YWDF8lN3qrnDH+BS7ZqWIeYJlAs8=
+github.com/ipfs/go-ipfs-cmds v0.1.1 h1:H9/BLf5rcsULHMj/x8gC0e5o+raYhqk1OQsfzbGMNM4=
+github.com/ipfs/go-ipfs-cmds v0.1.1/go.mod h1:k1zMXcOLtljA9iAnZHddbH69yVm5+weRL0snmMD/rK0=
 github.com/ipfs/go-ipfs-config v0.0.11 h1:5/4nas2CQXiKr2/MLxU24GDGTBvtstQIQezuk7ltOQQ=
 github.com/ipfs/go-ipfs-config v0.0.11/go.mod h1:wveA8UT5ywN26oKStByzmz1CO6cXwLKKM6Jn/Hfw08I=
 github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
@@ -192,6 +192,8 @@ github.com/ipfs/go-ipfs-exchange-offline v0.0.1/go.mod h1:WhHSFCVYX36H/anEKQboAz
 github.com/ipfs/go-ipfs-files v0.0.2/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4=
 github.com/ipfs/go-ipfs-files v0.0.3 h1:ME+QnC3uOyla1ciRPezDW0ynQYK2ikOh9OCKAEg4uUA=
 github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4=
+github.com/ipfs/go-ipfs-files v0.0.4 h1:WzRCivcybUQch/Qh6v8LBRhKtRsjnwyiuOV09mK7mrE=
+github.com/ipfs/go-ipfs-files v0.0.4/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4=
 github.com/ipfs/go-ipfs-flags v0.0.1/go.mod h1:RnXBb9WV53GSfTrSDVK61NLTFKvWc60n+K9EgCDh+rA=
 github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs=
 github.com/ipfs/go-ipfs-posinfo v0.0.1/go.mod h1:SwyeVP+jCwiDu0C313l/8jg6ZxM0qqtlt2a0vILTc1A=
@@ -570,8 +572,8 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z
 github.com/prometheus/procfs v0.0.0-20190519111021-9935e8e0588d h1:Z5QMcUKnQw7ouB1wDuyZM6TL/rm+brJcNk6Ai8ut3zM=
 github.com/prometheus/procfs v0.0.0-20190519111021-9935e8e0588d/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
-github.com/rs/cors v1.6.0 h1:G9tHG9lebljV9mfp9SNPDL36nCDxmo3zTlAf1YgvzmI=
-github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
+github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
+github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=

test/dependencies/pollEndpoint/main.go

@@ -3,10 +3,6 @@ package main
 
 import (
 	"flag"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"net/url"
 	"os"
 	"time"
 
@@ -16,11 +12,10 @@ import (
 )
 
 var (
-	host     = flag.String("host", "/ip4/127.0.0.1/tcp/5001", "the multiaddr host to dial on")
-	endpoint = flag.String("ep", "/version", "which http endpoint path to hit")
-	tries    = flag.Int("tries", 10, "how many tries to make before failing")
-	timeout  = flag.Duration("tout", time.Second, "how long to wait between attempts")
-	verbose  = flag.Bool("v", false, "verbose logging")
+	host    = flag.String("host", "/ip4/127.0.0.1/tcp/5001", "the multiaddr host to dial on")
+	tries   = flag.Int("tries", 10, "how many tries to make before failing")
+	timeout = flag.Duration("tout", time.Second, "how long to wait between attempts")
+	verbose = flag.Bool("v", false, "verbose logging")
 )
 
 var log = logging.Logger("pollEndpoint")
@@ -33,56 +28,27 @@ func main() {
 	if err != nil {
 		log.Fatal("NewMultiaddr() failed: ", err)
 	}
-	p := addr.Protocols()
-	if len(p) < 2 {
-		log.Fatal("need two protocols in host flag (/ip/tcp): ", addr)
-	}
-	_, host, err := manet.DialArgs(addr)
-	if err != nil {
-		log.Fatal("manet.DialArgs() failed: ", err)
-	}
 
 	if *verbose { // lower log level
 		logging.SetDebugLogging()
 	}
 
-	// construct url to dial
-	var u url.URL
-	u.Scheme = "http"
-	u.Host = host
-	u.Path = *endpoint
-
 	// show what we got
 	start := time.Now()
-	log.Debugf("starting at %s, tries: %d, timeout: %s, url: %s", start, *tries, *timeout, u)
+	log.Debugf("starting at %s, tries: %d, timeout: %s, addr: %s", start, *tries, *timeout, addr)
 
 	for *tries > 0 {
-
-		err := checkOK(http.Get(u.String()))
+		c, err := manet.Dial(addr)
 		if err == nil {
 			log.Debugf("ok -  endpoint reachable with %d tries remaining, took %s", *tries, time.Since(start))
+			c.Close()
 			os.Exit(0)
 		}
-		log.Debug("get failed: ", err)
+		log.Debug("connect failed: ", err)
 		time.Sleep(*timeout)
 		*tries--
 	}
 
 	log.Error("failed.")
 	os.Exit(1)
 }
-
-func checkOK(resp *http.Response, err error) error {
-	if err == nil { // request worked
-		defer resp.Body.Close()
-		if resp.StatusCode == http.StatusOK {
-			return nil
-		}
-		body, err := ioutil.ReadAll(resp.Body)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "pollEndpoint: ioutil.ReadAll() Error: %s", err)
-		}
-		return fmt.Errorf("response not OK. %d %s %q", resp.StatusCode, resp.Status, string(body))
-	}
-	return err
-}

test/sharness/lib/test-lib.sh

@@ -230,7 +230,7 @@ test_launch_ipfs_daemon() {
 
   # we say the daemon is ready when the API server is ready.
   test_expect_success "'ipfs daemon' is ready" '
-    pollEndpoint -ep=/version -host=$API_MADDR -v -tout=1s -tries=60 2>poll_apierr > poll_apiout ||
+    pollEndpoint -host=$API_MADDR -v -tout=1s -tries=60 2>poll_apierr > poll_apiout ||
     test_fsh cat actual_daemon || test_fsh cat daemon_err || test_fsh cat poll_apierr || test_fsh cat poll_apiout
   '
 }

test/sharness/t0060-daemon.sh

@@ -170,7 +170,7 @@ test_expect_success "'ipfs daemon' should be able to run with a pipe attached to
 '
 
 test_expect_success "daemon with pipe eventually becomes live" '
-  pollEndpoint -host='$API_MADDR' -ep=/version -v -tout=1s -tries=10 >stdin_poll_apiout 2>stdin_poll_apierr &&
+  pollEndpoint -host='$API_MADDR' -v -tout=1s -tries=10 >stdin_poll_apiout 2>stdin_poll_apierr &&
   test_kill_repeat_10_sec $DAEMON_PID ||
   test_fsh cat stdin_daemon_out || test_fsh cat stdin_daemon_err || test_fsh cat stdin_poll_apiout || test_fsh cat stdin_poll_apierr
 '

test/sharness/t0067-unix-api.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+#
+# MIT Licensed; see the LICENSE file in this repository.
+#
+
+test_description="Test unix API transport"
+
+. lib/test-lib.sh
+
+test_init_ipfs
+
+# We can't use the trash dir as the full name must be longer less than 108 bytes
+# long (because that's the max unix domain socket path length).
+SOCKDIR="$(mktemp -d "${TMPDIR:-/tmp}/unix-api-sharness.XXXXXX")"
+
+test_expect_success "configure" '
+  peerid=$(ipfs config Identity.PeerID) &&
+  ipfs config Addresses.API "/unix/$SOCKDIR/sock"
+'
+
+test_launch_ipfs_daemon
+
+test_expect_success "client works" '
+  printf "$peerid" >expected &&
+  ipfs --api="/unix/$SOCKDIR/sock" id -f="<id>" >actual &&
+  test_cmp expected actual
+'
+
+test_kill_ipfs_daemon
+test_done

test/sharness/t0111-gateway-writeable.sh

@@ -30,7 +30,7 @@ test_launch_ipfs_daemon
 port=$GWAY_PORT
 
 test_expect_success "ipfs daemon up" '
-  pollEndpoint -host $GWAY_MADDR -ep=/version -v -tout=1s -tries=60 2>poll_apierr > poll_apiout ||
+  pollEndpoint -host $GWAY_MADDR -v -tout=1s -tries=60 2>poll_apierr > poll_apiout ||
   test_fsh cat poll_apierr || test_fsh cat poll_apiout
 '