feat(tls): splitting tls_dir + making health point available on HTTP (#…

…6821)

* DGRAPH-1263 | DGRAPH-599 | splitting tls_dir + making health point available on http

conn/pool.go

@@ -167,7 +167,7 @@ func (p *Pools) Connect(addr string, tlsClientConf *tls.Config) *Pool {
 
 // newPool creates a new "pool" with one gRPC connection, refcount 0.
 func newPool(addr string, tlsClientConf *tls.Config) (*Pool, error) {
-	conOpts := []grpc.DialOption {
+	conOpts := []grpc.DialOption{
 		grpc.WithStatsHandler(&ocgrpc.ClientHandler{}),
 		grpc.WithDefaultCallOptions(
 			grpc.MaxCallRecvMsgSize(x.GrpcMaxSize),

contrib/manual_tests/test.sh

@@ -282,7 +282,7 @@ function test::manual_start_tls() {
   local -r n_alphas=4
 
   dgraph::start_zeros "$n_zeros"
-  dgraph::start_alphas "$n_alphas" --tls_dir "$TLS_PATH"
+  dgraph::start_alphas "$n_alphas" --tls_cacert "$TLS_PATH"/ca.crt --tls_node_cert "$TLS_PATH"/node.crt --tls_node_key "$TLS_PATH"/node.key
 
   for i in $(seq "$n_zeros"); do
     dgraph::healthcheck_zero "$i"
@@ -307,7 +307,7 @@ function test::manual_start_encryption_acl_tls() {
   dgraph::start_alphas "$n_alphas" \
     --acl_secret_file "$ACL_SECRET_PATH" \
     --encryption_key_file "$ENCRYPTION_KEY_PATH" \
-    --tls_dir "$TLS_PATH"
+    --tls_cacert "$TLS_PATH"/ca.crt --tls_node_cert "$TLS_PATH"/node.crt --tls_node_key "$TLS_PATH"/node.key
 
   for i in $(seq "$n_zeros"); do
     dgraph::healthcheck_zero "$i"

contrib/tlstest/alpha_tls.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 set -e
-$DGRAPH_BIN alpha --tls_dir $PWD/tls --zero 127.0.0.1:5081 &> alpha.log
+$DGRAPH_BIN alpha --tls_cacert $PWD/tls/ca.crt --tls_node_cert $PWD/tls/node.crt --tls_node_key $PWD/tls/node.key --zero 127.0.0.1:5081 &> alpha.log

contrib/tlstest/alpha_tls_auth.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 set -e
-$DGRAPH_BIN alpha --tls_dir $PWD/tls --tls_client_auth REQUIREANDVERIFY --zero 127.0.0.1:5081 &> alpha.log
+$DGRAPH_BIN alpha --tls_cacert $PWD/tls/ca.crt --tls_node_cert $PWD/tls/node.crt --tls_node_key $PWD/tls/node.key --tls_client_auth REQUIREANDVERIFY --zero 127.0.0.1:5081 &> alpha.log

dgraph/cmd/alpha/run.go

@@ -163,16 +163,6 @@ they form a Raft group and provide synchronous replication.
 	flag.Uint64("mutations_nquad_limit", 1e6,
 		"Limit for the maximum number of nquads that can be inserted in a mutation request")
 
-	// TLS configurations
-	flag.String("tls_dir", "", "Path to directory that has TLS certificates and keys.")
-	flag.Bool("tls_use_system_ca", true, "Include System CA into CA Certs.")
-	flag.String("tls_client_auth", "VERIFYIFGIVEN", "Enable TLS client authentication")
-	flag.Bool("tls_internal_port_enabled", false, "(optional) enable inter node TLS encryption between cluster nodes.")
-	flag.String("tls_cert", "", "(optional) The Cert file name in tls_dir which is needed to " +
-		"connect as a client with the other nodes in the cluster.")
-	flag.String("tls_key", "", "(optional) The private key file name "+
-		"in tls_dir needed to connect as a client with the other nodes in the cluster.")
-
 	//Custom plugins.
 	flag.String("custom_tokenizers", "",
 		"Comma separated list of tokenizer plugins")
@@ -196,6 +186,9 @@ they form a Raft group and provide synchronous replication.
 	flag.String("cache_percentage", "0,65,35,0",
 		`Cache percentages summing up to 100 for various caches (FORMAT:
 		PostingListCache,PstoreBlockCache,PstoreIndexCache,WAL).`)
+
+	// TLS configurations
+	x.RegisterServerTLSFlags(flag)
 }
 
 func setupCustomTokenizers() {
@@ -395,29 +388,6 @@ func serveGRPC(l net.Listener, tlsCfg *tls.Config, closer *z.Closer) {
 	s.Stop()
 }
 
-func serveHTTP(l net.Listener, tlsCfg *tls.Config, closer *z.Closer) {
-	defer closer.Done()
-	srv := &http.Server{
-		ReadTimeout:  10 * time.Second,
-		WriteTimeout: 600 * time.Second,
-		IdleTimeout:  2 * time.Minute,
-	}
-	var err error
-	switch {
-	case tlsCfg != nil:
-		srv.TLSConfig = tlsCfg
-		err = srv.ServeTLS(l, "", "")
-	default:
-		err = srv.Serve(l)
-	}
-	glog.Errorf("Stopped taking more http(s) requests. Err: %v", err)
-	ctx, cancel := context.WithTimeout(context.Background(), 630*time.Second)
-	defer cancel()
-	if err := srv.Shutdown(ctx); err != nil {
-		log.Printf("Http(s) shutdown err: %v", err.Error())
-	}
-}
-
 func setupServer(closer *z.Closer) {
 	go worker.RunServer(bindall) // For pb.communication.
 
@@ -426,7 +396,7 @@ func setupServer(closer *z.Closer) {
 		laddr = "0.0.0.0"
 	}
 
-	tlsCfg, err := x.LoadServerTLSConfig(Alpha.Conf, x.TLSNodeCert, x.TLSNodeKey)
+	tlsCfg, err := x.LoadServerTLSConfig(Alpha.Conf)
 	if err != nil {
 		log.Fatalf("Failed to setup TLS: %v\n", err)
 	}
@@ -549,7 +519,7 @@ func setupServer(closer *z.Closer) {
 	// Initialize the servers.
 	admin.ServerCloser = z.NewCloser(3)
 	go serveGRPC(grpcListener, tlsCfg, admin.ServerCloser)
-	go serveHTTP(httpListener, tlsCfg, admin.ServerCloser)
+	go x.StartListenHttpAndHttps(httpListener, tlsCfg, admin.ServerCloser)
 
 	if Alpha.Conf.GetBool("telemetry") {
 		go edgraph.PeriodicallyPostTelemetry()
@@ -651,8 +621,11 @@ func run() {
 	abortDur, err := time.ParseDuration(Alpha.Conf.GetString("abort_older_than"))
 	x.Check(err)
 
-	tlsConf, err := x.LoadClientTLSConfigForInternalPort(Alpha.Conf)
+	tlsClientConf, err := x.LoadClientTLSConfigForInternalPort(Alpha.Conf)
+	x.Check(err)
+	tlsServerConf, err := x.LoadServerTLSConfigForInternalPort(Alpha.Conf)
 	x.Check(err)
+
 	x.WorkerConfig = x.WorkerOptions{
 		ExportPath:           Alpha.Conf.GetString("export"),
 		NumPendingProposals:  Alpha.Conf.GetInt("pending_proposals"),
@@ -667,9 +640,8 @@ func run() {
 		StartTime:            startTime,
 		LudicrousMode:        Alpha.Conf.GetBool("ludicrous_mode"),
 		LudicrousConcurrency: Alpha.Conf.GetInt("ludicrous_concurrency"),
-		TLSClientConfig:      tlsConf,
-		TLSDir:               Alpha.Conf.GetString("tls_dir"),
-		TLSInterNodeEnabled: Alpha.Conf.GetBool("tls_internal_port_enabled"),
+		TLSClientConfig:      tlsClientConf,
+		TLSServerConfig:      tlsServerConf,
 	}
 	x.WorkerConfig.Parse(Alpha.Conf)
 

dgraph/cmd/zero/http.go

@@ -17,22 +17,16 @@
 package zero
 
 import (
-	"bufio"
 	"context"
-	"crypto/tls"
 	"fmt"
-	"io"
-	"net"
 	"net/http"
 	"strconv"
-	"strings"
 	"time"
 
 	"github.com/dgraph-io/dgraph/protos/pb"
 	"github.com/dgraph-io/dgraph/x"
 	"github.com/gogo/protobuf/jsonpb"
 	"github.com/golang/glog"
-	"github.com/soheilhy/cmux"
 )
 
 // intFromQueryParam checks for name as a query param, converts it to uint64 and returns it.
@@ -243,85 +237,3 @@ func (st *state) pingResponse(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusOK)
 	_, _ = w.Write([]byte("OK"))
 }
-
-func (st *state) startListenHttpAndHttps(l net.Listener) {
-	if Zero.Conf.GetString("tls_dir") == "" && Zero.Conf.GetString("tls_disabled_route") != "" {
-		glog.Fatal("--tls_disabled_route is provided as an option but tls_dir is empty. Please provide --tls_dir")
-	}
-
-	m := cmux.New(l)
-	startServers(m)
-
-	go func() {
-		defer st.zero.closer.Done()
-		err := m.Serve()
-		if err != nil {
-			glog.Errorf("error from cmux serve: %v", err)
-		}
-	}()
-}
-
-func startServers(m cmux.CMux) {
-	httpRule := m.Match(func(r io.Reader) bool {
-		//no tls config is provided. http is being used.
-		if opts.tlsDir == "" {
-			return true
-		}
-		//tls config is provided but none of the routes are disabled.
-		if len(opts.tlsDisabledRoutes) == 0 {
-			return false
-		}
-		path, ok := parseRequestPath(r)
-		// not able to parse the request. Let it be resolved via TLS
-		if !ok {
-			return false
-		}
-		for _, r := range opts.tlsDisabledRoutes {
-			if strings.HasPrefix(path, r) {
-				return true
-			}
-		}
-		return false
-	})
-	go startListen(httpRule)
-
-	// if tls is enabled, make tls encryption based connections as default
-	if Zero.Conf.GetString("tls_dir") != "" {
-		tlsCfg, err := x.LoadServerTLSConfig(Zero.Conf, x.TLSNodeCert, x.TLSNodeKey)
-		x.Check(err)
-		if tlsCfg == nil {
-			glog.Fatalf("tls_dir is set but tls config provided is not correct. Please define correct variable --tls_dir")
-		}
-
-		httpsRule := m.Match(cmux.Any())
-		//this is chained listener. tls listener will decrypt the message and send it in plain text to HTTP server
-		go startListen(tls.NewListener(httpsRule, tlsCfg))
-	}
-}
-
-func startListen(l net.Listener) {
-	srv := &http.Server{
-		ReadTimeout:  10 * time.Second,
-		WriteTimeout: 600 * time.Second,
-		IdleTimeout:  2 * time.Minute,
-	}
-
-	err := srv.Serve(l)
-	glog.Errorf("Stopped taking more http(s) requests. Err: %v", err)
-	ctx, cancel := context.WithTimeout(context.Background(), 630*time.Second)
-	defer cancel()
-	err = srv.Shutdown(ctx)
-	glog.Infoln("All http(s) requests finished.")
-	if err != nil {
-		glog.Errorf("Http(s) shutdown err: %v", err)
-	}
-}
-
-func parseRequestPath(r io.Reader) (path string, ok bool) {
-	request, err := http.ReadRequest(bufio.NewReader(r))
-	if err != nil {
-		return "", false
-	}
-
-	return request.URL.Path, true
-}

dgraph/cmd/zero/run.go

@@ -25,7 +25,6 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
-	"strings"
 	"syscall"
 	"time"
 
@@ -54,8 +53,6 @@ type options struct {
 	peer              string
 	w                 string
 	rebalanceInterval time.Duration
-	tlsDir            string
-	tlsDisabledRoutes []string
 	tlsClientConfig   *tls.Config
 	totalCache        int64
 }
@@ -94,17 +91,7 @@ instances to achieve high-availability.
 	flag.Duration("rebalance_interval", 8*time.Minute, "Interval for trying a predicate move.")
 	flag.String("enterprise_license", "", "Path to the enterprise license file.")
 	// TLS configurations
-	flag.String("tls_dir", "", "Path to directory that has TLS certificates and keys.")
-	flag.Bool("tls_use_system_ca", true, "Include System CA into CA Certs.")
-	flag.String("tls_client_auth", "VERIFYIFGIVEN", "Enable TLS client authentication")
-	flag.String("tls_disabled_route", "",
-		"comma separated zero endpoint which will be disabled from TLS encryption."+
-		"Valid values are /health,/state,/removeNode,/moveTablet,/assign,/enterpriseLicense,/debug.")
-	flag.Bool("tls_internal_port_enabled", false, "enable inter node TLS encryption between cluster nodes.")
-	flag.String("tls_cert", "", "(optional) The Cert file name in tls_dir which is needed to " +
-		"connect as a client with the other nodes in the cluster.")
-	flag.String("tls_key", "", "(optional) The private key file name "+
-		"in tls_dir which is needed to connect as a client with the other nodes in the cluster.")
+	x.RegisterServerTLSFlags(flag)
 }
 
 func setupListener(addr string, port int, kind string) (listener net.Listener, err error) {
@@ -128,7 +115,7 @@ func (st *state) serveGRPC(l net.Listener, store *raftwal.DiskStorage) {
 		grpc.StatsHandler(&ocgrpc.ServerHandler{}),
 	}
 
-	tlsConf, err := x.LoadServerTLSConfigForInternalPort(Zero.Conf.GetBool("tls_internal_port_enabled"), Zero.Conf.GetString("tls_dir"))
+	tlsConf, err := x.LoadServerTLSConfigForInternalPort(Zero.Conf)
 	x.Check(err)
 	if tlsConf != nil {
 		grpcOpts = append(grpcOpts, grpc.Creds(credentials.NewTLS(tlsConf)))
@@ -184,11 +171,6 @@ func run() {
 	}
 
 	x.PrintVersion()
-	var tlsDisRoutes []string
-	if Zero.Conf.GetString("tls_disabled_route") != "" {
-		tlsDisRoutes = strings.Split(Zero.Conf.GetString("tls_disabled_route"), ",")
-	}
-
 	tlsConf, err := x.LoadClientTLSConfigForInternalPort(Zero.Conf)
 	x.Check(err)
 	opts = options{
@@ -200,8 +182,6 @@ func run() {
 		w:                 Zero.Conf.GetString("wal"),
 		rebalanceInterval: Zero.Conf.GetDuration("rebalance_interval"),
 		totalCache:        int64(Zero.Conf.GetInt("cache_mb")),
-		tlsDir:            Zero.Conf.GetString("tls_dir"),
-		tlsDisabledRoutes: tlsDisRoutes,
 		tlsClientConfig:   tlsConf,
 	}
 	glog.Infof("Setting Config to: %+v", opts)
@@ -260,7 +240,10 @@ func run() {
 	// Initialize the servers.
 	var st state
 	st.serveGRPC(grpcListener, store)
-	st.startListenHttpAndHttps(httpListener)
+
+	tlsCfg, err := x.LoadServerTLSConfig(Zero.Conf)
+	x.Check(err)
+	go x.StartListenHttpAndHttps(httpListener, tlsCfg, st.zero.closer)
 
 	http.HandleFunc("/health", st.pingResponse)
 	http.HandleFunc("/state", st.getState)

dgraph/cmd/zero/zero.go

@@ -73,8 +73,8 @@ type Server struct {
 	// tls client config used to connect with zero internally
 	tlsClientConfig *tls.Config
 
-	moveOngoing     chan struct{}
-	blockCommitsOn  *sync.Map
+	moveOngoing    chan struct{}
+	blockCommitsOn *sync.Map
 }
 
 // Init initializes the zero server.

tlstest/acl/docker-compose.yml

@@ -23,7 +23,8 @@ services:
       target: /dgraph-tls
       read_only: true
     command: /gobin/dgraph alpha -o 100 --my=alpha1:7180 --zero=zero1:5180
-      --logtostderr -v=2 --whitelist=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16 --tls_dir /dgraph-tls
+      --logtostderr -v=2 --whitelist=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
+      --tls_cacert /dgraph-tls/ca.crt --tls_node_cert /dgraph-tls/node.crt --tls_node_key /dgraph-tls/node.key
       --tls_client_auth VERIFYIFGIVEN --acl_secret_file /dgraph-acl/hmac-secret
   zero1:
     image: dgraph/dgraph:latest

tlstest/certrequest/docker-compose.yml

@@ -18,9 +18,9 @@ services:
       source: ../tls
       target: /dgraph-tls
       read_only: true
-    command: /gobin/dgraph alpha -o 100 --my=alpha1:7180 --zero=zero1:5180
-      --logtostderr -v=2 --tls_dir /dgraph-tls --tls_client_auth REQUEST
-      --whitelist=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
+    command: /gobin/dgraph alpha -o 100 --my=alpha1:7180 --zero=zero1:5180 --logtostderr -v=2
+      --tls_cacert /dgraph-tls/ca.crt --tls_node_cert /dgraph-tls/node.crt --tls_node_key /dgraph-tls/node.key
+      --tls_client_auth REQUEST --whitelist=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
   zero1:
     image: dgraph/dgraph:latest
     container_name: zero1

tlstest/certrequireandverify/docker-compose.yml

@@ -18,7 +18,7 @@ services:
       source: ../tls
       target: /dgraph-tls
       read_only: true
-    command: /gobin/dgraph alpha -o 100 --my=alpha1:7180 --zero=zero1:5180 --logtostderr -v=2 --tls_dir /dgraph-tls --tls_client_auth REQUIREANDVERIFY --whitelist 10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
+    command: /gobin/dgraph alpha -o 100 --my=alpha1:7180 --zero=zero1:5180 --logtostderr -v=2 --tls_cacert /dgraph-tls/ca.crt --tls_node_cert /dgraph-tls/node.crt --tls_node_key /dgraph-tls/node.key --tls_client_auth REQUIREANDVERIFY --whitelist 10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
   zero1:
     image: dgraph/dgraph:latest
     container_name: zero1

tlstest/certverifyifgiven/docker-compose.yml

@@ -19,7 +19,8 @@ services:
       target: /dgraph-tls
       read_only: true
     command: /gobin/dgraph alpha -o 100 --my=alpha1:7180 --zero=zero1:5180
-      --logtostderr -v=2 --tls_dir /dgraph-tls --tls_client_auth VERIFYIFGIVEN
+      --logtostderr -v=2 --tls_cacert /dgraph-tls/ca.crt --tls_node_cert /dgraph-tls/node.crt --tls_node_key /dgraph-tls/node.key
+      --tls_client_auth VERIFYIFGIVEN
       --whitelist=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
   zero1:
     image: dgraph/dgraph:latest

tlstest/mtls_internal/acl/docker-compose.yml

@@ -22,8 +22,9 @@ services:
       source: ../tls/alpha1
       target: /dgraph-tls
       read_only: true
-    command: /gobin/dgraph alpha -o 100 --my=alpha1:7180 --zero=zero1:5180
-      --logtostderr -v=2 --tls_dir /dgraph-tls --tls_internal_port_enabled=true --tls_cert client.alpha1.crt --tls_key client.alpha1.key
+    command: /gobin/dgraph alpha -o 100 --my=alpha1:7180 --zero=zero1:5180 --logtostderr -v=2
+      --tls_cacert /dgraph-tls/ca.crt --tls_node_cert /dgraph-tls/node.crt --tls_node_key /dgraph-tls/node.key
+      --tls_internal_port_enabled=true --tls_cert /dgraph-tls/client.alpha1.crt --tls_key /dgraph-tls/client.alpha1.key
       --whitelist=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
       --tls_client_auth VERIFYIFGIVEN --acl_secret_file /dgraph-acl/hmac-secret
   zero1:
@@ -45,6 +46,7 @@ services:
       target: /dgraph-tls
       read_only: true
     command: /gobin/dgraph zero -o 100 --idx=1 --my=zero1:5180 --logtostderr
-      --tls_dir /dgraph-tls --tls_internal_port_enabled=true --tls_cert client.zero1.crt --tls_key client.zero1.key
+      --tls_cacert /dgraph-tls/ca.crt --tls_node_cert /dgraph-tls/node.crt --tls_node_key /dgraph-tls/node.key
+      --tls_internal_port_enabled=true --tls_cert /dgraph-tls/client.zero1.crt --tls_key /dgraph-tls/client.zero1.key
       -v=2 --bindall
 volumes: {}