Add optional TLS feature to gRPC servers

go/vt/grpcoptionaltls/certificates_test.go

@@ -0,0 +1,53 @@
+/*
+Copyright 2019 The Vitess Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package grpcoptionaltls
+
+import "strings"
+
+// LocalhostCert is a PEM-encoded TLS cert with SAN IPs
+// "127.0.0.1" and "[::1]", expiring at Jan 29 16:00:00 2084 GMT.
+// generated from src/crypto/tls:
+// go run generate_cert.go  --rsa-bits 1024 --host 127.0.0.1,::1,example.com --ca --start-date "Jan 1 00:00:00 1970" --duration=1000000h
+var localhostCert = []byte(`-----BEGIN CERTIFICATE-----
+MIICEzCCAXygAwIBAgIQMIMChMLGrR+QvmQvpwAU6zANBgkqhkiG9w0BAQsFADAS
+MRAwDgYDVQQKEwdBY21lIENvMCAXDTcwMDEwMTAwMDAwMFoYDzIwODQwMTI5MTYw
+MDAwWjASMRAwDgYDVQQKEwdBY21lIENvMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCB
+iQKBgQDuLnQAI3mDgey3VBzWnB2L39JUU4txjeVE6myuDqkM/uGlfjb9SjY1bIw4
+iA5sBBZzHi3z0h1YV8QPuxEbi4nW91IJm2gsvvZhIrCHS3l6afab4pZBl2+XsDul
+rKBxKKtD1rGxlG4LjncdabFn9gvLZad2bSysqz/qTAUStTvqJQIDAQABo2gwZjAO
+BgNVHQ8BAf8EBAMCAqQwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDwYDVR0TAQH/BAUw
+AwEB/zAuBgNVHREEJzAlggtleGFtcGxlLmNvbYcEfwAAAYcQAAAAAAAAAAAAAAAA
+AAAAATANBgkqhkiG9w0BAQsFAAOBgQCEcetwO59EWk7WiJsG4x8SY+UIAA+flUI9
+tyC4lNhbcF2Idq9greZwbYCqTTTr2XiRNSMLCOjKyI7ukPoPjo16ocHj+P3vZGfs
+h1fIw3cSS2OolhloGw/XM6RWPWtPAlGykKLciQrBru5NAPvCMsb/I1DAceTiotQM
+fblo6RBxUQ==
+-----END CERTIFICATE-----`)
+
+// LocalhostKey is the private key for localhostCert.
+var localhostKey = []byte(testingKey(`-----BEGIN RSA TESTING KEY-----
+MIICXgIBAAKBgQDuLnQAI3mDgey3VBzWnB2L39JUU4txjeVE6myuDqkM/uGlfjb9
+SjY1bIw4iA5sBBZzHi3z0h1YV8QPuxEbi4nW91IJm2gsvvZhIrCHS3l6afab4pZB
+l2+XsDulrKBxKKtD1rGxlG4LjncdabFn9gvLZad2bSysqz/qTAUStTvqJQIDAQAB
+AoGAGRzwwir7XvBOAy5tM/uV6e+Zf6anZzus1s1Y1ClbjbE6HXbnWWF/wbZGOpet
+3Zm4vD6MXc7jpTLryzTQIvVdfQbRc6+MUVeLKwZatTXtdZrhu+Jk7hx0nTPy8Jcb
+uJqFk541aEw+mMogY/xEcfbWd6IOkp+4xqjlFLBEDytgbIECQQDvH/E6nk+hgN4H
+qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQcj2PprIMmPcQrooz8vp
+jy4SHEg1AkEA/v13/5M47K9vCxmb8QeD/asydfsgS5TeuNi8DoUBEmiSJwma7FXY
+fFUtxuvL7XvjwjN5B30pNEbc6Iuyt7y4MQJBAIt21su4b3sjXNueLKH85Q+phy2U
+fQtuUE9txblTu14q3N7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xlp/DoCzjA0CQQDU
+y2ptGsuSmgUtWj3NM9xuwYPm+Z/F84K6+ARYiZ6PYj013sovGKUFfYAqVXVlxtIX
+qyUBnu3X9ps8ZfjLZO7BAkEAlT4R5Yl6cGhaJQYZHOde3JEMhNRcVFMO8dJDaFeo
+f9Oeos0UUothgiDktdQHxdNEwLjQf7lJJBzV+5OtwswCWA==
+-----END RSA TESTING KEY-----`))
+
+func testingKey(s string) string { return strings.ReplaceAll(s, "TESTING KEY", "PRIVATE KEY") }

go/vt/grpcoptionaltls/conn_wrapper.go

@@ -0,0 +1,38 @@
+/*
+Copyright 2019 The Vitess Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package grpcoptionaltls
+
+import (
+	"bytes"
+	"io"
+	"net"
+)
+
+// WrappedConn imitates MSG_PEEK behaviour
+// Unlike net.Conn is not thread-safe for reading already peeked bytes
+type WrappedConn struct {
+	net.Conn
+	rd io.Reader
+}
+
+func NewWrappedConn(conn net.Conn, peeked []byte) net.Conn {
+	var rd = io.MultiReader(bytes.NewReader(peeked), conn)
+	return &WrappedConn{
+		Conn: conn,
+		rd:   rd,
+	}
+}
+
+func (wc *WrappedConn) Read(b []byte) (n int, err error) {
+	return wc.rd.Read(b)
+}

go/vt/grpcoptionaltls/optionaltls.go

@@ -0,0 +1,58 @@
+/*
+Copyright 2019 The Vitess Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package grpcoptionaltls
+
+import (
+	"net"
+
+	"google.golang.org/grpc/credentials"
+)
+
+type optionalTLSCreds struct {
+	credentials.TransportCredentials
+}
+
+func (c *optionalTLSCreds) Clone() credentials.TransportCredentials {
+	return New(c.TransportCredentials.Clone())
+}
+
+func (c *optionalTLSCreds) ServerHandshake(conn net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	isTLS, bytes, err := DetectTLS(conn)
+	if err != nil {
+		conn.Close()
+		return nil, nil, err
+	}
+
+	var wc net.Conn = NewWrappedConn(conn, bytes)
+	if isTLS {
+		return c.TransportCredentials.ServerHandshake(wc)
+	}
+
+	var authInfo = info{
+		CommonAuthInfo: credentials.CommonAuthInfo{SecurityLevel: credentials.NoSecurity},
+	}
+
+	return wc, authInfo, nil
+}
+
+func New(tc credentials.TransportCredentials) credentials.TransportCredentials {
+	return &optionalTLSCreds{TransportCredentials: tc}
+}
+
+type info struct {
+	credentials.CommonAuthInfo
+}
+
+func (info) AuthType() string {
+	return "insecure"
+}

go/vt/grpcoptionaltls/server_test.go

@@ -0,0 +1,119 @@
+/*
+Copyright 2019 The Vitess Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package grpcoptionaltls
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"net"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+	pb "google.golang.org/grpc/examples/helloworld/helloworld"
+)
+
+// server is used to implement helloworld.GreeterServer.
+type server struct {
+	pb.UnimplementedGreeterServer
+}
+
+// SayHello implements helloworld.GreeterServer
+func (s *server) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) {
+	return &pb.HelloReply{Message: "Hello " + in.GetName()}, nil
+}
+
+func createUnstartedServer(creds credentials.TransportCredentials) *grpc.Server {
+	s := grpc.NewServer(grpc.Creds(creds))
+	pb.RegisterGreeterServer(s, &server{})
+	return s
+}
+
+type testCredentials struct {
+	client credentials.TransportCredentials
+	server credentials.TransportCredentials
+}
+
+func createCredentials() (*testCredentials, error) {
+	cert, err := tls.X509KeyPair(localhostCert, localhostKey)
+	if err != nil {
+		return nil, err
+	}
+
+	certificate, err := x509.ParseCertificate(cert.Certificate[0])
+	if err != nil {
+		return nil, err
+	}
+	certpool := x509.NewCertPool()
+	certpool.AddCert(certificate)
+
+	tc := &testCredentials{
+		client: credentials.NewClientTLSFromCert(certpool, "example.com"),
+		server: credentials.NewServerTLSFromCert(&cert),
+	}
+	return tc, nil
+}
+
+func TestOptionalTLS(t *testing.T) {
+	testCtx, testCancel := context.WithCancel(context.Background())
+	defer testCancel()
+
+	tc, err := createCredentials()
+	if err != nil {
+		t.Fatalf("failed to create credentials %v", err)
+	}
+
+	lis, err := net.Listen("tcp", "")
+	if err != nil {
+		t.Fatalf("failed to listen %v", err)
+	}
+	defer lis.Close()
+	addr := lis.Addr().String()
+
+	srv := createUnstartedServer(New(tc.server))
+	go func() {
+		srv.Serve(lis)
+	}()
+	defer srv.Stop()
+
+	testFunc := func(t *testing.T, dialOpt grpc.DialOption) {
+		ctx, cancel := context.WithTimeout(testCtx, 5*time.Second)
+		defer cancel()
+		conn, err := grpc.DialContext(ctx, addr, dialOpt)
+		if err != nil {
+			t.Fatalf("failed to connect to the server %v", err)
+		}
+		defer conn.Close()
+		c := pb.NewGreeterClient(conn)
+		resp, err := c.SayHello(ctx, &pb.HelloRequest{Name: "Vittes"})
+		if err != nil {
+			t.Fatalf("could not greet: %v", err)
+		}
+		if resp.Message != "Hello Vittes" {
+			t.Fatalf("unexpected reply %s", resp.Message)
+		}
+	}
+
+	t.Run("Plain2TLS", func(t *testing.T) {
+		for i := 0; i < 5; i += 1 {
+			testFunc(t, grpc.WithInsecure())
+		}
+	})
+	t.Run("TLS2TLS", func(t *testing.T) {
+		for i := 0; i < 5; i += 1 {
+			testFunc(t, grpc.WithTransportCredentials(tc.client))
+		}
+	})
+}

go/vt/grpcoptionaltls/tls_detector.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2019 The Vitess Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package grpcoptionaltls
+
+import "io"
+
+const TLSPeekedBytes = 6
+
+func looksLikeTLS(bytes []byte) bool {
+	if len(bytes) < TLSPeekedBytes {
+		return false
+	}
+	// TLS starts as
+	// 0: 0x16 - handshake protocol magic
+	// 1: 0x03 - SSL version major
+	// 2: 0x00 to 0x03 - SSL version minor (SSLv3 or TLS1.0 through TLS1.3)
+	// 3-4: length (2 bytes)
+	// 5: 0x01 - handshake type (ClientHello)
+	// 6-8: handshake len (3 bytes), equals value from offset 3-4 minus 4
+	// HTTP2 initial frame bytes
+	// https://tools.ietf.org/html/rfc7540#section-3.4
+
+	// Definitely not TLS
+	if bytes[0] != 0x16 || bytes[1] != 0x03 || bytes[5] != 0x01 {
+		return false
+	}
+	return true
+}
+
+// DetectTLS reads necessary number of bytes from io.Reader
+// returns result, bytes read from Reader and error
+// No matter if error happens or what flag value is
+// returned bytes should be checked
+func DetectTLS(r io.Reader) (bool, []byte, error) {
+	var bytes = make([]byte, TLSPeekedBytes)
+	if n, err := io.ReadFull(r, bytes); err != nil {
+		return false, bytes[:n], err
+	}
+	return looksLikeTLS(bytes), bytes, nil
+}

go/vt/servenv/grpc_server.go

@@ -35,6 +35,7 @@ import (
 	"context"
 
 	"vitess.io/vitess/go/vt/grpccommon"
+	"vitess.io/vitess/go/vt/grpcoptionaltls"
 	"vitess.io/vitess/go/vt/log"
 	"vitess.io/vitess/go/vt/vttls"
 )
@@ -64,6 +65,8 @@ var (
 	// GRPCCA is the CA to use if TLS is enabled
 	GRPCCA = flag.String("grpc_ca", "", "server CA to use for gRPC connections, requires TLS, and enforces client certificate check")
 
+	GRPCEnableOptionalTLS = flag.Bool("grpc_enable_optional_tls", false, "enable optional TLS mode when a server accepts both TLS and plain-text connections on the same port")
+
 	// GRPCServerCA if specified will combine server cert and server CA
 	GRPCServerCA = flag.String("grpc_server_ca", "", "path to server CA in PEM format, which will be combine with server cert, return full certificate chain to clients")
 
@@ -135,6 +138,9 @@ func createGRPCServer() {
 
 		// create the creds server options
 		creds := credentials.NewTLS(config)
+		if *GRPCEnableOptionalTLS {
+			creds = grpcoptionaltls.New(creds)
+		}
 		opts = []grpc.ServerOption{grpc.Creds(creds)}
 	}
 	// Override the default max message size for both send and receive