From f72131e1c0247cd0e2bf0d15c1ab0cc9dca137fb Mon Sep 17 00:00:00 2001 From: Taekyu Date: Thu, 17 Mar 2022 00:26:21 +0900 Subject: [PATCH 1/2] feature. . support grpc tls . add/mod logging intercepter for server/client side --- pkg/grpc_client/client.go | 59 +++++++++++++++++++++++++++++++++------ pkg/grpc_server/server.go | 57 +++++++++++++++++++++++++++++++++++++ pkg/helper/grpc.go | 31 -------------------- pkg/log/log.go | 25 ++++++++++++----- 4 files changed, 125 insertions(+), 47 deletions(-) create mode 100644 pkg/grpc_server/server.go delete mode 100644 pkg/helper/grpc.go diff --git a/pkg/grpc_client/client.go b/pkg/grpc_client/client.go index ac8c5e6..20efd30 100644 --- a/pkg/grpc_client/client.go +++ b/pkg/grpc_client/client.go @@ -1,15 +1,18 @@ package grpc_client import ( + "fmt" "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "github.com/grpc-ecosystem/go-grpc-middleware" "github.com/openinfradev/tks-common/pkg/log" - "github.com/openinfradev/tks-common/pkg/helper" pb "github.com/openinfradev/tks-proto/tks_pb" ) -func CreateCspInfoClient(address string, port int, caller string) (*grpc.ClientConn, pb.CspInfoServiceClient, error) { - cc, err := helper.CreateConnection(address, port, caller) +func CreateCspInfoClient(address string, port int, enabledTLS bool, certPath string ) (*grpc.ClientConn, pb.CspInfoServiceClient, error) { + cc, err := createConnection(address, port, enabledTLS, certPath) if err != nil { log.Fatal("Could not connect to gRPC server", err) return nil, nil, err @@ -18,8 +21,8 @@ func CreateCspInfoClient(address string, port int, caller string) (*grpc.ClientC return cc, sc, nil } -func CreateContractClient(address string, port int, caller string) (*grpc.ClientConn, pb.ContractServiceClient, error) { - cc, err := helper.CreateConnection(address, port, caller) +func CreateContractClient(address string, port int, enabledTLS bool, certPath string) (*grpc.ClientConn, pb.ContractServiceClient, error) { + cc, err := createConnection(address, port, enabledTLS, certPath) if err != nil { log.Fatal("Could not connect to gRPC server", err) return nil, nil, err @@ -28,8 +31,8 @@ func CreateContractClient(address string, port int, caller string) (*grpc.Client return cc, sc, nil } -func CreateClusterInfoClient(address string, port int, caller string) (*grpc.ClientConn, pb.ClusterInfoServiceClient, error) { - cc, err := helper.CreateConnection(address, port, caller) +func CreateClusterInfoClient(address string, port int, enabledTLS bool, certPath string) (*grpc.ClientConn, pb.ClusterInfoServiceClient, error) { + cc, err := createConnection(address, port, enabledTLS, certPath) if err != nil { log.Fatal("Could not connect to gRPC server", err) return nil, nil, err @@ -38,8 +41,8 @@ func CreateClusterInfoClient(address string, port int, caller string) (*grpc.Cli return cc, sc, nil } -func CreateAppInfoClient(address string, port int, caller string) (*grpc.ClientConn, pb.AppInfoServiceClient, error) { - cc, err := helper.CreateConnection(address, port, caller) +func CreateAppInfoClient(address string, port int, enabledTLS bool, certPath string) (*grpc.ClientConn, pb.AppInfoServiceClient, error) { + cc, err := createConnection(address, port, enabledTLS, certPath) if err != nil { log.Fatal("Could not connect to gRPC server", err) return nil, nil, err @@ -48,3 +51,41 @@ func CreateAppInfoClient(address string, port int, caller string) (*grpc.ClientC return cc, sc, nil } + +func createConnection(address string, port int, enabledTLS bool, certPath string) (*grpc.ClientConn, error) { + + creds := insecure.NewCredentials() + if enabledTLS { + _creds, err := loadTLSClientCredential( certPath ) + if err != nil { + return nil, err + } + creds = _creds + } + + host := fmt.Sprintf("%s:%d", address, port) + conn, err := grpc.Dial( + host, + grpc.WithTransportCredentials(creds), + grpc.WithUnaryInterceptor( + grpc_middleware.ChainUnaryClient( + log.IOLoggingForClientSide(), + ), + ), + ) + if err != nil { + return nil, err + } + return conn, nil +} + +func loadTLSClientCredential(clientCertPath string) (credentials.TransportCredentials, error) { + creds, err := credentials.NewClientTLSFromFile(clientCertPath, "") + if err != nil { + log.Error("Fail to load client credentials: ", err) + return nil, err + } + + return creds, nil +} + diff --git a/pkg/grpc_server/server.go b/pkg/grpc_server/server.go new file mode 100644 index 0000000..75a054f --- /dev/null +++ b/pkg/grpc_server/server.go @@ -0,0 +1,57 @@ +package grpc_server + +import ( + "net" + "strconv" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + + "github.com/grpc-ecosystem/go-grpc-middleware" + "github.com/grpc-ecosystem/go-grpc-middleware/recovery" + + "github.com/openinfradev/tks-common/pkg/log" + +) + +func CreateServer(port int, enabledTLS bool, certPath string, keyPath string) (*grpc.Server, net.Listener, error) { + log.Info("Starting to listen port ", port) + + lis, err := net.Listen("tcp", ":"+strconv.Itoa(port)) + if err != nil { + log.Error("failed to listen:", err) + return nil, nil, err + } + + serverOptions := []grpc.ServerOption{ + grpc.UnaryInterceptor( + grpc_middleware.ChainUnaryServer( + grpc_recovery.UnaryServerInterceptor(), + log.IOLoggingForServerSide(), + ), + ), + } + + if enabledTLS { + log.Info("TLS enabled!!!") + tlsCredentials, err := loadTLSCredentials(certPath, keyPath) + if err != nil { + log.Error("Cannot load TLS credentials: ", err) + return nil, nil, err + } + serverOptions = append(serverOptions, grpc.Creds(tlsCredentials)) + } + + return grpc.NewServer(serverOptions...), lis, nil +} + +func loadTLSCredentials(certPath string, keyPath string) (credentials.TransportCredentials, error) { + creds, err := credentials.NewServerTLSFromFile(certPath, keyPath) + if err != nil { + log.Error("Fail to load credentials: ", err) + return nil, err + } + + return creds, nil +} + diff --git a/pkg/helper/grpc.go b/pkg/helper/grpc.go deleted file mode 100644 index 77ba901..0000000 --- a/pkg/helper/grpc.go +++ /dev/null @@ -1,31 +0,0 @@ -package helper - -import ( - "fmt" - - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" - grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" - - "github.com/openinfradev/tks-common/pkg/log" -) - -func CreateConnection(address string, port int, caller string) (*grpc.ClientConn, error) { -insecure.NewCredentials() - - host := fmt.Sprintf("%s:%d", address, port) - conn, err := grpc.Dial( - host, - grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithUnaryInterceptor( - grpc_middleware.ChainUnaryClient( - log.IOLog(), - ), - ), - ) - if err != nil { - return nil, err - } - return conn, nil -} - diff --git a/pkg/log/log.go b/pkg/log/log.go index 6c9a992..da516b3 100644 --- a/pkg/log/log.go +++ b/pkg/log/log.go @@ -6,7 +6,6 @@ import ( "io/ioutil" "context" "fmt" - "time" "github.com/sirupsen/logrus" "google.golang.org/grpc" @@ -74,17 +73,29 @@ func Disable() { } -// for grpc IO logging -func IOLog() grpc.UnaryClientInterceptor { +// grpc IO logging for client-side +func IOLoggingForClientSide() grpc.UnaryClientInterceptor { return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { - start := time.Now() err := invoker(ctx, method, req, reply, cc, opts...) - end := time.Now() - Info(fmt.Sprintf("[GRPC:%s][START:%s][END:%s][ERR:%v]", method, start.Format(time.RFC3339), end.Format(time.RFC3339), err)) - Debug(fmt.Sprintf("[GRPC:%s][REQUEST %s][REPLY %s]", method, req, reply)) + Info(fmt.Sprintf("[INTERNAL_CALL:%s][REQUEST %s][RESPONSE %s]", method, req, reply)) return err } } +// grpc IO logging for server-side +func IOLoggingForServerSide() grpc.UnaryServerInterceptor { + return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (_ interface{}, err error) { + Info(fmt.Sprintf("[START:%s][REQUEST %s]", info.FullMethod, req)) + + res, err := handler(ctx, req) + if err != nil { + Error(err) + } + + Info(fmt.Sprintf("[END:%s][RESPONSE %s]", info.FullMethod, res)) + + return res, err + } +} From e2afd0c0e25b086146eb62f55db04d3a62cac398 Mon Sep 17 00:00:00 2001 From: Taekyu Date: Sun, 20 Mar 2022 00:04:28 +0900 Subject: [PATCH 2/2] trivial. rename variables --- pkg/grpc_client/client.go | 30 ++++++++++++++++-------------- pkg/grpc_server/server.go | 4 ++-- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/pkg/grpc_client/client.go b/pkg/grpc_client/client.go index 20efd30..0907de1 100644 --- a/pkg/grpc_client/client.go +++ b/pkg/grpc_client/client.go @@ -11,8 +11,8 @@ import ( pb "github.com/openinfradev/tks-proto/tks_pb" ) -func CreateCspInfoClient(address string, port int, enabledTLS bool, certPath string ) (*grpc.ClientConn, pb.CspInfoServiceClient, error) { - cc, err := createConnection(address, port, enabledTLS, certPath) +func CreateCspInfoClient(address string, port int, tlsEnabled bool, certPath string ) (*grpc.ClientConn, pb.CspInfoServiceClient, error) { + cc, err := createConnection(address, port, tlsEnabled, certPath) if err != nil { log.Fatal("Could not connect to gRPC server", err) return nil, nil, err @@ -21,8 +21,8 @@ func CreateCspInfoClient(address string, port int, enabledTLS bool, certPath str return cc, sc, nil } -func CreateContractClient(address string, port int, enabledTLS bool, certPath string) (*grpc.ClientConn, pb.ContractServiceClient, error) { - cc, err := createConnection(address, port, enabledTLS, certPath) +func CreateContractClient(address string, port int, tlsEnabled bool, certPath string) (*grpc.ClientConn, pb.ContractServiceClient, error) { + cc, err := createConnection(address, port, tlsEnabled, certPath) if err != nil { log.Fatal("Could not connect to gRPC server", err) return nil, nil, err @@ -31,8 +31,8 @@ func CreateContractClient(address string, port int, enabledTLS bool, certPath st return cc, sc, nil } -func CreateClusterInfoClient(address string, port int, enabledTLS bool, certPath string) (*grpc.ClientConn, pb.ClusterInfoServiceClient, error) { - cc, err := createConnection(address, port, enabledTLS, certPath) +func CreateClusterInfoClient(address string, port int, tlsEnabled bool, certPath string) (*grpc.ClientConn, pb.ClusterInfoServiceClient, error) { + cc, err := createConnection(address, port, tlsEnabled, certPath) if err != nil { log.Fatal("Could not connect to gRPC server", err) return nil, nil, err @@ -41,8 +41,8 @@ func CreateClusterInfoClient(address string, port int, enabledTLS bool, certPath return cc, sc, nil } -func CreateAppInfoClient(address string, port int, enabledTLS bool, certPath string) (*grpc.ClientConn, pb.AppInfoServiceClient, error) { - cc, err := createConnection(address, port, enabledTLS, certPath) +func CreateAppInfoClient(address string, port int, tlsEnabled bool, certPath string) (*grpc.ClientConn, pb.AppInfoServiceClient, error) { + cc, err := createConnection(address, port, tlsEnabled, certPath) if err != nil { log.Fatal("Could not connect to gRPC server", err) return nil, nil, err @@ -52,16 +52,18 @@ func CreateAppInfoClient(address string, port int, enabledTLS bool, certPath str } -func createConnection(address string, port int, enabledTLS bool, certPath string) (*grpc.ClientConn, error) { +func createConnection(address string, port int, tlsEnabled bool, certPath string) (*grpc.ClientConn, error) { + var err error + var creds credentials.TransportCredentials - creds := insecure.NewCredentials() - if enabledTLS { - _creds, err := loadTLSClientCredential( certPath ) + if tlsEnabled { + creds, err = loadTLSClientCredential( certPath ) if err != nil { return nil, err } - creds = _creds - } + } else { + creds = insecure.NewCredentials() + } host := fmt.Sprintf("%s:%d", address, port) conn, err := grpc.Dial( diff --git a/pkg/grpc_server/server.go b/pkg/grpc_server/server.go index 75a054f..df2059d 100644 --- a/pkg/grpc_server/server.go +++ b/pkg/grpc_server/server.go @@ -14,7 +14,7 @@ import ( ) -func CreateServer(port int, enabledTLS bool, certPath string, keyPath string) (*grpc.Server, net.Listener, error) { +func CreateServer(port int, tlsEnabled bool, certPath string, keyPath string) (*grpc.Server, net.Listener, error) { log.Info("Starting to listen port ", port) lis, err := net.Listen("tcp", ":"+strconv.Itoa(port)) @@ -32,7 +32,7 @@ func CreateServer(port int, enabledTLS bool, certPath string, keyPath string) (* ), } - if enabledTLS { + if tlsEnabled { log.Info("TLS enabled!!!") tlsCredentials, err := loadTLSCredentials(certPath, keyPath) if err != nil {