Ensure grpc keepalive on all clients

CHANGELOG.md

@@ -7,6 +7,7 @@
   - `cortex_ingester_tsdb_head_truncations_failed_total`
   - `cortex_ingester_tsdb_head_truncations_total`
   - `cortex_ingester_tsdb_head_gc_duration_seconds`
+* [ENHANCEMENT] Enforced keepalive on all gRPC clients used for inter-service communication. #3431
 * [ENHANCEMENT] Added `cortex_alertmanager_config_hash` metric to expose hash of Alertmanager Config loaded per user. #3388
 
 ## 1.5.0 in progress

pkg/chunk/grpc/grpc_client.go

@@ -22,7 +22,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 func connectToGrpcServer(serverAddress string) (GrpcStoreClient, *grpc.ClientConn, error) {
 	params := keepalive.ClientParameters{
 		Time:                time.Second * 20,
-		Timeout:             time.Minute * 10,
+		Timeout:             time.Second * 10,
 		PermitWithoutStream: true,
 	}
 	param := grpc.WithKeepaliveParams(params)

pkg/util/grpcclient/grpcclient.go

@@ -2,13 +2,15 @@ package grpcclient
 
 import (
 	"flag"
+	"time"
 
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
 	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
 	"github.com/pkg/errors"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/encoding/gzip"
+	"google.golang.org/grpc/keepalive"
 
 	"github.com/cortexproject/cortex/pkg/util"
 	"github.com/cortexproject/cortex/pkg/util/flagext"
@@ -90,6 +92,11 @@ func (cfg *Config) DialOption(unaryClientInterceptors []grpc.UnaryClientIntercep
 		grpc.WithDefaultCallOptions(cfg.CallOptions()...),
 		grpc.WithUnaryInterceptor(grpc_middleware.ChainUnaryClient(unaryClientInterceptors...)),
 		grpc.WithStreamInterceptor(grpc_middleware.ChainStreamClient(streamClientInterceptors...)),
+		grpc.WithKeepaliveParams(keepalive.ClientParameters{
+			Time:                time.Second * 20,
+			Timeout:             time.Second * 10,
+			PermitWithoutStream: true,
+		}),
 	}
 }