libcontainerd: use healthcheck to track containerd conn

Signed-off-by: Alexander Morozov <[email protected]>

Author: LK4D4

Dockerfile

@@ -243,7 +243,7 @@ RUN set -x \
 	&& rm -rf "$GOPATH"
 
 # Install containerd
-ENV CONTAINERD_COMMIT 35a736c471ccd3ebfc7b80ceeb0ee303129acd61
+ENV CONTAINERD_COMMIT 4c21ad662f71af56c0e6b29c0afef72df441d1ff
 RUN set -x \
 	&& export GOPATH="$(mktemp -d)" \
 	&& git clone https://github.com/docker/containerd.git "$GOPATH/src/github.com/docker/containerd" \

Dockerfile.aarch64

@@ -186,7 +186,7 @@ RUN set -x \
 	&& rm -rf "$GOPATH"
 
 # Install containerd
-ENV CONTAINERD_COMMIT 35a736c471ccd3ebfc7b80ceeb0ee303129acd61
+ENV CONTAINERD_COMMIT 4c21ad662f71af56c0e6b29c0afef72df441d1ff
 RUN set -x \
 	&& export GOPATH="$(mktemp -d)" \
 	&& git clone https://github.com/docker/containerd.git "$GOPATH/src/github.com/docker/containerd" \

Dockerfile.armhf

@@ -184,7 +184,7 @@ RUN set -x \
 	&& rm -rf "$GOPATH"
 
 # Install containerd
-ENV CONTAINERD_COMMIT 35a736c471ccd3ebfc7b80ceeb0ee303129acd61
+ENV CONTAINERD_COMMIT 4c21ad662f71af56c0e6b29c0afef72df441d1ff
 RUN set -x \
 	&& export GOPATH="$(mktemp -d)" \
 	&& git clone https://github.com/docker/containerd.git "$GOPATH/src/github.com/docker/containerd" \

Dockerfile.ppc64le

@@ -204,7 +204,7 @@ RUN set -x \
 	&& rm -rf "$GOPATH"
 
 # Install containerd
-ENV CONTAINERD_COMMIT 35a736c471ccd3ebfc7b80ceeb0ee303129acd61
+ENV CONTAINERD_COMMIT 4c21ad662f71af56c0e6b29c0afef72df441d1ff
 RUN set -x \
 	&& export GOPATH="$(mktemp -d)" \
 	&& git clone https://github.com/docker/containerd.git "$GOPATH/src/github.com/docker/containerd" \

Dockerfile.s390x

@@ -196,7 +196,7 @@ RUN set -x \
 	&& rm -rf "$GOPATH"
 
 # Install containerd
-ENV CONTAINERD_COMMIT 35a736c471ccd3ebfc7b80ceeb0ee303129acd61
+ENV CONTAINERD_COMMIT 4c21ad662f71af56c0e6b29c0afef72df441d1ff
 RUN set -x \
 	&& export GOPATH="$(mktemp -d)" \
 	&& git clone https://github.com/docker/containerd.git "$GOPATH/src/github.com/docker/containerd" \

Dockerfile.simple

@@ -68,7 +68,7 @@ RUN set -x \
 	&& rm -rf "$GOPATH"
 
 # Install containerd
-ENV CONTAINERD_COMMIT 35a736c471ccd3ebfc7b80ceeb0ee303129acd61
+ENV CONTAINERD_COMMIT 4c21ad662f71af56c0e6b29c0afef72df441d1ff
 RUN set -x \
 	&& export GOPATH="$(mktemp -d)" \
 	&& git clone https://github.com/docker/containerd.git "$GOPATH/src/github.com/docker/containerd" \

hack/vendor.sh

@@ -141,7 +141,7 @@ clone git google.golang.org/cloud dae7e3d993bc3812a2185af60552bb6b847e52a0 https
 clone git github.com/docker/docker-credential-helpers v0.3.0
 
 # containerd
-clone git github.com/docker/containerd 35a736c471ccd3ebfc7b80ceeb0ee303129acd61
+clone git github.com/docker/containerd 4c21ad662f71af56c0e6b29c0afef72df441d1ff
 
 # cluster
 clone git github.com/docker/swarmkit 27fbaef4ceed648bb575969ccc9083a6e104a719

libcontainerd/remote_linux.go

@@ -25,18 +25,20 @@ import (
 	"golang.org/x/net/context"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/grpclog"
+	"google.golang.org/grpc/health/grpc_health_v1"
 	"google.golang.org/grpc/transport"
 )
 
 const (
-	maxConnectionRetryCount   = 3
-	connectionRetryDelay      = 3 * time.Second
-	containerdShutdownTimeout = 15 * time.Second
-	containerdBinary          = "docker-containerd"
-	containerdPidFilename     = "docker-containerd.pid"
-	containerdSockFilename    = "docker-containerd.sock"
-	containerdStateDir        = "containerd"
-	eventTimestampFilename    = "event.ts"
+	maxConnectionRetryCount      = 3
+	connectionRetryDelay         = 3 * time.Second
+	containerdHealthCheckTimeout = 3 * time.Second
+	containerdShutdownTimeout    = 15 * time.Second
+	containerdBinary             = "docker-containerd"
+	containerdPidFilename        = "docker-containerd.pid"
+	containerdSockFilename       = "docker-containerd.sock"
+	containerdStateDir           = "containerd"
+	eventTimestampFilename       = "event.ts"
 )
 
 type remote struct {
@@ -134,37 +136,41 @@ func (r *remote) UpdateOptions(options ...RemoteOption) error {
 
 func (r *remote) handleConnectionChange() {
 	var transientFailureCount = 0
-	state := grpc.Idle
+
+	ticker := time.NewTicker(500 * time.Millisecond)
+	defer ticker.Stop()
+	healthClient := grpc_health_v1.NewHealthClient(r.rpcConn)
+
 	for {
-		s, err := r.rpcConn.WaitForStateChange(context.Background(), state)
-		if err != nil {
-			break
+		<-ticker.C
+		ctx, cancel := context.WithTimeout(context.Background(), containerdHealthCheckTimeout)
+		_, err := healthClient.Check(ctx, &grpc_health_v1.HealthCheckRequest{})
+		cancel()
+		if err == nil {
+			continue
 		}
-		state = s
-		logrus.Debugf("libcontainerd: containerd connection state change: %v", s)
+
+		logrus.Debugf("libcontainerd: containerd health check returned error: %v", err)
 
 		if r.daemonPid != -1 {
-			switch state {
-			case grpc.TransientFailure:
-				// Reset state to be notified of next failure
-				transientFailureCount++
-				if transientFailureCount >= maxConnectionRetryCount {
-					transientFailureCount = 0
-					if utils.IsProcessAlive(r.daemonPid) {
-						utils.KillProcess(r.daemonPid)
-					}
-					<-r.daemonWaitCh
-					if err := r.runContainerdDaemon(); err != nil { //FIXME: Handle error
-						logrus.Errorf("libcontainerd: error restarting containerd: %v", err)
-					}
-				} else {
-					state = grpc.Idle
-					time.Sleep(connectionRetryDelay)
-				}
-			case grpc.Shutdown:
+			if strings.Contains(err.Error(), "is closing") {
 				// Well, we asked for it to stop, just return
 				return
 			}
+			// all other errors are transient
+			// Reset state to be notified of next failure
+			transientFailureCount++
+			if transientFailureCount >= maxConnectionRetryCount {
+				transientFailureCount = 0
+				if utils.IsProcessAlive(r.daemonPid) {
+					utils.KillProcess(r.daemonPid)
+				}
+				<-r.daemonWaitCh
+				if err := r.runContainerdDaemon(); err != nil { //FIXME: Handle error
+					logrus.Errorf("libcontainerd: error restarting containerd: %v", err)
+				}
+				continue
+			}
 		}
 	}
 }

vendor/src/google.golang.org/grpc/health/grpc_health_v1/health.pb.go

@@ -0,0 +1,20 @@
+syntax = "proto3";
+
+package grpc.health.v1;
+
+message HealthCheckRequest {
+  string service = 1;
+}
+
+message HealthCheckResponse {
+  enum ServingStatus {
+ 	UNKNOWN = 0;
+	SERVING = 1;
+	NOT_SERVING = 2;
+  }
+  ServingStatus status = 1;
+}
+
+service Health{
+  rpc Check(HealthCheckRequest) returns (HealthCheckResponse);
+}