Add health check on API server

manifests/components/04d_argocd-server-deployment.yaml

@@ -33,6 +33,14 @@ spec:
         volumeMounts:
         - mountPath: /shared
           name: static-files
+
+        readinessProbe:
+          httpGet:
+            path: /healthz
+            port: 8080
+          initialDelaySeconds: 3
+          periodSeconds: 30
+
       - name: dex
         image: quay.io/coreos/dex:v2.10.0
         command: [/shared/argocd-util, rundex]

server/server.go

@@ -51,6 +51,7 @@ import (
 	"github.com/argoproj/argo-cd/util/dex"
 	dexutil "github.com/argoproj/argo-cd/util/dex"
 	grpc_util "github.com/argoproj/argo-cd/util/grpc"
+	"github.com/argoproj/argo-cd/util/healthz"
 	jsonutil "github.com/argoproj/argo-cd/util/json"
 	jwtutil "github.com/argoproj/argo-cd/util/jwt"
 	projectutil "github.com/argoproj/argo-cd/util/project"
@@ -415,6 +416,10 @@ func (a *ArgoCDServer) newHTTPServer(ctx context.Context, port int) *http.Server
 	mustRegisterGWHandler(project.RegisterProjectServiceHandlerFromEndpoint, ctx, gwmux, endpoint, dOpts)
 
 	swagger.ServeSwaggerUI(mux, packr.NewBox("."), "/swagger-ui")
+	healthz.ServeHealthCheck(mux, func() error {
+		_, err := a.KubeClientset.(*kubernetes.Clientset).ServerVersion()
+		return err
+	})
 
 	// Dex reverse proxy and client app and OAuth2 login/callback
 	a.registerDexHandlers(mux)

util/healthz/healthz.go

@@ -0,0 +1,21 @@
+package healthz
+
+import (
+	"fmt"
+	"net/http"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// ServeHealthCheck serves the health check endpoint.
+// ServeHealthCheck relies on the provided function to return an error if unhealthy and nil otherwise.
+func ServeHealthCheck(mux *http.ServeMux, f func() error) {
+	mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
+		if err := f(); err != nil {
+			w.WriteHeader(http.StatusServiceUnavailable)
+			log.Errorln(w, err)
+		} else {
+			fmt.Fprintln(w, "ok")
+		}
+	})
+}

util/healthz/healthz_test.go

@@ -0,0 +1,59 @@
+package healthz
+
+import (
+	"fmt"
+	"net"
+	"net/http"
+	"testing"
+)
+
+func TestHealthCheck(t *testing.T) {
+	sentinel := false
+
+	serve := func(c chan<- string) {
+		// listen on first available dynamic (unprivileged) port
+		listener, err := net.Listen("tcp", ":0")
+		if err != nil {
+			panic(err)
+		}
+
+		// send back the address so that it can be used
+		c <- listener.Addr().String()
+
+		mux := http.NewServeMux()
+		ServeHealthCheck(mux, func() error {
+			if sentinel {
+				return fmt.Errorf("This is a dummy error")
+			}
+			return nil
+		})
+		panic(http.Serve(listener, mux))
+	}
+
+	c := make(chan string, 1)
+
+	// run a local webserver to test data retrieval
+	go serve(c)
+
+	address := <-c
+	t.Logf("Listening at address: %s", address)
+
+	server := "http://" + address
+
+	resp, err := http.Get(server + "/healthz")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if resp.StatusCode != 200 {
+		t.Fatalf("Was expecting status code 200 from health check, but got %d instead", resp.StatusCode)
+	}
+
+	sentinel = true
+
+	resp, _ = http.Get(server + "/healthz")
+	if resp.StatusCode != 503 {
+		t.Fatalf("Was expecting status code 503 from health check, but got %d instead", resp.StatusCode)
+	}
+
+}