Bug 1912237: Rebase to v1.1.0 for OCP 4.7

Dockerfile.openshift.rhel7

@@ -3,6 +3,6 @@ WORKDIR /go/src/github.com/kubernetes-csi/external-resizer
 COPY . .
 RUN make build
 
-FROM registry.svc.ci.openshift.org/ocp/4.0:base
+FROM registry.svc.ci.openshift.org/ocp/4.7:base
 COPY --from=builder /go/src/github.com/kubernetes-csi/external-resizer/bin/csi-resizer /usr/bin/
 ENTRYPOINT ["/usr/bin/csi-resizer"]

README.md

@@ -57,7 +57,7 @@ Note that the external-resizer does not scale with more replicas. Only one exter
 
 * `--workers <num>`: Number of simultaneously running `ControllerExpandVolume` operations. Default value is `10`.
 
-* `--metrics-address`: The TCP network address where the prometheus metrics endpoint will run (example: `:8080` which corresponds to port 8080 on local host). The default is empty string, which means metrics endpoint is disabled.
+* `--http-endpoint`: The TCP network address where the HTTP server for diagnostics, including metrics and leader election health check, will listen (example: `:8080` which corresponds to port 8080 on local host). The default is empty string, which means the server is disabled.
 
 * `--metrics-path`: The HTTP path where prometheus metrics will be exposed. Default is `/metrics`.
 
@@ -69,10 +69,20 @@ Note that the external-resizer does not scale with more replicas. Only one exter
 
 * `--master <url>`: Master URL to build a client config from. When omitted, default token provided by Kubernetes will be used. This option is useful only when the external-resizer does not run as a Kubernetes pod, e.g. for debugging. Either this or `--kubeconfig` needs to be set if the external-resizer is being run out of cluster.
 
+* `--metrics-address`: (deprecated) The TCP network address where the prometheus metrics endpoint will run (example: `:8080` which corresponds to port 8080 on local host). The default is empty string, which means metrics endpoint is disabled.
+
 * `--version`: Prints current external-resizer version and quits.
 
 * All glog / klog arguments are supported, such as `-v <log level>` or `-alsologtostderr`.
 
+### HTTP endpoint
+
+The external-resizer optionally exposes an HTTP endpoint at address:port specified by `--http-endpoint` argument. When set, these two paths are exposed:
+
+* Metrics path, as set by `--metrics-path` argument (default is `/metrics`).
+* Leader election health check at `/healthz/leader-election`. It is recommended to run a liveness probe against this endpoint when leader election is used to kill external-resizer leader that fails to connect to the API server to renew its leadership. See https://github.com/kubernetes-csi/csi-lib-utils/issues/66 for details.
+
+
 ## Community, discussion, contribution, and support
 
 Learn how to engage with the Kubernetes community on the [community page](http://kubernetes.io/community/).

cmd/csi-resizer/main.go

@@ -20,9 +20,12 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"net/http"
 	"os"
 	"time"
 
+	"github.com/kubernetes-csi/csi-lib-utils/metrics"
+	"github.com/kubernetes-csi/external-resizer/pkg/csi"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
@@ -36,7 +39,7 @@ import (
 
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/informers"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 var (
@@ -56,7 +59,8 @@ var (
 	enableLeaderElection    = flag.Bool("leader-election", false, "Enable leader election.")
 	leaderElectionNamespace = flag.String("leader-election-namespace", "", "Namespace where the leader election resource lives. Defaults to the pod namespace if not set.")
 
-	metricsAddress = flag.String("metrics-address", "", "The TCP network address where the prometheus metrics endpoint will listen (example: `:8080`). The default is empty string, which means metrics endpoint is disabled.")
+	metricsAddress = flag.String("metrics-address", "", "(deprecated) The TCP network address where the prometheus metrics endpoint will listen (example: `:8080`). The default is empty string, which means metrics endpoint is disabled. Only one of `--metrics-address` and `--http-endpoint` can be set.")
+	httpEndpoint   = flag.String("http-endpoint", "", "The TCP network address where the HTTP server for diagnostics, including metrics and leader election health check, will listen (example: `:8080`). The default is empty string, which means the server is disabled. Only one of `--metrics-address` and `--http-endpoint` can be set.")
 	metricsPath    = flag.String("metrics-path", "/metrics", "The HTTP path where prometheus metrics will be exposed. Default is `/metrics`.")
 
 	kubeAPIQPS   = flag.Float64("kube-api-qps", 5, "QPS to use while communicating with the kubernetes apiserver. Defaults to 5.0.")
@@ -78,6 +82,15 @@ func main() {
 	}
 	klog.Infof("Version : %s", version)
 
+	if *metricsAddress != "" && *httpEndpoint != "" {
+		klog.Error("only one of `--metrics-address` and `--http-endpoint` can be set.")
+		os.Exit(1)
+	}
+	addr := *metricsAddress
+	if addr == "" {
+		addr = *httpEndpoint
+	}
+
 	var config *rest.Config
 	var err error
 	if *master != "" || *kubeConfig != "" {
@@ -99,17 +112,44 @@ func main() {
 
 	informerFactory := informers.NewSharedInformerFactory(kubeClient, *resyncPeriod)
 
-	csiResizer, err := resizer.NewResizer(
-		*csiAddress,
+	mux := http.NewServeMux()
+
+	metricsManager := metrics.NewCSIMetricsManager("" /* driverName */)
+
+	csiClient, err := csi.New(*csiAddress, *timeout, metricsManager)
+	if err != nil {
+		klog.Fatal(err.Error())
+	}
+
+	driverName, err := getDriverName(csiClient, *timeout)
+	if err != nil {
+		klog.Fatal(fmt.Errorf("get driver name failed: %v", err))
+	}
+	klog.V(2).Infof("CSI driver name: %q", driverName)
+
+	csiResizer, err := resizer.NewResizerFromClient(
+		csiClient,
 		*timeout,
 		kubeClient,
 		informerFactory,
-		*metricsAddress,
-		*metricsPath)
+		driverName)
 	if err != nil {
 		klog.Fatal(err.Error())
 	}
 
+	// Start HTTP server for metrics + leader election healthz
+	if addr != "" {
+		metricsManager.RegisterToServer(mux, *metricsPath)
+		metricsManager.SetDriverName(driverName)
+		go func() {
+			klog.Infof("ServeMux listening at %q", addr)
+			err := http.ListenAndServe(addr, mux)
+			if err != nil {
+				klog.Fatalf("Failed to start HTTP server at specified address (%q) and metrics path (%q): %s", addr, *metricsPath, err)
+			}
+		}()
+	}
+
 	resizerName := csiResizer.Name()
 	rc := controller.NewResizeController(resizerName, csiResizer, kubeClient, *resyncPeriod, informerFactory,
 		workqueue.NewItemExponentialFailureRateLimiter(*retryIntervalStart, *retryIntervalMax),
@@ -129,6 +169,9 @@ func main() {
 			klog.Fatal(err.Error())
 		}
 		le := leaderelection.NewLeaderElection(leKubeClient, lockName, run)
+		if *httpEndpoint != "" {
+			le.PrepareHealthCheck(mux, leaderelection.DefaultHealthCheckTimeout)
+		}
 
 		if *leaderElectionNamespace != "" {
 			le.WithNamespace(*leaderElectionNamespace)
@@ -139,3 +182,9 @@ func main() {
 		}
 	}
 }
+
+func getDriverName(client csi.Client, timeout time.Duration) (string, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+	return client.GetDriverName(ctx)
+}

deploy/kubernetes/deployment.yaml

@@ -19,15 +19,28 @@ spec:
       serviceAccount: csi-resizer
       containers:
         - name: csi-resizer
-          image: quay.io/k8scsi/csi-resizer:canary
+          image: gcr.io/k8s-staging-sig-storage/csi-resizer:canary
           args:
             - "--v=5"
             - "--csi-address=$(ADDRESS)"
             - "--leader-election"
+            - "--http-endpoint=:8080"
           env:
             - name: ADDRESS
               value: /var/lib/csi/sockets/pluginproxy/mock.socket
           imagePullPolicy: "IfNotPresent"
+          ports:
+            - containerPort: 8080
+              name: http-endpoint
+              protocol: TCP
+          livenessProbe:
+            failureThreshold: 1
+            httpGet:
+              path: /healthz/leader-election
+              port: http-endpoint
+            initialDelaySeconds: 10
+            timeoutSeconds: 10
+            periodSeconds: 20
           volumeMounts:
             - name: socket-dir
               mountPath: /var/lib/csi/sockets/pluginproxy/

go.mod

@@ -3,53 +3,58 @@ module github.com/kubernetes-csi/external-resizer
 go 1.15
 
 require (
-	github.com/container-storage-interface/spec v1.2.0
-	github.com/imdario/mergo v0.3.9 // indirect
-	github.com/kubernetes-csi/csi-lib-utils v0.7.0
-	google.golang.org/grpc v1.28.0
-	k8s.io/api v0.19.0
-	k8s.io/apimachinery v0.19.0
-	k8s.io/client-go v0.19.0
-	k8s.io/csi-translation-lib v0.19.0
-	k8s.io/klog v1.0.0
+	github.com/container-storage-interface/spec v1.3.0
+	github.com/go-logr/logr v0.3.0 // indirect
+	github.com/google/go-cmp v0.5.4 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/googleapis/gnostic v0.5.3 // indirect
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
+	github.com/imdario/mergo v0.3.11 // indirect
+	github.com/kubernetes-csi/csi-lib-utils v0.9.0
+	github.com/prometheus/client_golang v1.9.0 // indirect
+	golang.org/x/crypto v0.0.0-20201217014255-9d1352758620 // indirect
+	golang.org/x/net v0.0.0-20201216054612-986b41b23924 // indirect
+	golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5 // indirect
+	golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf // indirect
+	golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d // indirect
+	google.golang.org/grpc v1.34.0
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	k8s.io/api v0.20.0
+	k8s.io/apimachinery v0.21.0-alpha.0
+	k8s.io/client-go v0.20.0
+	k8s.io/component-base v0.20.0 // indirect
+	k8s.io/csi-translation-lib v0.20.0
+	k8s.io/klog/v2 v2.4.0
 )
 
-replace k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.19.0
-
-replace k8s.io/apiserver => k8s.io/apiserver v0.19.0
-
-replace k8s.io/cli-runtime => k8s.io/cli-runtime v0.19.0
-
-replace k8s.io/cloud-provider => k8s.io/cloud-provider v0.19.0
-
-replace k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.19.0
-
-replace k8s.io/code-generator => k8s.io/code-generator v0.19.0
-
-replace k8s.io/component-base => k8s.io/component-base v0.19.0
-
-replace k8s.io/cri-api => k8s.io/cri-api v0.19.0
-
-replace k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.19.0
-
-replace k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.19.0
-
-replace k8s.io/kube-proxy => k8s.io/kube-proxy v0.19.0
-
-replace k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.19.0
-
-replace k8s.io/kubectl => k8s.io/kubectl v0.19.0
-
-replace k8s.io/kubelet => k8s.io/kubelet v0.19.0
-
-replace k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.19.0
-
-replace k8s.io/metrics => k8s.io/metrics v0.19.0
-
-replace k8s.io/node-api => k8s.io/node-api v0.19.0
-
-replace k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.19.0
-
-replace k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.19.0
-
-replace k8s.io/sample-controller => k8s.io/sample-controller v0.19.0
+replace (
+	k8s.io/api => k8s.io/api v0.20.0
+	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.20.0
+	k8s.io/apimachinery => k8s.io/apimachinery v0.21.0-alpha.0
+	k8s.io/apiserver => k8s.io/apiserver v0.20.0
+	k8s.io/cli-runtime => k8s.io/cli-runtime v0.20.0
+	k8s.io/client-go => k8s.io/client-go v0.20.0
+	k8s.io/cloud-provider => k8s.io/cloud-provider v0.20.0
+	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.20.0
+	k8s.io/code-generator => k8s.io/code-generator v0.20.1-rc.1
+	k8s.io/component-base => k8s.io/component-base v0.20.0
+	k8s.io/component-helpers => k8s.io/component-helpers v0.20.0
+	k8s.io/controller-manager => k8s.io/controller-manager v0.20.0
+	k8s.io/cri-api => k8s.io/cri-api v0.20.1-rc.1
+	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.20.0
+	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.20.0
+	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.20.0
+	k8s.io/kube-proxy => k8s.io/kube-proxy v0.20.0
+	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.20.0
+	k8s.io/kubectl => k8s.io/kubectl v0.20.0
+	k8s.io/kubelet => k8s.io/kubelet v0.20.0
+	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.20.0
+	k8s.io/metrics => k8s.io/metrics v0.20.0
+	k8s.io/mount-utils => k8s.io/mount-utils v0.20.1-rc.1
+	k8s.io/node-api => k8s.io/node-api v0.19.0
+	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.20.0
+	k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.20.0
+	k8s.io/sample-controller => k8s.io/sample-controller v0.20.0
+)