diff --git a/README.md b/README.md index e172cf8be..62f7cbc75 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,10 @@ Note that the external-resizer does not scale with more replicas. Only one exter * `--csiTimeout `: Timeout of all calls to CSI driver. It should be set to value that accommodates majority of `ControllerExpandVolume` calls. 15 seconds is used by default. +* `--retry-interval-start`: The starting value of the exponential backoff for failures. 1 second is used by default. + +* `--retry-interval-max`: The exponential backoff maximum value. 5 minutes is used by default. + * `--workers `: Number of simultaneously running `ControllerExpandVolume` operations. Default value is `10`. * `--metrics-address`: The TCP network address where the prometheus metrics endpoint will run (example: `:8080` which corresponds to port 8080 on local host). The default is empty string, which means metrics endpoint is disabled. diff --git a/cmd/csi-resizer/main.go b/cmd/csi-resizer/main.go index 643a65315..76ad9ff0d 100644 --- a/cmd/csi-resizer/main.go +++ b/cmd/csi-resizer/main.go @@ -20,6 +20,7 @@ import ( "context" "flag" "fmt" + "k8s.io/client-go/util/workqueue" "os" "time" @@ -43,6 +44,9 @@ var ( csiTimeout = flag.Duration("csiTimeout", 15*time.Second, "Timeout for waiting for CSI driver socket.") showVersion = flag.Bool("version", false, "Show version") + retryIntervalStart = flag.Duration("retry-interval-start", time.Second, "Initial retry interval of failed volume resize. It exponentially increases with each failure, up to retry-interval-max.") + retryIntervalMax = flag.Duration("retry-interval-max", 5*time.Minute, "Maximum retry interval of failed volume resize.") + enableLeaderElection = flag.Bool("leader-election", false, "Enable leader election.") leaderElectionNamespace = flag.String("leader-election-namespace", "", "Namespace where the leader election resource lives. Defaults to the pod namespace if not set.") @@ -82,7 +86,9 @@ func main() { } resizerName := csiResizer.Name() - rc := controller.NewResizeController(resizerName, csiResizer, kubeClient, *resyncPeriod, informerFactory) + rc := controller.NewResizeController(resizerName, csiResizer, kubeClient, *resyncPeriod, informerFactory, + workqueue.NewItemExponentialFailureRateLimiter(*retryIntervalStart, *retryIntervalMax), + ) run := func(ctx context.Context) { informerFactory.Start(wait.NeverStop) rc.Run(*workers, ctx) diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 934e63271..931a3ba73 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -65,7 +65,8 @@ func NewResizeController( resizer resizer.Resizer, kubeClient kubernetes.Interface, resyncPeriod time.Duration, - informerFactory informers.SharedInformerFactory) ResizeController { + informerFactory informers.SharedInformerFactory, + pvcRateLimiter workqueue.RateLimiter) ResizeController { pvInformer := informerFactory.Core().V1().PersistentVolumes() pvcInformer := informerFactory.Core().V1().PersistentVolumeClaims() @@ -76,7 +77,7 @@ func NewResizeController( v1.EventSource{Component: fmt.Sprintf("external-resizer %s", name)}) claimQueue := workqueue.NewNamedRateLimitingQueue( - workqueue.DefaultControllerRateLimiter(), fmt.Sprintf("%s-pvc", name)) + pvcRateLimiter, fmt.Sprintf("%s-pvc", name)) ctrl := &resizeController{ name: name, diff --git a/pkg/controller/controller_test.go b/pkg/controller/controller_test.go index defece9a8..75ccc77e8 100644 --- a/pkg/controller/controller_test.go +++ b/pkg/controller/controller_test.go @@ -3,6 +3,7 @@ package controller import ( "context" "fmt" + "k8s.io/client-go/util/workqueue" "testing" "time" @@ -123,7 +124,7 @@ func TestController(t *testing.T) { t.Fatalf("Test %s: Unable to create resizer: %v", test.Name, err) } - controller := NewResizeController(driverName, csiResizer, kubeClient, time.Second, informerFactory) + controller := NewResizeController(driverName, csiResizer, kubeClient, time.Second, informerFactory, workqueue.DefaultControllerRateLimiter()) err = controller.(*resizeController).syncPVC(fmt.Sprintf("%s/%s", test.PVC.Namespace, test.PVC.Name)) if err != nil { t.Fatalf("Test %s: Unexpected error: %v", test.Name, err)