OCPCLOUD-492: Leader election for machineSets, nodelink, vsphere and healthchecks

cmd/machine-healthcheck/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"flag"
 	"runtime"
+	"time"
 
 	"github.com/openshift/machine-api-operator/pkg/controller/machinehealthcheck"
 	"github.com/openshift/machine-api-operator/pkg/metrics"
@@ -44,6 +45,24 @@ func main() {
 		"The address for health checking.",
 	)
 
+	leaderElectResourceNamespace := flag.String(
+		"leader-elect-resource-namespace",
+		"",
+		"The namespace of resource object that is used for locking during leader election. If unspecified and running in cluster, defaults to the service account namespace for the controller. Required for leader-election outside of a cluster.",
+	)
+
+	leaderElect := flag.Bool(
+		"leader-elect",
+		false,
+		"Start a leader election client and gain leadership before executing the main loop. Enable this when running replicated components for high availability.",
+	)
+
+	leaderElectLeaseDuration := flag.Duration(
+		"leader-elect-lease-duration",
+		15*time.Second,
+		"The duration that non-leader candidates will wait after observing a leadership renewal until attempting to acquire leadership of a led but unrenewed leader slot. This is effectively the maximum duration that a leader can be stopped before it is replaced by another candidate. This is only applicable if leader election is enabled.",
+	)
+
 	flag.Parse()
 	printVersion()
 
@@ -54,9 +73,14 @@ func main() {
 	}
 
 	opts := manager.Options{
-		MetricsBindAddress:     *metricsAddress,
-		HealthProbeBindAddress: *healthAddr,
+		MetricsBindAddress:      *metricsAddress,
+		HealthProbeBindAddress:  *healthAddr,
+		LeaderElection:          *leaderElect,
+		LeaderElectionNamespace: *leaderElectResourceNamespace,
+		LeaderElectionID:        "cluster-api-provider-healthcheck-leader",
+		LeaseDuration:           leaderElectLeaseDuration,
 	}
+
 	if *watchNamespace != "" {
 		opts.Namespace = *watchNamespace
 		glog.Infof("Watching machine-api objects only in namespace %q for reconciliation.", opts.Namespace)

cmd/machineset/main.go

@@ -61,6 +61,24 @@ func main() {
 		"The address for health checking.",
 	)
 
+	leaderElectResourceNamespace := flag.String(
+		"leader-elect-resource-namespace",
+		"",
+		"The namespace of resource object that is used for locking during leader election. If unspecified and running in cluster, defaults to the service account namespace for the controller. Required for leader-election outside of a cluster.",
+	)
+
+	leaderElect := flag.Bool(
+		"leader-elect",
+		false,
+		"Start a leader election client and gain leadership before executing the main loop. Enable this when running replicated components for high availability.",
+	)
+
+	leaderElectLeaseDuration := flag.Duration(
+		"leader-elect-lease-duration",
+		15*time.Second,
+		"The duration that non-leader candidates will wait after observing a leadership renewal until attempting to acquire leadership of a led but unrenewed leader slot. This is effectively the maximum duration that a leader can be stopped before it is replaced by another candidate. This is only applicable if leader election is enabled.",
+	)
+
 	flag.Parse()
 	if *watchNamespace != "" {
 		log.Printf("Watching cluster-api objects only in namespace %q for reconciliation.", *watchNamespace)
@@ -76,10 +94,14 @@ func main() {
 	// Create a new Cmd to provide shared dependencies and start components
 	syncPeriod := 10 * time.Minute
 	opts := manager.Options{
-		MetricsBindAddress:     *metricsAddress,
-		SyncPeriod:             &syncPeriod,
-		Namespace:              *watchNamespace,
-		HealthProbeBindAddress: *healthAddr,
+		MetricsBindAddress:      *metricsAddress,
+		SyncPeriod:              &syncPeriod,
+		Namespace:               *watchNamespace,
+		HealthProbeBindAddress:  *healthAddr,
+		LeaderElection:          *leaderElect,
+		LeaderElectionNamespace: *leaderElectResourceNamespace,
+		LeaderElectionID:        "cluster-api-provider-machineset-leader",
+		LeaseDuration:           leaderElectLeaseDuration,
 	}
 
 	mgr, err := manager.New(cfg, opts)

cmd/nodelink-controller/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"flag"
 	"runtime"
+	"time"
 
 	mapiv1 "github.com/openshift/machine-api-operator/pkg/apis/machine/v1beta1"
 	"github.com/openshift/machine-api-operator/pkg/controller"
@@ -23,7 +24,30 @@ func printVersion() {
 func main() {
 	printVersion()
 
-	watchNamespace := flag.String("namespace", "", "Namespace that the controller watches to reconcile machine-api objects. If unspecified, the controller watches for machine-api objects across all namespaces.")
+	watchNamespace := flag.String(
+		"namespace",
+		"",
+		"Namespace that the controller watches to reconcile machine-api objects. If unspecified, the controller watches for machine-api objects across all namespaces.",
+	)
+
+	leaderElectResourceNamespace := flag.String(
+		"leader-elect-resource-namespace",
+		"",
+		"The namespace of resource object that is used for locking during leader election. If unspecified and running in cluster, defaults to the service account namespace for the controller. Required for leader-election outside of a cluster.",
+	)
+
+	leaderElect := flag.Bool(
+		"leader-elect",
+		false,
+		"Start a leader election client and gain leadership before executing the main loop. Enable this when running replicated components for high availability.",
+	)
+
+	leaderElectLeaseDuration := flag.Duration(
+		"leader-elect-lease-duration",
+		15*time.Second,
+		"The duration that non-leader candidates will wait after observing a leadership renewal until attempting to acquire leadership of a led but unrenewed leader slot. This is effectively the maximum duration that a leader can be stopped before it is replaced by another candidate. This is only applicable if leader election is enabled.",
+	)
+
 	klog.InitFlags(nil)
 	flag.Set("logtostderr", "true")
 	flag.Parse()
@@ -36,7 +60,11 @@ func main() {
 
 	opts := manager.Options{
 		// Disable metrics serving
-		MetricsBindAddress: "0",
+		MetricsBindAddress:      "0",
+		LeaderElection:          *leaderElect,
+		LeaderElectionNamespace: *leaderElectResourceNamespace,
+		LeaderElectionID:        "cluster-api-provider-nodelink-leader",
+		LeaseDuration:           leaderElectLeaseDuration,
 	}
 	if *watchNamespace != "" {
 		opts.Namespace = *watchNamespace

cmd/vsphere/main.go

@@ -25,8 +25,36 @@ func main() {
 	flag.BoolVar(&printVersion, "version", false, "print version and exit")
 
 	klog.InitFlags(nil)
-	watchNamespace := flag.String("namespace", "", "Namespace that the controller watches to reconcile machine-api objects. If unspecified, the controller watches for machine-api objects across all namespaces.")
-	metricsAddress := flag.String("metrics-bind-address", metrics.DefaultMachineMetricsAddress, "Address for hosting metrics")
+	watchNamespace := flag.String(
+		"namespace",
+		"",
+		"Namespace that the controller watches to reconcile machine-api objects. If unspecified, the controller watches for machine-api objects across all namespaces.",
+	)
+
+	leaderElectResourceNamespace := flag.String(
+		"leader-elect-resource-namespace",
+		"",
+		"The namespace of resource object that is used for locking during leader election. If unspecified and running in cluster, defaults to the service account namespace for the controller. Required for leader-election outside of a cluster.",
+	)
+
+	leaderElect := flag.Bool(
+		"leader-elect",
+		false,
+		"Start a leader election client and gain leadership before executing the main loop. Enable this when running replicated components for high availability.",
+	)
+
+	leaderElectLeaseDuration := flag.Duration(
+		"leader-elect-lease-duration",
+		15*time.Second,
+		"The duration that non-leader candidates will wait after observing a leadership renewal until attempting to acquire leadership of a led but unrenewed leader slot. This is effectively the maximum duration that a leader can be stopped before it is replaced by another candidate. This is only applicable if leader election is enabled.",
+	)
+
+	metricsAddress := flag.String(
+		"metrics-bind-address",
+		metrics.DefaultMachineMetricsAddress,
+		"Address for hosting metrics",
+	)
+
 	flag.Set("logtostderr", "true")
 	healthAddr := flag.String(
 		"health-addr",
@@ -44,10 +72,15 @@ func main() {
 	syncPeriod := 10 * time.Minute
 
 	opts := manager.Options{
-		MetricsBindAddress:     *metricsAddress,
-		HealthProbeBindAddress: *healthAddr,
-		SyncPeriod:             &syncPeriod,
+		MetricsBindAddress:      *metricsAddress,
+		HealthProbeBindAddress:  *healthAddr,
+		SyncPeriod:              &syncPeriod,
+		LeaderElection:          *leaderElect,
+		LeaderElectionNamespace: *leaderElectResourceNamespace,
+		LeaderElectionID:        "cluster-api-provider-vsphere-leader",
+		LeaseDuration:           leaderElectLeaseDuration,
 	}
+
 	if *watchNamespace != "" {
 		opts.Namespace = *watchNamespace
 		klog.Infof("Watching machine-api objects only in namespace %q for reconciliation.", opts.Namespace)

pkg/operator/sync.go

@@ -368,6 +368,7 @@ func newContainers(config *OperatorConfig, features map[string]bool) []corev1.Co
 	args := []string{
 		"--logtostderr=true",
 		"--v=3",
+		"--leader-elect=true",
 		fmt.Sprintf("--namespace=%s", config.TargetNamespace),
 	}