From c685224f21edf4f67dee864e148df7eb4bbd09e2 Mon Sep 17 00:00:00 2001 From: Himanshu Sharma Date: Thu, 14 Jul 2022 18:10:51 +0530 Subject: [PATCH 1/2] mark machine failed during rollout also --- .../machinecontroller/machine_util.go | 27 ------------------- .../machinecontroller/machine_util_test.go | 21 --------------- 2 files changed, 48 deletions(-) diff --git a/pkg/util/provider/machinecontroller/machine_util.go b/pkg/util/provider/machinecontroller/machine_util.go index d001e92ef..b64ef2ed6 100644 --- a/pkg/util/provider/machinecontroller/machine_util.go +++ b/pkg/util/provider/machinecontroller/machine_util.go @@ -35,7 +35,6 @@ import ( machineapi "github.com/gardener/machine-controller-manager/pkg/apis/machine" "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" - "github.com/gardener/machine-controller-manager/pkg/controller/autoscaler" "github.com/gardener/machine-controller-manager/pkg/util/nodeops" "github.com/gardener/machine-controller-manager/pkg/util/provider/drain" "github.com/gardener/machine-controller-manager/pkg/util/provider/driver" @@ -739,11 +738,6 @@ func (c *controller) reconcileMachineHealth(ctx context.Context, machine *v1alph timeOutDuration, machine.Status.Conditions, ) - // ensuring rollout of machineDeployment is not happening - if c.isRollingOut(machine) { - klog.V(2).Infof("Skipping marking machine=%s Failed, as rollout for corresponding machineDeployment is happening, will retry in next sync", machine.Name) - return machineutils.MediumRetry, nil - } machineDeployName := getMachineDeploymentName(machine) // creating lock for machineDeployment, if not allocated @@ -1514,27 +1508,6 @@ func (c *controller) tryMarkingMachineFailed(ctx context.Context, machine, clone return machineutils.ShortRetry, err } -func (c *controller) isRollingOut(machine *v1alpha1.Machine) bool { - nodeName := machine.Labels["node"] - node, err := c.nodeLister.Get(nodeName) - if err != nil { - if apierrors.IsNotFound(err) { - return false - } - klog.Errorf("error fetching node object for machine %q", machine.Name) - return true - } - - nodeAnnotations := node.Annotations - if nodeAnnotations == nil { - return false - } else if _, exists := nodeAnnotations[autoscaler.ClusterAutoscalerScaleDownDisabledAnnotationByMCMKey]; exists { - return true - } - - return false -} - func getProviderID(machine *v1alpha1.Machine) string { return machine.Spec.ProviderID } diff --git a/pkg/util/provider/machinecontroller/machine_util_test.go b/pkg/util/provider/machinecontroller/machine_util_test.go index 3900ad3a8..f8e4af29a 100644 --- a/pkg/util/provider/machinecontroller/machine_util_test.go +++ b/pkg/util/provider/machinecontroller/machine_util_test.go @@ -24,7 +24,6 @@ import ( "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" machinev1 "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" - "github.com/gardener/machine-controller-manager/pkg/controller/autoscaler" "github.com/gardener/machine-controller-manager/pkg/fakeclient" "github.com/gardener/machine-controller-manager/pkg/util/permits" "github.com/gardener/machine-controller-manager/pkg/util/provider/machineutils" @@ -2299,26 +2298,6 @@ var _ = Describe("machine_util", func() { expectedPhase: v1alpha1.MachineFailed, }, }), - Entry("HealthTimedout machine shouldn't be marked Failed, if rolling update happening", &data{ - setup: setup{ - machines: []*v1alpha1.Machine{ - newMachine( - &machinev1.MachineTemplateSpec{ObjectMeta: *newObjectMeta(&metav1.ObjectMeta{GenerateName: machineSet1Deploy1}, 0)}, - &machinev1.MachineStatus{Node: "node", Conditions: nodeConditions(false, false, false, false, false), CurrentStatus: machinev1.CurrentStatus{Phase: v1alpha1.MachineUnknown, LastUpdateTime: metav1.NewTime(time.Now().Add(-15 * time.Minute))}}, - &metav1.OwnerReference{Name: machineSet1Deploy1}, - nil, map[string]string{"node": "node-0"}, true, metav1.Now()), - }, - nodes: []*v1.Node{ - newNode(1, nil, map[string]string{autoscaler.ClusterAutoscalerScaleDownDisabledAnnotationByMCMKey: "true"}, &v1.NodeSpec{}, &v1.NodeStatus{Phase: corev1.NodeRunning, Conditions: nodeConditions(false, false, false, false, false)}), - }, - targetMachineName: machineSet1Deploy1 + "-" + "0", - }, - expect: expect{ - retryPeriod: machineutils.MediumRetry, - err: nil, - expectedPhase: v1alpha1.MachineUnknown, - }, - }), Entry("simple machine WITHOUT creation Timeout(20 min) and Pending state", &data{ setup: setup{ machines: []*v1alpha1.Machine{ From 9256b551e42c38fe19c3e2e32d2cc71df355b3f8 Mon Sep 17 00:00:00 2001 From: Himanshu Sharma Date: Mon, 18 Jul 2022 17:06:15 +0530 Subject: [PATCH 2/2] muted unnecessary logs --- pkg/controller/controller_suite_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/controller/controller_suite_test.go b/pkg/controller/controller_suite_test.go index a230e9191..9ae1ecba2 100644 --- a/pkg/controller/controller_suite_test.go +++ b/pkg/controller/controller_suite_test.go @@ -18,7 +18,9 @@ package controller import ( "context" + "flag" "fmt" + "io" "testing" "time" @@ -45,6 +47,12 @@ import ( ) func TestMachineControllerManagerSuite(t *testing.T) { + //for filtering out warning logs. Reflector short watch warning logs won't print now + klog.SetOutput(io.Discard) + flags := &flag.FlagSet{} + klog.InitFlags(flags) + flags.Set("logtostderr", "false") + RegisterFailHandler(Fail) RunSpecs(t, "Machine Controller Manager Suite") }