Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Mark machine failed during rollout also #733

Merged
merged 2 commits into from
Jul 20, 2022
Merged

Mark machine failed during rollout also #733

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c685224
mark machine failed during rollout also
himanshu-kun Jul 14, 2022
9256b55
muted unnecessary logs
himanshu-kun Jul 18, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions pkg/controller/controller_suite_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,9 @@ package controller

import (
"context"
"flag"
"fmt"
"io"
"testing"
"time"

Expand All @@ -45,6 +47,12 @@ import (
)

func TestMachineControllerManagerSuite(t *testing.T) {
//for filtering out warning logs. Reflector short watch warning logs won't print now
klog.SetOutput(io.Discard)
flags := &flag.FlagSet{}
klog.InitFlags(flags)
flags.Set("logtostderr", "false")

RegisterFailHandler(Fail)
RunSpecs(t, "Machine Controller Manager Suite")
}
Expand Down
27 changes: 0 additions & 27 deletions pkg/util/provider/machinecontroller/machine_util.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ import (

machineapi "github.com/gardener/machine-controller-manager/pkg/apis/machine"
"github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1"
"github.com/gardener/machine-controller-manager/pkg/controller/autoscaler"
"github.com/gardener/machine-controller-manager/pkg/util/nodeops"
"github.com/gardener/machine-controller-manager/pkg/util/provider/drain"
"github.com/gardener/machine-controller-manager/pkg/util/provider/driver"
Expand Down Expand Up @@ -739,11 +738,6 @@ func (c *controller) reconcileMachineHealth(ctx context.Context, machine *v1alph
timeOutDuration,
machine.Status.Conditions,
)
// ensuring rollout of machineDeployment is not happening
if c.isRollingOut(machine) {
klog.V(2).Infof("Skipping marking machine=%s Failed, as rollout for corresponding machineDeployment is happening, will retry in next sync", machine.Name)
return machineutils.MediumRetry, nil
}

machineDeployName := getMachineDeploymentName(machine)
// creating lock for machineDeployment, if not allocated
Expand Down Expand Up @@ -1514,27 +1508,6 @@ func (c *controller) tryMarkingMachineFailed(ctx context.Context, machine, clone
return machineutils.ShortRetry, err
}

func (c *controller) isRollingOut(machine *v1alpha1.Machine) bool {
nodeName := machine.Labels["node"]
node, err := c.nodeLister.Get(nodeName)
if err != nil {
if apierrors.IsNotFound(err) {
return false
}
klog.Errorf("error fetching node object for machine %q", machine.Name)
return true
}

nodeAnnotations := node.Annotations
if nodeAnnotations == nil {
return false
} else if _, exists := nodeAnnotations[autoscaler.ClusterAutoscalerScaleDownDisabledAnnotationByMCMKey]; exists {
return true
}

return false
}

func getProviderID(machine *v1alpha1.Machine) string {
return machine.Spec.ProviderID
}
Expand Down
21 changes: 0 additions & 21 deletions pkg/util/provider/machinecontroller/machine_util_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import (

"github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1"
machinev1 "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1"
"github.com/gardener/machine-controller-manager/pkg/controller/autoscaler"
"github.com/gardener/machine-controller-manager/pkg/fakeclient"
"github.com/gardener/machine-controller-manager/pkg/util/permits"
"github.com/gardener/machine-controller-manager/pkg/util/provider/machineutils"
Expand Down Expand Up @@ -2299,26 +2298,6 @@ var _ = Describe("machine_util", func() {
expectedPhase: v1alpha1.MachineFailed,
},
}),
Entry("HealthTimedout machine shouldn't be marked Failed, if rolling update happening", &data{
setup: setup{
machines: []*v1alpha1.Machine{
newMachine(
&machinev1.MachineTemplateSpec{ObjectMeta: *newObjectMeta(&metav1.ObjectMeta{GenerateName: machineSet1Deploy1}, 0)},
&machinev1.MachineStatus{Node: "node", Conditions: nodeConditions(false, false, false, false, false), CurrentStatus: machinev1.CurrentStatus{Phase: v1alpha1.MachineUnknown, LastUpdateTime: metav1.NewTime(time.Now().Add(-15 * time.Minute))}},
&metav1.OwnerReference{Name: machineSet1Deploy1},
nil, map[string]string{"node": "node-0"}, true, metav1.Now()),
},
nodes: []*v1.Node{
newNode(1, nil, map[string]string{autoscaler.ClusterAutoscalerScaleDownDisabledAnnotationByMCMKey: "true"}, &v1.NodeSpec{}, &v1.NodeStatus{Phase: corev1.NodeRunning, Conditions: nodeConditions(false, false, false, false, false)}),
},
targetMachineName: machineSet1Deploy1 + "-" + "0",
},
expect: expect{
retryPeriod: machineutils.MediumRetry,
err: nil,
expectedPhase: v1alpha1.MachineUnknown,
},
}),
Entry("simple machine WITHOUT creation Timeout(20 min) and Pending state", &data{
setup: setup{
machines: []*v1alpha1.Machine{
Expand Down