diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index 474b2b17a8..cdd20c1aa9 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -681,7 +681,7 @@ func (dn *Daemon) prepUpdateFromCluster() (bool, error) { } // Grab the node instance - node, err := dn.kubeClient.CoreV1().Nodes().Get(dn.name, metav1.GetOptions{}) + node, err := GetNode(dn.kubeClient.CoreV1().Nodes(), dn.name) if err != nil { return false, err } @@ -719,7 +719,7 @@ func (dn *Daemon) executeUpdateFromCluster() error { // "transient state" file, which signifies that all of those prior steps have // been completed. func (dn *Daemon) completeUpdate(desiredConfigName string) error { - node, err := dn.kubeClient.CoreV1().Nodes().Get(dn.name, metav1.GetOptions{}) + node, err := GetNode(dn.kubeClient.CoreV1().Nodes(), dn.name) if err != nil { return err } diff --git a/pkg/daemon/node.go b/pkg/daemon/node.go index 3a700c8ccb..6007c7dc2e 100644 --- a/pkg/daemon/node.go +++ b/pkg/daemon/node.go @@ -8,6 +8,7 @@ import ( "github.com/golang/glog" "github.com/openshift/machine-config-operator/pkg/daemon/constants" + "github.com/pkg/errors" core_v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" @@ -49,19 +50,32 @@ func getNodeAnnotation(client corev1.NodeInterface, node string, k string) (stri return getNodeAnnotationExt(client, node, k, false) } -// getNodeAnnotationExt is like getNodeAnnotation, but allows one to customize ENOENT handling -func getNodeAnnotationExt(client corev1.NodeInterface, node string, k string, allowNoent bool) (string, error) { +// GetNode gets the node object. +func GetNode(client corev1.NodeInterface, node string) (*core_v1.Node, error) { + var lastErr error var n *core_v1.Node err := wait.PollImmediate(10*time.Second, 5*time.Minute, func() (bool, error) { - var err error - n, err = client.Get(node, metav1.GetOptions{}) - if err == nil { + n, lastErr = client.Get(node, metav1.GetOptions{}) + if lastErr == nil { return true, nil } + glog.Warningf("Failed to fetch node %s (%v); retrying...", node, lastErr) return false, nil }) if err != nil { - return "", err + if err == wait.ErrWaitTimeout { + return nil, errors.Wrapf(lastErr, "Timed out trying to fetch node %s", node) + } + return nil, err + } + return n, nil +} + +// getNodeAnnotationExt is like getNodeAnnotation, but allows one to customize ENOENT handling +func getNodeAnnotationExt(client corev1.NodeInterface, node string, k string, allowNoent bool) (string, error) { + n, err := GetNode(client, node) + if err != nil { + return "", fmt.Errorf("Failed fetching node %s: %v", node, err) } v, ok := n.Annotations[k] diff --git a/pkg/daemon/update.go b/pkg/daemon/update.go index 59ead42e78..7da205c981 100644 --- a/pkg/daemon/update.go +++ b/pkg/daemon/update.go @@ -75,7 +75,7 @@ func (dn *Daemon) updateOSAndReboot(newConfig *mcfgv1.MachineConfig) error { if dn.onceFrom == "" { glog.Info("Update prepared; draining the node") - node, err := dn.kubeClient.CoreV1().Nodes().Get(dn.name, metav1.GetOptions{}) + node, err := GetNode(dn.kubeClient.CoreV1().Nodes(), dn.name) if err != nil { return err } @@ -102,7 +102,7 @@ func (dn *Daemon) updateOSAndReboot(newConfig *mcfgv1.MachineConfig) error { if err != nil { return err } - glog.V(2).Info("Node successfully drained") + glog.Info("Node successfully drained") } // reboot. this function shouldn't actually return. diff --git a/pkg/daemon/writer.go b/pkg/daemon/writer.go index 4e3c4dfaca..28372142b8 100644 --- a/pkg/daemon/writer.go +++ b/pkg/daemon/writer.go @@ -6,7 +6,6 @@ import ( "github.com/golang/glog" "github.com/openshift/machine-config-operator/pkg/daemon/constants" "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" corev1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/util/retry" ) @@ -145,7 +144,7 @@ func (nw *NodeWriter) SetSSHAccessed(client corev1.NodeInterface, node string) e // a retry is necessary. func updateNodeRetry(client corev1.NodeInterface, node string, f func(*v1.Node)) error { err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { - n, getErr := client.Get(node, metav1.GetOptions{}) + n, getErr := GetNode(client, node) if getErr != nil { return getErr }