Skip to content

operator: fix status updates on ClusterOperator #351

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
openshift-merge-robot merged 3 commits into from
Feb 1, 2019
Merged

operator: fix status updates on ClusterOperator #351

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
70ad2d0
vendor: bump openshift/{api,client-go}
abhinavdahiya Jan 29, 2019
eb30ef6
operator: track versions for each major component
abhinavdahiya Jan 29, 2019
cf20795
operator: fix status updates on ClusterOperator
abhinavdahiya Jan 29, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 13 additions & 4 deletions Gopkg.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 5 additions & 1 deletion Gopkg.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,10 @@ required = [
name = "github.com/openshift/kubernetes-drain"
revision = "c2e51be1758efa30d71a4d30dc4e2db86b70a4df"

[[constraint]]
name = "github.com/openshift/library-go"
branch = "master"

[[constraint]]
name = "github.com/spf13/cobra"
version = "v0.0.3"
Expand Down Expand Up @@ -150,4 +154,4 @@ required = [

[[override]]
name = "k8s.io/apiserver"
version = "kubernetes-1.12.5"
version = "kubernetes-1.12.5"
6 changes: 6 additions & 0 deletions pkg/operator/operator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@ type Operator struct {

imagesFile string

vStore *versionStore

client mcfgclientset.Interface
kubeClient kubernetes.Interface
apiExtClient apiextclientset.Interface
Expand Down Expand Up @@ -113,6 +115,7 @@ func New(
namespace: namespace,
name: name,
imagesFile: imagesFile,
vStore: newVersionStore(),
client: client,
kubeClient: kubeClient,
apiExtClient: apiExtClient,
Expand All @@ -121,6 +124,9 @@ func New(
queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "machineconfigoperator"),
}

// set operator version in version store
optr.vStore.Set("operator", version.Version.String())

mcoconfigInformer.Informer().AddEventHandler(optr.eventHandler())
controllerConfigInformer.Informer().AddEventHandler(optr.eventHandler())
serviceAccountInfomer.Informer().AddEventHandler(optr.eventHandler())
Expand Down
151 changes: 39 additions & 112 deletions pkg/operator/status.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@ package operator
import (
"encoding/json"
"fmt"
"time"

"github.com/golang/glog"
configv1 "github.com/openshift/api/config/v1"
cov1helpers "github.com/openshift/library-go/pkg/config/clusteroperator/v1helpers"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand All @@ -26,16 +26,19 @@ func (optr *Operator) syncAvailableStatus() error {
if co == nil {
return nil
}
now := metav1.Now()

optrVersion, _ := optr.vStore.Get("operator")
// set available
SetClusterOperatorStatusCondition(&co.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorAvailable, Status: configv1.ConditionTrue, LastTransitionTime: now})
cov1helpers.SetStatusCondition(&co.Status.Conditions, configv1.ClusterOperatorStatusCondition{
Type: configv1.OperatorAvailable, Status: configv1.ConditionTrue,
Message: fmt.Sprintf("Cluster is available at %s", optrVersion),
Copy link
Member

@cgwalters cgwalters Jan 30, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

s/Cluster/MCO/? (or s/Cluster/Operator/, or...)

Copy link
Contributor Author

@abhinavdahiya abhinavdahiya Jan 30, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

MCO is reporting that its view (in terms of ownership) of the cluster is at that version.

Copy link
Contributor

@kikisdeliveryservice kikisdeliveryservice Jan 30, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

- lastTransitionTime: 2019-01-29T20:53:23Z
    message: Cluster is available at 3.11.0-521-g377bde3b
    status: "True"
    type: Available
  - lastTransitionTime: 2019-01-30T20:56:32Z
    message: Running resync for 3.11.0-521-g377bde3b
    status: "True"
    type: Progressing
  - lastTransitionTime: 2019-01-30T06:21:34Z
    status: "False"
    type: Failing

@cgwalters it ends up reading this way in the yaml ^^

Copy link
Member

@cgwalters cgwalters Jan 30, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK, sounds fine to me!

})
// clear progressing
SetClusterOperatorStatusCondition(&co.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse, LastTransitionTime: now})
cov1helpers.SetStatusCondition(&co.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse})
// clear failure
SetClusterOperatorStatusCondition(&co.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorFailing, Status: configv1.ConditionFalse, LastTransitionTime: now})

co.Status.Version = version.Version.String()
cov1helpers.SetStatusCondition(&co.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorFailing, Status: configv1.ConditionFalse})

co.Status.Versions = optr.vStore.GetAll()
optr.setMachineConfigPoolStatuses(&co.Status)
_, err = optr.configClient.ConfigV1().ClusterOperators().UpdateStatus(co)
return err
Expand All @@ -50,23 +53,20 @@ func (optr *Operator) syncProgressingStatus() error {
if co == nil {
return nil
}
now := metav1.Now()
// clear the available condition
SetClusterOperatorStatusCondition(&co.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorAvailable, Status: configv1.ConditionFalse, LastTransitionTime: now})

// preserve the most recent failing condition
if IsClusterOperatorStatusConditionNotIn(co.Status.Conditions, configv1.OperatorFailing, configv1.ConditionTrue) {
SetClusterOperatorStatusCondition(&co.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorFailing, Status: configv1.ConditionFalse, LastTransitionTime: now})
}

// set progressing
if c := FindClusterOperatorStatusCondition(co.Status.Conditions, configv1.OperatorFailing); c != nil && c.Status == configv1.ConditionTrue {
SetClusterOperatorStatusCondition(&co.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorProgressing, Status: configv1.ConditionTrue, Message: fmt.Sprintf("Unable to apply %s", version.Version.String()), LastTransitionTime: now})
optrVersion, _ := optr.vStore.Get("operator")
var message string
if optr.vStore.Equal(co.Status.Versions) {
// syncing the state to existing version.
message = fmt.Sprintf("Running resync for %s", optrVersion)
} else {
SetClusterOperatorStatusCondition(&co.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorProgressing, Status: configv1.ConditionTrue, Message: fmt.Sprintf("Progressing towards %s", version.Version.String()), LastTransitionTime: now})
message = fmt.Sprintf("Progressing towards %s", optrVersion)
}
cov1helpers.SetStatusCondition(&co.Status.Conditions, configv1.ClusterOperatorStatusCondition{
Type: configv1.OperatorProgressing, Status: configv1.ConditionTrue,
Message: message,
})

co.Status.Version = version.Version.String()
optr.setMachineConfigPoolStatuses(&co.Status)
_, err = optr.configClient.ConfigV1().ClusterOperators().UpdateStatus(co)
return err
Expand All @@ -84,21 +84,29 @@ func (optr *Operator) syncFailingStatus(ierr error) error {
if co == nil {
return nil
}
now := metav1.Now()
// clear the available condition
SetClusterOperatorStatusCondition(&co.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorAvailable, Status: configv1.ConditionFalse, LastTransitionTime: now})

optrVersion, _ := optr.vStore.Get("operator")
var message string
if optr.vStore.Equal(co.Status.Versions) {
// syncing the state to exiting version.
message = fmt.Sprintf("Failed to resync %s because: %v", optrVersion, ierr.Error())
} else {
message = fmt.Sprintf("Failed when progressing towards %s because: %v", optrVersion, ierr.Error())
}
// set failing condition
SetClusterOperatorStatusCondition(&co.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorFailing, Status: configv1.ConditionTrue, Message: ierr.Error(), LastTransitionTime: now})
cov1helpers.SetStatusCondition(&co.Status.Conditions, configv1.ClusterOperatorStatusCondition{
Type: configv1.OperatorFailing, Status: configv1.ConditionTrue,
Message: message,
Reason: ierr.Error(),
})

// set progressing
if IsClusterOperatorStatusConditionTrue(co.Status.Conditions, configv1.OperatorProgressing) {
SetClusterOperatorStatusCondition(&co.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorProgressing, Status: configv1.ConditionTrue, Message: fmt.Sprintf("Unable to apply %s", version.Version.String()), LastTransitionTime: now})
if cov1helpers.IsStatusConditionTrue(co.Status.Conditions, configv1.OperatorProgressing) {
cov1helpers.SetStatusCondition(&co.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorProgressing, Status: configv1.ConditionTrue, Message: fmt.Sprintf("Unable to apply %s", version.Version.String())})
} else {
SetClusterOperatorStatusCondition(&co.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse, Message: fmt.Sprintf("Error while reconciling %s", version.Version.String()), LastTransitionTime: now})
cov1helpers.SetStatusCondition(&co.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse, Message: fmt.Sprintf("Error while reconciling %s", version.Version.String())})
}

co.Status.Version = version.Version.String()
optr.setMachineConfigPoolStatuses(&co.Status)
_, err = optr.configClient.ConfigV1().ClusterOperators().UpdateStatus(co)
return err
Expand Down Expand Up @@ -127,10 +135,9 @@ func (optr *Operator) initializeClusterOperator() (*configv1.ClusterOperator, er
if err != nil {
return nil, err
}
now := metav1.Now()
SetClusterOperatorStatusCondition(&co.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorAvailable, Status: configv1.ConditionFalse, LastTransitionTime: now})
SetClusterOperatorStatusCondition(&co.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse, LastTransitionTime: now})
SetClusterOperatorStatusCondition(&co.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorFailing, Status: configv1.ConditionFalse, LastTransitionTime: now})
cov1helpers.SetStatusCondition(&co.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorAvailable, Status: configv1.ConditionFalse})
cov1helpers.SetStatusCondition(&co.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse})
cov1helpers.SetStatusCondition(&co.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorFailing, Status: configv1.ConditionFalse})
return optr.configClient.ConfigV1().ClusterOperators().UpdateStatus(co)
}

Expand Down Expand Up @@ -239,83 +246,3 @@ func machineConfigPoolStatus(pool *mcfgv1.MachineConfigPool) string {
return "<unknown>"
}
}

// From https://github.com/openshift/library-go/pull/97

// SetClusterOperatorStatusCondition sets the corresponding condition in conditions to newCondition.
func SetClusterOperatorStatusCondition(conditions *[]configv1.ClusterOperatorStatusCondition, newCondition configv1.ClusterOperatorStatusCondition) {
if conditions == nil {
conditions = &[]configv1.ClusterOperatorStatusCondition{}
}
existingCondition := FindClusterOperatorStatusCondition(*conditions, newCondition.Type)
if existingCondition == nil {
newCondition.LastTransitionTime = metav1.NewTime(time.Now())
*conditions = append(*conditions, newCondition)
return
}
if existingCondition.Status != newCondition.Status {
existingCondition.Status = newCondition.Status
existingCondition.LastTransitionTime = newCondition.LastTransitionTime
}
existingCondition.Reason = newCondition.Reason
existingCondition.Message = newCondition.Message
}

// RemoveClusterOperatorStatusCondition removes the corresponding conditionType from conditions.
func RemoveClusterOperatorStatusCondition(conditions *[]configv1.ClusterOperatorStatusCondition, conditionType configv1.ClusterStatusConditionType) {
if conditions == nil {
conditions = &[]configv1.ClusterOperatorStatusCondition{}
}
newConditions := []configv1.ClusterOperatorStatusCondition{}
for _, condition := range *conditions {
if condition.Type != conditionType {
newConditions = append(newConditions, condition)
}
}
*conditions = newConditions
}

// FindClusterOperatorStatusCondition finds the conditionType in conditions.
func FindClusterOperatorStatusCondition(conditions []configv1.ClusterOperatorStatusCondition, conditionType configv1.ClusterStatusConditionType) *configv1.ClusterOperatorStatusCondition {
for i := range conditions {
if conditions[i].Type == conditionType {
return &conditions[i]
}
}
return nil
}

// IsClusterOperatorStatusConditionTrue returns true when the conditionType is present and set to `configv1.ConditionTrue`
func IsClusterOperatorStatusConditionTrue(conditions []configv1.ClusterOperatorStatusCondition, conditionType configv1.ClusterStatusConditionType) bool {
return IsClusterOperatorStatusConditionPresentAndEqual(conditions, conditionType, configv1.ConditionTrue)
}

// IsClusterOperatorStatusConditionFalse returns true when the conditionType is present and set to `configv1.ConditionFalse`
func IsClusterOperatorStatusConditionFalse(conditions []configv1.ClusterOperatorStatusCondition, conditionType configv1.ClusterStatusConditionType) bool {
return IsClusterOperatorStatusConditionPresentAndEqual(conditions, conditionType, configv1.ConditionFalse)
}

// IsClusterOperatorStatusConditionPresentAndEqual returns true when conditionType is present and equal to status.
func IsClusterOperatorStatusConditionPresentAndEqual(conditions []configv1.ClusterOperatorStatusCondition, conditionType configv1.ClusterStatusConditionType, status configv1.ConditionStatus) bool {
for _, condition := range conditions {
if condition.Type == conditionType {
return condition.Status == status
}
}
return false
}

// IsClusterOperatorStatusConditionNotIn returns true when the conditionType does not match the status.
func IsClusterOperatorStatusConditionNotIn(conditions []configv1.ClusterOperatorStatusCondition, conditionType configv1.ClusterStatusConditionType, status ...configv1.ConditionStatus) bool {
for _, condition := range conditions {
if condition.Type == conditionType {
for _, s := range status {
if s == condition.Status {
return false
}
}
return true
}
}
return true
}
21 changes: 17 additions & 4 deletions pkg/operator/sync.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,10 @@ func (optr *Operator) syncMachineConfigController(config renderConfig) error {
return err
}
mcc := resourceread.ReadDeploymentV1OrDie(mccBytes)

// store version of machineconfigcontroller
optr.vStore.Set("machineconfigcontroller", imageForContainer("machine-config-controller", mcc.Spec.Template.Spec.Containers))

_, updated, err := resourceapply.ApplyDeployment(optr.kubeClient.AppsV1(), mcc)
if err != nil {
return err
Expand Down Expand Up @@ -218,6 +222,10 @@ func (optr *Operator) syncMachineConfigDaemon(config renderConfig) error {
return err
}
mcd := resourceread.ReadDaemonSetV1OrDie(mcdBytes)

// store version of machineconfigdaemon
optr.vStore.Set("machineconfigdaemon", imageForContainer("machine-config-daemon", mcd.Spec.Template.Spec.Containers))

_, updated, err := resourceapply.ApplyDaemonSet(optr.kubeClient.AppsV1(), mcd)
if err != nil {
return err
Expand Down Expand Up @@ -283,17 +291,22 @@ func (optr *Operator) syncMachineConfigServer(config renderConfig) error {
return err
}

mcdBytes, err := renderAsset(config, "manifests/machineconfigserver/daemonset.yaml")
mcsBytes, err := renderAsset(config, "manifests/machineconfigserver/daemonset.yaml")
if err != nil {
return err
}
mcd := resourceread.ReadDaemonSetV1OrDie(mcdBytes)
_, updated, err := resourceapply.ApplyDaemonSet(optr.kubeClient.AppsV1(), mcd)

mcs := resourceread.ReadDaemonSetV1OrDie(mcsBytes)

// store version of machineconfigserver
optr.vStore.Set("machineconfigserver", imageForContainer("machine-config-server", mcs.Spec.Template.Spec.Containers))

_, updated, err := resourceapply.ApplyDaemonSet(optr.kubeClient.AppsV1(), mcs)
if err != nil {
return err
}
if updated {
return optr.waitForDaemonsetRollout(mcd)
return optr.waitForDaemonsetRollout(mcs)
}
return nil
}
Expand Down
Loading