Skip to content

Commit f977475

Browse files
openshift-merge-robotopenshift-merge-robot
authored
Merge pull request #442 from runcom/clear-failing
MCO: clear out failing status on success and add tests
2 parents 0eb517a + 2a582a9 commit f977475

File tree

18 files changed

+4432
-52
lines changed

18 files changed

+4432
-52
lines changed

‎Gopkg.lock‎

Lines changed: 17 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Gopkg.toml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,10 @@ required = [
3939
go-tests = false
4040
unused-packages = false
4141

42+
[[constraint]]
43+
name = "github.com/stretchr/testify"
44+
version = "v1.3.0"
45+
4246
[[constraint]]
4347
name = "github.com/apparentlymart/go-cidr"
4448
version = "1.0.0"

‎cmd/common/client_builder.go‎

Lines changed: 41 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,10 @@ import (
44
"os"
55

66
"github.com/golang/glog"
7+
configv1 "github.com/openshift/api/config/v1"
78
configclientset "github.com/openshift/client-go/config/clientset/versioned"
89
apiext "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
10+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
911
"k8s.io/client-go/kubernetes"
1012
"k8s.io/client-go/rest"
1113
"k8s.io/client-go/tools/clientcmd"
@@ -19,23 +21,58 @@ type ClientBuilder struct {
1921
config *rest.Config
2022
}
2123

22-
// ClientOrDie returns the kubernetes client interface for machine config.
24+
// MachineConfigClientOrDie returns the kubernetes client interface for machine config.
2325
func (cb *ClientBuilder) MachineConfigClientOrDie(name string) mcfgclientset.Interface {
2426
return mcfgclientset.NewForConfigOrDie(rest.AddUserAgent(cb.config, name))
2527
}
2628

27-
// ClientOrDie returns the kubernetes client interface for general kubernetes objects.
29+
// KubeClientOrDie returns the kubernetes client interface for general kubernetes objects.
2830
func (cb *ClientBuilder) KubeClientOrDie(name string) kubernetes.Interface {
2931
return kubernetes.NewForConfigOrDie(rest.AddUserAgent(cb.config, name))
3032
}
3133

32-
// ClientOrDie returns the kubernetes client interface for security related kubernetes objects
34+
// clusterOperatorsClient is a wrapper around the ClusterOperators client
35+
// implementing the ClusterOperatorsClientInterface
36+
type clusterOperatorsClient struct {
37+
innerConfigClient configclientset.Interface
38+
}
39+
40+
// Create creates the cluster operator and returns it
41+
func (c *clusterOperatorsClient) Create(co *configv1.ClusterOperator) (*configv1.ClusterOperator, error) {
42+
return c.innerConfigClient.ConfigV1().ClusterOperators().Create(co)
43+
}
44+
45+
// UpdateStatus updates the status of the cluster operator and returns it
46+
func (c *clusterOperatorsClient) UpdateStatus(co *configv1.ClusterOperator) (*configv1.ClusterOperator, error) {
47+
return c.innerConfigClient.ConfigV1().ClusterOperators().UpdateStatus(co)
48+
}
49+
50+
// Get returns the cluster operator by name
51+
func (c *clusterOperatorsClient) Get(name string, options metav1.GetOptions) (*configv1.ClusterOperator, error) {
52+
return c.innerConfigClient.ConfigV1().ClusterOperators().Get(name, options)
53+
}
54+
55+
// ClusterOperatorsClientInterface is a controlled ClusterOperators client which is used
56+
// by the operator and allows us to mock it in tests.
57+
type ClusterOperatorsClientInterface interface {
58+
Create(*configv1.ClusterOperator) (*configv1.ClusterOperator, error)
59+
UpdateStatus(*configv1.ClusterOperator) (*configv1.ClusterOperator, error)
60+
Get(name string, options metav1.GetOptions) (*configv1.ClusterOperator, error)
61+
}
62+
63+
// ClusterOperatorsClientOrDie returns the controlleed ClusterOperators client used by the operator
64+
func (cb *ClientBuilder) ClusterOperatorsClientOrDie(name string) ClusterOperatorsClientInterface {
65+
cc := configclientset.NewForConfigOrDie(rest.AddUserAgent(cb.config, name))
66+
return &clusterOperatorsClient{innerConfigClient: cc}
67+
}
68+
69+
// ConfigClientOrDie returns the kubernetes client interface for security related kubernetes objects
3370
// such as pod security policy, security context.
3471
func (cb *ClientBuilder) ConfigClientOrDie(name string) configclientset.Interface {
3572
return configclientset.NewForConfigOrDie(rest.AddUserAgent(cb.config, name))
3673
}
3774

38-
// ClientOrDie returns the kubernetes client interface for extended kubernetes objects.
75+
// APIExtClientOrDie returns the kubernetes client interface for extended kubernetes objects.
3976
func (cb *ClientBuilder) APIExtClientOrDie(name string) apiext.Interface {
4077
return apiext.NewForConfigOrDie(rest.AddUserAgent(cb.config, name))
4178
}

‎cmd/machine-config-operator/start.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ func startControllers(ctx *common.ControllerContext) error {
9999
ctx.ClientBuilder.MachineConfigClientOrDie(componentName),
100100
ctx.ClientBuilder.KubeClientOrDie(componentName),
101101
ctx.ClientBuilder.APIExtClientOrDie(componentName),
102-
ctx.ClientBuilder.ConfigClientOrDie(componentName),
102+
ctx.ClientBuilder.ClusterOperatorsClientOrDie(componentName),
103103
).Run(2, ctx.Stop)
104104

105105
return nil

‎pkg/operator/operator.go‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,12 @@ import (
2929
"k8s.io/client-go/util/workqueue"
3030

3131
configv1 "github.com/openshift/api/config/v1"
32-
configclientset "github.com/openshift/client-go/config/clientset/versioned"
3332
configinformersv1 "github.com/openshift/client-go/config/informers/externalversions/config/v1"
3433
configlistersv1 "github.com/openshift/client-go/config/listers/config/v1"
3534
installertypes "github.com/openshift/installer/pkg/types"
3635

36+
// TODO(runcom): move to pkg
37+
"github.com/openshift/machine-config-operator/cmd/common"
3738
mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
3839
templatectrl "github.com/openshift/machine-config-operator/pkg/controller/template"
3940
mcfgclientset "github.com/openshift/machine-config-operator/pkg/generated/clientset/versioned"
@@ -68,7 +69,7 @@ type Operator struct {
6869
client mcfgclientset.Interface
6970
kubeClient kubernetes.Interface
7071
apiExtClient apiextclientset.Interface
71-
configClient configclientset.Interface
72+
configClient common.ClusterOperatorsClientInterface
7273
eventRecorder record.EventRecorder
7374

7475
syncHandler func(ic string) error
@@ -115,7 +116,7 @@ func New(
115116
client mcfgclientset.Interface,
116117
kubeClient kubernetes.Interface,
117118
apiExtClient apiextclientset.Interface,
118-
configClient configclientset.Interface,
119+
configClient common.ClusterOperatorsClientInterface,
119120
) *Operator {
120121
eventBroadcaster := record.NewBroadcaster()
121122
eventBroadcaster.StartLogging(glog.Infof)
@@ -329,7 +330,16 @@ func (optr *Operator) sync(key string) error {
329330

330331
// create renderConfig
331332
rc := getRenderConfig(namespace, spec, imgs, infra.Status.APIServerURL)
332-
return optr.syncAll(rc)
333+
// syncFuncs is the list of sync functions that are executed in order.
334+
// any error marks sync as failure but continues to next syncFunc
335+
var syncFuncs = []syncFunc{
336+
{"pools", optr.syncMachineConfigPools},
337+
{"mcc", optr.syncMachineConfigController},
338+
{"mcs", optr.syncMachineConfigServer},
339+
{"mcd", optr.syncMachineConfigDaemon},
340+
{"required-pools", optr.syncRequiredMachineConfigPools},
341+
}
342+
return optr.syncAll(rc, syncFuncs)
333343
}
334344

335345
func (optr *Operator) getOsImageURL(namespace string) (string, error) {

‎pkg/operator/status.go‎

Lines changed: 29 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,12 @@ func (optr *Operator) syncAvailableStatus() error {
2828
}
2929

3030
optrVersion, _ := optr.vStore.Get("operator")
31-
progressing := cov1helpers.IsStatusConditionTrue(co.Status.Conditions, configv1.OperatorProgressing)
3231
failing := cov1helpers.IsStatusConditionTrue(co.Status.Conditions, configv1.OperatorFailing)
3332
message := fmt.Sprintf("Cluster has deployed %s", optrVersion)
3433

3534
available := configv1.ConditionTrue
3635

37-
if failing && !progressing {
36+
if failing {
3837
available = configv1.ConditionFalse
3938
message = fmt.Sprintf("Cluster not available for %s", optrVersion)
4039
}
@@ -47,7 +46,7 @@ func (optr *Operator) syncAvailableStatus() error {
4746

4847
co.Status.Versions = optr.vStore.GetAll()
4948
optr.setMachineConfigPoolStatuses(&co.Status)
50-
_, err = optr.configClient.ConfigV1().ClusterOperators().UpdateStatus(co)
49+
_, err = optr.configClient.UpdateStatus(co)
5150
return err
5251
}
5352

@@ -67,6 +66,7 @@ func (optr *Operator) syncProgressingStatus() error {
6766

6867
if optr.vStore.Equal(co.Status.Versions) {
6968
if optr.inClusterBringup {
69+
message = fmt.Sprintf("Cluster is bootstrapping %s", optrVersion)
7070
progressing = configv1.ConditionTrue
7171
}
7272
} else {
@@ -80,15 +80,12 @@ func (optr *Operator) syncProgressingStatus() error {
8080
})
8181

8282
optr.setMachineConfigPoolStatuses(&co.Status)
83-
_, err = optr.configClient.ConfigV1().ClusterOperators().UpdateStatus(co)
83+
_, err = optr.configClient.UpdateStatus(co)
8484
return err
8585
}
8686

8787
// syncFailingStatus applies the new condition to the mco's ClusterOperator object.
88-
func (optr *Operator) syncFailingStatus(ierr error) error {
89-
if ierr == nil {
90-
return nil
91-
}
88+
func (optr *Operator) syncFailingStatus(ierr error) (err error) {
9289
co, err := optr.fetchClusterOperator()
9390
if err != nil {
9491
return err
@@ -98,34 +95,40 @@ func (optr *Operator) syncFailingStatus(ierr error) error {
9895
}
9996

10097
optrVersion, _ := optr.vStore.Get("operator")
101-
var message string
102-
if optr.vStore.Equal(co.Status.Versions) {
103-
// syncing the state to exiting version.
104-
message = fmt.Sprintf("Failed to resync %s because: %v", optrVersion, ierr.Error())
98+
failing := configv1.ConditionTrue
99+
var message, reason string
100+
if ierr == nil {
101+
failing = configv1.ConditionFalse
105102
} else {
106-
message = fmt.Sprintf("Unable to apply %s: %v", optrVersion, ierr.Error())
103+
if optr.vStore.Equal(co.Status.Versions) {
104+
// syncing the state to exiting version.
105+
message = fmt.Sprintf("Failed to resync %s because: %v", optrVersion, ierr.Error())
106+
} else {
107+
message = fmt.Sprintf("Unable to apply %s: %v", optrVersion, ierr.Error())
108+
}
109+
reason = ierr.Error()
110+
111+
// set progressing
112+
if cov1helpers.IsStatusConditionTrue(co.Status.Conditions, configv1.OperatorProgressing) {
113+
cov1helpers.SetStatusCondition(&co.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorProgressing, Status: configv1.ConditionTrue, Message: fmt.Sprintf("Unable to apply %s", version.Version.String())})
114+
} else {
115+
cov1helpers.SetStatusCondition(&co.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse, Message: fmt.Sprintf("Error while reconciling %s", version.Version.String())})
116+
}
107117
}
108118
// set failing condition
109119
cov1helpers.SetStatusCondition(&co.Status.Conditions, configv1.ClusterOperatorStatusCondition{
110-
Type: configv1.OperatorFailing, Status: configv1.ConditionTrue,
120+
Type: configv1.OperatorFailing, Status: failing,
111121
Message: message,
112-
Reason: ierr.Error(),
122+
Reason: reason,
113123
})
114124

115-
// set progressing
116-
if cov1helpers.IsStatusConditionTrue(co.Status.Conditions, configv1.OperatorProgressing) {
117-
cov1helpers.SetStatusCondition(&co.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorProgressing, Status: configv1.ConditionTrue, Message: fmt.Sprintf("Unable to apply %s", version.Version.String())})
118-
} else {
119-
cov1helpers.SetStatusCondition(&co.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse, Message: fmt.Sprintf("Error while reconciling %s", version.Version.String())})
120-
}
121-
122125
optr.setMachineConfigPoolStatuses(&co.Status)
123-
_, err = optr.configClient.ConfigV1().ClusterOperators().UpdateStatus(co)
126+
_, err = optr.configClient.UpdateStatus(co)
124127
return err
125128
}
126129

127130
func (optr *Operator) fetchClusterOperator() (*configv1.ClusterOperator, error) {
128-
co, err := optr.configClient.ConfigV1().ClusterOperators().Get(optr.name, metav1.GetOptions{})
131+
co, err := optr.configClient.Get(optr.name, metav1.GetOptions{})
129132
if meta.IsNoMatchError(err) {
130133
return nil, nil
131134
}
@@ -139,7 +142,7 @@ func (optr *Operator) fetchClusterOperator() (*configv1.ClusterOperator, error)
139142
}
140143

141144
func (optr *Operator) initializeClusterOperator() (*configv1.ClusterOperator, error) {
142-
co, err := optr.configClient.ConfigV1().ClusterOperators().Create(&configv1.ClusterOperator{
145+
co, err := optr.configClient.Create(&configv1.ClusterOperator{
143146
ObjectMeta: metav1.ObjectMeta{
144147
Name: optr.name,
145148
},
@@ -154,7 +157,7 @@ func (optr *Operator) initializeClusterOperator() (*configv1.ClusterOperator, er
154157
co.Status.RelatedObjects = []configv1.ObjectReference{
155158
{Resource: "namespaces", Name: "openshift-machine-config-operator"},
156159
}
157-
return optr.configClient.ConfigV1().ClusterOperators().UpdateStatus(co)
160+
return optr.configClient.UpdateStatus(co)
158161
}
159162

160163
func (optr *Operator) setMachineConfigPoolStatuses(status *configv1.ClusterOperatorStatus) {

0 commit comments

Comments
 (0)