diff --git a/src/assisted_installer_controller/assisted_installer_controller.go b/src/assisted_installer_controller/assisted_installer_controller.go index 5bf73744a4..107bae9a2e 100644 --- a/src/assisted_installer_controller/assisted_installer_controller.go +++ b/src/assisted_installer_controller/assisted_installer_controller.go @@ -20,6 +20,7 @@ import ( "github.com/hashicorp/go-version" metal3v1alpha1 "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1" + configv1 "github.com/openshift/api/config/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" certificatesv1 "k8s.io/api/certificates/v1" @@ -57,6 +58,8 @@ const ( ExitWaiting = true customManifestsFile = "custom_manifests.json" kubeconfigFileName = "kubeconfig-noingress" + + consoleCapabilityName = configv1.ClusterVersionCapability("Console") ) var ( @@ -985,12 +988,24 @@ func (c controller) waitingForClusterOperators(ctx context.Context) error { ctxWithTimeout, cancel := context.WithTimeout(ctx, CVOMaxTimeout) defer cancel() isClusterVersionAvailable := func(timer *time.Timer) bool { - result := c.isOperatorAvailable(NewClusterOperatorHandler(c.kc, consoleOperatorName)) - + clusterOperatorHandler := NewClusterOperatorHandler(c.kc, consoleOperatorName) + clusterVersionHandler := NewClusterVersionHandler(c.kc, timer) + isConsoleEnabled, err := c.kc.IsClusterCapabilityEnabled(consoleCapabilityName) + if err != nil { + c.log.WithError(err).Error("Failed to check if console is enabled") + return false + } + var result bool + if isConsoleEnabled { + c.log.Info("Console is enabled, will wait for the console operator to be available") + result = c.isOperatorAvailable(clusterOperatorHandler) + } else { + c.log.Info("Console is disabled, will not wait for the console operator to be available") + result = true + } if c.WaitForClusterVersion { - result = c.isOperatorAvailable(NewClusterVersionHandler(c.kc, timer)) && result + result = c.isOperatorAvailable(clusterVersionHandler) && result } - return result } return utils.WaitForPredicateWithTimer(ctxWithTimeout, WaitTimeout, GeneralProgressUpdateInt, isClusterVersionAvailable) diff --git a/src/assisted_installer_controller/assisted_installer_controller_test.go b/src/assisted_installer_controller/assisted_installer_controller_test.go index a675f887fb..b06f4cf240 100644 --- a/src/assisted_installer_controller/assisted_installer_controller_test.go +++ b/src/assisted_installer_controller/assisted_installer_controller_test.go @@ -183,7 +183,7 @@ var _ = Describe("installer HostRoleMaster role", func() { setCvoAsAvailable := func() { mockGetServiceOperators([]models.MonitoredOperator{{Name: cvoOperatorName, Status: models.OperatorStatusProgressing}}) - mockk8sclient.EXPECT().GetClusterVersion(clusterVersionName).Return(availableClusterVersionCondition, nil).Times(1) + mockk8sclient.EXPECT().GetClusterVersion().Return(availableClusterVersionCondition, nil).Times(1) mockbmclient.EXPECT().UpdateClusterOperator(gomock.Any(), gomock.Any(), cvoOperatorName, models.OperatorStatusAvailable, availableClusterVersionCondition.Status.Conditions[0].Message).Times(1) mockGetServiceOperators([]models.MonitoredOperator{{Name: cvoOperatorName, Status: models.OperatorStatusAvailable}}) @@ -247,6 +247,10 @@ var _ = Describe("installer HostRoleMaster role", func() { mockbmclient.EXPECT().DownloadClusterCredentials(gomock.Any(), kubeconfigFileName, gomock.Any()).Return(nil).Times(1) } + mockAllCapabilitiesEnabled := func() { + mockk8sclient.EXPECT().IsClusterCapabilityEnabled(gomock.Any()).Return(true, nil).AnyTimes() + } + Context("Waiting for 3 nodes", func() { It("Set ready event", func() { // fail to connect to assisted and then succeed @@ -590,6 +594,8 @@ var _ = Describe("installer HostRoleMaster role", func() { }) It("failure if console not available in service or failed to set status and success if available", func() { + mockAllCapabilitiesEnabled() + installing := models.ClusterStatusInstalling mockbmclient.EXPECT().GetCluster(gomock.Any(), false).Return(&models.Cluster{Status: &installing}, nil).Times(2) @@ -620,6 +626,8 @@ var _ = Describe("installer HostRoleMaster role", func() { }) It("success", func() { + mockAllCapabilitiesEnabled() + installing := models.ClusterStatusInstalling mockbmclient.EXPECT().GetCluster(gomock.Any(), false).Return(&models.Cluster{Status: &installing}, nil).Times(1) setControllerWaitForOLMOperators(assistedController.ClusterID) @@ -642,6 +650,8 @@ var _ = Describe("installer HostRoleMaster role", func() { }) It("lots of failures then success", func() { + mockAllCapabilitiesEnabled() + installing := models.ClusterStatusInstalling mockbmclient.EXPECT().GetCluster(gomock.Any(), false).Return(&models.Cluster{Status: &installing}, nil).Times(1) setClusterAsFinalizing() @@ -708,10 +718,10 @@ var _ = Describe("installer HostRoleMaster role", func() { // CVO errors mockGetServiceOperators([]models.MonitoredOperator{{Name: cvoOperatorName, Status: ""}}) - mockk8sclient.EXPECT().GetClusterVersion(clusterVersionName).Return(nil, fmt.Errorf("dummy")).Times(1) + mockk8sclient.EXPECT().GetClusterVersion().Return(nil, fmt.Errorf("dummy")).Times(1) mockGetServiceOperators([]models.MonitoredOperator{{Name: cvoOperatorName, Status: ""}}) - mockk8sclient.EXPECT().GetClusterVersion(clusterVersionName).Return(progressClusterVersionCondition, nil).Times(1) + mockk8sclient.EXPECT().GetClusterVersion().Return(progressClusterVersionCondition, nil).Times(1) mockbmclient.EXPECT().UpdateClusterOperator(gomock.Any(), gomock.Any(), cvoOperatorName, models.OperatorStatusProgressing, progressClusterVersionCondition.Status.Conditions[0].Message).Times(1) // Fail 8 more times when console fail @@ -719,7 +729,7 @@ var _ = Describe("installer HostRoleMaster role", func() { for i := 0; i < extraFailTimes; i++ { mockGetServiceOperators([]models.MonitoredOperator{{Name: cvoOperatorName, Status: ""}}) } - mockk8sclient.EXPECT().GetClusterVersion(clusterVersionName).Return(nil, fmt.Errorf("dummy")).Times(extraFailTimes) + mockk8sclient.EXPECT().GetClusterVersion().Return(nil, fmt.Errorf("dummy")).Times(extraFailTimes) setCvoAsAvailable() @@ -739,6 +749,8 @@ var _ = Describe("installer HostRoleMaster role", func() { }) It("failure", func() { + mockAllCapabilitiesEnabled() + setClusterAsFinalizing() mockbmclient.EXPECT().GetClusterMonitoredOperator(gomock.Any(), gomock.Any(), consoleOperatorName, gomock.Any()). @@ -746,7 +758,7 @@ var _ = Describe("installer HostRoleMaster role", func() { mockk8sclient.EXPECT().GetClusterOperator(consoleOperatorName).Return(nil, fmt.Errorf("dummy")).AnyTimes() mockbmclient.EXPECT().GetClusterMonitoredOperator(gomock.Any(), gomock.Any(), cvoOperatorName, gomock.Any()). Return(&models.MonitoredOperator{Status: "", StatusInfo: ""}, nil).AnyTimes() - mockk8sclient.EXPECT().GetClusterVersion(clusterVersionName).Return(nil, fmt.Errorf("dummy")).AnyTimes() + mockk8sclient.EXPECT().GetClusterVersion().Return(nil, fmt.Errorf("dummy")).AnyTimes() mockbmclient.EXPECT().CompleteInstallation(gomock.Any(), "cluster-id", false, gomock.Any()).Return(nil).Times(1) @@ -763,6 +775,8 @@ var _ = Describe("installer HostRoleMaster role", func() { GeneralWaitInterval = 10 * time.Millisecond }) It("success", func() { + mockAllCapabilitiesEnabled() + installing := models.ClusterStatusInstalling mockbmclient.EXPECT().GetCluster(gomock.Any(), false).Return(&models.Cluster{Status: &installing}, nil).Times(1) setControllerWaitForOLMOperators(assistedController.ClusterID) @@ -780,6 +794,8 @@ var _ = Describe("installer HostRoleMaster role", func() { Expect(assistedController.Status.HasError()).Should(Equal(false)) }) It("failure", func() { + mockAllCapabilitiesEnabled() + setClusterAsFinalizing() setConsoleAsAvailable("cluster-id") mockk8sclient.EXPECT().GetConfigMap(gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("aaa")).MinTimes(1) @@ -804,6 +820,8 @@ var _ = Describe("installer HostRoleMaster role", func() { }) It("waiting for single OLM operator", func() { + mockAllCapabilitiesEnabled() + By("setup", func() { setControllerWaitForOLMOperators(assistedController.ClusterID) operators := []models.MonitoredOperator{ @@ -857,6 +875,8 @@ var _ = Describe("installer HostRoleMaster role", func() { }) It("waiting for single OLM operator which timeouts", func() { + mockAllCapabilitiesEnabled() + By("setup", func() { setControllerWaitForOLMOperators(assistedController.ClusterID) operators := []models.MonitoredOperator{{SubscriptionName: "local-storage-operator", Namespace: "openshift-local-storage", OperatorType: models.OperatorTypeOlm, Name: "lso", Status: models.OperatorStatusProgressing, TimeoutSeconds: 0}} @@ -892,6 +912,8 @@ var _ = Describe("installer HostRoleMaster role", func() { GeneralWaitInterval = 10 * time.Millisecond }) It("success", func() { + mockAllCapabilitiesEnabled() + installing := models.ClusterStatusInstalling mockbmclient.EXPECT().GetCluster(gomock.Any(), false).Return(&models.Cluster{Status: &installing}, nil).Times(1) setControllerWaitForOLMOperators(assistedController.ClusterID) @@ -1505,6 +1527,8 @@ var _ = Describe("installer HostRoleMaster role", func() { for i := range tests { t := tests[i] It(t.name, func() { + mockAllCapabilitiesEnabled() + clusterVersionReport := &configv1.ClusterVersion{ Status: configv1.ClusterVersionStatus{ Conditions: []configv1.ClusterOperatorStatusCondition{t.newCVOCondition}, @@ -1529,7 +1553,7 @@ var _ = Describe("installer HostRoleMaster role", func() { if t.currentServiceCVOStatus.Status != models.OperatorStatusAvailable { amountOfSamples++ } - mockk8sclient.EXPECT().GetClusterVersion(clusterVersionName).Return(clusterVersionReport, nil).MinTimes(amountOfSamples) + mockk8sclient.EXPECT().GetClusterVersion().Return(clusterVersionReport, nil).MinTimes(amountOfSamples) if newServiceCVOStatus.Status == models.OperatorStatusAvailable { Expect(assistedController.waitingForClusterOperators(ctx)).ShouldNot(HaveOccurred()) @@ -1540,6 +1564,8 @@ var _ = Describe("installer HostRoleMaster role", func() { } It("service fail to sync - context cancel", func() { + mockAllCapabilitiesEnabled() + currentServiceCVOStatus := &models.MonitoredOperator{Status: models.OperatorStatusProgressing, StatusInfo: ""} clusterVersionReport := &configv1.ClusterVersion{ Status: configv1.ClusterVersionStatus{ @@ -1549,7 +1575,7 @@ var _ = Describe("installer HostRoleMaster role", func() { }, } - mockk8sclient.EXPECT().GetClusterVersion(clusterVersionName).Return(clusterVersionReport, nil).AnyTimes() + mockk8sclient.EXPECT().GetClusterVersion().Return(clusterVersionReport, nil).AnyTimes() mockbmclient.EXPECT().GetClusterMonitoredOperator(gomock.Any(), gomock.Any(), cvoOperatorName, gomock.Any()).Return(currentServiceCVOStatus, nil).AnyTimes() mockbmclient.EXPECT().UpdateClusterOperator(gomock.Any(), gomock.Any(), cvoOperatorName, gomock.Any(), gomock.Any()).AnyTimes() @@ -1563,6 +1589,8 @@ var _ = Describe("installer HostRoleMaster role", func() { }) It("service fail to sync - maxTimeout applied", func() { + mockAllCapabilitiesEnabled() + WaitTimeout = 1 * time.Second CVOMaxTimeout = 200 * time.Millisecond currentServiceCVOStatus := &models.MonitoredOperator{Status: models.OperatorStatusProgressing, StatusInfo: ""} @@ -1574,7 +1602,7 @@ var _ = Describe("installer HostRoleMaster role", func() { }, } - mockk8sclient.EXPECT().GetClusterVersion(clusterVersionName).Return(clusterVersionReport, nil).AnyTimes() + mockk8sclient.EXPECT().GetClusterVersion().Return(clusterVersionReport, nil).AnyTimes() mockbmclient.EXPECT().GetClusterMonitoredOperator(gomock.Any(), gomock.Any(), cvoOperatorName, gomock.Any()).Return(currentServiceCVOStatus, nil).AnyTimes() mockbmclient.EXPECT().UpdateClusterOperator(gomock.Any(), gomock.Any(), cvoOperatorName, gomock.Any(), gomock.Any()).AnyTimes() @@ -1586,6 +1614,8 @@ var _ = Describe("installer HostRoleMaster role", func() { }) It("service fail to sync - finally succeed", func() { + mockAllCapabilitiesEnabled() + currentServiceCVOStatus := &models.MonitoredOperator{Status: models.OperatorStatusProgressing, StatusInfo: ""} newServiceCVOStatus := &models.MonitoredOperator{Status: models.OperatorStatusAvailable, StatusInfo: ""} clusterVersionReport := &configv1.ClusterVersion{ @@ -1597,7 +1627,7 @@ var _ = Describe("installer HostRoleMaster role", func() { } // Fail twice - mockk8sclient.EXPECT().GetClusterVersion(clusterVersionName).Return(clusterVersionReport, nil).Times(2) + mockk8sclient.EXPECT().GetClusterVersion().Return(clusterVersionReport, nil).Times(2) mockbmclient.EXPECT().GetClusterMonitoredOperator(gomock.Any(), gomock.Any(), cvoOperatorName, gomock.Any()).Return(currentServiceCVOStatus, nil).Times(2) mockbmclient.EXPECT().UpdateClusterOperator(gomock.Any(), gomock.Any(), cvoOperatorName, gomock.Any(), gomock.Any()).Times(2) diff --git a/src/assisted_installer_controller/operator_handler.go b/src/assisted_installer_controller/operator_handler.go index f0d213f353..745e620605 100644 --- a/src/assisted_installer_controller/operator_handler.go +++ b/src/assisted_installer_controller/operator_handler.go @@ -10,8 +10,7 @@ import ( ) const ( - cvoOperatorName = "cvo" - clusterVersionName = "version" + cvoOperatorName = "cvo" ) type OperatorHandler interface { @@ -36,7 +35,7 @@ func (c controller) isOperatorAvailable(handler OperatorHandler) bool { return false } - if operatorStatusInService.Status != operatorStatus || (operatorStatusInService.StatusInfo != operatorMessage && operatorMessage != "") { + if operatorStatusInService != nil && (operatorStatusInService.Status != operatorStatus || (operatorStatusInService.StatusInfo != operatorMessage && operatorMessage != "")) { c.log.Infof("Operator <%s> updated, status: %s -> %s, message: %s -> %s.", operatorName, operatorStatusInService.Status, operatorStatus, operatorStatusInService.StatusInfo, operatorMessage) if !handler.OnChange(operatorStatus) { c.log.WithError(err).Warnf("<%s> operator's OnChange() returned false. Will skip an update.", operatorName) @@ -107,7 +106,7 @@ func (handler ClusterVersionHandler) GetName() string { return cvoOperatorName } func (handler ClusterVersionHandler) IsInitialized() bool { return true } func (handler ClusterVersionHandler) GetStatus() (models.OperatorStatus, string, error) { - co, err := handler.kc.GetClusterVersion(clusterVersionName) + co, err := handler.kc.GetClusterVersion() if err != nil { return "", "", err } diff --git a/src/k8s_client/k8s_client.go b/src/k8s_client/k8s_client.go index 3422e99a46..c38b2aebe2 100644 --- a/src/k8s_client/k8s_client.go +++ b/src/k8s_client/k8s_client.go @@ -10,8 +10,6 @@ import ( "strings" "time" - "gopkg.in/yaml.v2" - metal3v1alpha1 "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1" configv1 "github.com/openshift/api/config/v1" configv1client "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1" @@ -22,6 +20,8 @@ import ( olmv1client "github.com/operator-framework/operator-lifecycle-manager/pkg/api/client/clientset/versioned/typed/operators/v1alpha1" "github.com/pkg/errors" "github.com/sirupsen/logrus" + "github.com/thoas/go-funk" + "gopkg.in/yaml.v2" certificatesv1 "k8s.io/api/certificates/v1" v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -69,7 +69,7 @@ type K8SClient interface { UpdateBMHStatus(bmh *metal3v1alpha1.BareMetalHost) error UpdateBMH(bmh *metal3v1alpha1.BareMetalHost) error SetProxyEnvVars() error - GetClusterVersion(name string) (*configv1.ClusterVersion, error) + GetClusterVersion() (*configv1.ClusterVersion, error) GetNetworkType() (string, error) GetServiceNetworks() ([]string, error) GetControlPlaneReplicas() (int, error) @@ -83,6 +83,7 @@ type K8SClient interface { PatchNamespace(namespace string, data []byte) error GetNode(name string) (*v1.Node, error) PatchNodeLabels(nodeName string, nodeLabels string) error + IsClusterCapabilityEnabled(configv1.ClusterVersionCapability) (bool, error) } type K8SClientBuilder func(configPath string, logger logrus.FieldLogger) (K8SClient, error) @@ -551,12 +552,12 @@ func (c *k8sClient) UpdateBMH(bmh *metal3v1alpha1.BareMetalHost) error { return c.runtimeClient.Update(context.TODO(), bmh) } -func (c *k8sClient) GetClusterVersion(name string) (*configv1.ClusterVersion, error) { +func (c *k8sClient) GetClusterVersion() (*configv1.ClusterVersion, error) { result := &configv1.ClusterVersion{} err := c.client.RESTClient().Get(). AbsPath("/apis/config.openshift.io/v1"). Resource("clusterversions"). - Name(name). + Name("version"). Do(context.Background()). Into(result) return result, err @@ -613,3 +614,13 @@ func (c *k8sClient) PatchNodeLabels(nodeName string, nodeLabels string) error { _, err := c.client.CoreV1().Nodes().Patch(context.Background(), nodeName, types.MergePatchType, data, metav1.PatchOptions{}) return err } + +func (c *k8sClient) IsClusterCapabilityEnabled(capability configv1.ClusterVersionCapability) (bool, error) { + version, err := c.GetClusterVersion() + if err != nil { + return false, err + } + isExplicitlyEnabled := funk.Contains(version.Status.Capabilities.EnabledCapabilities, capability) + isKnown := funk.Contains(version.Status.Capabilities.KnownCapabilities, capability) + return isExplicitlyEnabled || !isKnown, nil +} diff --git a/src/k8s_client/mock_k8s_client.go b/src/k8s_client/mock_k8s_client.go index 86c293cc26..b9fbb581a0 100644 --- a/src/k8s_client/mock_k8s_client.go +++ b/src/k8s_client/mock_k8s_client.go @@ -378,18 +378,18 @@ func (mr *MockK8SClientMockRecorder) SetProxyEnvVars() *gomock.Call { } // GetClusterVersion mocks base method -func (m *MockK8SClient) GetClusterVersion(name string) (*v1.ClusterVersion, error) { +func (m *MockK8SClient) GetClusterVersion() (*v1.ClusterVersion, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetClusterVersion", name) + ret := m.ctrl.Call(m, "GetClusterVersion") ret0, _ := ret[0].(*v1.ClusterVersion) ret1, _ := ret[1].(error) return ret0, ret1 } // GetClusterVersion indicates an expected call of GetClusterVersion -func (mr *MockK8SClientMockRecorder) GetClusterVersion(name interface{}) *gomock.Call { +func (mr *MockK8SClientMockRecorder) GetClusterVersion() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetClusterVersion", reflect.TypeOf((*MockK8SClient)(nil).GetClusterVersion), name) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetClusterVersion", reflect.TypeOf((*MockK8SClient)(nil).GetClusterVersion)) } // GetNetworkType mocks base method @@ -582,3 +582,18 @@ func (mr *MockK8SClientMockRecorder) PatchNodeLabels(nodeName, nodeLabels interf mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PatchNodeLabels", reflect.TypeOf((*MockK8SClient)(nil).PatchNodeLabels), nodeName, nodeLabels) } + +// IsClusterCapabilityEnabled mocks base method +func (m *MockK8SClient) IsClusterCapabilityEnabled(arg0 v1.ClusterVersionCapability) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsClusterCapabilityEnabled", arg0) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IsClusterCapabilityEnabled indicates an expected call of IsClusterCapabilityEnabled +func (mr *MockK8SClientMockRecorder) IsClusterCapabilityEnabled(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsClusterCapabilityEnabled", reflect.TypeOf((*MockK8SClient)(nil).IsClusterCapabilityEnabled), arg0) +} diff --git a/src/main/drymock/dry_mode_k8s_mock.go b/src/main/drymock/dry_mode_k8s_mock.go index 36787ae87b..6c0075349a 100644 --- a/src/main/drymock/dry_mode_k8s_mock.go +++ b/src/main/drymock/dry_mode_k8s_mock.go @@ -150,7 +150,7 @@ func PrepareControllerDryMock(mockk8sclient *k8s_client.MockK8SClient, logger *l Conditions: availableConditions, }, } - mockk8sclient.EXPECT().GetClusterVersion(gomock.Any()).Return(&clusterVersion, nil).AnyTimes() + mockk8sclient.EXPECT().GetClusterVersion().Return(&clusterVersion, nil).AnyTimes() configMap := v1.ConfigMap{ Data: map[string]string{ @@ -273,6 +273,8 @@ dEFgad6P3hMZTOg7yVkMOd3QtgVQ9I8dXqS2nG9EMEh97WIhi6f5ztvcQvQ5tXjh } mockk8sclient.EXPECT().ListClusterOperators().Return(clusterOperatorList, nil).AnyTimes() + + mockk8sclient.EXPECT().IsClusterCapabilityEnabled(gomock.Any()).Return(true, nil).AnyTimes() } // PrepareInstallerDryK8sMock utilizes k8s_client.MockK8SClient to fake the k8s API to make the