From 33f41849d6b27d937ecfa8cf97f7b57d53b0c58d Mon Sep 17 00:00:00 2001 From: danmanor Date: Mon, 11 Nov 2024 17:02:58 +0200 Subject: [PATCH] MGMT-19080: Enable streched cluster installation in day1 --- .../v1beta1/agentclusterinstall_types.go | 6 +- .../models/cluster_create_params.go | 3 + .../models/v2_cluster_update_params.go | 3 + .../models/cluster_create_params.go | 3 + .../models/v2_cluster_update_params.go | 3 + ...ive.openshift.io_agentclusterinstalls.yaml | 2 +- config/crd/resources.yaml | 2 +- ...ive.openshift.io_agentclusterinstalls.yaml | 2 +- internal/bminventory/inventory.go | 99 +++- internal/bminventory/inventory_test.go | 533 +++++++++++++++++- internal/cluster/cluster.go | 19 +- internal/cluster/cluster_test.go | 106 ++-- internal/cluster/common.go | 4 - internal/cluster/mock_cluster_api.go | 15 + internal/cluster/progress_test.go | 2 +- .../cluster/refresh_status_preprocessor.go | 2 +- internal/cluster/statemachine.go | 3 +- internal/cluster/transition.go | 40 +- internal/cluster/transition_test.go | 529 +++++++++++++---- internal/cluster/validator.go | 75 ++- internal/cluster/validator_test.go | 318 +++++++++++ internal/common/common.go | 84 ++- internal/common/common_test.go | 125 ++++ internal/common/db.go | 34 ++ internal/common/db_test.go | 288 ++++++++++ internal/common/validations.go | 17 +- .../clusterdeployments_controller.go | 141 ++++- .../clusterdeployments_controller_test.go | 266 ++++++++- internal/host/host.go | 64 ++- internal/host/host_test.go | 6 +- internal/host/hostrole_test.go | 2 +- internal/host/mock_host_api.go | 24 +- internal/host/monitor.go | 7 +- internal/host/transition_test.go | 23 +- internal/host/validations_test.go | 2 +- internal/ignition/installmanifests.go | 16 +- internal/ignition/installmanifests_test.go | 5 + internal/operators/common/common.go | 1 + internal/operators/lvm/lvm_operator.go | 4 +- internal/operators/odf/odf_operator.go | 14 + internal/operators/odf/odf_operator_test.go | 36 ++ internal/operators/odf/validation_test.go | 9 +- internal/operators/odf/validations.go | 8 + internal/testing/common.go | 15 + models/cluster_create_params.go | 3 + models/v2_cluster_update_params.go | 3 + restapi/embedded_spec.go | 20 + subsystem/cluster_test.go | 43 ++ subsystem/kubeapi_test.go | 110 ++++ swagger.yaml | 8 + .../v1beta1/agentclusterinstall_types.go | 6 +- .../models/cluster_create_params.go | 3 + .../models/v2_cluster_update_params.go | 3 + 53 files changed, 2768 insertions(+), 391 deletions(-) create mode 100644 internal/common/db_test.go create mode 100644 internal/testing/common.go diff --git a/api/hiveextension/v1beta1/agentclusterinstall_types.go b/api/hiveextension/v1beta1/agentclusterinstall_types.go index 8ef92a34ada6..327020fd25bc 100644 --- a/api/hiveextension/v1beta1/agentclusterinstall_types.go +++ b/api/hiveextension/v1beta1/agentclusterinstall_types.go @@ -22,13 +22,11 @@ const ( ClusterInstallationStoppedReason string = "ClusterInstallationStopped" ClusterInstallationStoppedMsg string = "The cluster installation stopped" ClusterInsufficientAgentsReason string = "InsufficientAgents" - ClusterInsufficientAgentsMsg string = "The cluster currently requires %d agents but only %d have registered" + ClusterInsufficientAgentsMsg string = "The cluster currently requires exactly %d master agents and %d worker agents, but currently registered %d master agents and %d worker agents" ClusterUnapprovedAgentsReason string = "UnapprovedAgents" ClusterUnapprovedAgentsMsg string = "The installation is pending on the approval of %d agents" ClusterUnsyncedAgentsReason string = "UnsyncedAgents" ClusterUnsyncedAgentsMsg string = "The cluster currently has %d agents with spec error" - ClusterAdditionalAgentsReason string = "AdditionalAgents" - ClusterAdditionalAgentsMsg string = "The cluster currently requires exactly %d agents but have %d registered" ClusterValidatedCondition hivev1.ClusterInstallConditionType = "Validated" ClusterValidationsOKMsg string = "The cluster's validations are passing" @@ -355,7 +353,7 @@ type ClusterNetworkEntry struct { type ProvisionRequirements struct { // ControlPlaneAgents is the number of matching approved and ready Agents with the control plane role - // required to launch the install. Must be either 1 or 3. + // required to launch the install. Must be either 1 or 3-5. ControlPlaneAgents int `json:"controlPlaneAgents"` // WorkerAgents is the minimum number of matching approved and ready Agents with the worker role diff --git a/api/vendor/github.com/openshift/assisted-service/models/cluster_create_params.go b/api/vendor/github.com/openshift/assisted-service/models/cluster_create_params.go index 3eeade9fcdc5..1351a6d02ee1 100644 --- a/api/vendor/github.com/openshift/assisted-service/models/cluster_create_params.go +++ b/api/vendor/github.com/openshift/assisted-service/models/cluster_create_params.go @@ -42,6 +42,9 @@ type ClusterCreateParams struct { // Cluster networks that are associated with this cluster. ClusterNetworks []*ClusterNetwork `json:"cluster_networks"` + // The amount of control planes which should be part of the cluster. + ControlPlaneCount *int64 `json:"control_plane_count,omitempty"` + // The CPU architecture of the image (x86_64/arm64/etc). // Enum: [x86_64 aarch64 arm64 ppc64le s390x multi] CPUArchitecture string `json:"cpu_architecture,omitempty"` diff --git a/api/vendor/github.com/openshift/assisted-service/models/v2_cluster_update_params.go b/api/vendor/github.com/openshift/assisted-service/models/v2_cluster_update_params.go index cf3edc1efcff..437f8dbed610 100644 --- a/api/vendor/github.com/openshift/assisted-service/models/v2_cluster_update_params.go +++ b/api/vendor/github.com/openshift/assisted-service/models/v2_cluster_update_params.go @@ -45,6 +45,9 @@ type V2ClusterUpdateParams struct { // Cluster networks that are associated with this cluster. ClusterNetworks []*ClusterNetwork `json:"cluster_networks"` + // The amount of control planes which should be part of the cluster. + ControlPlaneCount *int64 `json:"control_plane_count,omitempty"` + // Installation disks encryption mode and host roles to be applied. DiskEncryption *DiskEncryption `json:"disk_encryption,omitempty" gorm:"embedded;embeddedPrefix:disk_encryption_"` diff --git a/client/vendor/github.com/openshift/assisted-service/models/cluster_create_params.go b/client/vendor/github.com/openshift/assisted-service/models/cluster_create_params.go index 3eeade9fcdc5..1351a6d02ee1 100644 --- a/client/vendor/github.com/openshift/assisted-service/models/cluster_create_params.go +++ b/client/vendor/github.com/openshift/assisted-service/models/cluster_create_params.go @@ -42,6 +42,9 @@ type ClusterCreateParams struct { // Cluster networks that are associated with this cluster. ClusterNetworks []*ClusterNetwork `json:"cluster_networks"` + // The amount of control planes which should be part of the cluster. + ControlPlaneCount *int64 `json:"control_plane_count,omitempty"` + // The CPU architecture of the image (x86_64/arm64/etc). // Enum: [x86_64 aarch64 arm64 ppc64le s390x multi] CPUArchitecture string `json:"cpu_architecture,omitempty"` diff --git a/client/vendor/github.com/openshift/assisted-service/models/v2_cluster_update_params.go b/client/vendor/github.com/openshift/assisted-service/models/v2_cluster_update_params.go index cf3edc1efcff..437f8dbed610 100644 --- a/client/vendor/github.com/openshift/assisted-service/models/v2_cluster_update_params.go +++ b/client/vendor/github.com/openshift/assisted-service/models/v2_cluster_update_params.go @@ -45,6 +45,9 @@ type V2ClusterUpdateParams struct { // Cluster networks that are associated with this cluster. ClusterNetworks []*ClusterNetwork `json:"cluster_networks"` + // The amount of control planes which should be part of the cluster. + ControlPlaneCount *int64 `json:"control_plane_count,omitempty"` + // Installation disks encryption mode and host roles to be applied. DiskEncryption *DiskEncryption `json:"disk_encryption,omitempty" gorm:"embedded;embeddedPrefix:disk_encryption_"` diff --git a/config/crd/bases/extensions.hive.openshift.io_agentclusterinstalls.yaml b/config/crd/bases/extensions.hive.openshift.io_agentclusterinstalls.yaml index 39ae7bf57dd1..39ae9ee69a7c 100644 --- a/config/crd/bases/extensions.hive.openshift.io_agentclusterinstalls.yaml +++ b/config/crd/bases/extensions.hive.openshift.io_agentclusterinstalls.yaml @@ -437,7 +437,7 @@ spec: controlPlaneAgents: description: |- ControlPlaneAgents is the number of matching approved and ready Agents with the control plane role - required to launch the install. Must be either 1 or 3. + required to launch the install. Must be either 1 or 3-5. type: integer workerAgents: description: |- diff --git a/config/crd/resources.yaml b/config/crd/resources.yaml index db3c5b5b0fc3..d2934f909b82 100644 --- a/config/crd/resources.yaml +++ b/config/crd/resources.yaml @@ -552,7 +552,7 @@ spec: controlPlaneAgents: description: |- ControlPlaneAgents is the number of matching approved and ready Agents with the control plane role - required to launch the install. Must be either 1 or 3. + required to launch the install. Must be either 1 or 3-5. type: integer workerAgents: description: |- diff --git a/deploy/olm-catalog/manifests/extensions.hive.openshift.io_agentclusterinstalls.yaml b/deploy/olm-catalog/manifests/extensions.hive.openshift.io_agentclusterinstalls.yaml index e59ce84d088d..eea4afa720ad 100644 --- a/deploy/olm-catalog/manifests/extensions.hive.openshift.io_agentclusterinstalls.yaml +++ b/deploy/olm-catalog/manifests/extensions.hive.openshift.io_agentclusterinstalls.yaml @@ -434,7 +434,7 @@ spec: controlPlaneAgents: description: |- ControlPlaneAgents is the number of matching approved and ready Agents with the control plane role - required to launch the install. Must be either 1 or 3. + required to launch the install. Must be either 1 or 3-5. type: integer workerAgents: description: |- diff --git a/internal/bminventory/inventory.go b/internal/bminventory/inventory.go index ea34a9c07972..91b29358e7dd 100644 --- a/internal/bminventory/inventory.go +++ b/internal/bminventory/inventory.go @@ -317,6 +317,7 @@ func (b *bareMetalInventory) setDefaultRegisterClusterParams(ctx context.Context {Cidr: models.Subnet(b.Config.DefaultServiceNetworkCidr)}, } } + if params.NewClusterParams.MachineNetworks == nil { params.NewClusterParams.MachineNetworks = []*models.MachineNetwork{} } @@ -324,15 +325,18 @@ func (b *bareMetalInventory) setDefaultRegisterClusterParams(ctx context.Context if params.NewClusterParams.VipDhcpAllocation == nil { params.NewClusterParams.VipDhcpAllocation = swag.Bool(false) } + if params.NewClusterParams.Hyperthreading == nil { params.NewClusterParams.Hyperthreading = swag.String(models.ClusterHyperthreadingAll) } + if params.NewClusterParams.SchedulableMasters == nil { params.NewClusterParams.SchedulableMasters = swag.Bool(false) } - if params.NewClusterParams.HighAvailabilityMode == nil { - params.NewClusterParams.HighAvailabilityMode = swag.String(models.ClusterHighAvailabilityModeFull) - } + + params.NewClusterParams.HighAvailabilityMode, params.NewClusterParams.ControlPlaneCount = common.GetDefaultHighAvailabilityAndMasterCountParams( + params.NewClusterParams.HighAvailabilityMode, params.NewClusterParams.ControlPlaneCount, + ) log.Infof("Verifying cluster platform and user-managed-networking, got platform=%s and userManagedNetworking=%s", getPlatformType(params.NewClusterParams.Platform), common.BoolPtrForLog(params.NewClusterParams.UserManagedNetworking)) platform, userManagedNetworking, err := provider.GetActualCreateClusterPlatformParams(params.NewClusterParams.Platform, params.NewClusterParams.UserManagedNetworking, params.NewClusterParams.HighAvailabilityMode, params.NewClusterParams.CPUArchitecture) @@ -441,6 +445,15 @@ func (b *bareMetalInventory) validateRegisterClusterInternalParams(params *insta } } + // should be called after defaults were set + if err := validateHighAvailabilityWithControlPlaneCount( + swag.StringValue(params.NewClusterParams.HighAvailabilityMode), + swag.Int64Value(params.NewClusterParams.ControlPlaneCount), + swag.StringValue(params.NewClusterParams.OpenshiftVersion), + ); err != nil { + return common.NewApiError(http.StatusBadRequest, err) + } + return nil } @@ -641,6 +654,7 @@ func (b *bareMetalInventory) RegisterClusterInternal( KubeKeyNamespace: kubeKey.Namespace, TriggerMonitorTimestamp: time.Now(), MachineNetworkCidrUpdatedAt: time.Now(), + ControlPlaneCount: swag.Int64Value(params.NewClusterParams.ControlPlaneCount), } newOLMOperators, err := b.getOLMMonitoredOperators(log, cluster, params, *releaseImage.Version) @@ -1308,13 +1322,20 @@ func (b *bareMetalInventory) InstallClusterInternal(ctx context.Context, params sortedHosts, canRefreshRoles := host.SortHosts(cluster.Hosts) if canRefreshRoles { for i := range sortedHosts { - updated, err = b.hostApi.AutoAssignRole(ctx, cluster.Hosts[i], tx) + updated, err = b.hostApi.AutoAssignRole(ctx, cluster.Hosts[i], tx, swag.Int(int(cluster.ControlPlaneCount))) if err != nil { return err } autoAssigned = autoAssigned || updated } } + + // Refresh schedulable masters again after all roles are assigned + if internalErr := b.clusterApi.RefreshSchedulableMastersForcedTrue(ctx, *cluster.ID); internalErr != nil { + log.WithError(internalErr).Errorf("Failed to refresh SchedulableMastersForcedTrue while installing cluster <%s>", cluster.ID) + return internalErr + } + hasIgnoredValidations := common.IgnoredValidationsAreSet(cluster) if hasIgnoredValidations { eventgen.SendValidationsIgnoredEvent(ctx, b.eventsHandler, *cluster.ID) @@ -1322,7 +1343,7 @@ func (b *bareMetalInventory) InstallClusterInternal(ctx context.Context, params //usage for auto role selection is measured only for day1 clusters with more than //3 hosts (which would automatically be assigned as masters if the hw is sufficient) if usages, u_err := usage.Unmarshal(cluster.Cluster.FeatureUsage); u_err == nil { - report := cluster.Cluster.TotalHostCount > common.MinMasterHostsNeededForInstallation && autoAssigned + report := cluster.Cluster.TotalHostCount > common.MinMasterHostsNeededForInstallationInHaMode && autoAssigned if hasIgnoredValidations { b.setUsage(true, usage.ValidationsIgnored, nil, usages) } @@ -1435,9 +1456,11 @@ func (b *bareMetalInventory) InstallSingleDay2HostInternal(ctx context.Context, if h, err = b.getHost(ctx, infraEnvId.String(), hostId.String()); err != nil { return err } + // auto select host roles if not selected yet. err = b.db.Transaction(func(tx *gorm.DB) error { - if _, err = b.hostApi.AutoAssignRole(ctx, &h.Host, tx); err != nil { + // no need to specify expected master count for day2 hosts, as their suggested role will always be worker + if _, err = b.hostApi.AutoAssignRole(ctx, &h.Host, tx, nil); err != nil { return err } return nil @@ -1509,7 +1532,8 @@ func (b *bareMetalInventory) V2InstallHost(ctx context.Context, params installer return common.NewApiError(http.StatusConflict, fmt.Errorf("cannot install host in state %s", swag.StringValue(h.Status))) } - _, err = b.hostApi.AutoAssignRole(ctx, h, b.db) + // no need to specify expected master count for day2 hosts, as their suggested role will always be worker + _, err = b.hostApi.AutoAssignRole(ctx, h, b.db, nil) if err != nil { log.Errorf("Failed to update role for host %s", params.HostID) return common.GenerateErrorResponder(err) @@ -1911,9 +1935,66 @@ func (b *bareMetalInventory) validateAndUpdateClusterParams(ctx context.Context, return installer.V2UpdateClusterParams{}, err } + // We don't want to update ControlPlaneCount in day2 clusters as we don't know the previous hosts + if params.ClusterUpdateParams.ControlPlaneCount != nil && swag.StringValue(cluster.Kind) != models.ClusterKindAddHostsCluster { + if err := validateHighAvailabilityWithControlPlaneCount( + swag.StringValue(cluster.HighAvailabilityMode), + swag.Int64Value(params.ClusterUpdateParams.ControlPlaneCount), + cluster.OpenshiftVersion, + ); err != nil { + return installer.V2UpdateClusterParams{}, err + } + } + return *params, nil } +func validateHighAvailabilityWithControlPlaneCount(highAvailabilityMode string, controlPlaneCount int64, openshiftVersion string) error { + if highAvailabilityMode == models.ClusterCreateParamsHighAvailabilityModeNone && + controlPlaneCount != 1 { + return common.NewApiError( + http.StatusBadRequest, + errors.New("single-node clusters must have a single control plane node"), + ) + } + + stretchedClustersNotSuported, err := common.BaseVersionLessThan(common.MinimumVersionForStretchedControlPlanesCluster, openshiftVersion) + if err != nil { + return err + } + + if highAvailabilityMode == models.ClusterCreateParamsHighAvailabilityModeFull && + controlPlaneCount != common.AllowedNumberOfMasterHostsForInstallationInHaModeOfOCP417OrOlder && + stretchedClustersNotSuported { + return common.NewApiError( + http.StatusBadRequest, + fmt.Errorf( + "there should be exactly %d dedicated control plane nodes for high availability mode %s in openshift version older than %s", + common.AllowedNumberOfMasterHostsForInstallationInHaModeOfOCP417OrOlder, + highAvailabilityMode, + common.MinimumVersionForStretchedControlPlanesCluster, + ), + ) + } + + if highAvailabilityMode == models.ClusterCreateParamsHighAvailabilityModeFull && + (controlPlaneCount < common.MinMasterHostsNeededForInstallationInHaMode || + controlPlaneCount > common.MaxMasterHostsNeededForInstallationInHaModeOfOCP418OrNewer) { + return common.NewApiError( + http.StatusBadRequest, + fmt.Errorf( + "there should be %d-%d dedicated control plane nodes for high availability mode %s in openshift version %s or newer", + common.MinMasterHostsNeededForInstallationInHaMode, + common.MaxMasterHostsNeededForInstallationInHaModeOfOCP418OrNewer, + highAvailabilityMode, + common.MinimumVersionForStretchedControlPlanesCluster, + ), + ) + } + + return nil +} + func (b *bareMetalInventory) V2UpdateCluster(ctx context.Context, params installer.V2UpdateClusterParams) middleware.Responder { c, err := b.v2UpdateClusterInternal(ctx, params, Interactive) if err != nil { @@ -2358,6 +2439,10 @@ func (b *bareMetalInventory) updateClusterData(_ context.Context, cluster *commo b.setUserManagedNetworkingAndMultiNodeUsage(swag.BoolValue(params.ClusterUpdateParams.UserManagedNetworking), *cluster.HighAvailabilityMode, usages) } + if params.ClusterUpdateParams.ControlPlaneCount != nil && *params.ClusterUpdateParams.ControlPlaneCount != cluster.ControlPlaneCount { + updates["control_plane_count"] = *params.ClusterUpdateParams.ControlPlaneCount + } + if len(updates) > 0 { updates["trigger_monitor_timestamp"] = time.Now() err = db.Model(&common.Cluster{}).Where("id = ?", cluster.ID.String()).Updates(updates).Error diff --git a/internal/bminventory/inventory_test.go b/internal/bminventory/inventory_test.go index f9d7aced26bd..d6ff6c1c08d6 100644 --- a/internal/bminventory/inventory_test.go +++ b/internal/bminventory/inventory_test.go @@ -61,6 +61,7 @@ import ( "github.com/openshift/assisted-service/internal/provider/registry" "github.com/openshift/assisted-service/internal/provider/vsphere" "github.com/openshift/assisted-service/internal/stream" + testutils "github.com/openshift/assisted-service/internal/testing" "github.com/openshift/assisted-service/internal/usage" "github.com/openshift/assisted-service/internal/versions" "github.com/openshift/assisted-service/models" @@ -1493,6 +1494,10 @@ func mockDetectAndStoreCollidingIPsForCluster(mockClusterApi *cluster.MockAPI, t mockClusterApi.EXPECT().DetectAndStoreCollidingIPsForCluster(gomock.Any(), gomock.Any()).Return(nil).Times(times) } +func mockRefreshSchedulableMastersForcedTrue(mockClusterApi *cluster.MockAPI, times int) { + mockClusterApi.EXPECT().RefreshSchedulableMastersForcedTrue(gomock.Any(), gomock.Any()).Return(nil).Times(times) +} + var _ = Describe("cluster", func() { masterHostId1 := strfmt.UUID(uuid.New().String()) masterHostId2 := strfmt.UUID(uuid.New().String()) @@ -1605,14 +1610,14 @@ var _ = Describe("cluster", func() { mockClusterApi.EXPECT().ResetCluster(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(common.NewApiError(http.StatusInternalServerError, nil)).Times(1) } mockAutoAssignFailed := func() { - mockHostApi.EXPECT().AutoAssignRole(gomock.Any(), gomock.Any(), gomock.Any()). + mockHostApi.EXPECT().AutoAssignRole(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). Return(false, errors.Errorf("")).Times(1) } mockAutoAssignSuccess := func(times int) { - mockHostApi.EXPECT().AutoAssignRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).Times(times) + mockHostApi.EXPECT().AutoAssignRole(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).Times(times) } mockFalseAutoAssignSuccess := func(times int) { - mockHostApi.EXPECT().AutoAssignRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil).Times(times) + mockHostApi.EXPECT().AutoAssignRole(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil).Times(times) } mockClusterRefreshStatusSuccess := func() { mockClusterApi.EXPECT().RefreshStatus(gomock.Any(), gomock.Any(), gomock.Any()). @@ -5307,6 +5312,7 @@ var _ = Describe("cluster", func() { mockClusterRefreshStatus(mockClusterApi) mockClusterDeleteLogsSuccess(mockClusterApi) mockSetConnectivityMajorityGroupsForCluster(mockClusterApi) + mockRefreshSchedulableMastersForcedTrue(mockClusterApi, 1) mockEvents.EXPECT().SendInfraEnvEvent(gomock.Any(), eventstest.NewEventMatcher( eventstest.WithNameMatcher(eventgen.IgnitionConfigImageGeneratedEventName), eventstest.WithClusterIdMatcher(clusterID.String()), @@ -5355,6 +5361,7 @@ var _ = Describe("cluster", func() { mockClusterRefreshStatus(mockClusterApi) mockClusterDeleteLogsSuccess(mockClusterApi) mockSetConnectivityMajorityGroupsForCluster(mockClusterApi) + mockRefreshSchedulableMastersForcedTrue(mockClusterApi, 1) mockEvents.EXPECT().SendInfraEnvEvent(gomock.Any(), eventstest.NewEventMatcher( eventstest.WithNameMatcher(eventgen.IgnitionConfigImageGeneratedEventName), eventstest.WithClusterIdMatcher(clusterID.String()), @@ -5479,6 +5486,7 @@ var _ = Describe("cluster", func() { mockClusterRefreshStatus(mockClusterApi) mockClusterDeleteLogsSuccess(mockClusterApi) mockSetConnectivityMajorityGroupsForCluster(mockClusterApi) + mockRefreshSchedulableMastersForcedTrue(mockClusterApi, 1) mockEvents.EXPECT().SendInfraEnvEvent(gomock.Any(), eventstest.NewEventMatcher( eventstest.WithNameMatcher(eventgen.IgnitionConfigImageGeneratedEventName), eventstest.WithClusterIdMatcher(clusterID.String()), @@ -5538,6 +5546,7 @@ var _ = Describe("cluster", func() { mockClusterRefreshStatus(mockClusterApi) mockClusterDeleteLogsSuccess(mockClusterApi) mockSetConnectivityMajorityGroupsForCluster(mockClusterApi) + mockRefreshSchedulableMastersForcedTrue(mockClusterApi, 1) mockClusterApi.EXPECT().HandlePreInstallError(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Do(func(ctx, c, err interface{}) { DoneChannel <- 1 }) _ = bm.V2InstallCluster(ctx, installer.V2InstallClusterParams{ @@ -5576,6 +5585,7 @@ var _ = Describe("cluster", func() { mockClusterRefreshStatus(mockClusterApi) setIsReadyForInstallationTrue(mockClusterApi) mockSetConnectivityMajorityGroupsForCluster(mockClusterApi) + mockRefreshSchedulableMastersForcedTrue(mockClusterApi, 1) reply := bm.V2InstallCluster(ctx, installer.V2InstallClusterParams{ ClusterID: clusterID, @@ -5592,6 +5602,7 @@ var _ = Describe("cluster", func() { mockClusterRefreshStatus(mockClusterApi) setIsReadyForInstallationFalse(mockClusterApi) mockSetConnectivityMajorityGroupsForCluster(mockClusterApi) + mockRefreshSchedulableMastersForcedTrue(mockClusterApi, 1) Expect(db.Model(&common.Cluster{Cluster: models.Cluster{ID: &clusterID}}).UpdateColumn("status", "insufficient").Error).To(Not(HaveOccurred())) reply := bm.V2InstallCluster(ctx, installer.V2InstallClusterParams{ @@ -5604,6 +5615,7 @@ var _ = Describe("cluster", func() { createCluster(defaultCluster) mockFalseAutoAssignSuccess(3) setIsReadyForInstallationFalse(mockClusterApi) + mockRefreshSchedulableMastersForcedTrue(mockClusterApi, 1) Expect(db.Model(&common.Cluster{Cluster: models.Cluster{ID: &clusterID}}).UpdateColumn("status", "insufficient").Error).To(Not(HaveOccurred())) reply := bm.V2InstallCluster(ctx, installer.V2InstallClusterParams{ @@ -5621,6 +5633,7 @@ var _ = Describe("cluster", func() { mockClusterPrepareForInstallationSuccess(mockClusterApi) setIsReadyForInstallationTrue(mockClusterApi) mockSetConnectivityMajorityGroupsForCluster(mockClusterApi) + mockRefreshSchedulableMastersForcedTrue(mockClusterApi, 1) mockClusterApi.EXPECT().GetMasterNodesIds(gomock.Any(), gomock.Any(), gomock.Any()). Return([]*strfmt.UUID{}, nil).Times(1) @@ -5642,6 +5655,7 @@ var _ = Describe("cluster", func() { mockClusterPrepareForInstallationSuccess(mockClusterApi) setIsReadyForInstallationTrue(mockClusterApi) mockSetConnectivityMajorityGroupsForCluster(mockClusterApi) + mockRefreshSchedulableMastersForcedTrue(mockClusterApi, 1) reply := bm.V2InstallCluster(ctx, installer.V2InstallClusterParams{ ClusterID: clusterID, @@ -5658,6 +5672,7 @@ var _ = Describe("cluster", func() { mockClusterPrepareForInstallationSuccess(mockClusterApi) setIsReadyForInstallationTrue(mockClusterApi) mockSetConnectivityMajorityGroupsForCluster(mockClusterApi) + mockRefreshSchedulableMastersForcedTrue(mockClusterApi, 1) mockClusterApi.EXPECT().GetMasterNodesIds(gomock.Any(), gomock.Any(), gomock.Any()). Return([]*strfmt.UUID{&masterHostId1, &masterHostId2, &masterHostId3}, errors.Errorf("nop")) @@ -5687,6 +5702,8 @@ var _ = Describe("cluster", func() { mockClusterDeleteLogsFailure(mockClusterApi) mockHandlePreInstallationSuccess(mockClusterApi, DoneChannel) mockSetConnectivityMajorityGroupsForCluster(mockClusterApi) + mockRefreshSchedulableMastersForcedTrue(mockClusterApi, 1) + mockEvents.EXPECT().SendInfraEnvEvent(gomock.Any(), eventstest.NewEventMatcher( eventstest.WithNameMatcher(eventgen.IgnitionConfigImageGeneratedEventName), eventstest.WithClusterIdMatcher(clusterID.String()), @@ -5840,7 +5857,7 @@ var _ = Describe("cluster", func() { }) }) -var _ = Describe("V2ClusterUpdate cluster", func() { +var _ = Describe("V2UpdateCluster", func() { masterHostId1 := strfmt.UUID(uuid.New().String()) masterHostId3 := strfmt.UUID(uuid.New().String()) @@ -5910,12 +5927,12 @@ var _ = Describe("V2ClusterUpdate cluster", func() { mockClusterUpdateSuccess(1, 0) } - Context("Update", func() { + Context("update", func() { BeforeEach(func() { mockDurationsSuccess() }) - Context("Single node", func() { + Context("single node cluster", func() { var cluster *common.Cluster BeforeEach(func() { clusterID = strfmt.UUID(uuid.New().String()) @@ -7828,6 +7845,259 @@ var _ = Describe("V2ClusterUpdate cluster", func() { }) }) + + Context("control_plane_count", func() { + var ( + clusterID = strfmt.UUID(uuid.New().String()) + ) + + Context("should succeed", func() { + + BeforeEach(func() { + mockSetConnectivityMajorityGroupsForCluster(mockClusterApi) + mockDetectAndStoreCollidingIPsForCluster(mockClusterApi, 1) + mockClusterApi.EXPECT().RefreshStatus(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).Times(1) + }) + + It("when using nil value", func() { + cluster := &common.Cluster{ + Cluster: models.Cluster{ + ID: &clusterID, + HighAvailabilityMode: swag.String(models.ClusterCreateParamsHighAvailabilityModeFull), + OpenshiftVersion: "4.16", + }} + + err := db.Create(cluster).Error + Expect(err).ShouldNot(HaveOccurred()) + mockClusterApi.EXPECT().VerifyClusterUpdatability(createClusterIdMatcher(cluster)).Return(nil).Times(1) + + reply := bm.V2UpdateCluster(ctx, installer.V2UpdateClusterParams{ + ClusterID: clusterID, + ClusterUpdateParams: &models.V2ClusterUpdateParams{}, + }) + + Expect(reply).To(BeAssignableToTypeOf(installer.NewV2UpdateClusterCreated())) + }) + + It("not changing the value", func() { + cluster := &common.Cluster{ + Cluster: models.Cluster{ + ID: &clusterID, + HighAvailabilityMode: swag.String(models.ClusterCreateParamsHighAvailabilityModeFull), + OpenshiftVersion: "4.16", + }, + ControlPlaneCount: 3, + } + + err := db.Create(cluster).Error + Expect(err).ShouldNot(HaveOccurred()) + + mockClusterApi.EXPECT().VerifyClusterUpdatability(createClusterIdMatcher(cluster)).Return(nil).Times(1) + + reply := bm.V2UpdateCluster(ctx, installer.V2UpdateClusterParams{ + ClusterID: clusterID, + ClusterUpdateParams: &models.V2ClusterUpdateParams{ + ControlPlaneCount: swag.Int64(3), + }, + }) + + Expect(reply).To(BeAssignableToTypeOf(installer.NewV2UpdateClusterCreated())) + + var newCluster *common.Cluster + err = db.Where("id = ?", clusterID).Take(&newCluster).Error + Expect(err).ToNot(HaveOccurred()) + + Expect(newCluster.ControlPlaneCount).To(BeEquivalentTo(3)) + }) + + It("increasing to 4 control planes with OCP >= 4.18 the value and multi-node", func() { + cluster := &common.Cluster{ + Cluster: models.Cluster{ + ID: &clusterID, + HighAvailabilityMode: swag.String(models.ClusterCreateParamsHighAvailabilityModeFull), + OpenshiftVersion: "4.18", + }, + ControlPlaneCount: 3, + } + + err := db.Create(cluster).Error + Expect(err).ShouldNot(HaveOccurred()) + + mockClusterApi.EXPECT().VerifyClusterUpdatability(createClusterIdMatcher(cluster)).Return(nil).Times(1) + + reply := bm.V2UpdateCluster(ctx, installer.V2UpdateClusterParams{ + ClusterID: clusterID, + ClusterUpdateParams: &models.V2ClusterUpdateParams{ + ControlPlaneCount: swag.Int64(4), + }, + }) + + Expect(reply).To(BeAssignableToTypeOf(installer.NewV2UpdateClusterCreated())) + + var newCluster *common.Cluster + err = db.Where("id = ?", clusterID).Take(&newCluster).Error + Expect(err).ToNot(HaveOccurred()) + + Expect(newCluster.ControlPlaneCount).To(BeEquivalentTo(4)) + }) + + It(fmt.Sprintf("descreasing to 3 control planes with OCP >= %s the value and multi-node", common.MinimumVersionForStretchedControlPlanesCluster), func() { + cluster := &common.Cluster{ + Cluster: models.Cluster{ + ID: &clusterID, + HighAvailabilityMode: swag.String(models.ClusterCreateParamsHighAvailabilityModeFull), + OpenshiftVersion: "4.18", + }, + ControlPlaneCount: 4, + } + + err := db.Create(cluster).Error + Expect(err).ShouldNot(HaveOccurred()) + + mockClusterApi.EXPECT().VerifyClusterUpdatability(createClusterIdMatcher(cluster)).Return(nil).Times(1) + + reply := bm.V2UpdateCluster(ctx, installer.V2UpdateClusterParams{ + ClusterID: clusterID, + ClusterUpdateParams: &models.V2ClusterUpdateParams{ + ControlPlaneCount: swag.Int64(3), + }, + }) + + Expect(reply).To(BeAssignableToTypeOf(installer.NewV2UpdateClusterCreated())) + + var newCluster *common.Cluster + err = db.Where("id = ?", clusterID).Take(&newCluster).Error + Expect(err).ToNot(HaveOccurred()) + + Expect(newCluster.ControlPlaneCount).To(BeEquivalentTo(3)) + }) + }) + + Context("should fail", func() { + It("update to invalid value, stretched clusters not supported", func() { + cluster := &common.Cluster{ + Cluster: models.Cluster{ + ID: &clusterID, + HighAvailabilityMode: swag.String(models.ClusterCreateParamsHighAvailabilityModeFull), + OpenshiftVersion: "4.16", + }, + ControlPlaneCount: 3, + } + + err := db.Create(cluster).Error + Expect(err).ShouldNot(HaveOccurred()) + + reply := bm.V2UpdateCluster(ctx, installer.V2UpdateClusterParams{ + ClusterID: clusterID, + ClusterUpdateParams: &models.V2ClusterUpdateParams{ + ControlPlaneCount: swag.Int64(6), + }, + }) + + verifyApiErrorString(reply, http.StatusBadRequest, "there should be exactly 3 dedicated control plane nodes for high availability mode Full in openshift version older than 4.18") + }) + + It("update to invalid value, stretched clusters supported", func() { + cluster := &common.Cluster{ + Cluster: models.Cluster{ + ID: &clusterID, + HighAvailabilityMode: swag.String(models.ClusterCreateParamsHighAvailabilityModeFull), + OpenshiftVersion: "4.18", + }, + ControlPlaneCount: 3, + } + + err := db.Create(cluster).Error + Expect(err).ShouldNot(HaveOccurred()) + + reply := bm.V2UpdateCluster(ctx, installer.V2UpdateClusterParams{ + ClusterID: clusterID, + ClusterUpdateParams: &models.V2ClusterUpdateParams{ + ControlPlaneCount: swag.Int64(6), + }, + }) + + verifyApiErrorString(reply, http.StatusBadRequest, "there should be 3-5 dedicated control plane nodes for high availability mode Full in openshift version 4.18 or newer") + }) + + It("update amount to != 1 when SNO", func() { + cluster := &common.Cluster{ + Cluster: models.Cluster{ + ID: &clusterID, + HighAvailabilityMode: swag.String(models.ClusterCreateParamsHighAvailabilityModeNone), + OpenshiftVersion: "4.16", + }, + ControlPlaneCount: 1, + } + + err := db.Create(cluster).Error + Expect(err).ShouldNot(HaveOccurred()) + + reply := bm.V2UpdateCluster(ctx, installer.V2UpdateClusterParams{ + ClusterID: clusterID, + ClusterUpdateParams: &models.V2ClusterUpdateParams{ + ControlPlaneCount: swag.Int64(3), + }, + }) + + verifyApiErrorString(reply, http.StatusBadRequest, "single-node clusters must have a single control plane node") + }) + + It(fmt.Sprintf("update amount to != 3 when multi-node, OCP version < %s", common.MinimumVersionForStretchedControlPlanesCluster), func() { + cluster := &common.Cluster{ + Cluster: models.Cluster{ + ID: &clusterID, + HighAvailabilityMode: swag.String(models.ClusterCreateParamsHighAvailabilityModeFull), + OpenshiftVersion: "4.16", + }, + ControlPlaneCount: 3, + } + + err := db.Create(cluster).Error + Expect(err).ShouldNot(HaveOccurred()) + + reply := bm.V2UpdateCluster(ctx, installer.V2UpdateClusterParams{ + ClusterID: clusterID, + ClusterUpdateParams: &models.V2ClusterUpdateParams{ + ControlPlaneCount: swag.Int64(4), + }, + }) + + verifyApiErrorString( + reply, + http.StatusBadRequest, + "there should be exactly 3 dedicated control plane nodes for high availability mode Full in openshift version older than 4.18", + ) + }) + + It(fmt.Sprintf("update amount to != 3 when multi-node, OCP version >= %s", common.MinimumVersionForStretchedControlPlanesCluster), func() { + cluster := &common.Cluster{ + Cluster: models.Cluster{ + ID: &clusterID, + HighAvailabilityMode: swag.String(models.ClusterCreateParamsHighAvailabilityModeFull), + OpenshiftVersion: "4.18", + }, + ControlPlaneCount: 4, + } + + err := db.Create(cluster).Error + Expect(err).ShouldNot(HaveOccurred()) + + reply := bm.V2UpdateCluster(ctx, installer.V2UpdateClusterParams{ + ClusterID: clusterID, + ClusterUpdateParams: &models.V2ClusterUpdateParams{ + ControlPlaneCount: swag.Int64(1), + }, + }) + + verifyApiErrorString( + reply, + http.StatusBadRequest, + "there should be 3-5 dedicated control plane nodes for high availability mode Full in openshift version 4.18 or newer", + ) + }) + }) + }) }) }) @@ -12831,7 +13101,7 @@ var _ = Describe("Install Host test", func() { HostID: hostID, } addHost(hostID, models.HostRoleWorker, models.HostStatusKnown, models.HostKindAddToExistingClusterHost, clusterID, clusterID, getInventoryStr("hostname0", "bootMode", "1.2.3.4/24", "10.11.50.90/16"), db) - mockHostApi.EXPECT().AutoAssignRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).Times(1) + mockHostApi.EXPECT().AutoAssignRole(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).Times(1) mockHostApi.EXPECT().RefreshStatus(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) mockHostApi.EXPECT().Install(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) mockS3Client.EXPECT().Upload(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) @@ -12849,7 +13119,7 @@ var _ = Describe("Install Host test", func() { HostID: hostID, } addHost(hostID, models.HostRoleWorker, models.HostStatusKnown, models.HostKindAddToExistingClusterHost, clusterID, clusterID, getInventoryStr("hostname0", "bootMode", "1.2.3.4/24", "10.11.50.90/16"), db) - mockHostApi.EXPECT().AutoAssignRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).Times(1) + mockHostApi.EXPECT().AutoAssignRole(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).Times(1) mockHostApi.EXPECT().RefreshStatus(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) mockHostApi.EXPECT().Install(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) mockS3Client.EXPECT().Upload(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) @@ -12898,7 +13168,7 @@ var _ = Describe("Install Host test", func() { HostID: hostID, } addHost(hostID, models.HostRoleWorker, models.HostStatusKnown, models.HostKindAddToExistingClusterHost, clusterID, clusterID, getInventoryStr("hostname0", "bootMode", "1.2.3.4/24", "10.11.50.90/16"), db) - mockHostApi.EXPECT().AutoAssignRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).Times(1) + mockHostApi.EXPECT().AutoAssignRole(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).Times(1) mockHostApi.EXPECT().RefreshStatus(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) mockS3Client.EXPECT().Upload(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("some error")).Times(0) mockIgnitionBuilder.EXPECT().FormatSecondDayWorkerIgnitionFile(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("ign failure")).Times(1) @@ -12913,7 +13183,7 @@ var _ = Describe("Install Host test", func() { HostID: hostID, } addHost(hostID, models.HostRoleWorker, models.HostStatusKnown, models.HostKindAddToExistingClusterHost, clusterID, clusterID, getInventoryStr("hostname0", "bootMode", "1.2.3.4/24", "10.11.50.90/16"), db) - mockHostApi.EXPECT().AutoAssignRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).Times(1) + mockHostApi.EXPECT().AutoAssignRole(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).Times(1) mockHostApi.EXPECT().RefreshStatus(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) mockS3Client.EXPECT().Upload(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("some error")).Times(1) mockIgnitionBuilder.EXPECT().FormatSecondDayWorkerIgnitionFile(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(secondDayWorkerIgnition, nil).Times(1) @@ -12961,7 +13231,7 @@ var _ = Describe("InstallSingleDay2Host test", func() { eventstest.WithHostIdMatcher(hostId.String()), eventstest.WithInfraEnvIdMatcher(clusterID.String()), eventstest.WithClusterIdMatcher(clusterID.String()))) - mockHostApi.EXPECT().AutoAssignRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).Times(1) + mockHostApi.EXPECT().AutoAssignRole(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).Times(1) mockHostApi.EXPECT().RefreshStatus(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) mockHostApi.EXPECT().Install(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) mockS3Client.EXPECT().Upload(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) @@ -12974,7 +13244,7 @@ var _ = Describe("InstallSingleDay2Host test", func() { expectedErrMsg := "some-internal-error" hostId := strfmt.UUID(uuid.New().String()) addHost(hostId, models.HostRoleWorker, models.HostStatusKnown, models.HostKindAddToExistingClusterHost, clusterID, clusterID, getInventoryStr("hostname0", "bootMode", "1.2.3.4/24", "10.11.50.90/16"), db) - mockHostApi.EXPECT().AutoAssignRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).Times(1) + mockHostApi.EXPECT().AutoAssignRole(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).Times(1) mockHostApi.EXPECT().RefreshStatus(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) mockHostApi.EXPECT().Install(gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New(expectedErrMsg)).Times(1) mockS3Client.EXPECT().Upload(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) @@ -13026,7 +13296,7 @@ var _ = Describe("Transform day1 cluster to a day2 cluster test", func() { }) }) -var _ = Describe("TestRegisterCluster", func() { +var _ = Describe("RegisterCluster", func() { var ( bm *bareMetalInventory cfg Config @@ -15257,6 +15527,242 @@ var _ = Describe("TestRegisterCluster", func() { }) }) }) + + Context("registering control_plane_count", func() { + Context("should succeed", func() { + BeforeEach(func() { + mockAMSSubscription(ctx) + }) + + Context("using defaults", func() { + It("high_availability mode is set to Full", func() { + mockClusterRegisterSuccessWithVersion(common.X86CPUArchitecture, testutils.ValidOCPVersionForNonStretchedClusters) + + reply := bm.V2RegisterCluster(ctx, installer.V2RegisterClusterParams{ + NewClusterParams: &models.ClusterCreateParams{ + OpenshiftVersion: swag.String(testutils.ValidOCPVersionForNonStretchedClusters), + HighAvailabilityMode: swag.String(models.ClusterCreateParamsHighAvailabilityModeFull), + }, + }) + + Expect(reply).To(BeAssignableToTypeOf(installer.NewV2RegisterClusterCreated())) + actual := reply.(*installer.V2RegisterClusterCreated) + clusterID := actual.Payload.ID + + var dbCluster common.Cluster + db.Where("id = ?", clusterID.String()).Take(&dbCluster) + + Expect(dbCluster.ControlPlaneCount).To(BeEquivalentTo(3)) + }) + + It("high_availability mode is set to None", func() { + mockClusterRegisterSuccessWithVersion(common.X86CPUArchitecture, testutils.ValidOCPVersionForNonStretchedClusters) + + reply := bm.V2RegisterCluster(ctx, installer.V2RegisterClusterParams{ + NewClusterParams: &models.ClusterCreateParams{ + OpenshiftVersion: swag.String(testutils.ValidOCPVersionForNonStretchedClusters), + HighAvailabilityMode: swag.String(models.ClusterCreateParamsHighAvailabilityModeNone), + }, + }) + + Expect(reply).To(BeAssignableToTypeOf(installer.NewV2RegisterClusterCreated())) + actual := reply.(*installer.V2RegisterClusterCreated) + clusterID := actual.Payload.ID + + var dbCluster common.Cluster + db.Where("id = ?", clusterID.String()).Take(&dbCluster) + + Expect(dbCluster.ControlPlaneCount).To(BeEquivalentTo(1)) + }) + + It("control_plane_count is set to 3", func() { + mockClusterRegisterSuccessWithVersion(common.X86CPUArchitecture, testutils.ValidOCPVersionForNonStretchedClusters) + + reply := bm.V2RegisterCluster(ctx, installer.V2RegisterClusterParams{ + NewClusterParams: &models.ClusterCreateParams{ + OpenshiftVersion: swag.String(testutils.ValidOCPVersionForNonStretchedClusters), + ControlPlaneCount: swag.Int64(3), + }, + }) + + Expect(reply).To(BeAssignableToTypeOf(installer.NewV2RegisterClusterCreated())) + actual := reply.(*installer.V2RegisterClusterCreated) + clusterID := actual.Payload.ID + + var dbCluster common.Cluster + db.Where("id = ?", clusterID.String()).Take(&dbCluster) + + Expect(*dbCluster.HighAvailabilityMode).To(BeEquivalentTo(models.ClusterCreateParamsHighAvailabilityModeFull)) + }) + + It("control_plane_count is set to 1", func() { + mockClusterRegisterSuccessWithVersion(common.X86CPUArchitecture, testutils.ValidOCPVersionForNonStretchedClusters) + + reply := bm.V2RegisterCluster(ctx, installer.V2RegisterClusterParams{ + NewClusterParams: &models.ClusterCreateParams{ + OpenshiftVersion: swag.String(testutils.ValidOCPVersionForNonStretchedClusters), + ControlPlaneCount: swag.Int64(1), + }, + }) + + Expect(reply).To(BeAssignableToTypeOf(installer.NewV2RegisterClusterCreated())) + actual := reply.(*installer.V2RegisterClusterCreated) + clusterID := actual.Payload.ID + + var dbCluster common.Cluster + db.Where("id = ?", clusterID.String()).Take(&dbCluster) + + Expect(*dbCluster.HighAvailabilityMode).To(BeEquivalentTo(models.ClusterCreateParamsHighAvailabilityModeNone)) + }) + + It("not set", func() { + mockClusterRegisterSuccessWithVersion(common.X86CPUArchitecture, testutils.ValidOCPVersionForNonStretchedClusters) + + reply := bm.V2RegisterCluster(ctx, installer.V2RegisterClusterParams{ + NewClusterParams: &models.ClusterCreateParams{ + OpenshiftVersion: swag.String(testutils.ValidOCPVersionForNonStretchedClusters), + }, + }) + + Expect(reply).To(BeAssignableToTypeOf(installer.NewV2RegisterClusterCreated())) + actual := reply.(*installer.V2RegisterClusterCreated) + clusterID := actual.Payload.ID + + var dbCluster common.Cluster + db.Where("id = ?", clusterID.String()).Take(&dbCluster) + + Expect(*dbCluster.HighAvailabilityMode).To(BeEquivalentTo(models.ClusterCreateParamsHighAvailabilityModeFull)) + Expect(dbCluster.ControlPlaneCount).To(BeEquivalentTo(3)) + }) + }) + + It(fmt.Sprintf("setting 5 control planes, multi-node with OCP version >= %s", common.MinimumVersionForStretchedControlPlanesCluster), func() { + mockClusterRegisterSuccessWithVersion(common.X86CPUArchitecture, common.MinimumVersionForStretchedControlPlanesCluster) + + reply := bm.V2RegisterCluster(ctx, installer.V2RegisterClusterParams{ + NewClusterParams: &models.ClusterCreateParams{ + OpenshiftVersion: swag.String(common.MinimumVersionForStretchedControlPlanesCluster), + ControlPlaneCount: swag.Int64(5), + HighAvailabilityMode: swag.String(models.ClusterCreateParamsHighAvailabilityModeFull), + }, + }) + + Expect(reply).To(BeAssignableToTypeOf(installer.NewV2RegisterClusterCreated())) + actual := reply.(*installer.V2RegisterClusterCreated) + clusterID := actual.Payload.ID + + var dbCluster common.Cluster + db.Where("id = ?", clusterID.String()).Take(&dbCluster) + + Expect(dbCluster.ControlPlaneCount).To(BeEquivalentTo(5)) + Expect(*dbCluster.HighAvailabilityMode).To(BeEquivalentTo(models.ClusterCreateParamsHighAvailabilityModeFull)) + }) + + It("setting 1 control plane, single-node", func() { + mockClusterRegisterSuccessWithVersion(common.X86CPUArchitecture, testutils.ValidOCPVersionForNonStretchedClusters) + + reply := bm.V2RegisterCluster(ctx, installer.V2RegisterClusterParams{ + NewClusterParams: &models.ClusterCreateParams{ + OpenshiftVersion: swag.String(testutils.ValidOCPVersionForNonStretchedClusters), + ControlPlaneCount: swag.Int64(1), + HighAvailabilityMode: swag.String(models.ClusterCreateParamsHighAvailabilityModeNone), + }, + }) + + Expect(reply).To(BeAssignableToTypeOf(installer.NewV2RegisterClusterCreated())) + actual := reply.(*installer.V2RegisterClusterCreated) + clusterID := actual.Payload.ID + + var dbCluster common.Cluster + db.Where("id = ?", clusterID.String()).Take(&dbCluster) + + Expect(dbCluster.ControlPlaneCount).To(BeEquivalentTo(1)) + Expect(*dbCluster.HighAvailabilityMode).To(BeEquivalentTo(models.ClusterCreateParamsHighAvailabilityModeNone)) + }) + }) + + Context("should fail", func() { + It("setting 6 control planes, multi-node, stretched clusters not supported", func() { + reply := bm.V2RegisterCluster(ctx, installer.V2RegisterClusterParams{ + NewClusterParams: &models.ClusterCreateParams{ + OpenshiftVersion: swag.String(testutils.ValidOCPVersionForNonStretchedClusters), + ControlPlaneCount: swag.Int64(6), + HighAvailabilityMode: swag.String(models.ClusterCreateParamsHighAvailabilityModeFull), + }, + }) + + verifyApiErrorString( + reply, + http.StatusBadRequest, + "there should be exactly 3 dedicated control plane nodes for high availability mode Full in openshift version older than 4.18", + ) + }) + + It("setting 6 control planes, multi-node, stretched clusters supported", func() { + reply := bm.V2RegisterCluster(ctx, installer.V2RegisterClusterParams{ + NewClusterParams: &models.ClusterCreateParams{ + OpenshiftVersion: swag.String(common.MinimumVersionForStretchedControlPlanesCluster), + ControlPlaneCount: swag.Int64(6), + HighAvailabilityMode: swag.String(models.ClusterCreateParamsHighAvailabilityModeFull), + }, + }) + + verifyApiErrorString( + reply, + http.StatusBadRequest, + "there should be 3-5 dedicated control plane nodes for high availability mode Full in openshift version 4.18 or newer", + ) + }) + + It("setting 3 control planes, single-node", func() { + reply := bm.V2RegisterCluster(ctx, installer.V2RegisterClusterParams{ + NewClusterParams: &models.ClusterCreateParams{ + OpenshiftVersion: swag.String(testutils.ValidOCPVersionForNonStretchedClusters), + ControlPlaneCount: swag.Int64(3), + HighAvailabilityMode: swag.String(models.ClusterCreateParamsHighAvailabilityModeNone), + }, + }) + + verifyApiErrorString( + reply, + http.StatusBadRequest, + "single-node clusters must have a single control plane node", + ) + }) + + It("setting 1 control plane, mutli-node", func() { + reply := bm.V2RegisterCluster(ctx, installer.V2RegisterClusterParams{ + NewClusterParams: &models.ClusterCreateParams{ + OpenshiftVersion: swag.String(testutils.ValidOCPVersionForNonStretchedClusters), + ControlPlaneCount: swag.Int64(1), + HighAvailabilityMode: swag.String(models.ClusterCreateParamsHighAvailabilityModeFull), + }, + }) + + verifyApiErrorString( + reply, + http.StatusBadRequest, + "there should be exactly 3 dedicated control plane nodes for high availability mode Full in openshift version older than 4.18", + ) + }) + + It("setting 4 control planes, multi-node, stretched clusters not supported", func() { + reply := bm.V2RegisterCluster(ctx, installer.V2RegisterClusterParams{ + NewClusterParams: &models.ClusterCreateParams{ + OpenshiftVersion: swag.String(testutils.ValidOCPVersionForNonStretchedClusters), + ControlPlaneCount: swag.Int64(4), + HighAvailabilityMode: swag.String(models.ClusterCreateParamsHighAvailabilityModeFull), + }, + }) + + verifyApiErrorString( + reply, + http.StatusBadRequest, + "there should be exactly 3 dedicated control plane nodes for high availability mode Full in openshift version older than 4.18", + ) + }) + }) + }) }) var _ = Describe("AMS subscriptions", func() { @@ -15508,6 +16014,7 @@ var _ = Describe("AMS subscriptions", func() { mockClusterApi.EXPECT().DeleteClusterLogs(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) mockGetInstallConfigSuccess(mockInstallConfigBuilder) mockGenerateInstallConfigSuccess(mockGenerator, mockVersions) + mockRefreshSchedulableMastersForcedTrue(mockClusterApi, 1) mockS3Client.EXPECT().Download(gomock.Any(), gomock.Any()).Return(ignitionReader, int64(0), nil).MinTimes(0) if test.status == "succeed" { mockAccountsMgmt.EXPECT().UpdateSubscriptionOpenshiftClusterID(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) diff --git a/internal/cluster/cluster.go b/internal/cluster/cluster.go index e72ffc8a343b..84469826754e 100644 --- a/internal/cluster/cluster.go +++ b/internal/cluster/cluster.go @@ -49,8 +49,7 @@ import ( ) const ( - DhcpLeaseTimeoutMinutes = 2 - ForceSchedulableMastersMaxHostCount = 5 + DhcpLeaseTimeoutMinutes = 2 ) var S3FileNames = []string{ @@ -122,6 +121,7 @@ type API interface { ResetClusterFiles(ctx context.Context, c *common.Cluster, objectHandler s3wrapper.API) error UpdateLogsProgress(ctx context.Context, c *common.Cluster, progress string) error GetClusterByKubeKey(key types.NamespacedName) (*common.Cluster, error) + GetHostCountByRole(clusterID strfmt.UUID, role models.HostRole, suggested bool) (*int64, error) UpdateAmsSubscriptionID(ctx context.Context, clusterID, amsSubscriptionID strfmt.UUID) *common.ApiErrorResponse GenerateAdditionalManifests(ctx context.Context, cluster *common.Cluster) error CompleteInstallation(ctx context.Context, db *gorm.DB, cluster *common.Cluster, reason string) (*common.Cluster, error) @@ -620,6 +620,11 @@ func (m *Manager) ClusterMonitoring() { if !m.SkipMonitoring(cluster) { monitored += 1 _ = m.autoAssignMachineNetworkCidr(cluster) + if cluster.ID == nil { + log.WithError(err).Error("cluster ID is nil") + continue + } + if err = m.setConnectivityMajorityGroupsForClusterInternal(cluster, m.db); err != nil { log.WithError(err).Error("failed to set majority group for clusters") } @@ -641,6 +646,10 @@ func (m *Manager) ClusterMonitoring() { if m.shouldTriggerLeaseTimeoutEvent(cluster, curMonitorInvokedAt) { m.triggerLeaseTimeoutEvent(ctx, cluster) } + + if err := m.RefreshSchedulableMastersForcedTrue(ctx, *cluster.ID); err != nil { + log.WithError(err).Errorf("failed to refresh cluster with ID '%s' masters schedulability", string(*cluster.ID)) + } } } offset += limit @@ -1713,7 +1722,7 @@ func (m *Manager) RefreshSchedulableMastersForcedTrue(ctx context.Context, clust return err } - newSchedulableMastersForcedTrue := len(cluster.Hosts) < ForceSchedulableMastersMaxHostCount + newSchedulableMastersForcedTrue := common.ShouldMastersBeSchedulable(&cluster.Cluster) if cluster.SchedulableMastersForcedTrue == nil || newSchedulableMastersForcedTrue != *cluster.SchedulableMastersForcedTrue { err = m.updateSchedulableMastersForcedTrue(ctx, clusterID, newSchedulableMastersForcedTrue) } @@ -1820,3 +1829,7 @@ func (m *Manager) UpdateFinalizingStage(ctx context.Context, clusterID strfmt.UU } return nil } + +func (m *Manager) GetHostCountByRole(clusterID strfmt.UUID, role models.HostRole, suggested bool) (*int64, error) { + return common.GetHostCountByRole(m.db, clusterID, role, suggested) +} diff --git a/internal/cluster/cluster_test.go b/internal/cluster/cluster_test.go index 8a36ef8d9467..18f9fb0bc946 100644 --- a/internal/cluster/cluster_test.go +++ b/internal/cluster/cluster_test.go @@ -32,6 +32,7 @@ import ( "github.com/openshift/assisted-service/internal/network" "github.com/openshift/assisted-service/internal/operators" "github.com/openshift/assisted-service/internal/operators/api" + "github.com/openshift/assisted-service/internal/testing" "github.com/openshift/assisted-service/internal/uploader" "github.com/openshift/assisted-service/models" "github.com/openshift/assisted-service/pkg/auth" @@ -195,6 +196,7 @@ var _ = Describe("TestClusterMonitoring", func() { PullSecretSet: true, MonitoredOperators: []*models.MonitoredOperator{&common.TestDefaultConfig.MonitoredOperator}, StatusUpdatedAt: strfmt.DateTime(time.Now()), + OpenshiftVersion: testing.ValidOCPVersionForNonStretchedClusters, }, TriggerMonitorTimestamp: time.Now(), } @@ -208,7 +210,7 @@ var _ = Describe("TestClusterMonitoring", func() { BeforeEach(func() { c = createCluster(&id, "installing", statusInfoInstalling) mockMetric.EXPECT().ClusterInstallationFinished(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() - mockHostAPI.EXPECT().IsValidMasterCandidate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).AnyTimes() + mockHostAPI.EXPECT().IsValidMasterCandidate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).AnyTimes() }) It("installing -> installing", func() { @@ -455,17 +457,18 @@ var _ = Describe("TestClusterMonitoring", func() { c = common.Cluster{ Cluster: models.Cluster{ - ID: &id, - Status: swag.String("insufficient"), - ClusterNetworks: common.TestIPv4Networking.ClusterNetworks, - ServiceNetworks: common.TestIPv4Networking.ServiceNetworks, - MachineNetworks: common.TestIPv4Networking.MachineNetworks, - APIVips: common.TestIPv4Networking.APIVips, - IngressVips: common.TestIPv4Networking.IngressVips, - BaseDNSDomain: "test.com", - PullSecretSet: true, - StatusInfo: swag.String(StatusInfoInsufficient), - NetworkType: swag.String(models.ClusterNetworkTypeOVNKubernetes), + ID: &id, + Status: swag.String("insufficient"), + ClusterNetworks: common.TestIPv4Networking.ClusterNetworks, + ServiceNetworks: common.TestIPv4Networking.ServiceNetworks, + MachineNetworks: common.TestIPv4Networking.MachineNetworks, + APIVips: common.TestIPv4Networking.APIVips, + IngressVips: common.TestIPv4Networking.IngressVips, + BaseDNSDomain: "test.com", + PullSecretSet: true, + StatusInfo: swag.String(StatusInfoInsufficient), + NetworkType: swag.String(models.ClusterNetworkTypeOVNKubernetes), + OpenshiftVersion: testing.ValidOCPVersionForNonStretchedClusters, }, TriggerMonitorTimestamp: time.Now(), } @@ -542,17 +545,18 @@ var _ = Describe("TestClusterMonitoring", func() { BeforeEach(func() { c = common.Cluster{ Cluster: models.Cluster{ - ID: &id, - Status: swag.String(models.ClusterStatusReady), - StatusInfo: swag.String(StatusInfoReady), - ClusterNetworks: common.TestIPv4Networking.ClusterNetworks, - ServiceNetworks: common.TestIPv4Networking.ServiceNetworks, - MachineNetworks: common.TestIPv4Networking.MachineNetworks, - APIVips: common.TestIPv4Networking.APIVips, - IngressVips: common.TestIPv4Networking.IngressVips, - BaseDNSDomain: "test.com", - PullSecretSet: true, - NetworkType: swag.String(models.ClusterNetworkTypeOVNKubernetes), + ID: &id, + Status: swag.String(models.ClusterStatusReady), + StatusInfo: swag.String(StatusInfoReady), + ClusterNetworks: common.TestIPv4Networking.ClusterNetworks, + ServiceNetworks: common.TestIPv4Networking.ServiceNetworks, + MachineNetworks: common.TestIPv4Networking.MachineNetworks, + APIVips: common.TestIPv4Networking.APIVips, + IngressVips: common.TestIPv4Networking.IngressVips, + BaseDNSDomain: "test.com", + PullSecretSet: true, + NetworkType: swag.String(models.ClusterNetworkTypeOVNKubernetes), + OpenshiftVersion: testing.ValidOCPVersionForNonStretchedClusters, }, TriggerMonitorTimestamp: time.Now(), } @@ -643,7 +647,7 @@ var _ = Describe("TestClusterMonitoring", func() { monitorKnownToInsufficient := func(nClusters int) { mockEvents.EXPECT().SendClusterEvent(gomock.Any(), gomock.Any()).AnyTimes() mockHostAPI.EXPECT().IsRequireUserActionReset(gomock.Any()).Return(false).AnyTimes() - mockHostAPI.EXPECT().IsValidMasterCandidate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + mockHostAPI.EXPECT().IsValidMasterCandidate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). Return(true, nil).AnyTimes() for i := 0; i < nClusters; i++ { @@ -708,7 +712,7 @@ var _ = Describe("TestClusterMonitoring", func() { mockEvents.EXPECT().SendClusterEvent(gomock.Any(), eventstest.NewEventMatcher( eventstest.WithNameMatcher(eventgen.ClusterStatusUpdatedEventName))).Times(0) mockHostAPI.EXPECT().IsRequireUserActionReset(gomock.Any()).Return(false).Times(0) - mockHostAPI.EXPECT().IsValidMasterCandidate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).Times(0) + mockHostAPI.EXPECT().IsValidMasterCandidate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).Times(0) }) It("empty log info (no logs expected or arrived)", func() { @@ -1402,7 +1406,7 @@ var _ = Describe("Auto assign machine CIDR", func() { eventstest.WithClusterIdMatcher(c.ID.String()))).AnyTimes() } if len(t.hosts) > 0 { - mockHostAPI.EXPECT().IsValidMasterCandidate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + mockHostAPI.EXPECT().IsValidMasterCandidate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() } if t.userActionResetExpected { mockHostAPI.EXPECT().IsRequireUserActionReset(gomock.Any()).AnyTimes() @@ -2250,16 +2254,17 @@ var _ = Describe("Majority groups", func() { ingressVip := "1.2.3.6" verificationSuccess := models.VipVerificationSucceeded cluster = common.Cluster{Cluster: models.Cluster{ - ID: &id, - Status: swag.String(models.ClusterStatusReady), - ClusterNetworks: common.TestIPv4Networking.ClusterNetworks, - ServiceNetworks: common.TestIPv4Networking.ServiceNetworks, - MachineNetworks: common.TestIPv4Networking.MachineNetworks, - APIVips: []*models.APIVip{{IP: models.IP(apiVip), ClusterID: id, Verification: &verificationSuccess}}, - IngressVips: []*models.IngressVip{{IP: models.IP(ingressVip), ClusterID: id, Verification: &verificationSuccess}}, - BaseDNSDomain: "test.com", - PullSecretSet: true, - NetworkType: swag.String(models.ClusterNetworkTypeOVNKubernetes), + ID: &id, + Status: swag.String(models.ClusterStatusReady), + ClusterNetworks: common.TestIPv4Networking.ClusterNetworks, + ServiceNetworks: common.TestIPv4Networking.ServiceNetworks, + MachineNetworks: common.TestIPv4Networking.MachineNetworks, + APIVips: []*models.APIVip{{IP: models.IP(apiVip), ClusterID: id, Verification: &verificationSuccess}}, + IngressVips: []*models.IngressVip{{IP: models.IP(ingressVip), ClusterID: id, Verification: &verificationSuccess}}, + BaseDNSDomain: "test.com", + PullSecretSet: true, + NetworkType: swag.String(models.ClusterNetworkTypeOVNKubernetes), + OpenshiftVersion: testing.ValidOCPVersionForNonStretchedClusters, }} Expect(db.Create(&cluster).Error).ShouldNot(HaveOccurred()) @@ -2568,16 +2573,17 @@ var _ = Describe("ready_state", func() { apiVip := "1.2.3.5" ingressVip := "1.2.3.6" cluster = common.Cluster{Cluster: models.Cluster{ - ID: &id, - Status: swag.String(models.ClusterStatusReady), - ClusterNetworks: common.TestIPv4Networking.ClusterNetworks, - ServiceNetworks: common.TestIPv4Networking.ServiceNetworks, - MachineNetworks: common.TestIPv4Networking.MachineNetworks, - APIVips: []*models.APIVip{{IP: models.IP(apiVip), ClusterID: id, Verification: common.VipVerificationPtr(models.VipVerificationSucceeded)}}, - IngressVips: []*models.IngressVip{{IP: models.IP(ingressVip), ClusterID: id, Verification: common.VipVerificationPtr(models.VipVerificationSucceeded)}}, - BaseDNSDomain: "test.com", - PullSecretSet: true, - NetworkType: swag.String(models.ClusterNetworkTypeOVNKubernetes), + ID: &id, + Status: swag.String(models.ClusterStatusReady), + ClusterNetworks: common.TestIPv4Networking.ClusterNetworks, + ServiceNetworks: common.TestIPv4Networking.ServiceNetworks, + MachineNetworks: common.TestIPv4Networking.MachineNetworks, + APIVips: []*models.APIVip{{IP: models.IP(apiVip), ClusterID: id, Verification: common.VipVerificationPtr(models.VipVerificationSucceeded)}}, + IngressVips: []*models.IngressVip{{IP: models.IP(ingressVip), ClusterID: id, Verification: common.VipVerificationPtr(models.VipVerificationSucceeded)}}, + BaseDNSDomain: "test.com", + PullSecretSet: true, + NetworkType: swag.String(models.ClusterNetworkTypeOVNKubernetes), + OpenshiftVersion: testing.ValidOCPVersionForNonStretchedClusters, }} Expect(db.Create(&cluster).Error).ShouldNot(HaveOccurred()) addInstallationRequirements(id, db) @@ -2759,7 +2765,7 @@ var _ = Describe("prepare-for-installation refresh status", func() { It("timeout - assisted pod failure", func() { // In the case of assisted pod failure, all of the hosts are likely to be successful // This is detecting the case where assisted pod failure is the only reason that the cluster failed to prepare. - mockHostAPI.EXPECT().IsValidMasterCandidate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).AnyTimes() + mockHostAPI.EXPECT().IsValidMasterCandidate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).AnyTimes() h1ID := strfmt.UUID(uuid.New().String()) h1 := common.Host{ Host: models.Host{ @@ -3827,8 +3833,8 @@ var _ = Describe("Test RefreshSchedulableMastersForcedTrue", func() { It("schedulableMastersForcedTrue should be set to false when MinHostsDisableSchedulableMasters hosts or more are registered with the cluster", func() { cluster := createCluster(swag.Bool(true)) - for hostCount := 0; hostCount < ForceSchedulableMastersMaxHostCount; hostCount++ { - createHost(*cluster.ID, "", db) + for hostCount := 0; hostCount < common.MinimumNumberOfWorkersForNonSchedulableMastersClusterInHaMode; hostCount++ { + createWorkerHost(*cluster.ID, "", db) } err := clusterApi.RefreshSchedulableMastersForcedTrue(ctx, *cluster.ID) @@ -3840,7 +3846,7 @@ var _ = Describe("Test RefreshSchedulableMastersForcedTrue", func() { It("schedulableMastersForcedTrue should be set to false less then MinHostsDisableSchedulableMasters hosts are registered with the cluster", func() { cluster := createCluster(swag.Bool(false)) - for hostCount := 0; hostCount < ForceSchedulableMastersMaxHostCount-1; hostCount++ { + for hostCount := 0; hostCount < common.MinimumNumberOfWorkersForNonSchedulableMastersClusterInHaMode-1; hostCount++ { createHost(*cluster.ID, "", db) } diff --git a/internal/cluster/common.go b/internal/cluster/common.go index 4d0b8c2efe6d..2472f59b67c8 100644 --- a/internal/cluster/common.go +++ b/internal/cluster/common.go @@ -19,10 +19,6 @@ import ( "gorm.io/gorm" ) -const ( - MinMastersNeededForInstallation = 3 -) - const ( StatusInfoReady = "Cluster ready to be installed" StatusInfoInsufficient = "Cluster is not ready for install" diff --git a/internal/cluster/mock_cluster_api.go b/internal/cluster/mock_cluster_api.go index 9d82d945de54..a507dd3ed6c8 100644 --- a/internal/cluster/mock_cluster_api.go +++ b/internal/cluster/mock_cluster_api.go @@ -348,6 +348,21 @@ func (mr *MockAPIMockRecorder) GetClusterByKubeKey(key interface{}) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetClusterByKubeKey", reflect.TypeOf((*MockAPI)(nil).GetClusterByKubeKey), key) } +// GetHostCountByRole mocks base method. +func (m *MockAPI) GetHostCountByRole(clusterID strfmt.UUID, role models.HostRole, suggested bool) (*int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetHostCountByRole", clusterID, role, suggested) + ret0, _ := ret[0].(*int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetHostCountByRole indicates an expected call of GetHostCountByRole. +func (mr *MockAPIMockRecorder) GetHostCountByRole(clusterID, role, suggested interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetHostCountByRole", reflect.TypeOf((*MockAPI)(nil).GetHostCountByRole), clusterID, role, suggested) +} + // GetMasterNodesIds mocks base method. func (m *MockAPI) GetMasterNodesIds(ctx context.Context, c *common.Cluster, db *gorm.DB) ([]*strfmt.UUID, error) { m.ctrl.T.Helper() diff --git a/internal/cluster/progress_test.go b/internal/cluster/progress_test.go index 0e39e5712260..53d13bac2dab 100644 --- a/internal/cluster/progress_test.go +++ b/internal/cluster/progress_test.go @@ -358,7 +358,7 @@ var _ = Describe("Progress bar test", func() { hid1 = strfmt.UUID(uuid.New().String()) none = models.ClusterHighAvailabilityModeNone - mockHostAPI.EXPECT().IsValidMasterCandidate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).AnyTimes() + mockHostAPI.EXPECT().IsValidMasterCandidate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).AnyTimes() mockDnsApi.EXPECT().CreateDNSRecordSets(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mockOperatorApi.EXPECT().ValidateCluster(gomock.Any(), gomock.Any()).Return([]api.ValidationResult{}, nil).AnyTimes() mockMetric.EXPECT().InstallationStarted().AnyTimes() diff --git a/internal/cluster/refresh_status_preprocessor.go b/internal/cluster/refresh_status_preprocessor.go index cbaa949c6d23..57d59d1cb320 100644 --- a/internal/cluster/refresh_status_preprocessor.go +++ b/internal/cluster/refresh_status_preprocessor.go @@ -167,7 +167,7 @@ func newValidations(v *clusterValidator) []validation { }, { id: SufficientMastersCount, - condition: v.sufficientMastersCount, + condition: v.SufficientMastersCount, }, { id: IsDNSDomainDefined, diff --git a/internal/cluster/statemachine.go b/internal/cluster/statemachine.go index 3aa3cbc68347..25e8b5600b1b 100644 --- a/internal/cluster/statemachine.go +++ b/internal/cluster/statemachine.go @@ -107,7 +107,6 @@ func NewClusterStateMachine(th TransitionHandler) stateswitch.StateMachine { var vipsDefinedConditions = stateswitch.And( If(AreIngressVipsDefined), If(AreApiVipsDefined), - If(AreIngressVipsDefined), ) var requiredForInstall = stateswitch.And( @@ -175,7 +174,7 @@ func NewClusterStateMachine(th TransitionHandler) stateswitch.StateMachine { PostTransition: th.PostRefreshCluster(StatusInfoInsufficient), Documentation: stateswitch.TransitionRuleDoc{ Name: "Refresh discovering cluster - detect insufficient", - Description: "In order for this transition to be fired at least one of the validations in isSufficientForInstallNonDhcp must fail. This transition handles the case that one of the required validations that are required in order for the cluster to be in ready state has failed", + Description: "In order for this transition to be fired at least one of the validations in isSufficientForInstallNonDhcp must fail. This transition handles the case that one of the required validations that are required in order for the cluster to be in ready state has failed", }, }) diff --git a/internal/cluster/transition.go b/internal/cluster/transition.go index 437f64c52f41..24056ed7dcde 100644 --- a/internal/cluster/transition.go +++ b/internal/cluster/transition.go @@ -415,7 +415,6 @@ func (th *transitionHandler) IsFinalizing(sw stateswitch.StateSwitch, args state sCluster, ok := sw.(*stateCluster) installedStatus := []string{models.HostStatusInstalled} - // Move to finalizing state when 3 masters and 0 or 2 worker (if workers are given) moved to installed state if ok && th.enoughMastersAndWorkers(sCluster, installedStatus) { th.log.Infof("Cluster %s has at least required number of installed hosts, "+ "cluster is finalizing.", sCluster.cluster.ID) @@ -505,31 +504,36 @@ func (th *transitionHandler) PostUpdateFinalizingAMSConsoleUrl(sw stateswitch.St return nil } +// enoughMastersAndWorkers returns whether the number of master and worker nodes in the specified cluster with the given status +// meets the required criteria. The conditions are as follows: +// - For SNO (Single Node OpenShift), there must be exactly one master node and zero worker nodes. +// - For High Availability cluster, the number of master nodes should match the user's request, and not less than the minimum. The worker node requirement depends on this request: +// If the user requested at least two workers, there must be at least two, indicating non-schedulable masters were intended. +// If the user requested fewer than two workers, any number of workers is acceptable. func (th *transitionHandler) enoughMastersAndWorkers(sCluster *stateCluster, statuses []string) bool { mastersInSomeInstallingStatus, workersInSomeInstallingStatus := HostsInStatus(sCluster.cluster, statuses) - minRequiredMasterNodes := MinMastersNeededForInstallation if swag.StringValue(sCluster.cluster.HighAvailabilityMode) == models.ClusterHighAvailabilityModeNone { - minRequiredMasterNodes = 1 + return mastersInSomeInstallingStatus == common.AllowedNumberOfMasterHostsInNoneHaMode && + workersInSomeInstallingStatus == common.AllowedNumberOfWorkersInNoneHaMode } - numberOfExpectedWorkers := common.NumberOfWorkers(sCluster.cluster) - minWorkersNeededForInstallation := 0 - if numberOfExpectedWorkers > 1 { - minWorkersNeededForInstallation = 2 - } + // hosts roles are known at this stage + masters, workers, _ := common.GetHostsByEachRole(&sCluster.cluster.Cluster, false) + numberOfExpectedMasters := len(masters) - // to be installed cluster need 3 master - // As for the workers, we need at least 2 workers when a cluster with 5 or more hosts is created - // otherwise no minimum workers are required. This is because in the case of 4 or less hosts the - // masters are set as schedulable and the workload can be shared across the available hosts. In the - // case of a 5 nodes cluster, masters are not schedulable so we depend on the workers to run the - // workload. - if mastersInSomeInstallingStatus >= minRequiredMasterNodes && - (numberOfExpectedWorkers == 0 || workersInSomeInstallingStatus >= minWorkersNeededForInstallation) { - return true + // validate masters + if numberOfExpectedMasters < common.MinMasterHostsNeededForInstallationInHaMode || + mastersInSomeInstallingStatus < numberOfExpectedMasters { + return false } - return false + + numberOfExpectedWorkers := len(workers) + + // validate workers + return numberOfExpectedWorkers < common.MinimumNumberOfWorkersForNonSchedulableMastersClusterInHaMode || + numberOfExpectedWorkers >= common.MinimumNumberOfWorkersForNonSchedulableMastersClusterInHaMode && + workersInSomeInstallingStatus >= common.MinimumNumberOfWorkersForNonSchedulableMastersClusterInHaMode } // check if installation reach to timeout diff --git a/internal/cluster/transition_test.go b/internal/cluster/transition_test.go index 877f25f54ee3..1b261aed3f3f 100644 --- a/internal/cluster/transition_test.go +++ b/internal/cluster/transition_test.go @@ -17,7 +17,7 @@ import ( . "github.com/onsi/gomega" "github.com/openshift/assisted-service/internal/common" eventgen "github.com/openshift/assisted-service/internal/common/events" - "github.com/openshift/assisted-service/internal/common/testing" + commontesting "github.com/openshift/assisted-service/internal/common/testing" "github.com/openshift/assisted-service/internal/constants" "github.com/openshift/assisted-service/internal/dns" "github.com/openshift/assisted-service/internal/events" @@ -26,6 +26,7 @@ import ( "github.com/openshift/assisted-service/internal/host" "github.com/openshift/assisted-service/internal/metrics" "github.com/openshift/assisted-service/internal/operators" + "github.com/openshift/assisted-service/internal/testing" "github.com/openshift/assisted-service/internal/uploader" "github.com/openshift/assisted-service/models" "github.com/openshift/assisted-service/pkg/ocm" @@ -55,7 +56,7 @@ var _ = Describe("Transition tests", func() { BeforeEach(func() { db, dbName = common.PrepareTestDB() ctrl = gomock.NewController(GinkgoT()) - eventsHandler = events.New(db, nil, testing.GetDummyNotificationStream(ctrl), logrus.New()) + eventsHandler = events.New(db, nil, commontesting.GetDummyNotificationStream(ctrl), logrus.New()) uploadClient = uploader.NewClient(&uploader.Config{EnableDataCollection: false}, nil, logrus.New(), nil) mockMetric = metrics.NewMockAPI(ctrl) mockS3Api = s3wrapper.NewMockAPI(ctrl) @@ -70,7 +71,7 @@ var _ = Describe("Transition tests", func() { Context("cancel_installation", func() { BeforeEach(func() { - capi = NewManager(getDefaultConfig(), common.GetTestLog(), db, testing.GetDummyNotificationStream(ctrl), eventsHandler, uploadClient, nil, mockMetric, nil, nil, operatorsManager, nil, nil, nil, nil, nil, false) + capi = NewManager(getDefaultConfig(), common.GetTestLog(), db, commontesting.GetDummyNotificationStream(ctrl), eventsHandler, uploadClient, nil, mockMetric, nil, nil, operatorsManager, nil, nil, nil, nil, nil, false) }) It("cancel_installation", func() { @@ -332,7 +333,7 @@ var _ = Describe("Transition tests", func() { } Expect(common.LoadTableFromDB(db, common.MonitoredOperatorsTable).Create(&c).Error).ShouldNot(HaveOccurred()) if t.withWorkers { - for i := 0; i < MinMastersNeededForInstallation; i++ { + for i := 0; i < common.AllowedNumberOfMasterHostsForInstallationInHaModeOfOCP417OrOlder; i++ { createHost(clusterId, models.HostStatusInstalled, db) } for i := 0; i < 2; i++ { @@ -367,7 +368,7 @@ var _ = Describe("Transition tests", func() { mockMetric.EXPECT().ClusterInstallationFinished(gomock.Any(), models.ClusterStatusInstalled, models.ClusterStatusFinalizing, gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()) } - capi = NewManager(getDefaultConfig(), common.GetTestLog(), db, testing.GetDummyNotificationStream(ctrl), eventsHandler, uploadClient, nil, mockMetric, nil, nil, operatorsManager, ocmClient, mockS3Api, nil, nil, nil, false) + capi = NewManager(getDefaultConfig(), common.GetTestLog(), db, commontesting.GetDummyNotificationStream(ctrl), eventsHandler, uploadClient, nil, mockMetric, nil, nil, operatorsManager, ocmClient, mockS3Api, nil, nil, nil, false) // Test clusterAfterRefresh, err := capi.RefreshStatus(ctx, &c, db) @@ -415,7 +416,7 @@ var _ = Describe("Cancel cluster installation", func() { mockMetric = metrics.NewMockAPI(ctrl) operatorsManager := operators.NewManager(common.GetTestLog(), nil, operators.Options{}, nil) uploadClient = uploader.NewClient(&uploader.Config{EnableDataCollection: false}, nil, logrus.New(), nil) - capi = NewManager(getDefaultConfig(), common.GetTestLog(), db, testing.GetDummyNotificationStream(ctrl), mockEventsHandler, uploadClient, nil, mockMetric, nil, nil, operatorsManager, nil, nil, nil, nil, nil, false) + capi = NewManager(getDefaultConfig(), common.GetTestLog(), db, commontesting.GetDummyNotificationStream(ctrl), mockEventsHandler, uploadClient, nil, mockMetric, nil, nil, operatorsManager, nil, nil, nil, nil, nil, false) }) AfterEach(func() { @@ -492,7 +493,7 @@ var _ = Describe("Reset cluster", func() { ctrl = gomock.NewController(GinkgoT()) mockEventsHandler = eventsapi.NewMockHandler(ctrl) operatorsManager := operators.NewManager(common.GetTestLog(), nil, operators.Options{}, nil) - capi = NewManager(getDefaultConfig(), common.GetTestLog(), db, testing.GetDummyNotificationStream(ctrl), mockEventsHandler, nil, nil, nil, nil, nil, operatorsManager, nil, nil, nil, nil, nil, false) + capi = NewManager(getDefaultConfig(), common.GetTestLog(), db, commontesting.GetDummyNotificationStream(ctrl), mockEventsHandler, nil, nil, nil, nil, nil, operatorsManager, nil, nil, nil, nil, nil, false) }) AfterEach(func() { @@ -648,17 +649,17 @@ func makeJsonChecker(expected map[ValidationID]validationCheckResult) *validatio var _ = Describe("Refresh Cluster - No DHCP", func() { var ( - ctx = context.Background() - db *gorm.DB - clusterId, hid1, hid2, hid3, hid4, hid5 strfmt.UUID - cluster common.Cluster - clusterApi *Manager - mockEvents *eventsapi.MockHandler - mockHostAPI *host.MockAPI - mockMetric *metrics.MockAPI - ctrl *gomock.Controller - dbName string - mockS3Api *s3wrapper.MockAPI + ctx = context.Background() + db *gorm.DB + clusterId, hid1, hid2, hid3, hid4, hid5, hid6 strfmt.UUID + cluster common.Cluster + clusterApi *Manager + mockEvents *eventsapi.MockHandler + mockHostAPI *host.MockAPI + mockMetric *metrics.MockAPI + ctrl *gomock.Controller + dbName string + mockS3Api *s3wrapper.MockAPI ) type candidateChecker func() @@ -669,7 +670,7 @@ var _ = Describe("Refresh Cluster - No DHCP", func() { checkMasterCandidates := func(times int) candidateChecker { return func() { - mockHostAPI.EXPECT().IsValidMasterCandidate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).Times(times) + mockHostAPI.EXPECT().IsValidMasterCandidate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).Times(times) } } @@ -682,7 +683,7 @@ var _ = Describe("Refresh Cluster - No DHCP", func() { operatorsManager := operators.NewManager(common.GetTestLog(), nil, operators.Options{}, nil) mockS3Api = s3wrapper.NewMockAPI(ctrl) mockS3Api.EXPECT().DoesObjectExist(gomock.Any(), gomock.Any()).Return(false, nil).AnyTimes() - clusterApi = NewManager(getDefaultConfig(), common.GetTestLog().WithField("pkg", "cluster-monitor"), db, testing.GetDummyNotificationStream(ctrl), + clusterApi = NewManager(getDefaultConfig(), common.GetTestLog().WithField("pkg", "cluster-monitor"), db, commontesting.GetDummyNotificationStream(ctrl), mockEvents, nil, mockHostAPI, mockMetric, nil, nil, operatorsManager, nil, mockS3Api, nil, nil, nil, false) hid1 = strfmt.UUID(uuid.New().String()) @@ -715,6 +716,8 @@ var _ = Describe("Refresh Cluster - No DHCP", func() { validationsChecker *validationsChecker candidateChecker candidateChecker errorExpected bool + openshiftVersion string + controlPlaneCount int64 }{ { name: "pending-for-input to pending-for-input", @@ -727,8 +730,6 @@ var _ = Describe("Refresh Cluster - No DHCP", func() { hosts: []models.Host{ {ID: &hid1, Status: swag.String(models.HostStatusKnown), Role: models.HostRoleMaster}, {ID: &hid2, Status: swag.String(models.HostStatusKnown), Role: models.HostRoleMaster}, - {ID: &hid3, Status: swag.String(models.HostStatusKnown), Role: models.HostRoleMaster}, - {ID: &hid4, Status: swag.String(models.HostStatusKnown), Role: models.HostRoleMaster}, }, statusInfoChecker: makeValueChecker(statusInfoPendingForInput), validationsChecker: makeJsonChecker(map[ValidationID]validationCheckResult{ @@ -743,8 +744,10 @@ var _ = Describe("Refresh Cluster - No DHCP", func() { IsPullSecretSet: {status: ValidationSuccess, messagePattern: "The pull secret is set."}, isNetworkTypeValid: {status: ValidationSuccess, messagePattern: "The cluster has a valid network type"}, SufficientMastersCount: {status: ValidationFailure, - messagePattern: fmt.Sprintf("Clusters must have exactly %d dedicated control plane nodes. Add or remove hosts, or change their roles configurations to meet the requirement.", - common.MinMasterHostsNeededForInstallation)}, + messagePattern: fmt.Sprintf( + "The cluster must have exactly %d dedicated control plane nodes. Add or remove hosts, or change their roles configurations to meet the requirement.", + common.AllowedNumberOfMasterHostsForInstallationInHaModeOfOCP417OrOlder, + )}, }), errorExpected: false, }, @@ -759,8 +762,6 @@ var _ = Describe("Refresh Cluster - No DHCP", func() { hosts: []models.Host{ {ID: &hid1, Status: swag.String(models.HostStatusKnown), Role: models.HostRoleMaster}, {ID: &hid2, Status: swag.String(models.HostStatusKnown), Role: models.HostRoleMaster}, - {ID: &hid3, Status: swag.String(models.HostStatusKnown), Role: models.HostRoleMaster}, - {ID: &hid4, Status: swag.String(models.HostStatusKnown), Role: models.HostRoleMaster}, }, statusInfoChecker: makeValueChecker(statusInfoPendingForInput), validationsChecker: makeJsonChecker(map[ValidationID]validationCheckResult{ @@ -775,8 +776,10 @@ var _ = Describe("Refresh Cluster - No DHCP", func() { IsPullSecretSet: {status: ValidationSuccess, messagePattern: "The pull secret is set."}, isNetworkTypeValid: {status: ValidationSuccess, messagePattern: "The cluster has a valid network type"}, SufficientMastersCount: {status: ValidationFailure, - messagePattern: fmt.Sprintf("Clusters must have exactly %d dedicated control plane nodes. Add or remove hosts, or change their roles configurations to meet the requirement.", - common.MinMasterHostsNeededForInstallation)}, + messagePattern: fmt.Sprintf( + "The cluster must have exactly %d dedicated control plane nodes. Add or remove hosts, or change their roles configurations to meet the requirement.", + common.AllowedNumberOfMasterHostsForInstallationInHaModeOfOCP417OrOlder, + )}, }), errorExpected: false, }, @@ -792,8 +795,6 @@ var _ = Describe("Refresh Cluster - No DHCP", func() { hosts: []models.Host{ {ID: &hid1, Status: swag.String(models.HostStatusKnown), Role: models.HostRoleMaster}, {ID: &hid2, Status: swag.String(models.HostStatusKnown), Role: models.HostRoleMaster}, - {ID: &hid3, Status: swag.String(models.HostStatusKnown), Role: models.HostRoleMaster}, - {ID: &hid4, Status: swag.String(models.HostStatusKnown), Role: models.HostRoleMaster}, }, statusInfoChecker: makeValueChecker(statusInfoPendingForInput), validationsChecker: makeJsonChecker(map[ValidationID]validationCheckResult{ @@ -808,8 +809,10 @@ var _ = Describe("Refresh Cluster - No DHCP", func() { IsPullSecretSet: {status: ValidationSuccess, messagePattern: "The pull secret is set."}, isNetworkTypeValid: {status: ValidationSuccess, messagePattern: "The cluster has a valid network type"}, SufficientMastersCount: {status: ValidationFailure, - messagePattern: fmt.Sprintf("Clusters must have exactly %d dedicated control plane nodes. Add or remove hosts, or change their roles configurations to meet the requirement.", - common.MinMasterHostsNeededForInstallation)}, + messagePattern: fmt.Sprintf( + "The cluster must have exactly %d dedicated control plane nodes. Add or remove hosts, or change their roles configurations to meet the requirement.", + common.AllowedNumberOfMasterHostsForInstallationInHaModeOfOCP417OrOlder, + )}, }), errorExpected: false, }, @@ -825,8 +828,6 @@ var _ = Describe("Refresh Cluster - No DHCP", func() { hosts: []models.Host{ {ID: &hid1, Status: swag.String(models.HostStatusKnown), Role: models.HostRoleMaster}, {ID: &hid2, Status: swag.String(models.HostStatusKnown), Role: models.HostRoleMaster}, - {ID: &hid3, Status: swag.String(models.HostStatusKnown), Role: models.HostRoleMaster}, - {ID: &hid4, Status: swag.String(models.HostStatusKnown), Role: models.HostRoleMaster}, }, statusInfoChecker: makeValueChecker(statusInfoPendingForInput), validationsChecker: makeJsonChecker(map[ValidationID]validationCheckResult{ @@ -841,8 +842,10 @@ var _ = Describe("Refresh Cluster - No DHCP", func() { IsPullSecretSet: {status: ValidationSuccess, messagePattern: "The pull secret is set."}, isNetworkTypeValid: {status: ValidationSuccess, messagePattern: "The cluster has a valid network type"}, SufficientMastersCount: {status: ValidationFailure, - messagePattern: fmt.Sprintf("Clusters must have exactly %d dedicated control plane nodes. Add or remove hosts, or change their roles configurations to meet the requirement.", - common.MinMasterHostsNeededForInstallation)}, + messagePattern: fmt.Sprintf( + "The cluster must have exactly %d dedicated control plane nodes. Add or remove hosts, or change their roles configurations to meet the requirement.", + common.AllowedNumberOfMasterHostsForInstallationInHaModeOfOCP417OrOlder, + )}, }), errorExpected: false, }, @@ -857,8 +860,6 @@ var _ = Describe("Refresh Cluster - No DHCP", func() { hosts: []models.Host{ {ID: &hid1, Status: swag.String(models.HostStatusKnown), Role: models.HostRoleMaster, Inventory: common.GenerateTestDefaultInventory()}, {ID: &hid2, Status: swag.String(models.HostStatusKnown), Role: models.HostRoleMaster}, - {ID: &hid3, Status: swag.String(models.HostStatusKnown), Role: models.HostRoleMaster}, - {ID: &hid4, Status: swag.String(models.HostStatusKnown), Role: models.HostRoleMaster}, }, statusInfoChecker: makeValueChecker(statusInfoPendingForInput), validationsChecker: makeJsonChecker(map[ValidationID]validationCheckResult{ @@ -873,8 +874,10 @@ var _ = Describe("Refresh Cluster - No DHCP", func() { IsPullSecretSet: {status: ValidationSuccess, messagePattern: "The pull secret is set."}, isNetworkTypeValid: {status: ValidationSuccess, messagePattern: "The cluster has a valid network type"}, SufficientMastersCount: {status: ValidationFailure, - messagePattern: fmt.Sprintf("Clusters must have exactly %d dedicated control plane nodes. Add or remove hosts, or change their roles configurations to meet the requirement.", - common.MinMasterHostsNeededForInstallation)}, + messagePattern: fmt.Sprintf( + "The cluster must have exactly %d dedicated control plane nodes. Add or remove hosts, or change their roles configurations to meet the requirement.", + common.AllowedNumberOfMasterHostsForInstallationInHaModeOfOCP417OrOlder, + )}, }), errorExpected: false, }, @@ -911,7 +914,7 @@ var _ = Describe("Refresh Cluster - No DHCP", func() { errorExpected: false, }, { - name: "pending-for-input to pending-for-input with 3 master 2 workers candidates in auto-assign mode", + name: "pending-for-input to pending-for-input with 5 hosts in auto-assign mode", srcState: models.ClusterStatusPendingForInput, dstState: models.ClusterStatusPendingForInput, apiVips: nil, @@ -944,7 +947,7 @@ var _ = Describe("Refresh Cluster - No DHCP", func() { errorExpected: false, }, { - name: "pending-for-input to insufficient - masters > 3", + name: "pending-for-input to insufficient, too much masters - stretched masters cluster not available", srcState: models.ClusterStatusPendingForInput, dstState: models.ClusterStatusInsufficient, machineNetworks: common.TestIPv4Networking.MachineNetworks, @@ -971,13 +974,111 @@ var _ = Describe("Refresh Cluster - No DHCP", func() { IsPullSecretSet: {status: ValidationSuccess, messagePattern: "The pull secret is set"}, isNetworkTypeValid: {status: ValidationSuccess, messagePattern: "The cluster has a valid network type"}, SufficientMastersCount: {status: ValidationFailure, - messagePattern: fmt.Sprintf("Clusters must have exactly %d dedicated control plane nodes. Add or remove hosts, or change their roles configurations to meet the requirement.", - common.MinMasterHostsNeededForInstallation)}, + messagePattern: fmt.Sprintf( + "The cluster must have exactly %d dedicated control plane nodes. Add or remove hosts, or change their roles configurations to meet the requirement.", + common.AllowedNumberOfMasterHostsForInstallationInHaModeOfOCP417OrOlder, + )}, + }), + errorExpected: false, + }, + { + name: "pending-for-input to insufficient, not enough masters - stretched masters cluster not available", + srcState: models.ClusterStatusPendingForInput, + dstState: models.ClusterStatusInsufficient, + machineNetworks: common.TestIPv4Networking.MachineNetworks, + apiVips: common.TestIPv4Networking.APIVips, + ingressVips: common.TestIPv4Networking.IngressVips, + dnsDomain: "test.com", + pullSecretSet: true, + hosts: []models.Host{ + {ID: &hid1, Status: swag.String(models.HostStatusKnown), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, + {ID: &hid2, Status: swag.String(models.HostStatusKnown), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, + }, + statusInfoChecker: makeValueChecker(StatusInfoInsufficient), + validationsChecker: makeJsonChecker(map[ValidationID]validationCheckResult{ + IsMachineCidrDefined: {status: ValidationSuccess, messagePattern: "Machine Network CIDR is defined"}, + IsMachineCidrEqualsToCalculatedCidr: {status: ValidationSuccess, messagePattern: "Cluster Machine CIDR is equivalent to the calculated CIDR"}, + AreApiVipsDefined: {status: ValidationSuccess, messagePattern: "API virtual IPs are defined"}, + AreApiVipsValid: {status: ValidationSuccess, messagePattern: "belongs to the Machine CIDR and is not in use"}, + AreIngressVipsDefined: {status: ValidationSuccess, messagePattern: "Ingress virtual IPs are defined"}, + AreIngressVipsValid: {status: ValidationSuccess, messagePattern: "belongs to the Machine CIDR and is not in use"}, + AllHostsAreReadyToInstall: {status: ValidationSuccess, messagePattern: "All hosts in the cluster are ready to install"}, + IsDNSDomainDefined: {status: ValidationSuccess, messagePattern: "The base domain is defined"}, + IsPullSecretSet: {status: ValidationSuccess, messagePattern: "The pull secret is set"}, + isNetworkTypeValid: {status: ValidationSuccess, messagePattern: "The cluster has a valid network type"}, + SufficientMastersCount: {status: ValidationFailure, + messagePattern: fmt.Sprintf( + "The cluster must have exactly %d dedicated control plane nodes. Add or remove hosts, or change their roles configurations to meet the requirement.", + common.AllowedNumberOfMasterHostsForInstallationInHaModeOfOCP417OrOlder, + )}, }), errorExpected: false, }, { - name: "pending-for-input to insufficient - worker = 1 with auto-assign", + name: "pending-for-input to insufficient, too much masters - stretched masters cluster available", + srcState: models.ClusterStatusPendingForInput, + dstState: models.ClusterStatusInsufficient, + machineNetworks: common.TestIPv4Networking.MachineNetworks, + apiVips: common.TestIPv4Networking.APIVips, + ingressVips: common.TestIPv4Networking.IngressVips, + dnsDomain: "test.com", + pullSecretSet: true, + hosts: []models.Host{ + {ID: &hid1, Status: swag.String(models.HostStatusKnown), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, + {ID: &hid2, Status: swag.String(models.HostStatusKnown), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, + {ID: &hid3, Status: swag.String(models.HostStatusKnown), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, + {ID: &hid4, Status: swag.String(models.HostStatusKnown), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, + {ID: &hid5, Status: swag.String(models.HostStatusKnown), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, + {ID: &hid6, Status: swag.String(models.HostStatusKnown), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}}, + statusInfoChecker: makeValueChecker(StatusInfoInsufficient), + validationsChecker: makeJsonChecker(map[ValidationID]validationCheckResult{ + IsMachineCidrDefined: {status: ValidationSuccess, messagePattern: "Machine Network CIDR is defined"}, + IsMachineCidrEqualsToCalculatedCidr: {status: ValidationSuccess, messagePattern: "Cluster Machine CIDR is equivalent to the calculated CIDR"}, + AreApiVipsDefined: {status: ValidationSuccess, messagePattern: "API virtual IPs are defined"}, + AreApiVipsValid: {status: ValidationSuccess, messagePattern: "belongs to the Machine CIDR and is not in use"}, + AreIngressVipsDefined: {status: ValidationSuccess, messagePattern: "Ingress virtual IPs are defined"}, + AreIngressVipsValid: {status: ValidationSuccess, messagePattern: "belongs to the Machine CIDR and is not in use"}, + AllHostsAreReadyToInstall: {status: ValidationSuccess, messagePattern: "All hosts in the cluster are ready to install"}, + IsDNSDomainDefined: {status: ValidationSuccess, messagePattern: "The base domain is defined"}, + IsPullSecretSet: {status: ValidationSuccess, messagePattern: "The pull secret is set"}, + isNetworkTypeValid: {status: ValidationSuccess, messagePattern: "The cluster has a valid network type"}, + SufficientMastersCount: {status: ValidationFailure, messagePattern: fmt.Sprintf("The cluster must have exactly %d dedicated control plane nodes. Add or remove hosts, or change their roles configurations to meet the requirement.", common.AllowedNumberOfMasterHostsForInstallationInHaModeOfOCP417OrOlder)}, + }), + errorExpected: false, + openshiftVersion: common.MinimumVersionForStretchedControlPlanesCluster, + }, + { + name: "pending-for-input to insufficient, not enough masters - stretched masters cluster available", + srcState: models.ClusterStatusPendingForInput, + dstState: models.ClusterStatusInsufficient, + machineNetworks: common.TestIPv4Networking.MachineNetworks, + apiVips: common.TestIPv4Networking.APIVips, + ingressVips: common.TestIPv4Networking.IngressVips, + dnsDomain: "test.com", + pullSecretSet: true, + hosts: []models.Host{ + {ID: &hid1, Status: swag.String(models.HostStatusKnown), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, + {ID: &hid2, Status: swag.String(models.HostStatusKnown), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, + }, + statusInfoChecker: makeValueChecker(StatusInfoInsufficient), + validationsChecker: makeJsonChecker(map[ValidationID]validationCheckResult{ + IsMachineCidrDefined: {status: ValidationSuccess, messagePattern: "Machine Network CIDR is defined"}, + IsMachineCidrEqualsToCalculatedCidr: {status: ValidationSuccess, messagePattern: "Cluster Machine CIDR is equivalent to the calculated CIDR"}, + AreApiVipsDefined: {status: ValidationSuccess, messagePattern: "API virtual IPs are defined"}, + AreApiVipsValid: {status: ValidationSuccess, messagePattern: "belongs to the Machine CIDR and is not in use"}, + AreIngressVipsDefined: {status: ValidationSuccess, messagePattern: "Ingress virtual IPs are defined"}, + AreIngressVipsValid: {status: ValidationSuccess, messagePattern: "belongs to the Machine CIDR and is not in use"}, + AllHostsAreReadyToInstall: {status: ValidationSuccess, messagePattern: "All hosts in the cluster are ready to install"}, + IsDNSDomainDefined: {status: ValidationSuccess, messagePattern: "The base domain is defined"}, + IsPullSecretSet: {status: ValidationSuccess, messagePattern: "The pull secret is set"}, + isNetworkTypeValid: {status: ValidationSuccess, messagePattern: "The cluster has a valid network type"}, + SufficientMastersCount: {status: ValidationFailure, messagePattern: fmt.Sprintf("The cluster must have exactly %d dedicated control plane nodes. Add or remove hosts, or change their roles configurations to meet the requirement.", common.AllowedNumberOfMasterHostsForInstallationInHaModeOfOCP417OrOlder)}, + }), + errorExpected: false, + openshiftVersion: common.MinimumVersionForStretchedControlPlanesCluster, + }, + { + name: "pending-for-input to ready, sufficient amount of potential masters - stretched masters cluster not available", srcState: models.ClusterStatusPendingForInput, dstState: models.ClusterStatusReady, machineNetworks: common.TestIPv4Networking.MachineNetworks, @@ -986,12 +1087,10 @@ var _ = Describe("Refresh Cluster - No DHCP", func() { dnsDomain: "test.com", pullSecretSet: true, hosts: []models.Host{ - {ID: &hid1, Status: swag.String(models.HostStatusKnown), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleAutoAssign}, - {ID: &hid2, Status: swag.String(models.HostStatusKnown), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleAutoAssign}, - {ID: &hid3, Status: swag.String(models.HostStatusKnown), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleAutoAssign}, - {ID: &hid4, Status: swag.String(models.HostStatusKnown), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleAutoAssign}, + {ID: &hid1, Status: swag.String(models.HostStatusKnown), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, + {ID: &hid2, Status: swag.String(models.HostStatusKnown), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, + {ID: &hid3, Status: swag.String(models.HostStatusKnown), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, }, - candidateChecker: checkMasterCandidates(3), statusInfoChecker: makeValueChecker(StatusInfoReady), validationsChecker: makeJsonChecker(map[ValidationID]validationCheckResult{ IsMachineCidrDefined: {status: ValidationSuccess, messagePattern: "The Machine Network CIDR is defined"}, @@ -1009,7 +1108,41 @@ var _ = Describe("Refresh Cluster - No DHCP", func() { errorExpected: false, }, { - name: "pending-for-input to insufficient - worker = 1", + name: "pending-for-input to ready, sufficient amount of potential masters - stretched masters cluster available", + srcState: models.ClusterStatusPendingForInput, + dstState: models.ClusterStatusReady, + machineNetworks: common.TestIPv4Networking.MachineNetworks, + apiVips: common.TestIPv4Networking.APIVips, + ingressVips: common.TestIPv4Networking.IngressVips, + dnsDomain: "test.com", + pullSecretSet: true, + hosts: []models.Host{ + {ID: &hid1, Status: swag.String(models.HostStatusKnown), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, + {ID: &hid2, Status: swag.String(models.HostStatusKnown), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, + {ID: &hid3, Status: swag.String(models.HostStatusKnown), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, + {ID: &hid4, Status: swag.String(models.HostStatusKnown), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, + {ID: &hid5, Status: swag.String(models.HostStatusKnown), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, + }, + statusInfoChecker: makeValueChecker(StatusInfoReady), + validationsChecker: makeJsonChecker(map[ValidationID]validationCheckResult{ + IsMachineCidrDefined: {status: ValidationSuccess, messagePattern: "The Machine Network CIDR is defined"}, + IsMachineCidrEqualsToCalculatedCidr: {status: ValidationSuccess, messagePattern: "The Cluster Machine CIDR is equivalent to the calculated CIDR"}, + AreApiVipsDefined: {status: ValidationSuccess, messagePattern: "API virtual IPs are defined"}, + AreApiVipsValid: {status: ValidationSuccess, messagePattern: "belongs to the Machine CIDR and is not in use"}, + AreIngressVipsDefined: {status: ValidationSuccess, messagePattern: "Ingress virtual IPs are defined"}, + AreIngressVipsValid: {status: ValidationSuccess, messagePattern: "belongs to the Machine CIDR and is not in use"}, + AllHostsAreReadyToInstall: {status: ValidationSuccess, messagePattern: "All hosts in the cluster are ready to install"}, + IsDNSDomainDefined: {status: ValidationSuccess, messagePattern: "The base domain is defined"}, + IsPullSecretSet: {status: ValidationSuccess, messagePattern: "The pull secret is set"}, + isNetworkTypeValid: {status: ValidationSuccess, messagePattern: "The cluster has a valid network type"}, + SufficientMastersCount: {status: ValidationSuccess, messagePattern: "The cluster has the exact amount of dedicated control plane nodes."}, + }), + errorExpected: false, + openshiftVersion: common.MinimumVersionForStretchedControlPlanesCluster, + controlPlaneCount: 5, + }, + { + name: "pending-for-input to ready, sufficient amount of masters - 1 worker", srcState: models.ClusterStatusPendingForInput, dstState: models.ClusterStatusReady, machineNetworks: common.TestIPv4Networking.MachineNetworks, @@ -1350,20 +1483,27 @@ var _ = Describe("Refresh Cluster - No DHCP", func() { It(t.name, func() { cluster = common.Cluster{ Cluster: models.Cluster{ - APIVips: t.apiVips, - ID: &clusterId, - IngressVips: t.ingressVips, - MachineNetworks: t.machineNetworks, - Status: &t.srcState, - StatusInfo: &t.srcStatusInfo, - BaseDNSDomain: t.dnsDomain, - PullSecretSet: t.pullSecretSet, - ClusterNetworks: common.TestIPv4Networking.ClusterNetworks, - ServiceNetworks: common.TestIPv4Networking.ServiceNetworks, - NetworkType: swag.String(models.ClusterNetworkTypeOVNKubernetes), - StatusUpdatedAt: strfmt.DateTime(time.Now()), + APIVips: t.apiVips, + ID: &clusterId, + IngressVips: t.ingressVips, + MachineNetworks: t.machineNetworks, + Status: &t.srcState, + StatusInfo: &t.srcStatusInfo, + BaseDNSDomain: t.dnsDomain, + PullSecretSet: t.pullSecretSet, + ClusterNetworks: common.TestIPv4Networking.ClusterNetworks, + ServiceNetworks: common.TestIPv4Networking.ServiceNetworks, + NetworkType: swag.String(models.ClusterNetworkTypeOVNKubernetes), + StatusUpdatedAt: strfmt.DateTime(time.Now()), + OpenshiftVersion: t.openshiftVersion, }, + ControlPlaneCount: t.controlPlaneCount, } + + if cluster.Cluster.OpenshiftVersion == "" { + cluster.Cluster.OpenshiftVersion = testing.ValidOCPVersionForNonStretchedClusters + } + Expect(db.Create(&cluster).Error).ShouldNot(HaveOccurred()) for i := range t.hosts { t.hosts[i].InfraEnvID = clusterId @@ -1433,7 +1573,7 @@ var _ = Describe("Refresh Cluster - Same networks", func() { mockS3Api = s3wrapper.NewMockAPI(ctrl) mockS3Api.EXPECT().DoesObjectExist(gomock.Any(), gomock.Any()).Return(false, nil).AnyTimes() - clusterApi = NewManager(getDefaultConfig(), common.GetTestLog().WithField("pkg", "cluster-monitor"), db, testing.GetDummyNotificationStream(ctrl), + clusterApi = NewManager(getDefaultConfig(), common.GetTestLog().WithField("pkg", "cluster-monitor"), db, commontesting.GetDummyNotificationStream(ctrl), mockEvents, nil, mockHostAPI, mockMetric, nil, nil, operatorsManager, nil, mockS3Api, nil, nil, nil, false) hid1 = strfmt.UUID(uuid.New().String()) @@ -1620,18 +1760,19 @@ var _ = Describe("Refresh Cluster - Same networks", func() { It(t.name, func() { cluster = common.Cluster{ Cluster: models.Cluster{ - APIVips: t.apiVips, - ID: &clusterId, - IngressVips: t.ingressVips, - MachineNetworks: t.machineNetworks, - Status: &t.srcState, - StatusInfo: &t.srcStatusInfo, - BaseDNSDomain: t.dnsDomain, - PullSecretSet: t.pullSecretSet, - ClusterNetworks: t.clusterNetworks, - ServiceNetworks: t.serviceNetworks, - NetworkType: swag.String(models.ClusterNetworkTypeOVNKubernetes), - StatusUpdatedAt: strfmt.DateTime(time.Now()), + APIVips: t.apiVips, + ID: &clusterId, + IngressVips: t.ingressVips, + MachineNetworks: t.machineNetworks, + Status: &t.srcState, + StatusInfo: &t.srcStatusInfo, + BaseDNSDomain: t.dnsDomain, + PullSecretSet: t.pullSecretSet, + ClusterNetworks: t.clusterNetworks, + ServiceNetworks: t.serviceNetworks, + NetworkType: swag.String(models.ClusterNetworkTypeOVNKubernetes), + StatusUpdatedAt: strfmt.DateTime(time.Now()), + OpenshiftVersion: testing.ValidOCPVersionForNonStretchedClusters, }, } Expect(db.Create(&cluster).Error).ShouldNot(HaveOccurred()) @@ -1698,10 +1839,10 @@ var _ = Describe("RefreshCluster - preparing for install", func() { mockMetric = metrics.NewMockAPI(ctrl) operatorsManager := operators.NewManager(common.GetTestLog(), nil, operators.Options{}, nil) dnsApi := dns.NewDNSHandler(nil, common.GetTestLog()) - clusterApi = NewManager(getDefaultConfig(), common.GetTestLog().WithField("pkg", "cluster-monitor"), db, testing.GetDummyNotificationStream(ctrl), + clusterApi = NewManager(getDefaultConfig(), common.GetTestLog().WithField("pkg", "cluster-monitor"), db, commontesting.GetDummyNotificationStream(ctrl), mockEvents, nil, mockHostAPI, mockMetric, nil, nil, operatorsManager, nil, nil, dnsApi, nil, nil, false) - mockHostAPI.EXPECT().IsValidMasterCandidate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).AnyTimes() + mockHostAPI.EXPECT().IsValidMasterCandidate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).AnyTimes() hid1 = strfmt.UUID(uuid.New().String()) hid2 = strfmt.UUID(uuid.New().String()) hid3 = strfmt.UUID(uuid.New().String()) @@ -1875,6 +2016,7 @@ var _ = Describe("RefreshCluster - preparing for install", func() { Status: t.installationStatus, Reason: "", }, + OpenshiftVersion: testing.ValidOCPVersionForNonStretchedClusters, }, } Expect(db.Create(&cluster).Error).ShouldNot(HaveOccurred()) @@ -1932,7 +2074,7 @@ var _ = Describe("Refresh Cluster - Advanced networking validations", func() { mockHostAPI = host.NewMockAPI(ctrl) mockMetric = metrics.NewMockAPI(ctrl) operatorsManager := operators.NewManager(common.GetTestLog(), nil, operators.Options{}, nil) - clusterApi = NewManager(getDefaultConfig(), common.GetTestLog().WithField("pkg", "cluster-monitor"), db, testing.GetDummyNotificationStream(ctrl), + clusterApi = NewManager(getDefaultConfig(), common.GetTestLog().WithField("pkg", "cluster-monitor"), db, commontesting.GetDummyNotificationStream(ctrl), mockEvents, nil, mockHostAPI, mockMetric, nil, nil, operatorsManager, nil, nil, nil, nil, nil, false) hid1 = strfmt.UUID(uuid.New().String()) @@ -2419,6 +2561,7 @@ var _ = Describe("Refresh Cluster - Advanced networking validations", func() { UserManagedNetworking: &t.userManagedNetworking, NetworkType: &t.networkType, VipDhcpAllocation: &t.vipDhcpAllocation, + OpenshiftVersion: testing.ValidOCPVersionForNonStretchedClusters, }, } if t.sno { @@ -2582,7 +2725,7 @@ var _ = Describe("Refresh Cluster - Advanced networking validations", func() { userManagedNetworking: true, }, { - name: "pending-for-input to ready user-managed-networking testing_now", + name: "pending-for-input to ready user-managed-networking commontesting_now", srcState: models.ClusterStatusPendingForInput, dstState: models.ClusterStatusReady, clusterNetworks: common.TestIPv6Networking.ClusterNetworks, @@ -2886,6 +3029,7 @@ var _ = Describe("Refresh Cluster - Advanced networking validations", func() { BaseDNSDomain: "test.com", UserManagedNetworking: &t.userManagedNetworking, NetworkType: &t.networkType, + OpenshiftVersion: testing.ValidOCPVersionForNonStretchedClusters, }, } Expect(db.Create(&cluster).Error).ShouldNot(HaveOccurred()) @@ -2926,17 +3070,17 @@ var _ = Describe("Refresh Cluster - Advanced networking validations", func() { var _ = Describe("Refresh Cluster - With DHCP", func() { var ( - ctx = context.Background() - db *gorm.DB - clusterId, hid1, hid2, hid3, hid4, hid5 strfmt.UUID - cluster common.Cluster - clusterApi *Manager - mockEvents *eventsapi.MockHandler - mockHostAPI *host.MockAPI - mockMetric *metrics.MockAPI - ctrl *gomock.Controller - dbName string - mockS3Api *s3wrapper.MockAPI + ctx = context.Background() + db *gorm.DB + clusterId, hid1, hid2, hid3, hid4, hid5, hid6 strfmt.UUID + cluster common.Cluster + clusterApi *Manager + mockEvents *eventsapi.MockHandler + mockHostAPI *host.MockAPI + mockMetric *metrics.MockAPI + ctrl *gomock.Controller + dbName string + mockS3Api *s3wrapper.MockAPI ) mockHostAPIIsRequireUserActionResetFalse := func() { @@ -2952,7 +3096,7 @@ var _ = Describe("Refresh Cluster - With DHCP", func() { operatorsManager := operators.NewManager(common.GetTestLog(), nil, operators.Options{}, nil) mockS3Api = s3wrapper.NewMockAPI(ctrl) mockS3Api.EXPECT().DoesObjectExist(gomock.Any(), gomock.Any()).Return(false, nil).AnyTimes() - clusterApi = NewManager(getDefaultConfig(), common.GetTestLog().WithField("pkg", "cluster-monitor"), db, testing.GetDummyNotificationStream(ctrl), + clusterApi = NewManager(getDefaultConfig(), common.GetTestLog().WithField("pkg", "cluster-monitor"), db, commontesting.GetDummyNotificationStream(ctrl), mockEvents, nil, mockHostAPI, mockMetric, nil, nil, operatorsManager, nil, mockS3Api, nil, nil, nil, false) hid1 = strfmt.UUID(uuid.New().String()) @@ -2960,6 +3104,7 @@ var _ = Describe("Refresh Cluster - With DHCP", func() { hid3 = strfmt.UUID(uuid.New().String()) hid4 = strfmt.UUID(uuid.New().String()) hid5 = strfmt.UUID(uuid.New().String()) + hid6 = strfmt.UUID(uuid.New().String()) clusterId = strfmt.UUID(uuid.New().String()) }) @@ -3006,6 +3151,7 @@ var _ = Describe("Refresh Cluster - With DHCP", func() { setMachineCidrUpdatedAt bool vipDhcpAllocation bool errorExpected bool + openshiftVersion string }{ { name: "pending-for-input to pending-for-input", @@ -3021,8 +3167,6 @@ var _ = Describe("Refresh Cluster - With DHCP", func() { hosts: []models.Host{ {ID: &hid1, Status: swag.String(models.HostStatusKnown), Role: models.HostRoleMaster}, {ID: &hid2, Status: swag.String(models.HostStatusKnown), Role: models.HostRoleMaster}, - {ID: &hid3, Status: swag.String(models.HostStatusKnown), Role: models.HostRoleMaster}, - {ID: &hid4, Status: swag.String(models.HostStatusKnown), Role: models.HostRoleMaster}, }, statusInfoChecker: makeValueChecker(statusInfoPendingForInput), validationsChecker: makeJsonChecker(map[ValidationID]validationCheckResult{ @@ -3036,13 +3180,15 @@ var _ = Describe("Refresh Cluster - With DHCP", func() { IsDNSDomainDefined: {status: ValidationSuccess, messagePattern: "The base domain is defined"}, IsPullSecretSet: {status: ValidationSuccess, messagePattern: "The pull secret is set."}, SufficientMastersCount: {status: ValidationFailure, - messagePattern: fmt.Sprintf("Clusters must have exactly %d dedicated control plane nodes. Add or remove hosts, or change their roles configurations to meet the requirement.", - common.MinMasterHostsNeededForInstallation)}, + messagePattern: fmt.Sprintf( + "The cluster must have exactly %d dedicated control plane nodes. Add or remove hosts, or change their roles configurations to meet the requirement.", + common.AllowedNumberOfMasterHostsForInstallationInHaModeOfOCP417OrOlder, + )}, }), errorExpected: false, }, { - name: "pending-for-input to insufficient - masters > 3", + name: "pending-for-input to insufficient, too much masters - stretched masters cluster not available", srcState: models.ClusterStatusPendingForInput, dstState: models.ClusterStatusInsufficient, machineNetworks: common.TestIPv4Networking.MachineNetworks, @@ -3068,11 +3214,107 @@ var _ = Describe("Refresh Cluster - With DHCP", func() { IsDNSDomainDefined: {status: ValidationSuccess, messagePattern: "The base domain is defined"}, IsPullSecretSet: {status: ValidationSuccess, messagePattern: "The pull secret is set."}, SufficientMastersCount: {status: ValidationFailure, - messagePattern: fmt.Sprintf("Clusters must have exactly %d dedicated control plane nodes. Add or remove hosts, or change their roles configurations to meet the requirement.", - common.MinMasterHostsNeededForInstallation)}, + messagePattern: fmt.Sprintf( + "The cluster must have exactly %d dedicated control plane nodes. Add or remove hosts, or change their roles configurations to meet the requirement.", + common.AllowedNumberOfMasterHostsForInstallationInHaModeOfOCP417OrOlder, + )}, + }), + errorExpected: false, + }, + { + name: "pending-for-input to insufficient, not enough masters - stretched masters cluster not available", + srcState: models.ClusterStatusPendingForInput, + dstState: models.ClusterStatusInsufficient, + machineNetworks: common.TestIPv4Networking.MachineNetworks, + apiVips: common.TestIPv4Networking.APIVips, + ingressVips: common.TestIPv4Networking.IngressVips, + dnsDomain: "test.com", + pullSecretSet: true, + hosts: []models.Host{ + {ID: &hid1, Status: swag.String(models.HostStatusKnown), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, + {ID: &hid2, Status: swag.String(models.HostStatusKnown), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, + }, + statusInfoChecker: makeValueChecker(StatusInfoInsufficient), + validationsChecker: makeJsonChecker(map[ValidationID]validationCheckResult{ + IsMachineCidrDefined: {status: ValidationSuccess, messagePattern: "Machine Network CIDR is defined"}, + IsMachineCidrEqualsToCalculatedCidr: {status: ValidationSuccess, messagePattern: "Cluster Machine CIDR is equivalent to the calculated CIDR"}, + AreApiVipsDefined: {status: ValidationSuccess, messagePattern: "API virtual IPs are defined"}, + AreApiVipsValid: {status: ValidationSuccess, messagePattern: "belongs to the Machine CIDR and is not in use."}, + AreIngressVipsDefined: {status: ValidationSuccess, messagePattern: "Ingress virtual IPs are defined"}, + AreIngressVipsValid: {status: ValidationSuccess, messagePattern: "belongs to the Machine CIDR and is not in use."}, + AllHostsAreReadyToInstall: {status: ValidationSuccess, messagePattern: "All hosts in the cluster are ready to install"}, + IsDNSDomainDefined: {status: ValidationSuccess, messagePattern: "The base domain is defined"}, + IsPullSecretSet: {status: ValidationSuccess, messagePattern: "The pull secret is set."}, + SufficientMastersCount: {status: ValidationFailure, + messagePattern: fmt.Sprintf( + "The cluster must have exactly %d dedicated control plane nodes. Add or remove hosts, or change their roles configurations to meet the requirement.", + common.AllowedNumberOfMasterHostsForInstallationInHaModeOfOCP417OrOlder, + )}, }), errorExpected: false, }, + { + name: "pending-for-input to insufficient, too much masters - stretched masters cluster available", + srcState: models.ClusterStatusPendingForInput, + dstState: models.ClusterStatusInsufficient, + machineNetworks: common.TestIPv4Networking.MachineNetworks, + apiVips: common.TestIPv4Networking.APIVips, + ingressVips: common.TestIPv4Networking.IngressVips, + dnsDomain: "test.com", + pullSecretSet: true, + hosts: []models.Host{ + {ID: &hid1, Status: swag.String(models.HostStatusKnown), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, + {ID: &hid2, Status: swag.String(models.HostStatusKnown), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, + {ID: &hid3, Status: swag.String(models.HostStatusKnown), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, + {ID: &hid4, Status: swag.String(models.HostStatusKnown), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, + {ID: &hid5, Status: swag.String(models.HostStatusKnown), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, + {ID: &hid6, Status: swag.String(models.HostStatusKnown), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, + }, + statusInfoChecker: makeValueChecker(StatusInfoInsufficient), + validationsChecker: makeJsonChecker(map[ValidationID]validationCheckResult{ + IsMachineCidrDefined: {status: ValidationSuccess, messagePattern: "Machine Network CIDR is defined"}, + IsMachineCidrEqualsToCalculatedCidr: {status: ValidationSuccess, messagePattern: "Cluster Machine CIDR is equivalent to the calculated CIDR"}, + AreApiVipsDefined: {status: ValidationSuccess, messagePattern: "API virtual IPs are defined"}, + AreApiVipsValid: {status: ValidationSuccess, messagePattern: "belongs to the Machine CIDR and is not in use."}, + AreIngressVipsDefined: {status: ValidationSuccess, messagePattern: "Ingress virtual IPs are defined"}, + AreIngressVipsValid: {status: ValidationSuccess, messagePattern: "belongs to the Machine CIDR and is not in use."}, + AllHostsAreReadyToInstall: {status: ValidationSuccess, messagePattern: "All hosts in the cluster are ready to install"}, + IsDNSDomainDefined: {status: ValidationSuccess, messagePattern: "The base domain is defined"}, + IsPullSecretSet: {status: ValidationSuccess, messagePattern: "The pull secret is set."}, + SufficientMastersCount: {status: ValidationFailure, messagePattern: fmt.Sprintf("The cluster must have exactly %d dedicated control plane nodes. Add or remove hosts, or change their roles configurations to meet the requirement.", common.AllowedNumberOfMasterHostsForInstallationInHaModeOfOCP417OrOlder)}, + }), + errorExpected: false, + openshiftVersion: common.MinimumVersionForStretchedControlPlanesCluster, + }, + { + name: "pending-for-input to insufficient, not enough masters - stretched masters cluster available", + srcState: models.ClusterStatusPendingForInput, + dstState: models.ClusterStatusInsufficient, + machineNetworks: common.TestIPv4Networking.MachineNetworks, + apiVips: common.TestIPv4Networking.APIVips, + ingressVips: common.TestIPv4Networking.IngressVips, + dnsDomain: "test.com", + pullSecretSet: true, + hosts: []models.Host{ + {ID: &hid1, Status: swag.String(models.HostStatusKnown), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, + {ID: &hid2, Status: swag.String(models.HostStatusKnown), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, + }, + statusInfoChecker: makeValueChecker(StatusInfoInsufficient), + validationsChecker: makeJsonChecker(map[ValidationID]validationCheckResult{ + IsMachineCidrDefined: {status: ValidationSuccess, messagePattern: "Machine Network CIDR is defined"}, + IsMachineCidrEqualsToCalculatedCidr: {status: ValidationSuccess, messagePattern: "Cluster Machine CIDR is equivalent to the calculated CIDR"}, + AreApiVipsDefined: {status: ValidationSuccess, messagePattern: "API virtual IPs are defined"}, + AreApiVipsValid: {status: ValidationSuccess, messagePattern: "belongs to the Machine CIDR and is not in use."}, + AreIngressVipsDefined: {status: ValidationSuccess, messagePattern: "Ingress virtual IPs are defined"}, + AreIngressVipsValid: {status: ValidationSuccess, messagePattern: "belongs to the Machine CIDR and is not in use."}, + AllHostsAreReadyToInstall: {status: ValidationSuccess, messagePattern: "All hosts in the cluster are ready to install"}, + IsDNSDomainDefined: {status: ValidationSuccess, messagePattern: "The base domain is defined"}, + IsPullSecretSet: {status: ValidationSuccess, messagePattern: "The pull secret is set."}, + SufficientMastersCount: {status: ValidationFailure, messagePattern: fmt.Sprintf("The cluster must have exactly %d dedicated control plane nodes. Add or remove hosts, or change their roles configurations to meet the requirement.", common.AllowedNumberOfMasterHostsForInstallationInHaModeOfOCP417OrOlder)}, + }), + errorExpected: false, + openshiftVersion: common.MinimumVersionForStretchedControlPlanesCluster, + }, { name: "pending-for-input to insufficient - not all hosts are ready to install - not enough workers", srcState: models.ClusterStatusPendingForInput, @@ -3490,8 +3732,14 @@ var _ = Describe("Refresh Cluster - With DHCP", func() { ServiceNetworks: common.TestIPv4Networking.ServiceNetworks, NetworkType: swag.String(models.ClusterNetworkTypeOVNKubernetes), StatusUpdatedAt: strfmt.DateTime(time.Now()), + OpenshiftVersion: t.openshiftVersion, }, } + + if cluster.Cluster.OpenshiftVersion == "" { + cluster.Cluster.OpenshiftVersion = testing.ValidOCPVersionForNonStretchedClusters + } + if t.setMachineCidrUpdatedAt { cluster.MachineNetworkCidrUpdatedAt = time.Now() } else { @@ -3565,7 +3813,7 @@ var _ = Describe("Refresh Cluster - Installing Cases", func() { mockMetric = metrics.NewMockAPI(ctrl) mockS3Api = s3wrapper.NewMockAPI(ctrl) operatorsManager = operators.NewManager(common.GetTestLog(), nil, operators.Options{}, nil) - clusterApi = NewManager(getDefaultConfig(), common.GetTestLog().WithField("pkg", "cluster-monitor"), db, testing.GetDummyNotificationStream(ctrl), + clusterApi = NewManager(getDefaultConfig(), common.GetTestLog().WithField("pkg", "cluster-monitor"), db, commontesting.GetDummyNotificationStream(ctrl), mockEvents, nil, mockHostAPI, mockMetric, nil, nil, operatorsManager, nil, mockS3Api, nil, nil, nil, false) hid1 = strfmt.UUID(uuid.New().String()) @@ -3597,9 +3845,10 @@ var _ = Describe("Refresh Cluster - Installing Cases", func() { installationTimeout bool vipDhcpAllocation bool operators []*models.MonitoredOperator + openshiftVersion string }{ { - name: "installing to installing", + name: "installing to installing - non stretched cluster", srcState: models.ClusterStatusInstalling, srcStatusInfo: statusInfoInstalling, dstState: models.ClusterStatusInstalling, @@ -3612,6 +3861,22 @@ var _ = Describe("Refresh Cluster - Installing Cases", func() { }, statusInfoChecker: makeValueChecker(statusInfoInstalling), }, + { + name: "installing to installing - stretched cluster", + srcState: models.ClusterStatusInstalling, + srcStatusInfo: statusInfoInstalling, + dstState: models.ClusterStatusInstalling, + hosts: []models.Host{ + {ID: &hid1, Status: swag.String(models.ClusterStatusInstalling), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, + {ID: &hid2, Status: swag.String(models.ClusterStatusInstalling), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, + {ID: &hid3, Status: swag.String(models.ClusterStatusInstalling), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, + {ID: &hid4, Status: swag.String(models.ClusterStatusInstalling), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, + {ID: &hid5, Status: swag.String(models.ClusterStatusInstalling), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleWorker}, + {ID: &hid6, Status: swag.String(models.ClusterStatusInstalling), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleWorker}, + }, + statusInfoChecker: makeValueChecker(statusInfoInstalling), + openshiftVersion: common.MinimumVersionForStretchedControlPlanesCluster, + }, { name: "installing to installing-pending-user-action", srcState: models.ClusterStatusInstalling, @@ -3771,7 +4036,7 @@ var _ = Describe("Refresh Cluster - Installing Cases", func() { statusInfoChecker: makeValueChecker(statusInfoFinalizing), }, { - name: "installing to finalizing", + name: "installing to finalizing - non stretched cluster", srcState: models.ClusterStatusInstalling, srcStatusInfo: statusInfoInstalling, dstState: models.ClusterStatusFinalizing, @@ -3784,6 +4049,21 @@ var _ = Describe("Refresh Cluster - Installing Cases", func() { }, statusInfoChecker: makeValueChecker(statusInfoFinalizing), }, + { + name: "installing to finalizing - stretched cluster", + srcState: models.ClusterStatusInstalling, + srcStatusInfo: statusInfoInstalling, + dstState: models.ClusterStatusFinalizing, + hosts: []models.Host{ + {ID: &hid1, Status: swag.String(models.HostStatusInstalled), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, + {ID: &hid2, Status: swag.String(models.HostStatusInstalled), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, + {ID: &hid3, Status: swag.String(models.HostStatusInstalled), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleMaster}, + {ID: &hid4, Status: swag.String(models.HostStatusInstalled), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleWorker}, + {ID: &hid5, Status: swag.String(models.HostStatusInstalled), Inventory: common.GenerateTestDefaultInventory(), Role: models.HostRoleWorker}, + }, + statusInfoChecker: makeValueChecker(statusInfoFinalizing), + openshiftVersion: common.MinimumVersionForStretchedControlPlanesCluster, + }, { name: "installing to error - failing master", srcState: models.ClusterStatusInstalling, @@ -3894,12 +4174,16 @@ var _ = Describe("Refresh Cluster - Installing Cases", func() { PullSecretSet: t.pullSecretSet, MonitoredOperators: t.operators, StatusUpdatedAt: strfmt.DateTime(time.Now()), + OpenshiftVersion: testing.ValidOCPVersionForNonStretchedClusters, }, } + if t.openshiftVersion != "" { + cluster.Cluster.OpenshiftVersion = t.openshiftVersion + } if t.withOCMClient { mockAccountsMgmt = ocm.NewMockOCMAccountsMgmt(ctrl) ocmClient := &ocm.Client{AccountsMgmt: mockAccountsMgmt, Config: &ocm.Config{}} - clusterApi = NewManager(getDefaultConfig(), common.GetTestLog().WithField("pkg", "cluster-monitor"), db, testing.GetDummyNotificationStream(ctrl), + clusterApi = NewManager(getDefaultConfig(), common.GetTestLog().WithField("pkg", "cluster-monitor"), db, commontesting.GetDummyNotificationStream(ctrl), mockEvents, nil, mockHostAPI, mockMetric, nil, nil, operatorsManager, ocmClient, mockS3Api, nil, nil, nil, false) if !t.requiresAMSUpdate { cluster.IsAmsSubscriptionConsoleUrlSet = true @@ -4003,7 +4287,7 @@ var _ = Describe("Log Collection - refresh cluster", func() { mockHostAPI = host.NewMockAPI(ctrl) mockMetric = metrics.NewMockAPI(ctrl) operatorsManager := operators.NewManager(common.GetTestLog(), nil, operators.Options{}, nil) - clusterApi = NewManager(logTimeoutConfig(), common.GetTestLog().WithField("pkg", "cluster-monitor"), db, testing.GetDummyNotificationStream(ctrl), + clusterApi = NewManager(logTimeoutConfig(), common.GetTestLog().WithField("pkg", "cluster-monitor"), db, commontesting.GetDummyNotificationStream(ctrl), mockEvents, nil, mockHostAPI, mockMetric, nil, nil, operatorsManager, nil, nil, nil, nil, nil, false) clusterId = strfmt.UUID(uuid.New().String()) }) @@ -4145,7 +4429,7 @@ var _ = Describe("NTP refresh cluster", func() { mockHostAPI = host.NewMockAPI(ctrl) mockMetric = metrics.NewMockAPI(ctrl) operatorsManager := operators.NewManager(common.GetTestLog(), nil, operators.Options{}, nil) - clusterApi = NewManager(getDefaultConfig(), common.GetTestLog().WithField("pkg", "cluster-monitor"), db, testing.GetDummyNotificationStream(ctrl), + clusterApi = NewManager(getDefaultConfig(), common.GetTestLog().WithField("pkg", "cluster-monitor"), db, commontesting.GetDummyNotificationStream(ctrl), mockEvents, nil, mockHostAPI, mockMetric, nil, nil, operatorsManager, nil, nil, nil, nil, nil, false) hid1 = strfmt.UUID(uuid.New().String()) @@ -4369,18 +4653,18 @@ var _ = Describe("NTP refresh cluster", func() { It(t.name, func() { cluster = common.Cluster{ Cluster: models.Cluster{ - ClusterNetworks: common.TestIPv4Networking.ClusterNetworks, - ServiceNetworks: common.TestIPv4Networking.ServiceNetworks, - MachineNetworks: common.TestIPv4Networking.MachineNetworks, - APIVips: common.TestIPv4Networking.APIVips, - IngressVips: common.TestIPv4Networking.IngressVips, - ID: &clusterId, - Status: &t.srcState, - StatusInfo: &t.srcStatusInfo, - BaseDNSDomain: "test.com", - PullSecretSet: t.pullSecretSet, - - NetworkType: swag.String(models.ClusterNetworkTypeOVNKubernetes), + ClusterNetworks: common.TestIPv4Networking.ClusterNetworks, + ServiceNetworks: common.TestIPv4Networking.ServiceNetworks, + MachineNetworks: common.TestIPv4Networking.MachineNetworks, + APIVips: common.TestIPv4Networking.APIVips, + IngressVips: common.TestIPv4Networking.IngressVips, + ID: &clusterId, + Status: &t.srcState, + StatusInfo: &t.srcStatusInfo, + BaseDNSDomain: "test.com", + PullSecretSet: t.pullSecretSet, + OpenshiftVersion: testing.ValidOCPVersionForNonStretchedClusters, + NetworkType: swag.String(models.ClusterNetworkTypeOVNKubernetes), }, } Expect(db.Create(&cluster).Error).ShouldNot(HaveOccurred()) @@ -4437,7 +4721,7 @@ var _ = Describe("Single node", func() { mockHostAPI.EXPECT().IsRequireUserActionReset(gomock.Any()).Return(false).AnyTimes() } mockIsValidMasterCandidate := func() { - mockHostAPI.EXPECT().IsValidMasterCandidate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).AnyTimes() + mockHostAPI.EXPECT().IsValidMasterCandidate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).AnyTimes() } BeforeEach(func() { @@ -4448,7 +4732,7 @@ var _ = Describe("Single node", func() { mockMetric = metrics.NewMockAPI(ctrl) operatorsManager := operators.NewManager(common.GetTestLog(), nil, operators.Options{}, nil) dnsApi := dns.NewDNSHandler(nil, common.GetTestLog()) - clusterApi = NewManager(getDefaultConfig(), common.GetTestLog().WithField("pkg", "cluster-monitor"), db, testing.GetDummyNotificationStream(ctrl), + clusterApi = NewManager(getDefaultConfig(), common.GetTestLog().WithField("pkg", "cluster-monitor"), db, commontesting.GetDummyNotificationStream(ctrl), mockEvents, nil, mockHostAPI, mockMetric, nil, nil, operatorsManager, nil, nil, dnsApi, nil, nil, false) hid1 = strfmt.UUID(uuid.New().String()) hid2 = strfmt.UUID(uuid.New().String()) @@ -4664,6 +4948,7 @@ var _ = Describe("Single node", func() { PullSecretSet: t.pullSecretSet, NetworkType: swag.String(models.ClusterNetworkTypeOVNKubernetes), HighAvailabilityMode: &haMode, + OpenshiftVersion: testing.ValidOCPVersionForNonStretchedClusters, }, } if t.srcState == models.ClusterStatusPreparingForInstallation && t.dstState == models.ClusterStatusInstalling { @@ -4750,7 +5035,7 @@ var _ = Describe("installation timeout", func() { mockS3Api = s3wrapper.NewMockAPI(ctrl) operatorsManager = operators.NewManager(common.GetTestLog(), nil, operators.Options{}, nil) clusterId = strfmt.UUID(uuid.New().String()) - clusterApi = NewManager(getDefaultConfig(), common.GetTestLog().WithField("pkg", "cluster-monitor"), db, testing.GetDummyNotificationStream(ctrl), + clusterApi = NewManager(getDefaultConfig(), common.GetTestLog().WithField("pkg", "cluster-monitor"), db, commontesting.GetDummyNotificationStream(ctrl), mockEvents, nil, mockHostAPI, mockMetric, nil, nil, operatorsManager, nil, mockS3Api, nil, nil, nil, true) }) createCluster := func(status, statusInfo string, installStartedAt time.Time) *common.Cluster { @@ -4868,7 +5153,7 @@ var _ = Describe("finalizing timeouts", func() { } Context("soft timeouts disabled", func() { BeforeEach(func() { - clusterApi = NewManager(getDefaultConfig(), common.GetTestLog().WithField("pkg", "cluster-monitor"), db, testing.GetDummyNotificationStream(ctrl), + clusterApi = NewManager(getDefaultConfig(), common.GetTestLog().WithField("pkg", "cluster-monitor"), db, commontesting.GetDummyNotificationStream(ctrl), mockEvents, nil, mockHostAPI, mockMetric, nil, nil, operatorsManager, nil, mockS3Api, nil, nil, nil, false) }) for _, st := range finalizingStages { @@ -4906,7 +5191,7 @@ var _ = Describe("finalizing timeouts", func() { }) Context("soft timeouts enabled", func() { BeforeEach(func() { - clusterApi = NewManager(getDefaultConfig(), common.GetTestLog().WithField("pkg", "cluster-monitor"), db, testing.GetDummyNotificationStream(ctrl), + clusterApi = NewManager(getDefaultConfig(), common.GetTestLog().WithField("pkg", "cluster-monitor"), db, commontesting.GetDummyNotificationStream(ctrl), mockEvents, nil, mockHostAPI, mockMetric, nil, nil, operatorsManager, nil, mockS3Api, nil, nil, nil, true) }) diff --git a/internal/cluster/validator.go b/internal/cluster/validator.go index 553418ded5bd..92260add4a87 100644 --- a/internal/cluster/validator.go +++ b/internal/cluster/validator.go @@ -305,49 +305,30 @@ func (v *clusterValidator) areIngressVipsValid(c *clusterPreprocessContext) (Val return v.areVipsValid(c, &IngressVipsWrapper{c: c}) } -// conditions to have a valid number of masters -// 1. have exactly three masters -// 2. have less then 3 masters but enough to auto-assign hosts that can become masters -// 3. have at least 2 workers or auto-assign hosts that can become workers, if workers configured -// 4. having more then 3 known masters is illegal -func (v *clusterValidator) sufficientMastersCount(c *clusterPreprocessContext) (ValidationStatus, string) { +// SufficientMastersCount validates that there is a sufficient amount of hosts to satisfy requirements of both masters and workers. +// The requirements are - +// - For none-high-availablity cluster (SNO), exactly 1 master and 0 workers are required. +// - For high-availablity cluster, the master count should match the expected master count (ControlPlaneCount). +func (v *clusterValidator) SufficientMastersCount(c *clusterPreprocessContext) (ValidationStatus, string) { status := ValidationSuccess - var message string - - minMastersNeededForInstallation := common.MinMasterHostsNeededForInstallation - nonHAMode := swag.StringValue(c.cluster.HighAvailabilityMode) == models.ClusterHighAvailabilityModeNone - if nonHAMode { - minMastersNeededForInstallation = common.AllowedNumberOfMasterHostsInNoneHaMode - } - - hosts := make([]*models.Host, 0) - for _, h := range MapHostsByStatus(c.cluster) { - hosts = append(hosts, h...) - } - masters := make([]*models.Host, 0) - workers := make([]*models.Host, 0) - candidates := make([]*models.Host, 0) - - for _, host := range hosts { - switch role := common.GetEffectiveRole(host); role { - case models.HostRoleMaster: - //add pre-assigned/suggested master hosts to the masters list - masters = append(masters, host) - case models.HostRoleWorker: - //add pre-assigned/suggested worker hosts to the worker list - workers = append(workers, host) - default: - //auto-assign hosts and other types go to the candidate list - candidates = append(candidates, host) - } + var ( + message string + ) + + // Might be the case for already existing records (default value). In this case high availablity mode is set, we will get + // the corresponding control planes count + if c.cluster.ControlPlaneCount == 0 { + _, count := common.GetDefaultHighAvailabilityAndMasterCountParams(c.cluster.HighAvailabilityMode, nil) + c.cluster.ControlPlaneCount = swag.Int64Value(count) } - for _, h := range candidates { + masters, workers, autoAssignHosts := common.GetHostsByEachRole(&c.cluster.Cluster, true) + for _, h := range autoAssignHosts { //if allocated masters count is less than the desired count, find eligible hosts - //from the candidate pool to match the master count criteria, up to 3 - if len(masters) < minMastersNeededForInstallation { + //from the candidate pool to match the master count criteria + if len(masters) < int(c.cluster.ControlPlaneCount) { candidate := *h - if isValid, err := v.hostAPI.IsValidMasterCandidate(&candidate, c.cluster, c.db, v.log); isValid && err == nil { + if isValid, err := v.hostAPI.IsValidMasterCandidate(&candidate, c.cluster, c.db, v.log, false); isValid && err == nil { masters = append(masters, h) continue } @@ -357,8 +338,15 @@ func (v *clusterValidator) sufficientMastersCount(c *clusterPreprocessContext) ( } numWorkers := len(workers) - if len(masters) != minMastersNeededForInstallation || - nonHAMode && numWorkers != common.AllowedNumberOfWorkersInNoneHaMode { + numMasters := len(masters) + + // validate masters + if numMasters != int(c.cluster.ControlPlaneCount) { + status = ValidationFailure + } + + // validate workers (for SNO) + if c.cluster.ControlPlaneCount == 1 && numWorkers != common.AllowedNumberOfWorkersInNoneHaMode { status = ValidationFailure } @@ -366,8 +354,11 @@ func (v *clusterValidator) sufficientMastersCount(c *clusterPreprocessContext) ( case ValidationSuccess: message = "The cluster has the exact amount of dedicated control plane nodes." case ValidationFailure: - message = fmt.Sprintf("Clusters must have exactly %d dedicated control plane nodes. Add or remove hosts, or change their roles configurations to meet the requirement.", common.MinMasterHostsNeededForInstallation) - if nonHAMode { + message = fmt.Sprintf( + "The cluster must have exactly %d dedicated control plane nodes. Add or remove hosts, or change their roles configurations to meet the requirement.", + c.cluster.ControlPlaneCount, + ) + if c.cluster.ControlPlaneCount == 1 { message = "Single-node clusters must have a single control plane node and no workers." } default: diff --git a/internal/cluster/validator_test.go b/internal/cluster/validator_test.go index c50fb0dd8cd0..22ad864c2b34 100644 --- a/internal/cluster/validator_test.go +++ b/internal/cluster/validator_test.go @@ -6,10 +6,13 @@ import ( "github.com/go-openapi/strfmt" "github.com/go-openapi/swag" + "github.com/golang/mock/gomock" "github.com/google/uuid" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/openshift/assisted-service/internal/common" + "github.com/openshift/assisted-service/internal/host" + "github.com/openshift/assisted-service/internal/testing" "github.com/openshift/assisted-service/models" "github.com/sirupsen/logrus" "github.com/thoas/go-funk" @@ -668,3 +671,318 @@ var _ = Describe("skipNetworkHostPrefixCheck", func() { Expect(skipped).Should(Equal(true)) }) }) + +var _ = Describe("SufficientMastersCount", func() { + var ( + validator clusterValidator + clusterID strfmt.UUID + mockHostAPI *host.MockAPI + ctrl *gomock.Controller + ) + + BeforeEach(func() { + clusterID = strfmt.UUID(uuid.New().String()) + ctrl = gomock.NewController(GinkgoT()) + mockHostAPI = host.NewMockAPI(ctrl) + validator = clusterValidator{log: logrus.New(), hostAPI: mockHostAPI} + }) + + AfterEach(func() { + ctrl.Finish() + }) + + Context("pass validation", func() { + It("with matching counts, default ControlPlaneCount", func() { + mockHostAPI.EXPECT(). + IsValidMasterCandidate( + gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), + ).Return(true, nil).AnyTimes() + + preprocessContext := &clusterPreprocessContext{ + clusterId: clusterID, + cluster: &common.Cluster{Cluster: models.Cluster{ + ID: &clusterID, + OpenshiftVersion: testing.ValidOCPVersionForNonStretchedClusters, + HighAvailabilityMode: swag.String(models.ClusterCreateParamsHighAvailabilityModeFull), + Hosts: []*models.Host{ + { + Role: models.HostRoleMaster, + }, + { + Role: models.HostRoleMaster, + }, + { + Role: models.HostRoleMaster, + }, + }, + }}, + } + + status, message := validator.SufficientMastersCount(preprocessContext) + Expect(status).To(Equal(ValidationSuccess)) + Expect(message).To(Equal("The cluster has the exact amount of dedicated control plane nodes.")) + }) + + It("with matching counts, set ControlPlaneCount", func() { + mockHostAPI.EXPECT(). + IsValidMasterCandidate( + gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), + ).Return(true, nil).AnyTimes() + + preprocessContext := &clusterPreprocessContext{ + clusterId: clusterID, + cluster: &common.Cluster{ + ControlPlaneCount: 3, + Cluster: models.Cluster{ + ID: &clusterID, + OpenshiftVersion: testing.ValidOCPVersionForNonStretchedClusters, + HighAvailabilityMode: swag.String(models.ClusterCreateParamsHighAvailabilityModeFull), + Hosts: []*models.Host{ + { + Role: models.HostRoleMaster, + }, + { + Role: models.HostRoleMaster, + }, + { + Role: models.HostRoleMaster, + }, + }, + }}, + } + + status, message := validator.SufficientMastersCount(preprocessContext) + Expect(status).To(Equal(ValidationSuccess)) + Expect(message).To(Equal("The cluster has the exact amount of dedicated control plane nodes.")) + }) + + It("with SNO cluster, default controlPlaneCount", func() { + mockHostAPI.EXPECT(). + IsValidMasterCandidate( + gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), + ).Return(true, nil).AnyTimes() + + preprocessContext := &clusterPreprocessContext{ + clusterId: clusterID, + cluster: &common.Cluster{Cluster: models.Cluster{ + ID: &clusterID, + OpenshiftVersion: testing.ValidOCPVersionForNonStretchedClusters, + HighAvailabilityMode: swag.String(models.ClusterCreateParamsHighAvailabilityModeNone), + Hosts: []*models.Host{ + { + Role: models.HostRoleMaster, + }, + }, + }}, + } + + status, message := validator.SufficientMastersCount(preprocessContext) + Expect(status).To(Equal(ValidationSuccess)) + Expect(message).To(Equal("The cluster has the exact amount of dedicated control plane nodes.")) + }) + + It("with SNO cluster, set controlPlaneCount", func() { + mockHostAPI.EXPECT(). + IsValidMasterCandidate( + gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), + ).Return(true, nil).AnyTimes() + + preprocessContext := &clusterPreprocessContext{ + clusterId: clusterID, + cluster: &common.Cluster{ + ControlPlaneCount: 1, + Cluster: models.Cluster{ + ID: &clusterID, + OpenshiftVersion: testing.ValidOCPVersionForNonStretchedClusters, + HighAvailabilityMode: swag.String(models.ClusterCreateParamsHighAvailabilityModeNone), + Hosts: []*models.Host{ + { + Role: models.HostRoleMaster, + }, + }, + }}, + } + + status, message := validator.SufficientMastersCount(preprocessContext) + Expect(status).To(Equal(ValidationSuccess)) + Expect(message).To(Equal("The cluster has the exact amount of dedicated control plane nodes.")) + }) + + It("with multi-node cluster, 5 masters", func() { + mockHostAPI.EXPECT(). + IsValidMasterCandidate( + gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), + ).Return(true, nil).AnyTimes() + + preprocessContext := &clusterPreprocessContext{ + clusterId: clusterID, + cluster: &common.Cluster{ + ControlPlaneCount: 5, + Cluster: models.Cluster{ + ID: &clusterID, + OpenshiftVersion: common.MinimumVersionForStretchedControlPlanesCluster, + HighAvailabilityMode: swag.String(models.ClusterCreateParamsHighAvailabilityModeFull), + Hosts: []*models.Host{ + { + Role: models.HostRoleMaster, + }, + { + Role: models.HostRoleMaster, + }, + { + Role: models.HostRoleMaster, + }, + { + Role: models.HostRoleMaster, + }, + { + Role: models.HostRoleMaster, + }, + }, + }}, + } + + status, message := validator.SufficientMastersCount(preprocessContext) + Expect(status).To(Equal(ValidationSuccess)) + Expect(message).To(Equal("The cluster has the exact amount of dedicated control plane nodes.")) + }) + }) + + Context("fails validation", func() { + It("with multi node cluster, 5 masters but expected 3 by default", func() { + mockHostAPI.EXPECT(). + IsValidMasterCandidate( + gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), + ).Return(true, nil).AnyTimes() + + preprocessContext := &clusterPreprocessContext{ + clusterId: clusterID, + cluster: &common.Cluster{ + Cluster: models.Cluster{ + ID: &clusterID, + OpenshiftVersion: testing.ValidOCPVersionForNonStretchedClusters, + HighAvailabilityMode: swag.String(models.ClusterCreateParamsHighAvailabilityModeFull), + Hosts: []*models.Host{ + { + Role: models.HostRoleMaster, + }, + { + Role: models.HostRoleMaster, + }, + { + Role: models.HostRoleMaster, + }, + { + Role: models.HostRoleMaster, + }, + { + Role: models.HostRoleMaster, + }, + }, + }}, + } + + status, message := validator.SufficientMastersCount(preprocessContext) + Expect(status).To(Equal(ValidationFailure)) + Expect(message).To(Equal(fmt.Sprintf( + "The cluster must have exactly %d dedicated control plane nodes. Add or remove hosts, or change their roles configurations to meet the requirement.", + common.MinMasterHostsNeededForInstallationInHaMode, + ))) + }) + + It("with multi node cluster, 5 masters but expected 3", func() { + mockHostAPI.EXPECT(). + IsValidMasterCandidate( + gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), + ).Return(true, nil).AnyTimes() + + preprocessContext := &clusterPreprocessContext{ + clusterId: clusterID, + cluster: &common.Cluster{ + ControlPlaneCount: 3, + Cluster: models.Cluster{ + ID: &clusterID, + OpenshiftVersion: testing.ValidOCPVersionForNonStretchedClusters, + HighAvailabilityMode: swag.String(models.ClusterCreateParamsHighAvailabilityModeFull), + Hosts: []*models.Host{ + { + Role: models.HostRoleMaster, + }, + { + Role: models.HostRoleMaster, + }, + { + Role: models.HostRoleMaster, + }, + { + Role: models.HostRoleMaster, + }, + { + Role: models.HostRoleMaster, + }, + }, + }}, + } + + status, message := validator.SufficientMastersCount(preprocessContext) + Expect(status).To(Equal(ValidationFailure)) + Expect(message).To(Equal("The cluster must have exactly 3 dedicated control plane nodes. Add or remove hosts, or change their roles configurations to meet the requirement.")) + }) + + It("with SNO cluster, 2 masters 0 workers", func() { + mockHostAPI.EXPECT(). + IsValidMasterCandidate( + gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), + ).Return(true, nil).AnyTimes() + + preprocessContext := &clusterPreprocessContext{ + clusterId: clusterID, + cluster: &common.Cluster{Cluster: models.Cluster{ + ID: &clusterID, + OpenshiftVersion: testing.ValidOCPVersionForNonStretchedClusters, + HighAvailabilityMode: swag.String(models.ClusterCreateParamsHighAvailabilityModeNone), + Hosts: []*models.Host{ + { + Role: models.HostRoleMaster, + }, + { + Role: models.HostRoleMaster, + }, + }, + }}, + } + + status, message := validator.SufficientMastersCount(preprocessContext) + Expect(status).To(Equal(ValidationFailure)) + Expect(message).To(Equal("Single-node clusters must have a single control plane node and no workers.")) + }) + + It("with SNO cluster, 1 masters 1 workers", func() { + mockHostAPI.EXPECT(). + IsValidMasterCandidate( + gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), + ).Return(true, nil).AnyTimes() + + preprocessContext := &clusterPreprocessContext{ + clusterId: clusterID, + cluster: &common.Cluster{Cluster: models.Cluster{ + ID: &clusterID, + OpenshiftVersion: testing.ValidOCPVersionForNonStretchedClusters, + HighAvailabilityMode: swag.String(models.ClusterCreateParamsHighAvailabilityModeNone), + Hosts: []*models.Host{ + { + Role: models.HostRoleMaster, + }, + { + Role: models.HostRoleWorker, + }, + }, + }}, + } + + status, message := validator.SufficientMastersCount(preprocessContext) + Expect(status).To(Equal(ValidationFailure)) + Expect(message).To(Equal("Single-node clusters must have a single control plane node and no workers.")) + }) + }) +}) diff --git a/internal/common/common.go b/internal/common/common.go index 6a8c62720306..7032eccc3b27 100644 --- a/internal/common/common.go +++ b/internal/common/common.go @@ -22,10 +22,6 @@ import ( const ( EnvConfigPrefix = "myapp" - MinMasterHostsNeededForInstallation = 3 - AllowedNumberOfMasterHostsInNoneHaMode = 1 - AllowedNumberOfWorkersInNoneHaMode = 0 - HostCACertPath = "/etc/assisted-service/service-ca-cert.crt" AdditionalTrustBundlePath = "/etc/pki/ca-trust/source/anchors/assisted-infraenv-additional-trust-bundle.pem" @@ -52,6 +48,14 @@ const ( MultiCPUArchitecture = "multi" ExternalPlatformNameOci = "oci" + + MaxMasterHostsNeededForInstallationInHaModeOfOCP418OrNewer = 5 + MinMasterHostsNeededForInstallationInHaMode = 3 + AllowedNumberOfMasterHostsForInstallationInHaModeOfOCP417OrOlder = 3 + AllowedNumberOfMasterHostsInNoneHaMode = 1 + AllowedNumberOfWorkersInNoneHaMode = 0 + MinimumVersionForStretchedControlPlanesCluster = "4.18" + MinimumNumberOfWorkersForNonSchedulableMastersClusterInHaMode = 2 ) type AddressFamily int @@ -190,6 +194,8 @@ func IsImportedCluster(cluster *Cluster) bool { return swag.BoolValue(cluster.Imported) } +// AreMastersSchedulable returns whether a given cluster masters will be schedulable +// It will get correct result only when all hosts roles are assigned. func AreMastersSchedulable(cluster *Cluster) bool { return swag.BoolValue(cluster.SchedulableMastersForcedTrue) || swag.BoolValue(cluster.SchedulableMasters) } @@ -640,3 +646,73 @@ func GetEffectiveRole(host *models.Host) models.HostRole { } return host.Role } + +// GetHostsByEachRole returns the 3 slices of hosts by their effective role for a given cluster: +// 1. bootstrap/master hosts +// 2. worker hosts +// 3. auto-assign hosts +// Note - The hosts should be preloaded in the cluster +func GetHostsByEachRole(cluster *models.Cluster, effectiveRoles bool) ([]*models.Host, []*models.Host, []*models.Host) { + masterHosts := make([]*models.Host, 0) + workerHosts := make([]*models.Host, 0) + autoAssignHosts := make([]*models.Host, 0) + + roleFunction := func(host *models.Host) models.HostRole { + if effectiveRoles { + return GetEffectiveRole(host) + } + + return host.Role + } + + for _, host := range cluster.Hosts { + switch role := roleFunction(host); role { + case models.HostRoleMaster, models.HostRoleBootstrap: + masterHosts = append(masterHosts, host) + case models.HostRoleWorker: + workerHosts = append(workerHosts, host) + case models.HostRoleAutoAssign: + autoAssignHosts = append(autoAssignHosts, host) + } + } + + return masterHosts, workerHosts, autoAssignHosts +} + +func ShouldMastersBeSchedulable(cluster *models.Cluster) bool { + if swag.StringValue(cluster.HighAvailabilityMode) == models.ClusterCreateParamsHighAvailabilityModeNone { + return true + } + + _, workers, _ := GetHostsByEachRole(cluster, true) + return len(workers) < MinimumNumberOfWorkersForNonSchedulableMastersClusterInHaMode +} + +func GetDefaultHighAvailabilityAndMasterCountParams(highAvailabilityMode *string, controlPlaneCount *int64) (*string, *int64) { + // Both not set, multi node by default + if highAvailabilityMode == nil && controlPlaneCount == nil { + return swag.String(models.ClusterCreateParamsHighAvailabilityModeFull), + swag.Int64(MinMasterHostsNeededForInstallationInHaMode) + } + + // only highAvailabilityMode set + if controlPlaneCount == nil { + if *highAvailabilityMode == models.ClusterHighAvailabilityModeNone { + return highAvailabilityMode, swag.Int64(AllowedNumberOfMasterHostsInNoneHaMode) + } else { + return highAvailabilityMode, swag.Int64(MinMasterHostsNeededForInstallationInHaMode) + } + } + + // only controlPlaneCount set + if highAvailabilityMode == nil { + if *controlPlaneCount == AllowedNumberOfMasterHostsInNoneHaMode { + return swag.String(models.ClusterHighAvailabilityModeNone), controlPlaneCount + } else { + return swag.String(models.ClusterHighAvailabilityModeFull), controlPlaneCount + } + } + + // both are set + return highAvailabilityMode, controlPlaneCount +} diff --git a/internal/common/common_test.go b/internal/common/common_test.go index 15369eacb3af..7a10d7ee79ff 100644 --- a/internal/common/common_test.go +++ b/internal/common/common_test.go @@ -376,6 +376,131 @@ var _ = Describe("JSON serialization checks", func() { }) }) +var _ = Describe("GetHostsByEachRole", func() { + Context("should categorize hosts based on Role", func() { + cluster := &models.Cluster{ + Hosts: []*models.Host{ + { // Host that has been automatically assigned + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleMaster, + }, + { // Host that has been automatically assigned + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleMaster, + }, + { // Host that has been automatically assigned + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleMaster, + }, + { // Host that has been automatically assigned + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleWorker, + }, + { // Host that has been automatically assigned + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleWorker, + }, + { // Host that has been manually assigned to worker + Role: models.HostRoleWorker, + SuggestedRole: models.HostRoleWorker, + }, + { // Host that has not been assigned + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleAutoAssign, + }, + }, + } + + It("with effective roles", func() { + effectiveRoles := true + masterHosts, workerHosts, autoAssignHosts := GetHostsByEachRole(cluster, effectiveRoles) + + Expect(masterHosts).To(HaveLen(3)) + Expect(workerHosts).To(HaveLen(3)) + Expect(autoAssignHosts).To(HaveLen(1)) + }) + + It("with non-effective roles", func() { + effectiveRoles := false + masterHosts, workerHosts, autoAssignHosts := GetHostsByEachRole(cluster, effectiveRoles) + + Expect(masterHosts).To(HaveLen(0)) + Expect(workerHosts).To(HaveLen(1)) + Expect(autoAssignHosts).To(HaveLen(6)) + }) + }) +}) + +var _ = Describe("ShouldMastersBeSchedulable", func() { + Context("should return false", func() { + It("when the cluster is composed from more than 1 worker, multi-node", func() { + cluster := &models.Cluster{ + HighAvailabilityMode: swag.String(models.ClusterCreateParamsHighAvailabilityModeFull), + Hosts: []*models.Host{ + { + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleMaster, + }, + { + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleMaster, + }, + { + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleMaster, + }, + { + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleWorker, + }, + { + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleWorker, + }, + }, + } + + Expect(ShouldMastersBeSchedulable(cluster)).To(BeFalse()) + }) + }) + + Context("should return true", func() { + It("when SNO cluster", func() { + cluster := &models.Cluster{ + HighAvailabilityMode: swag.String(models.ClusterCreateParamsHighAvailabilityModeNone), + } + + Expect(ShouldMastersBeSchedulable(cluster)).To(BeTrue()) + }) + + It("when less than 2 workers, multi-node", func() { + cluster := &models.Cluster{ + HighAvailabilityMode: swag.String(models.ClusterCreateParamsHighAvailabilityModeFull), + Hosts: []*models.Host{ + { + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleMaster, + }, + { + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleMaster, + }, + { + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleMaster, + }, + { + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleWorker, + }, + }, + } + + Expect(ShouldMastersBeSchedulable(cluster)).To(BeTrue()) + }) + }) +}) + func createHost(hostRole models.HostRole, state string) *models.Host { hostId := strfmt.UUID(uuid.New().String()) clusterId := strfmt.UUID(uuid.New().String()) diff --git a/internal/common/db.go b/internal/common/db.go index cc1f49aa7f62..2a97745f52b7 100644 --- a/internal/common/db.go +++ b/internal/common/db.go @@ -1,6 +1,7 @@ package common import ( + "fmt" "time" "github.com/go-openapi/strfmt" @@ -81,6 +82,9 @@ type Cluster struct { // A JSON blob in which cluster UI settings will be stored. UISettings string `json:"ui_settings"` + + // The amount of control planes which should be part of the cluster in high availability 'Full' mode. + ControlPlaneCount int64 `json:"control_plane_count"` } func (c *Cluster) GetClusterID() *strfmt.UUID { @@ -305,6 +309,36 @@ func GetClusterFromDBWithVips(db *gorm.DB, clusterId strfmt.UUID) (*Cluster, err return GetClusterFromDB(db, clusterId, SkipEagerLoading) } +func GetHostCountByRole(db *gorm.DB, clusterID strfmt.UUID, role models.HostRole, suggested bool) (*int64, error) { + var count int64 + + field := "role" + errStr := " " + if suggested { + field = "suggested_role" + errStr = " suggested " + } + condition := fmt.Sprintf("hosts.%s = ?", field) + + err := db.Model(&Host{}). + Joins("INNER JOIN clusters ON hosts.cluster_id = clusters.id"). + Where("clusters.id = ?", clusterID.String()). + Where(condition, string(role)). + Count(&count).Error + + if err != nil { + return nil, errors.Wrapf( + err, + "failed to count the number of hosts in cluster with ID '%s' and%srole '%s'", + clusterID.String(), + errStr, + string(role), + ) + } + + return &count, nil +} + func prepareClusterDB(db *gorm.DB, eagerLoading EagerLoadingState, includeDeleted DeleteRecordsState, conditions ...interface{}) *gorm.DB { if includeDeleted { db = db.Unscoped() diff --git a/internal/common/db_test.go b/internal/common/db_test.go new file mode 100644 index 000000000000..009514bbc9e5 --- /dev/null +++ b/internal/common/db_test.go @@ -0,0 +1,288 @@ +package common + +import ( + "github.com/go-openapi/strfmt" + "github.com/google/uuid" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/openshift/assisted-service/models" + "gorm.io/gorm" +) + +var _ = Describe("GetHostCountByRole", func() { + var ( + db *gorm.DB + dbName string + clusterID = strfmt.UUID(uuid.New().String()) + HostID1 = strfmt.UUID(uuid.New().String()) + HostID2 = strfmt.UUID(uuid.New().String()) + HostID3 = strfmt.UUID(uuid.New().String()) + HostID4 = strfmt.UUID(uuid.New().String()) + HostID5 = strfmt.UUID(uuid.New().String()) + HostID6 = strfmt.UUID(uuid.New().String()) + HostID7 = strfmt.UUID(uuid.New().String()) + HostID8 = strfmt.UUID(uuid.New().String()) + ) + + BeforeEach(func() { + db, dbName = PrepareTestDB() + }) + + AfterEach(func() { + DeleteTestDB(db, dbName) + }) + + Context("should succeed", func() { + Context("with suggested role", func() { + It("and some records found", func() { + cluster := Cluster{ + Cluster: models.Cluster{ + ID: &clusterID, + Hosts: []*models.Host{ + { + ID: &HostID1, + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleAutoAssign, + }, + { + ID: &HostID2, + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleMaster, + }, + { + ID: &HostID3, + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleMaster, + }, + { + ID: &HostID4, + Role: models.HostRoleMaster, + SuggestedRole: models.HostRoleMaster, + }, + { + ID: &HostID5, + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleWorker, + }, + { + ID: &HostID6, + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleWorker, + }, + { + ID: &HostID7, + Role: models.HostRoleWorker, + SuggestedRole: models.HostRoleWorker, + }, + { + ID: &HostID8, + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleAutoAssign, + }, + }, + }, + } + + err := db.Create(&cluster).Error + Expect(err).ToNot(HaveOccurred()) + + masterCount, err := GetHostCountByRole(db, clusterID, models.HostRoleMaster, true) + Expect(err).ToNot(HaveOccurred()) + + workerCount, err := GetHostCountByRole(db, clusterID, models.HostRoleWorker, true) + Expect(err).ToNot(HaveOccurred()) + + autoAssignCount, err := GetHostCountByRole(db, clusterID, models.HostRoleAutoAssign, true) + Expect(err).ToNot(HaveOccurred()) + + Expect(*masterCount).To(BeEquivalentTo(3)) + Expect(*workerCount).To(BeEquivalentTo(3)) + Expect(*autoAssignCount).To(BeEquivalentTo(2)) + }) + + It("no records found", func() { + cluster := Cluster{ + Cluster: models.Cluster{ + ID: &clusterID, + Hosts: []*models.Host{ + { + ID: &HostID1, + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleAutoAssign, + }, + { + ID: &HostID2, + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleMaster, + }, + { + ID: &HostID3, + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleMaster, + }, + { + ID: &HostID4, + Role: models.HostRoleMaster, + SuggestedRole: models.HostRoleMaster, + }, + { + ID: &HostID5, + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleWorker, + }, + { + ID: &HostID6, + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleWorker, + }, + { + ID: &HostID7, + Role: models.HostRoleWorker, + SuggestedRole: models.HostRoleWorker, + }, + { + ID: &HostID8, + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleAutoAssign, + }, + }, + }, + } + + err := db.Create(&cluster).Error + Expect(err).ToNot(HaveOccurred()) + + bootstrapHostCount, err := GetHostCountByRole(db, clusterID, models.HostRoleBootstrap, true) + Expect(err).ToNot(HaveOccurred()) + + Expect(*bootstrapHostCount).To(BeEquivalentTo(0)) + }) + }) + + Context("with non-suggested role", func() { + It("and some records found", func() { + cluster := Cluster{ + Cluster: models.Cluster{ + ID: &clusterID, + Hosts: []*models.Host{ + { + ID: &HostID1, + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleAutoAssign, + }, + { + ID: &HostID2, + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleMaster, + }, + { + ID: &HostID3, + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleMaster, + }, + { + ID: &HostID4, + Role: models.HostRoleMaster, + SuggestedRole: models.HostRoleMaster, + }, + { + ID: &HostID5, + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleWorker, + }, + { + ID: &HostID6, + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleWorker, + }, + { + ID: &HostID7, + Role: models.HostRoleWorker, + SuggestedRole: models.HostRoleWorker, + }, + { + ID: &HostID8, + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleAutoAssign, + }, + }, + }, + } + + err := db.Create(&cluster).Error + Expect(err).ToNot(HaveOccurred()) + + masterCount, err := GetHostCountByRole(db, clusterID, models.HostRoleMaster, false) + Expect(err).ToNot(HaveOccurred()) + + workerCount, err := GetHostCountByRole(db, clusterID, models.HostRoleWorker, false) + Expect(err).ToNot(HaveOccurred()) + + autoAssignCount, err := GetHostCountByRole(db, clusterID, models.HostRoleAutoAssign, false) + Expect(err).ToNot(HaveOccurred()) + + Expect(*masterCount).To(BeEquivalentTo(1)) + Expect(*workerCount).To(BeEquivalentTo(1)) + Expect(*autoAssignCount).To(BeEquivalentTo(6)) + }) + + It("no records found", func() { + cluster := Cluster{ + Cluster: models.Cluster{ + ID: &clusterID, + Hosts: []*models.Host{ + { + ID: &HostID1, + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleAutoAssign, + }, + { + ID: &HostID2, + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleMaster, + }, + { + ID: &HostID3, + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleMaster, + }, + { + ID: &HostID4, + Role: models.HostRoleMaster, + SuggestedRole: models.HostRoleMaster, + }, + { + ID: &HostID5, + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleWorker, + }, + { + ID: &HostID6, + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleWorker, + }, + { + ID: &HostID7, + Role: models.HostRoleWorker, + SuggestedRole: models.HostRoleWorker, + }, + { + ID: &HostID8, + Role: models.HostRoleAutoAssign, + SuggestedRole: models.HostRoleAutoAssign, + }, + }, + }, + } + + err := db.Create(&cluster).Error + Expect(err).ToNot(HaveOccurred()) + + bootstrapHostCount, err := GetHostCountByRole(db, clusterID, models.HostRoleBootstrap, false) + Expect(err).ToNot(HaveOccurred()) + + Expect(*bootstrapHostCount).To(BeEquivalentTo(0)) + }) + }) + }) +}) diff --git a/internal/common/validations.go b/internal/common/validations.go index 93dca02c316e..86fba3fcb296 100644 --- a/internal/common/validations.go +++ b/internal/common/validations.go @@ -5,6 +5,7 @@ package common import ( "github.com/go-openapi/swag" + "github.com/openshift/assisted-service/models" ) // IsAgentCompatible checks if the given agent image is compatible with what the service expects. @@ -12,8 +13,20 @@ func IsAgentCompatible(expectedImage, agentImage string) bool { return agentImage == expectedImage } -var NonIgnorableHostValidations []string = []string{"connected", "has-inventory", "machine-cidr-defined", "hostname-unique", "hostname-valid"} -var NonIgnorableClusterValidations []string = []string{"api-vips-defined", "ingress-vips-defined", "all-hosts-are-ready-to-install", "sufficient-masters-count", "pull-secret-set", "cluster-preparation-succeeded"} +var NonIgnorableHostValidations []string = []string{ + string(models.HostValidationIDConnected), + string(models.HostValidationIDHasInventory), + string(models.HostValidationIDMachineCidrDefined), + string(models.HostValidationIDHostnameUnique), + string(models.HostValidationIDHostnameValid), +} +var NonIgnorableClusterValidations []string = []string{ + string(models.ClusterValidationIDAPIVipsDefined), + string(models.ClusterValidationIDIngressVipsDefined), + string(models.ClusterValidationIDAllHostsAreReadyToInstall), + string(models.ClusterValidationIDSufficientMastersCount), + string(models.ClusterValidationIDPullSecretSet), +} func ShouldIgnoreValidation(ignoredValidations []string, validationId string, nonIgnoribles []string) bool { if !MayIgnoreValidation(validationId, nonIgnoribles) { diff --git a/internal/controller/controllers/clusterdeployments_controller.go b/internal/controller/controllers/clusterdeployments_controller.go index 0237b343a35a..cb89f69b06db 100644 --- a/internal/controller/controllers/clusterdeployments_controller.go +++ b/internal/controller/controllers/clusterdeployments_controller.go @@ -20,6 +20,7 @@ import ( "context" "encoding/base64" "encoding/json" + errpkg "errors" "fmt" "io" "net/http" @@ -223,6 +224,10 @@ func (r *ClusterDeploymentsReconciler) Reconcile(origCtx context.Context, req ct return r.updateStatus(ctx, log, clusterInstall, clusterDeployment, cluster, err) } + if cluster.ID == nil { + return ctrl.Result{}, fmt.Errorf("cluster associated with namespace '%s', name '%s' is missing ID", req.Namespace, req.Name) + } + // check for updates from user, compare spec and update if needed cluster, err = r.updateIfNeeded(ctx, log, clusterDeployment, clusterInstall, cluster) if err != nil { @@ -383,13 +388,41 @@ func isInstalled(clusterDeployment *hivev1.ClusterDeployment, clusterInstall *hi return cond != nil && cond.Reason == hiveext.ClusterInstalledReason } +// getHostSuggestedRoleCount returns the amount of suggested masters and workers for the given cluster. +// It counts from DB as the hosts are not loaded to memory, and we want to avoid loading them. +func getHostSuggestedRoleCount(clusterAPI cluster.API, clusterID strfmt.UUID) (*int64, *int64, error) { + var combinedErr error + + mastersCountPtr, err := clusterAPI.GetHostCountByRole(clusterID, models.HostRoleMaster, true) + if err != nil { + combinedErr = errpkg.Join(combinedErr, err) + } + + workersCountPtr, err := clusterAPI.GetHostCountByRole(clusterID, models.HostRoleWorker, true) + if err != nil { + combinedErr = errpkg.Join(combinedErr, err) + } + + if combinedErr != nil { + // convert the err to one line + combinedErr = errors.New(strings.ReplaceAll(combinedErr.Error(), "\n", ", ")) + return nil, nil, combinedErr + } + + return mastersCountPtr, workersCountPtr, nil +} + func (r *ClusterDeploymentsReconciler) installDay1(ctx context.Context, log logrus.FieldLogger, clusterDeployment *hivev1.ClusterDeployment, clusterInstall *hiveext.AgentClusterInstall, cluster *common.Cluster) (ctrl.Result, error) { - ready, err := r.isReadyForInstallation(ctx, log, clusterInstall, cluster) + + ready, err := r.isReadyForInstallation( + ctx, log, clusterInstall, cluster, + ) if err != nil { log.WithError(err).Error("failed to check if cluster ready for installation") return r.updateStatus(ctx, log, clusterInstall, clusterDeployment, cluster, err) } + if ready { // create custom manifests if needed before installation err = r.addCustomManifests(ctx, log, clusterInstall, cluster) @@ -413,6 +446,7 @@ func (r *ClusterDeploymentsReconciler) installDay1(ctx context.Context, log logr } return r.updateStatus(ctx, log, clusterInstall, clusterDeployment, ic, err) } + return r.updateStatus(ctx, log, clusterInstall, clusterDeployment, cluster, nil) } @@ -636,7 +670,12 @@ func (r *ClusterDeploymentsReconciler) createClusterCredentialSecret(ctx context return s, r.Create(ctx, s) } -func (r *ClusterDeploymentsReconciler) isReadyForInstallation(ctx context.Context, log logrus.FieldLogger, clusterInstall *hiveext.AgentClusterInstall, c *common.Cluster) (bool, error) { +func (r *ClusterDeploymentsReconciler) isReadyForInstallation( + ctx context.Context, + log logrus.FieldLogger, + clusterInstall *hiveext.AgentClusterInstall, + c *common.Cluster, +) (bool, error) { if ready, _ := r.ClusterApi.IsReadyForInstallation(c); !ready { return false, nil } @@ -655,9 +694,23 @@ func (r *ClusterDeploymentsReconciler) isReadyForInstallation(ctx context.Contex unsyncedHosts := getNumOfUnsyncedAgents(agents) log.Debugf("Calculating installation readiness, found %d unsynced agents out of total of %d agents", unsyncedHosts, len(agents)) - expectedHosts := clusterInstall.Spec.ProvisionRequirements.ControlPlaneAgents + - clusterInstall.Spec.ProvisionRequirements.WorkerAgents - return approvedHosts == expectedHosts && registered == approvedHosts && unsyncedHosts == 0, nil + + expectedMasterCount := clusterInstall.Spec.ProvisionRequirements.ControlPlaneAgents + expectedWorkerCount := clusterInstall.Spec.ProvisionRequirements.WorkerAgents + expectedHostCount := expectedMasterCount + expectedWorkerCount + + masterCountPrt, workerCountPtr, err := getHostSuggestedRoleCount(r.ClusterApi, *c.ID) + if err != nil { + // will be shown as a SpecSynced error + log.WithError(err).Error("failed to fetch host suggested role count") + return false, err + } + + return approvedHosts == expectedHostCount && + registered == expectedHostCount && + int(swag.Int64Value(masterCountPrt)) == expectedMasterCount && + int(swag.Int64Value(workerCountPtr)) == expectedWorkerCount && + unsyncedHosts == 0, nil } func isSupportedPlatform(cluster *hivev1.ClusterDeployment) bool { @@ -988,11 +1041,13 @@ func (r *ClusterDeploymentsReconciler) updateNetworkParams(clusterDeployment *hi return swag.Bool(update), nil } -func (r *ClusterDeploymentsReconciler) updateIfNeeded(ctx context.Context, +func (r *ClusterDeploymentsReconciler) updateIfNeeded( + ctx context.Context, log logrus.FieldLogger, clusterDeployment *hivev1.ClusterDeployment, clusterInstall *hiveext.AgentClusterInstall, - cluster *common.Cluster) (*common.Cluster, error) { + cluster *common.Cluster, +) (*common.Cluster, error) { update := false params := &models.V2ClusterUpdateParams{} @@ -1079,6 +1134,11 @@ func (r *ClusterDeploymentsReconciler) updateIfNeeded(ctx context.Context, update = true } + if clusterInstall.Spec.ProvisionRequirements.ControlPlaneAgents != int(cluster.ControlPlaneCount) { + params.ControlPlaneCount = swag.Int64(int64(clusterInstall.Spec.ProvisionRequirements.ControlPlaneAgents)) + update = true + } + if !update { return cluster, nil } @@ -1299,6 +1359,7 @@ func CreateClusterParams(clusterDeployment *hivev1.ClusterDeployment, clusterIns UserManagedNetworking: swag.Bool(isUserManagedNetwork(clusterInstall)), Platform: platform, SchedulableMasters: swag.Bool(clusterInstall.Spec.MastersSchedulable), + ControlPlaneCount: swag.Int64(int64(clusterInstall.Spec.ProvisionRequirements.ControlPlaneAgents)), } if len(clusterInstall.Spec.Networking.ClusterNetwork) > 0 { @@ -1746,6 +1807,10 @@ func (r *ClusterDeploymentsReconciler) updateStatus(ctx context.Context, log log clusterInstall *hiveext.AgentClusterInstall, clusterDeployment *hivev1.ClusterDeployment, c *common.Cluster, syncErr error) (ctrl.Result, error) { + var ( + mastersCountPtr, workersCountPtr *int64 + ) + clusterSpecSynced(clusterInstall, syncErr) if c != nil { clusterInstall.Status.ConnectivityMajorityGroups = c.ConnectivityMajorityGroups @@ -1766,8 +1831,8 @@ func (r *ClusterDeploymentsReconciler) updateStatus(ctx context.Context, log log clusterInstall.Status.UserManagedNetworking = c.UserManagedNetworking clusterInstall.Status.PlatformType = getPlatformType(c.Platform) status := *c.Status - var err error - err = r.populateEventsURL(log, clusterInstall, c) + + err := r.populateEventsURL(log, clusterInstall, c) if err != nil { return ctrl.Result{Requeue: true}, nil } @@ -1775,22 +1840,38 @@ func (r *ClusterDeploymentsReconciler) updateStatus(ctx context.Context, log log if err != nil { return ctrl.Result{Requeue: true}, nil } - var registeredHosts, approvedHosts, unsyncedHosts int + var approvedHosts, unsyncedHosts int if status == models.ClusterStatusReady { - registeredHosts, approvedHosts, err = r.getNumOfClusterAgents(c) + _, approvedHosts, err = r.getNumOfClusterAgents(c) if err != nil { log.WithError(err).Error("failed to fetch cluster's agents") return ctrl.Result{Requeue: true}, nil } - agents, err := findAgentsByAgentClusterInstall(r.Client, ctx, log, clusterInstall) + var agents []*aiv1beta1.Agent + agents, err = findAgentsByAgentClusterInstall(r.Client, ctx, log, clusterInstall) if err != nil { log.WithError(err).Error("failed to fetch ACI's agents") return ctrl.Result{Requeue: true}, nil } + + mastersCountPtr, workersCountPtr, err = getHostSuggestedRoleCount(r.ClusterApi, *c.ID) + if err != nil { + log.WithError(err).Error("failed to fetch host suggested role count") + // proceed to update the conditions (the counts will be 0) + } + unsyncedHosts = getNumOfUnsyncedAgents(agents) log.Debugf("Updating ACI conditions, found %d unsynced agents out of total of %d agents", unsyncedHosts, len(agents)) } - clusterRequirementsMet(clusterInstall, status, registeredHosts, approvedHosts, unsyncedHosts) + + clusterRequirementsMet( + clusterInstall, + status, + approvedHosts, + unsyncedHosts, + int(swag.Int64Value(mastersCountPtr)), + int(swag.Int64Value(workersCountPtr)), + ) clusterValidated(clusterInstall, status, c) clusterCompleted(clusterInstall, clusterDeployment, status, swag.StringValue(c.StatusInfo), c.MonitoredOperators) clusterFailed(clusterInstall, status, swag.StringValue(c.StatusInfo)) @@ -1929,31 +2010,43 @@ func clusterSpecSynced(cluster *hiveext.AgentClusterInstall, syncErr error) { }) } -func clusterRequirementsMet(clusterInstall *hiveext.AgentClusterInstall, status string, registeredHosts, approvedHosts, unsyncedHosts int) { +func clusterRequirementsMet( + clusterInstall *hiveext.AgentClusterInstall, + status string, + approvedHosts, + unsyncedHosts int, + masterCount int, + workerCount int, +) { var condStatus corev1.ConditionStatus var reason string var msg string switch status { case models.ClusterStatusReady: - expectedHosts := clusterInstall.Spec.ProvisionRequirements.ControlPlaneAgents + - clusterInstall.Spec.ProvisionRequirements.WorkerAgents - if registeredHosts < expectedHosts { + expectedMasterCount := clusterInstall.Spec.ProvisionRequirements.ControlPlaneAgents + expectedWorkerCount := clusterInstall.Spec.ProvisionRequirements.WorkerAgents + expectedHostCount := expectedMasterCount + expectedWorkerCount + + if masterCount != expectedMasterCount || + workerCount != expectedWorkerCount { condStatus = corev1.ConditionFalse reason = hiveext.ClusterInsufficientAgentsReason - msg = fmt.Sprintf(hiveext.ClusterInsufficientAgentsMsg, expectedHosts, approvedHosts) + msg = fmt.Sprintf( + hiveext.ClusterInsufficientAgentsMsg, + expectedMasterCount, + expectedWorkerCount, + masterCount, + workerCount, + ) } else if unsyncedHosts != 0 { condStatus = corev1.ConditionFalse reason = hiveext.ClusterUnsyncedAgentsReason msg = fmt.Sprintf(hiveext.ClusterUnsyncedAgentsMsg, unsyncedHosts) - } else if approvedHosts < expectedHosts { + } else if approvedHosts < expectedHostCount { condStatus = corev1.ConditionFalse reason = hiveext.ClusterUnapprovedAgentsReason - msg = fmt.Sprintf(hiveext.ClusterUnapprovedAgentsMsg, expectedHosts-approvedHosts) - } else if registeredHosts > expectedHosts { - condStatus = corev1.ConditionFalse - reason = hiveext.ClusterAdditionalAgentsReason - msg = fmt.Sprintf(hiveext.ClusterAdditionalAgentsMsg, expectedHosts, registeredHosts) + msg = fmt.Sprintf(hiveext.ClusterUnapprovedAgentsMsg, expectedHostCount-approvedHosts) } else { condStatus = corev1.ConditionTrue reason = hiveext.ClusterReadyReason diff --git a/internal/controller/controllers/clusterdeployments_controller_test.go b/internal/controller/controllers/clusterdeployments_controller_test.go index 8e501d0b2870..8fda3812477c 100644 --- a/internal/controller/controllers/clusterdeployments_controller_test.go +++ b/internal/controller/controllers/clusterdeployments_controller_test.go @@ -298,6 +298,7 @@ var _ = Describe("cluster reconcile", func() { Expect(swag.StringValue(params.NewClusterParams.OpenshiftVersion)).To(Equal(*releaseImage.Version)) }).Return(clusterReply, nil) mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(releaseImage, nil) + mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) cluster := newClusterDeployment(clusterName, testNamespace, defaultClusterSpec) @@ -318,6 +319,7 @@ var _ = Describe("cluster reconcile", func() { Expect(swag.StringValue(params.NewClusterParams.NoProxy)).To(Equal(noProxy)) }).Return(clusterReply, nil) mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(releaseImage, nil) + mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) cluster := newClusterDeployment(clusterName, testNamespace, defaultClusterSpec) @@ -335,6 +337,7 @@ var _ = Describe("cluster reconcile", func() { Expect(swag.StringValue(params.NewClusterParams.OpenshiftVersion)).To(Equal(*releaseImage.Version)) }).Return(clusterReply, nil) mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(releaseImage, nil) + mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) cluster := newClusterDeployment(clusterName, testNamespace, defaultClusterSpec) @@ -375,6 +378,7 @@ var _ = Describe("cluster reconcile", func() { Expect(params.NewClusterParams.CPUArchitecture).To(Equal(CpuArchitectureArm)) }).Return(clusterReply, nil) mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(armReleaseImage, nil) + mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) cluster := newClusterDeployment(clusterName, testNamespace, defaultClusterSpec) @@ -409,6 +413,7 @@ var _ = Describe("cluster reconcile", func() { Expect(params.NewClusterParams.DiskEncryption.TangServers).To(Equal(tangServersConfig)) }).Return(clusterReply, nil) mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(releaseImage, nil) + mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) cluster := newClusterDeployment(clusterName, testNamespace, defaultClusterSpec) @@ -430,6 +435,7 @@ var _ = Describe("cluster reconcile", func() { Expect(swag.StringValue(params.NewClusterParams.OpenshiftVersion)).To(Equal(ocpReleaseVersion)) }).Return(clusterReply, nil) mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(releaseImage, nil) + mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) cluster := newClusterDeployment(clusterName, testNamespace, @@ -451,7 +457,10 @@ var _ = Describe("cluster reconcile", func() { To(Equal(HighAvailabilityModeNone)) }).Return(clusterReply, nil) mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(releaseImage, nil) + mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) + mockClusterApi.EXPECT().GetHostCountByRole(gomock.Any(), models.HostRoleMaster, true).Return(swag.Int64(3), nil).AnyTimes() + mockClusterApi.EXPECT().GetHostCountByRole(gomock.Any(), models.HostRoleWorker, true).Return(swag.Int64(2), nil).AnyTimes() cluster := newClusterDeployment(clusterName, testNamespace, defaultClusterSpec) Expect(c.Create(ctx, cluster)).ShouldNot(HaveOccurred()) @@ -472,6 +481,7 @@ var _ = Describe("cluster reconcile", func() { To(BeTrue()) }).Return(clusterReply, nil) mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(releaseImage, nil) + mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) cluster := newClusterDeployment(clusterName, testNamespace, defaultClusterSpec) @@ -561,6 +571,7 @@ var _ = Describe("cluster reconcile", func() { It("fail to get openshift version when trying to create a cluster", func() { mockMirrorRegistries.EXPECT().IsMirrorRegistriesConfigured().AnyTimes().Return(false) mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.Errorf("some-error")) + mockInstallerInternal.EXPECT().ValidatePullSecret(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() cluster := newClusterDeployment(clusterName, testNamespace, defaultClusterSpec) @@ -598,6 +609,7 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().ValidatePullSecret(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mockMirrorRegistries.EXPECT().IsMirrorRegistriesConfigured().AnyTimes().Return(true) mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.Errorf("some-error")) + request := newClusterDeploymentRequest(cluster) result, err := cr.Reconcile(ctx, request) Expect(err).To(BeNil()) @@ -619,6 +631,7 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().ValidatePullSecret(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mockMirrorRegistries.EXPECT().IsMirrorRegistriesConfigured().AnyTimes().Return(true) mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.Errorf("some-error")) + request := newClusterDeploymentRequest(cluster) result, err := cr.Reconcile(ctx, request) Expect(err).To(BeNil()) @@ -640,6 +653,7 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().ValidatePullSecret(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mockMirrorRegistries.EXPECT().IsMirrorRegistriesConfigured().AnyTimes().Return(false) mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.Errorf("some-error")) + request := newClusterDeploymentRequest(cluster) result, err := cr.Reconcile(ctx, request) Expect(err).To(BeNil()) @@ -703,7 +717,10 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) mockInstallerInternal.EXPECT().ValidatePullSecret(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(releaseImage, nil) + mockMirrorRegistries.EXPECT().IsMirrorRegistriesConfigured().AnyTimes().Return(false) + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleMaster, true).Return(swag.Int64(1), nil).AnyTimes() + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleWorker, true).Return(swag.Int64(0), nil).AnyTimes() }) setupTestForStatusAndReason := func(status string, reason string) { @@ -718,16 +735,20 @@ var _ = Describe("cluster reconcile", func() { }, }, } + mockInstallerInternal.EXPECT().RegisterClusterInternal(gomock.Any(), gomock.Any(), gomock.Any()). Do(func(ctx, kubeKey interface{}, params installer.V2RegisterClusterParams) { Expect(swag.StringValue(params.NewClusterParams.HighAvailabilityMode)). To(Equal(HighAvailabilityModeNone)) }).Return(clusterReply, nil) + cluster := newClusterDeployment(clusterName, testNamespace, defaultClusterSpec) Expect(c.Create(ctx, cluster)).ShouldNot(HaveOccurred()) + aci := newAgentClusterInstall(agentClusterInstallName, testNamespace, defaultAgentClusterInstallSpec, cluster) aci.Spec.ProvisionRequirements.WorkerAgents = 0 aci.Spec.ProvisionRequirements.ControlPlaneAgents = 1 + Expect(c.Create(ctx, aci)).ShouldNot(HaveOccurred()) request := newClusterDeploymentRequest(cluster) _, err := cr.Reconcile(ctx, request) @@ -794,6 +815,7 @@ var _ = Describe("cluster reconcile", func() { Expect(c.Create(ctx, imageSet)).To(BeNil()) mockMirrorRegistries.EXPECT().IsMirrorRegistriesConfigured().AnyTimes().Return(false) mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(releaseImage, nil).AnyTimes() + mockInstallerInternal.EXPECT().ValidatePullSecret(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() }) @@ -835,6 +857,9 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(dbCluster, nil) mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil).AnyTimes() mockClusterApi.EXPECT().IsReadyForInstallation(gomock.Any()).Return(false, "").AnyTimes() + mockClusterApi.EXPECT().GetHostCountByRole(gomock.Any(), models.HostRoleMaster, true).Return(swag.Int64(1), nil).AnyTimes() + mockClusterApi.EXPECT().GetHostCountByRole(gomock.Any(), models.HostRoleWorker, true).Return(swag.Int64(0), nil).AnyTimes() + request := newClusterDeploymentRequest(cluster) result, err := cr.Reconcile(ctx, request) Expect(err).To(BeNil()) @@ -951,6 +976,9 @@ var _ = Describe("cluster reconcile", func() { mockClusterApi.EXPECT().IsReadyForInstallation(gomock.Any()).Return(false, "").Times(1) mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(releaseImage, nil).AnyTimes() + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleMaster, true).Return(swag.Int64(3), nil).AnyTimes() + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleWorker, true).Return(swag.Int64(2), nil).AnyTimes() + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) cluster := newClusterDeployment(clusterName, testNamespace, defaultClusterSpec) Expect(c.Create(ctx, cluster)).ShouldNot(HaveOccurred()) @@ -1003,6 +1031,7 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil) mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(releaseImage, nil).AnyTimes() + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) pullSecret := getDefaultTestPullSecret("pull-secret", testNamespace) Expect(c.Create(ctx, pullSecret)).To(BeNil()) @@ -1030,6 +1059,7 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil).AnyTimes() mockInstallerInternal.EXPECT().ValidatePullSecret(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(releaseImage, nil).AnyTimes() + pullSecret := getDefaultTestPullSecret("pull-secret", testNamespace) Expect(c.Create(ctx, pullSecret)).To(BeNil()) imageSet := getDefaultTestImageSet(imageSetName, releaseImageUrl) @@ -1063,6 +1093,8 @@ var _ = Describe("cluster reconcile", func() { PullSecret: testPullSecretVal, } + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) + // caCertificateReference := aci.Spec.IgnitionEndpoint.CaCertificateReference caCertificateData := map[string][]byte{ corev1.TLSCertKey: ignitionCert, @@ -1108,6 +1140,7 @@ var _ = Describe("cluster reconcile", func() { mockMirrorRegistries.EXPECT().IsMirrorRegistriesConfigured().AnyTimes().Return(false) mockInstallerInternal.EXPECT().ValidatePullSecret(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(releaseImage, nil).AnyTimes() + mockInstallerInternal.EXPECT().ValidatePullSecret(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() kubeconfig := "kubeconfig content" mockInstallerInternal.EXPECT().V2DownloadClusterCredentialsInternal(gomock.Any(), gomock.Any()).Return(io.NopCloser(strings.NewReader(kubeconfig)), int64(len(kubeconfig)), nil).AnyTimes() @@ -1191,6 +1224,7 @@ var _ = Describe("cluster reconcile", func() { mockMirrorRegistries.EXPECT().IsMirrorRegistriesConfigured().AnyTimes().Return(false) mockInstallerInternal.EXPECT().ValidatePullSecret(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(releaseImage, nil).AnyTimes() + kubeconfig := "kubeconfig content" mockInstallerInternal.EXPECT().V2DownloadClusterCredentialsInternal(gomock.Any(), gomock.Any()).Return(io.NopCloser(strings.NewReader(kubeconfig)), int64(len(kubeconfig)), nil).AnyTimes() serviceBaseURL := "http://acme.com" @@ -1508,6 +1542,7 @@ var _ = Describe("cluster reconcile", func() { backEndCluster.Hosts = hosts mockMirrorRegistries.EXPECT().IsMirrorRegistriesConfigured().AnyTimes().Return(false) mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(releaseImage, nil).AnyTimes() + }) It("success", func() { @@ -1517,7 +1552,9 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().GetKnownHostApprovedCounts(gomock.Any()).Return(5, 5, nil).Times(1) mockManifestsApi.EXPECT().ListClusterManifestsInternal(gomock.Any(), gomock.Any()).Return(models.ListManifests{}, nil).Times(1) mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) - mockInstallerInternal.EXPECT().ValidatePullSecret(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + mockInstallerInternal.EXPECT().ValidatePullSecret(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(2) + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleMaster, true).Return(swag.Int64(3), nil).AnyTimes() + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleWorker, true).Return(swag.Int64(2), nil).AnyTimes() installClusterReply := &common.Cluster{ Cluster: models.Cluster{ @@ -1528,7 +1565,7 @@ var _ = Describe("cluster reconcile", func() { } mockInstallerInternal.EXPECT().InstallClusterInternal(gomock.Any(), gomock.Any()). Return(installClusterReply, nil) - + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil).AnyTimes() request := newClusterDeploymentRequest(cluster) result, err := cr.Reconcile(ctx, request) Expect(err).To(BeNil()) @@ -1548,6 +1585,9 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().GetKnownHostApprovedCounts(gomock.Any()).Return(5, 5, nil).Times(2) mockManifestsApi.EXPECT().ListClusterManifestsInternal(gomock.Any(), gomock.Any()).Return(models.ListManifests{}, nil).Times(1) mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil).Times(2) + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleMaster, true).Return(swag.Int64(3), nil).AnyTimes() + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleWorker, true).Return(swag.Int64(2), nil).AnyTimes() + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil).AnyTimes() installClusterReply := &common.Cluster{ Cluster: models.Cluster{ @@ -1597,6 +1637,9 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().GetKnownHostApprovedCounts(gomock.Any()).Return(5, 5, nil).Times(2) mockManifestsApi.EXPECT().ListClusterManifestsInternal(gomock.Any(), gomock.Any()).Return(models.ListManifests{}, nil).Times(1) mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil).Times(2) + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleMaster, true).Return(swag.Int64(3), nil).AnyTimes() + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleWorker, true).Return(swag.Int64(2), nil).AnyTimes() + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil).AnyTimes() installClusterReply := &common.Cluster{ Cluster: models.Cluster{ @@ -1646,6 +1689,9 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().GetKnownHostApprovedCounts(gomock.Any()).Return(5, 5, nil).Times(1) mockManifestsApi.EXPECT().ListClusterManifestsInternal(gomock.Any(), gomock.Any()).Return(models.ListManifests{}, nil).Times(1) mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleMaster, true).Return(swag.Int64(3), nil).AnyTimes() + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleWorker, true).Return(swag.Int64(2), nil).AnyTimes() + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) cvoStatusInfo := "Working towards 4.8.0-rc.0: 654 of 676 done (96% complete)" oper := make([]*models.MonitoredOperator, 1) @@ -1687,6 +1733,9 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().GetKnownHostApprovedCounts(gomock.Any()).Return(5, 5, nil).Times(1) mockManifestsApi.EXPECT().ListClusterManifestsInternal(gomock.Any(), gomock.Any()).Return(models.ListManifests{}, nil).Times(1) mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleMaster, true).Return(swag.Int64(3), nil).AnyTimes() + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleWorker, true).Return(swag.Int64(2), nil).AnyTimes() + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) oper := make([]*models.MonitoredOperator, 1) oper[0] = &models.MonitoredOperator{ @@ -1728,6 +1777,8 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().GetCredentialsInternal(gomock.Any(), gomock.Any()).Return(&models.Credentials{Password: "foo", Username: "bar"}, nil).Times(1) mockInstallerInternal.EXPECT().V2DownloadClusterCredentialsInternal(gomock.Any(), gomock.Any()).Return(io.NopCloser(strings.NewReader(kubeconfig)), int64(len(kubeconfig)), nil).Times(1) mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil).AnyTimes() + request := newClusterDeploymentRequest(cluster) result, err := cr.Reconcile(ctx, request) Expect(err).To(BeNil()) @@ -1766,6 +1817,8 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().ValidatePullSecret(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mockClusterApi.EXPECT().IsReadyForInstallation(gomock.Any()).Return(false, "").Times(1) mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil).AnyTimes() + password := "test" username := "admin" kubeconfigNoIngress := "kubeconfig content NOINGRESS" @@ -1811,6 +1864,7 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil).Times(2) mockInstallerInternal.EXPECT().ValidatePullSecret(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil).AnyTimes() password := "test" username := "admin" kubeconfig := "kubeconfig content" @@ -1834,6 +1888,7 @@ var _ = Describe("cluster reconcile", func() { }, PullSecret: testPullSecretVal, } + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(day2backEndCluster, nil).AnyTimes() mockInstallerInternal.EXPECT().TransformClusterToDay2Internal(gomock.Any(), gomock.Any()).Times(1).Return(day2backEndCluster, nil) cluster = newClusterDeployment("test-cluster-sno", testNamespace, defaultClusterSpec) cluster.Spec.BaseDomain = "hive.example.com" @@ -1874,6 +1929,7 @@ var _ = Describe("cluster reconcile", func() { backEndCluster.Kind = swag.String(models.ClusterKindCluster) mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil).Times(1) mockInstallerInternal.EXPECT().ValidatePullSecret(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) day2backEndCluster := &common.Cluster{ Cluster: models.Cluster{ @@ -1922,6 +1978,7 @@ var _ = Describe("cluster reconcile", func() { backEndCluster.Kind = swag.String(models.ClusterKindCluster) mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil).Times(1) mockInstallerInternal.EXPECT().ValidatePullSecret(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) password := "test" username := "admin" cred := &models.Credentials{ @@ -1953,6 +2010,7 @@ var _ = Describe("cluster reconcile", func() { backEndCluster.Kind = swag.String(models.ClusterKindCluster) mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil).Times(1) mockInstallerInternal.EXPECT().ValidatePullSecret(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) expectedErr := "internal error" mockInstallerInternal.EXPECT().GetCredentialsInternal(gomock.Any(), gomock.Any()).Return(nil, errors.New(expectedErr)).Times(1) request := newClusterDeploymentRequest(cluster) @@ -1981,6 +2039,9 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().GetKnownHostApprovedCounts(gomock.Any()).Return(5, 5, nil).Times(2) mockManifestsApi.EXPECT().ListClusterManifestsInternal(gomock.Any(), gomock.Any()).Return(models.ListManifests{}, nil).Times(1) mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleMaster, true).Return(swag.Int64(3), nil).AnyTimes() + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleWorker, true).Return(swag.Int64(2), nil).AnyTimes() + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) request := newClusterDeploymentRequest(cluster) result, err := cr.Reconcile(ctx, request) @@ -2004,6 +2065,7 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil) mockInstallerInternal.EXPECT().ValidatePullSecret(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) request := newClusterDeploymentRequest(cluster) result, err := cr.Reconcile(ctx, request) Expect(err).To(BeNil()) @@ -2021,6 +2083,9 @@ var _ = Describe("cluster reconcile", func() { mockClusterApi.EXPECT().IsReadyForInstallation(gomock.Any()).Return(true, "").Times(1) mockInstallerInternal.EXPECT().GetKnownHostApprovedCounts(gomock.Any()).Return(5, 0, nil).Times(1) mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleMaster, true).Return(swag.Int64(3), nil).AnyTimes() + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleWorker, true).Return(swag.Int64(2), nil).AnyTimes() + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) Expect(c.Update(ctx, cluster)).Should(BeNil()) mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil) @@ -2042,6 +2107,9 @@ var _ = Describe("cluster reconcile", func() { mockClusterApi.EXPECT().IsReadyForInstallation(gomock.Any()).Return(true, "").Times(1) mockInstallerInternal.EXPECT().GetKnownHostApprovedCounts(gomock.Any()).Return(5, 0, nil).Times(2) mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleMaster, true).Return(swag.Int64(3), nil).AnyTimes() + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleWorker, true).Return(swag.Int64(2), nil).AnyTimes() + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) Expect(c.Update(ctx, cluster)).Should(BeNil()) mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil) @@ -2069,29 +2137,56 @@ var _ = Describe("cluster reconcile", func() { Expect(c.Update(ctx, cluster)).Should(BeNil()) mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil) - mockInstallerInternal.EXPECT().ValidatePullSecret(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + mockInstallerInternal.EXPECT().ValidatePullSecret(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(2) + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleMaster, true).Return(swag.Int64(3), nil).Times(2) + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleWorker, true).Return(swag.Int64(1), nil).Times(2) + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) + request := newClusterDeploymentRequest(cluster) result, err := cr.Reconcile(ctx, request) Expect(err).To(BeNil()) Expect(result).To(Equal(ctrl.Result{})) aci = getTestClusterInstall() - expectedHosts := defaultAgentClusterInstallSpec.ProvisionRequirements.ControlPlaneAgents + - defaultAgentClusterInstallSpec.ProvisionRequirements.WorkerAgents - msg := fmt.Sprintf(hiveext.ClusterInsufficientAgentsMsg, expectedHosts, 0) + + expectedMasterCount := defaultAgentClusterInstallSpec.ProvisionRequirements.ControlPlaneAgents + expectedWorkerCount := defaultAgentClusterInstallSpec.ProvisionRequirements.WorkerAgents + + actualMasterCount := 3 + actualWorkerCount := 1 + + msg := fmt.Sprintf(hiveext.ClusterInsufficientAgentsMsg, expectedMasterCount, expectedWorkerCount, actualMasterCount, actualWorkerCount) + Expect(FindStatusCondition(aci.Status.Conditions, hiveext.ClusterSpecSyncedCondition).Reason).To(Equal(hiveext.ClusterSyncedOkReason)) Expect(FindStatusCondition(aci.Status.Conditions, hiveext.ClusterRequirementsMetCondition).Reason).To(Equal(hiveext.ClusterInsufficientAgentsReason)) Expect(FindStatusCondition(aci.Status.Conditions, hiveext.ClusterRequirementsMetCondition).Message).To(Equal(msg)) Expect(FindStatusCondition(aci.Status.Conditions, hiveext.ClusterRequirementsMetCondition).Status).To(Equal(corev1.ConditionFalse)) }) - It("ready for installation - but too much approved hosts", func() { + It("ready for installation - but not all hosts are ready, failed counting masters and workers, fail updating status", func() { backEndCluster.Status = swag.String(models.ClusterStatusReady) mockClusterApi.EXPECT().IsReadyForInstallation(gomock.Any()).Return(true, "").Times(1) - mockInstallerInternal.EXPECT().GetKnownHostApprovedCounts(gomock.Any()).Return(5, 5, nil).Times(2) + mockInstallerInternal.EXPECT().GetKnownHostApprovedCounts(gomock.Any()).Return(0, 0, nil).Times(2) mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) - aci.Spec.ProvisionRequirements.WorkerAgents = 0 + // succeed in 'isReadyForInstallation' + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleMaster, true). + Return(swag.Int64(3), nil). + Times(1) + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleWorker, true). + Return(swag.Int64(2), nil). + Times(1) + + // fail in 'clusterRequirementsMet' + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleMaster, true). + Return(nil, fmt.Errorf("failed to count the number of hosts in cluster with ID '%s' and suggested role 'master'", string(*backEndCluster.ID))). + Times(1) + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleWorker, true). + Return(nil, fmt.Errorf("failed to count the number of hosts in cluster with ID '%s' and suggested role 'worker'", string(*backEndCluster.ID))). + Times(1) + + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) + Expect(c.Update(ctx, aci)).Should(BeNil()) mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil) mockInstallerInternal.EXPECT().ValidatePullSecret(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() @@ -2101,21 +2196,77 @@ var _ = Describe("cluster reconcile", func() { Expect(result).To(Equal(ctrl.Result{})) aci = getTestClusterInstall() - expectedHosts := aci.Spec.ProvisionRequirements.ControlPlaneAgents + aci.Spec.ProvisionRequirements.WorkerAgents - msg := fmt.Sprintf(hiveext.ClusterAdditionalAgentsMsg, expectedHosts, 5) + + expectedMasterCount := defaultAgentClusterInstallSpec.ProvisionRequirements.ControlPlaneAgents + expectedWorkerCount := defaultAgentClusterInstallSpec.ProvisionRequirements.WorkerAgents + + actualMasterCount := 0 + actualWorkerCount := 0 + + msg := fmt.Sprintf(hiveext.ClusterInsufficientAgentsMsg, expectedMasterCount, expectedWorkerCount, actualMasterCount, actualWorkerCount) + Expect(FindStatusCondition(aci.Status.Conditions, hiveext.ClusterSpecSyncedCondition).Reason).To(Equal(hiveext.ClusterSyncedOkReason)) - Expect(FindStatusCondition(aci.Status.Conditions, hiveext.ClusterRequirementsMetCondition).Reason).To(Equal(hiveext.ClusterAdditionalAgentsReason)) + Expect(FindStatusCondition(aci.Status.Conditions, hiveext.ClusterRequirementsMetCondition).Status).To(Equal(corev1.ConditionFalse)) Expect(FindStatusCondition(aci.Status.Conditions, hiveext.ClusterRequirementsMetCondition).Message).To(Equal(msg)) + }) + + It("ready for installation - but not all hosts are ready, failed counting masters and workers, fail checking installation readiness", func() { + backEndCluster.Status = swag.String(models.ClusterStatusReady) + mockClusterApi.EXPECT().IsReadyForInstallation(gomock.Any()).Return(true, "").Times(1) + mockInstallerInternal.EXPECT().GetKnownHostApprovedCounts(gomock.Any()).Return(0, 0, nil).Times(2) + mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) + + // fail in 'isReadyForInstallation' + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleMaster, true). + Return(nil, fmt.Errorf("failed to count the number of hosts in cluster with ID '%s' and suggested role 'master'", string(*backEndCluster.ID))). + Times(1) + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleWorker, true). + Return(nil, fmt.Errorf("failed to count the number of hosts in cluster with ID '%s' and suggested role 'worker'", string(*backEndCluster.ID))). + Times(1) + + // succeed in 'clusterRequirementsMet' + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleMaster, true). + Return(swag.Int64(3), nil). + Times(1) + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleWorker, true). + Return(swag.Int64(2), nil). + Times(1) + + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) + + Expect(c.Update(ctx, aci)).Should(BeNil()) + mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil) + mockInstallerInternal.EXPECT().ValidatePullSecret(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + request := newClusterDeploymentRequest(cluster) + result, err := cr.Reconcile(ctx, request) + Expect(err).To(BeNil()) + Expect(result).To(Equal(ctrl.Result{RequeueAfter: defaultRequeueAfterOnError})) + + aci = getTestClusterInstall() + + expectedErr := fmt.Sprintf( + "failed to count the number of hosts in cluster with ID '%s' and suggested role 'master', failed to count the number of hosts in cluster with ID '%s' and suggested role 'worker'", + string(*backEndCluster.ID), + string(*backEndCluster.ID), + ) + expectedSyncMsg := fmt.Sprintf("%s %s", hiveext.ClusterBackendErrorMsg, expectedErr) + + Expect(FindStatusCondition(aci.Status.Conditions, hiveext.ClusterSpecSyncedCondition).Reason).To(Equal(hiveext.ClusterBackendErrorReason)) + Expect(FindStatusCondition(aci.Status.Conditions, hiveext.ClusterSpecSyncedCondition).Message).To(Equal(expectedSyncMsg)) Expect(FindStatusCondition(aci.Status.Conditions, hiveext.ClusterRequirementsMetCondition).Status).To(Equal(corev1.ConditionFalse)) + Expect(FindStatusCondition(aci.Status.Conditions, hiveext.ClusterRequirementsMetCondition).Reason).To(Equal(hiveext.ClusterUnapprovedAgentsReason)) }) - It("ready for installation - but too much registered hosts", func() { + It("ready for installation - accurate amount of masters and workers", func() { backEndCluster.Status = swag.String(models.ClusterStatusReady) mockClusterApi.EXPECT().IsReadyForInstallation(gomock.Any()).Return(true, "").Times(1) - mockInstallerInternal.EXPECT().GetKnownHostApprovedCounts(gomock.Any()).Return(5, 3, nil).Times(2) + mockInstallerInternal.EXPECT().GetKnownHostApprovedCounts(gomock.Any()).Return(5, 5, nil).Times(2) mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleMaster, true).Return(swag.Int64(3), nil).Times(2) + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleWorker, true).Return(swag.Int64(1), nil).Times(2) + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) - aci.Spec.ProvisionRequirements.WorkerAgents = 0 + aci.Spec.ProvisionRequirements.WorkerAgents = 1 Expect(c.Update(ctx, aci)).Should(BeNil()) mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil) mockInstallerInternal.EXPECT().ValidatePullSecret(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() @@ -2125,12 +2276,8 @@ var _ = Describe("cluster reconcile", func() { Expect(result).To(Equal(ctrl.Result{})) aci = getTestClusterInstall() - expectedHosts := aci.Spec.ProvisionRequirements.ControlPlaneAgents + aci.Spec.ProvisionRequirements.WorkerAgents - msg := fmt.Sprintf(hiveext.ClusterAdditionalAgentsMsg, expectedHosts, 5) - Expect(FindStatusCondition(aci.Status.Conditions, hiveext.ClusterSpecSyncedCondition).Reason).To(Equal(hiveext.ClusterSyncedOkReason)) - Expect(FindStatusCondition(aci.Status.Conditions, hiveext.ClusterRequirementsMetCondition).Reason).To(Equal(hiveext.ClusterAdditionalAgentsReason)) - Expect(FindStatusCondition(aci.Status.Conditions, hiveext.ClusterRequirementsMetCondition).Message).To(Equal(msg)) - Expect(FindStatusCondition(aci.Status.Conditions, hiveext.ClusterRequirementsMetCondition).Status).To(Equal(corev1.ConditionFalse)) + + Expect(FindStatusCondition(aci.Status.Conditions, hiveext.ClusterRequirementsMetCondition).Status).To(Equal(corev1.ConditionTrue)) }) It("install day2 host", func() { @@ -2157,6 +2304,8 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().GetKnownApprovedHosts(gomock.Any()).Return(commonHosts, nil) mockInstallerInternal.EXPECT().InstallSingleDay2HostInternal(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) + request := newClusterDeploymentRequest(cluster) result, err := cr.Reconcile(ctx, request) Expect(err).To(BeNil()) @@ -2188,6 +2337,8 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) mockInstallerInternal.EXPECT().GetKnownApprovedHosts(gomock.Any()).Return(commonHosts, nil) mockInstallerInternal.EXPECT().InstallSingleDay2HostInternal(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New(expectedErr)) + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) + request := newClusterDeploymentRequest(cluster) result, err := cr.Reconcile(ctx, request) Expect(err).To(BeNil()) @@ -2214,6 +2365,9 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().GetKnownHostApprovedCounts(gomock.Any()).Return(5, 5, nil).Times(2) mockManifestsApi.EXPECT().ListClusterManifestsInternal(gomock.Any(), gomock.Any()).Return(models.ListManifests{}, nil).Times(1) mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleMaster, true).Return(swag.Int64(3), nil).AnyTimes() + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleWorker, true).Return(swag.Int64(2), nil).AnyTimes() + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) request := newClusterDeploymentRequest(cluster) result, err := cr.Reconcile(ctx, request) @@ -2252,6 +2406,10 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().GetKnownHostApprovedCounts(gomock.Any()).Return(5, 5, nil).Times(2) mockManifestsApi.EXPECT().ListClusterManifestsInternal(gomock.Any(), gomock.Any()).Return(models.ListManifests{}, nil).Times(1) mockManifestsApi.EXPECT().CreateClusterManifestInternal(gomock.Any(), gomock.Any(), true).Return(nil, errors.Errorf("error")).Times(1) + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleMaster, true).Return(swag.Int64(3), nil).AnyTimes() + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleWorker, true).Return(swag.Int64(2), nil).AnyTimes() + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) + request := newClusterDeploymentRequest(cluster) aci = getTestClusterInstall() aci.Spec.ManifestsConfigMapRef = ref @@ -2278,6 +2436,9 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) mockInstallerInternal.EXPECT().GetKnownHostApprovedCounts(gomock.Any()).Return(5, 5, nil).Times(2) mockManifestsApi.EXPECT().ListClusterManifestsInternal(gomock.Any(), gomock.Any()).Return(nil, errors.Errorf("error")).Times(1) + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleMaster, true).Return(swag.Int64(3), nil).AnyTimes() + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleWorker, true).Return(swag.Int64(2), nil).AnyTimes() + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) request := newClusterDeploymentRequest(cluster) cluster = getTestCluster() @@ -2320,6 +2481,10 @@ var _ = Describe("cluster reconcile", func() { mockManifestsApi.EXPECT().ListClusterManifestsInternal(gomock.Any(), gomock.Any()).Return(models.ListManifests{}, nil).Times(1) mockInstallerInternal.EXPECT().GetKnownHostApprovedCounts(gomock.Any()).Return(5, 5, nil).Times(1) mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleMaster, true).Return(swag.Int64(3), nil).AnyTimes() + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleWorker, true).Return(swag.Int64(2), nil).AnyTimes() + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) + installClusterReply := &common.Cluster{ Cluster: models.Cluster{ ID: backEndCluster.ID, @@ -2362,6 +2527,10 @@ var _ = Describe("cluster reconcile", func() { mockManifestsApi.EXPECT().ListClusterManifestsInternal(gomock.Any(), gomock.Any()).Return(nil, nil).Times(1) mockClusterApi.EXPECT().IsReadyForInstallation(gomock.Any()).Return(true, "").Times(1) mockInstallerInternal.EXPECT().GetKnownHostApprovedCounts(gomock.Any()).Return(5, 5, nil).Times(1) + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleMaster, true).Return(swag.Int64(3), nil).AnyTimes() + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleWorker, true).Return(swag.Int64(2), nil).AnyTimes() + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) + request := newClusterDeploymentRequest(cluster) result, err := cr.Reconcile(ctx, request) Expect(err).To(BeNil()) @@ -2382,6 +2551,10 @@ var _ = Describe("cluster reconcile", func() { mockClusterApi.EXPECT().IsReadyForInstallation(gomock.Any()).Return(true, "").Times(1) mockInstallerInternal.EXPECT().GetKnownHostApprovedCounts(gomock.Any()).Return(5, 5, nil).Times(1) mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleMaster, true).Return(swag.Int64(3), nil).AnyTimes() + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleWorker, true).Return(swag.Int64(2), nil).AnyTimes() + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) + installClusterReply := &common.Cluster{ Cluster: models.Cluster{ ID: backEndCluster.ID, @@ -2445,6 +2618,10 @@ var _ = Describe("cluster reconcile", func() { mockManifestsApi.EXPECT().ListClusterManifestsInternal(gomock.Any(), gomock.Any()).Return(models.ListManifests{}, nil).Times(1) mockInstallerInternal.EXPECT().GetKnownHostApprovedCounts(gomock.Any()).Return(5, 5, nil).Times(1) mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleMaster, true).Return(swag.Int64(3), nil).AnyTimes() + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleWorker, true).Return(swag.Int64(2), nil).AnyTimes() + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) + installClusterReply := &common.Cluster{ Cluster: models.Cluster{ ID: backEndCluster.ID, @@ -2509,6 +2686,9 @@ var _ = Describe("cluster reconcile", func() { mockManifestsApi.EXPECT().ListClusterManifestsInternal(gomock.Any(), gomock.Any()).Return(models.ListManifests{}, nil).Times(1) mockInstallerInternal.EXPECT().GetKnownHostApprovedCounts(gomock.Any()).Return(5, 5, nil).Times(2) mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleMaster, true).Return(swag.Int64(3), nil).AnyTimes() + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleWorker, true).Return(swag.Int64(2), nil).AnyTimes() + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) cluster = getTestCluster() aci.Spec.ManifestsConfigMapRefs = refs @@ -2568,6 +2748,9 @@ var _ = Describe("cluster reconcile", func() { mockManifestsApi.EXPECT().ListClusterManifestsInternal(gomock.Any(), gomock.Any()).Return(models.ListManifests{}, nil).Times(1) mockInstallerInternal.EXPECT().GetKnownHostApprovedCounts(gomock.Any()).Return(5, 5, nil).Times(2) mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleMaster, true).Return(swag.Int64(3), nil).AnyTimes() + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleWorker, true).Return(swag.Int64(2), nil).AnyTimes() + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) cluster = getTestCluster() aci.Spec.ManifestsConfigMapRefs = refs @@ -2595,6 +2778,9 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().GetKnownHostApprovedCounts(gomock.Any()).Return(5, 5, nil).Times(1) mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) mockManifestsApi.EXPECT().ListClusterManifestsInternal(gomock.Any(), gomock.Any()).Return(models.ListManifests{}, nil).Times(1) + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleMaster, true).Return(swag.Int64(3), nil).AnyTimes() + mockClusterApi.EXPECT().GetHostCountByRole(*backEndCluster.ID, models.HostRoleWorker, true).Return(swag.Int64(2), nil).AnyTimes() + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) installClusterReply := &common.Cluster{ Cluster: models.Cluster{ @@ -2700,6 +2886,7 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().ValidatePullSecret(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(releaseImage, nil) + updateReply := &common.Cluster{ Cluster: models.Cluster{ ID: &sId, @@ -2752,6 +2939,7 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().ValidatePullSecret(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(releaseImage, nil) + updateReply := getDefaultTestCluster() mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()). @@ -2798,6 +2986,7 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().ValidatePullSecret(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(releaseImage, nil) + updateReply := &common.Cluster{ Cluster: models.Cluster{ ID: &sId, @@ -2916,6 +3105,8 @@ var _ = Describe("cluster reconcile", func() { Expect(param.ClusterUpdateParams.MachineNetworks).Should( Equal(machineNetworksEntriesToArray(test.expectedMachineNetworks))) }).Return(updateReply, nil) + } else { + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) } request := newClusterDeploymentRequest(cluster) @@ -2932,6 +3123,7 @@ var _ = Describe("cluster reconcile", func() { It("only state changed", func() { mockMirrorRegistries.EXPECT().IsMirrorRegistriesConfigured().AnyTimes().Return(false) mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(releaseImage, nil) + backEndCluster := &common.Cluster{ Cluster: models.Cluster{ ID: &sId, @@ -2954,6 +3146,7 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil) mockInstallerInternal.EXPECT().ValidatePullSecret(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) mockClusterApi.EXPECT().IsReadyForInstallation(gomock.Any()).Return(false, "").Times(1) request := newClusterDeploymentRequest(cluster) result, err := cr.Reconcile(ctx, request) @@ -2971,6 +3164,7 @@ var _ = Describe("cluster reconcile", func() { It("failed getting cluster", func() { mockInstallerInternal.EXPECT().ValidatePullSecret(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(releaseImage, nil) + mockMirrorRegistries.EXPECT().IsMirrorRegistriesConfigured().AnyTimes().Return(false) expectedErr := "some internal error" mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()). @@ -2990,6 +3184,7 @@ var _ = Describe("cluster reconcile", func() { It("update internal error", func() { mockMirrorRegistries.EXPECT().IsMirrorRegistriesConfigured().AnyTimes().Return(false) mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(releaseImage, nil) + backEndCluster := &common.Cluster{ Cluster: models.Cluster{ ID: &sId, @@ -3024,6 +3219,7 @@ var _ = Describe("cluster reconcile", func() { It("add install config overrides annotation", func() { mockMirrorRegistries.EXPECT().IsMirrorRegistriesConfigured().AnyTimes().Return(false) mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(releaseImage, nil) + backEndCluster := &common.Cluster{ Cluster: models.Cluster{ ID: &sId, @@ -3044,6 +3240,7 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil) mockInstallerInternal.EXPECT().ValidatePullSecret(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) installConfigOverrides := `{"controlPlane": {"hyperthreading": "Disabled"}}` updateReply := &common.Cluster{ Cluster: models.Cluster{ @@ -3070,6 +3267,7 @@ var _ = Describe("cluster reconcile", func() { It("Remove existing install config overrides annotation", func() { mockMirrorRegistries.EXPECT().IsMirrorRegistriesConfigured().AnyTimes().Return(false) mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(releaseImage, nil) + backEndCluster := &common.Cluster{ Cluster: models.Cluster{ ID: &sId, @@ -3098,6 +3296,7 @@ var _ = Describe("cluster reconcile", func() { }, PullSecret: testPullSecretVal, } + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) mockInstallerInternal.EXPECT().UpdateClusterInstallConfigInternal(gomock.Any(), gomock.Any()). Do(func(ctx context.Context, param installer.V2UpdateClusterInstallConfigParams) { Expect(param.ClusterID).To(Equal(sId)) @@ -3113,6 +3312,7 @@ var _ = Describe("cluster reconcile", func() { It("Update install config overrides annotation", func() { mockMirrorRegistries.EXPECT().IsMirrorRegistriesConfigured().AnyTimes().Return(false) mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(releaseImage, nil) + backEndCluster := &common.Cluster{ Cluster: models.Cluster{ ID: &sId, @@ -3134,6 +3334,7 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil) mockInstallerInternal.EXPECT().ValidatePullSecret(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) installConfigOverrides := `{"controlPlane": {"hyperthreading": "Enabled"}}` updateReply := &common.Cluster{ Cluster: models.Cluster{ @@ -3160,6 +3361,7 @@ var _ = Describe("cluster reconcile", func() { It("invalid install config overrides annotation", func() { mockMirrorRegistries.EXPECT().IsMirrorRegistriesConfigured().AnyTimes().Return(false) mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(releaseImage, nil) + backEndCluster := &common.Cluster{ Cluster: models.Cluster{ ID: &sId, @@ -3180,6 +3382,7 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil) mockInstallerInternal.EXPECT().ValidatePullSecret(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) installConfigOverrides := `{{{"controlPlane": ""` mockInstallerInternal.EXPECT().UpdateClusterInstallConfigInternal(gomock.Any(), gomock.Any()). Do(func(ctx context.Context, param installer.V2UpdateClusterInstallConfigParams) { @@ -3234,6 +3437,8 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().ValidatePullSecret(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(releaseImage, nil) + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) + pullSecret := getDefaultTestPullSecret("pull-secret", testNamespace) Expect(c.Create(ctx, pullSecret)).To(BeNil()) imageSetName := getDefaultTestImageSet(imageSetName, releaseImageUrl) @@ -3277,6 +3482,7 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().V2DownloadClusterCredentialsInternal(gomock.Any(), gomock.Any()).Return(io.NopCloser(strings.NewReader("kubeconfig")), int64(len("kubeconfig")), nil).Times(1) mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(releaseImage, nil) + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()). Do(func(ctx context.Context, param installer.V2UpdateClusterParams) { Expect(len(param.ClusterUpdateParams.APIVips)).To(Equal(1)) @@ -3328,6 +3534,8 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().V2DownloadClusterCredentialsInternal(gomock.Any(), gomock.Any()).Return(io.NopCloser(strings.NewReader("kubeconfig")), int64(len("kubeconfig")), nil).Times(1) mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(releaseImage, nil) + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(backEndCluster, nil) + pullSecret := getDefaultTestPullSecret("pull-secret", testNamespace) Expect(c.Create(ctx, pullSecret)).To(BeNil()) imageSetName := getDefaultTestImageSet(imageSetName, releaseImageUrl) @@ -3367,6 +3575,7 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(nil, gorm.ErrRecordNotFound) mockMirrorRegistries.EXPECT().IsMirrorRegistriesConfigured().AnyTimes().Return(false) mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(releaseImage, nil) + }) It("success", func() { @@ -3384,6 +3593,8 @@ var _ = Describe("cluster reconcile", func() { }, } + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(clusterReply, nil).AnyTimes() + V2ImportClusterInternal := func(ctx context.Context, kubeKey *types.NamespacedName, id *strfmt.UUID, params installer.V2ImportClusterParams) (*common.Cluster, error) { Expect(string(*params.NewImportClusterParams.OpenshiftClusterID)).To(Equal(cid)) @@ -3394,6 +3605,7 @@ var _ = Describe("cluster reconcile", func() { DoAndReturn(V2ImportClusterInternal) mockInstallerInternal.EXPECT().HostWithCollectedLogsExists(gomock.Any()).Return(false, nil) + request := newClusterDeploymentRequest(cluster) result, err := cr.Reconcile(ctx, request) Expect(err).To(BeNil()) @@ -3443,6 +3655,7 @@ var _ = Describe("cluster reconcile", func() { BeforeEach(func() { mockMirrorRegistries.EXPECT().IsMirrorRegistriesConfigured().AnyTimes().Return(false) mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(releaseImage, nil) + pullSecret := getDefaultTestPullSecret("pull-secret", testNamespace) Expect(c.Create(ctx, pullSecret)).To(BeNil()) imageSet := getDefaultTestImageSet(imageSetName, releaseImageUrl) @@ -3601,6 +3814,7 @@ var _ = Describe("cluster reconcile", func() { mockClientFactory.EXPECT().CreateFromSecret(gomock.Any()).Return(mockClient, nil).AnyTimes() mockClient.EXPECT().PatchMachineConfigPoolPaused(gomock.Any(), true, "worker").Return(nil) mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(releaseImage, nil).AnyTimes() + result, err := cr.Reconcile(ctx, request) Expect(err).To(BeNil()) Expect(result).To(Equal(ctrl.Result{})) @@ -3683,6 +3897,7 @@ var _ = Describe("TestConditions", func() { URL: &releaseImageUrl, Version: &ocpReleaseVersion, } + mockClusterApi *cluster.MockAPI ) BeforeEach(func() { @@ -3693,7 +3908,7 @@ var _ = Describe("TestConditions", func() { mockMirrorRegistries = mirrorregistries.NewMockMirrorRegistriesConfigBuilder(mockCtrl) mockMirrorRegistries.EXPECT().IsMirrorRegistriesConfigured().AnyTimes().Return(false) mockInstallerInternal = bminventory.NewMockInstallerInternals(mockCtrl) - mockClusterApi := cluster.NewMockAPI(mockCtrl) + mockClusterApi = cluster.NewMockAPI(mockCtrl) cr = &ClusterDeploymentsReconciler{ Client: c, APIReader: c, @@ -3733,6 +3948,7 @@ var _ = Describe("TestConditions", func() { kubeconfigNoIngress := "kubeconfig content NOINGRESS" mockInstallerInternal.EXPECT().V2DownloadClusterCredentialsInternal(gomock.Any(), gomock.Any()).Return(io.NopCloser(strings.NewReader(kubeconfigNoIngress)), int64(len(kubeconfigNoIngress)), nil).AnyTimes() mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(releaseImage, nil) + pullSecret := getDefaultTestPullSecret("pull-secret", testNamespace) Expect(c.Create(ctx, pullSecret)).To(BeNil()) imageSetName := getDefaultTestImageSet(imageSetName, releaseImageUrl) @@ -4032,6 +4248,10 @@ var _ = Describe("TestConditions", func() { if t.clusterStatus == models.ClusterStatusReady { mockInstallerInternal.EXPECT().GetKnownHostApprovedCounts(gomock.Any()).Return(0, 0, nil).Times(1) } + + mockClusterApi.EXPECT().GetHostCountByRole(gomock.Any(), models.HostRoleMaster, true).Return(swag.Int64(0), nil).AnyTimes() + mockClusterApi.EXPECT().GetHostCountByRole(gomock.Any(), models.HostRoleWorker, true).Return(swag.Int64(0), nil).AnyTimes() + _, err := cr.Reconcile(ctx, clusterRequest) Expect(err).To(BeNil()) cluster := &hivev1.ClusterDeployment{} @@ -4494,6 +4714,8 @@ var _ = Describe("day2 cluster", func() { mockInstallerInternal.EXPECT().ValidatePullSecret(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mockVersions.EXPECT().GetReleaseImageByURL(gomock.Any(), gomock.Any(), gomock.Any()).Return(releaseImage, nil).AnyTimes() + mockInstallerInternal.EXPECT().UpdateClusterNonInteractive(gomock.Any(), gomock.Any()).Return(dbCluster, nil) + By("first time will create the cluster") request := newClusterDeploymentRequest(cd) result, err := cr.Reconcile(ctx, request) diff --git a/internal/host/host.go b/internal/host/host.go index 904fc8a020d8..8e7e929897a0 100644 --- a/internal/host/host.go +++ b/internal/host/host.go @@ -91,9 +91,9 @@ type API interface { IndexOfStage(element models.HostStage, data []models.HostStage) int IsInstallable(h *models.Host) bool // auto assign host role - AutoAssignRole(ctx context.Context, h *models.Host, db *gorm.DB) (bool, error) - RefreshRole(ctx context.Context, h *models.Host, db *gorm.DB) error - IsValidMasterCandidate(h *models.Host, c *common.Cluster, db *gorm.DB, log logrus.FieldLogger) (bool, error) + AutoAssignRole(ctx context.Context, h *models.Host, db *gorm.DB, expectedMasterCount *int) (bool, error) + RefreshRole(ctx context.Context, h *models.Host, db *gorm.DB, expectedMasterCount *int) error + IsValidMasterCandidate(h *models.Host, c *common.Cluster, db *gorm.DB, log logrus.FieldLogger, validateAgainstOperators bool) (bool, error) SetUploadLogsAt(ctx context.Context, h *models.Host, db *gorm.DB) error UpdateLogsProgress(ctx context.Context, h *models.Host, progress string) error PermanentHostsDeletion(olderThan strfmt.DateTime) error @@ -428,7 +428,7 @@ func (m *Manager) UpdateMediaConnected(ctx context.Context, h *models.Host) erro return m.updateHostAndNotify(ctx, m.db, h, updates).Error } -func (m *Manager) refreshRoleInternal(ctx context.Context, h *models.Host, db *gorm.DB, forceRefresh bool) error { +func (m *Manager) refreshRoleInternal(ctx context.Context, h *models.Host, db *gorm.DB, forceRefresh bool, expectedMasterCount *int) error { //update suggested role, if not yet set var suggestedRole models.HostRole var err error @@ -439,7 +439,7 @@ func (m *Manager) refreshRoleInternal(ctx context.Context, h *models.Host, db *g if h.Role == models.HostRoleAutoAssign && funk.ContainsString(hostStatusesBeforeInstallation[:], *h.Status) { host := *h //must have a defensive copy becuase selectRole changes the host object - if suggestedRole, err = m.selectRole(ctx, &host, db); err == nil { + if suggestedRole, err = m.selectRole(ctx, &host, db, expectedMasterCount); err == nil { m.log.Debugf("calculated role for host %s is %s (original suggested = %s)", hostutil.GetHostnameForMsg(h), suggestedRole, h.SuggestedRole) if h.SuggestedRole != suggestedRole { if err = updateRole(m.log, h, h.Role, suggestedRole, db, string(h.Role)); err == nil { @@ -504,11 +504,11 @@ func (m *Manager) refreshStatusInternal(ctx context.Context, h *models.Host, c * return nil } -func (m *Manager) RefreshRole(ctx context.Context, h *models.Host, db *gorm.DB) error { +func (m *Manager) RefreshRole(ctx context.Context, h *models.Host, db *gorm.DB, expectedMasterCount *int) error { if db == nil { db = m.db } - return m.refreshRoleInternal(ctx, h, db, true) + return m.refreshRoleInternal(ctx, h, db, true, expectedMasterCount) } func (m *Manager) RefreshStatus(ctx context.Context, h *models.Host, db *gorm.DB) error { @@ -1204,12 +1204,12 @@ func (m *Manager) updateValidationsInDB(ctx context.Context, db *gorm.DB, h *mod return hostutil.UpdateHost(logutil.FromContext(ctx, m.log), db, h.InfraEnvID, *h.ID, *h.Status, "validations_info", string(b)) } -func (m *Manager) AutoAssignRole(ctx context.Context, h *models.Host, db *gorm.DB) (bool, error) { +func (m *Manager) AutoAssignRole(ctx context.Context, h *models.Host, db *gorm.DB, expectedMasterCount *int) (bool, error) { if h.Role == models.HostRoleAutoAssign { log := logutil.FromContext(ctx, m.log) // If role is auto-assigned calculate the suggested roles // to make sure the suggestion is fresh - if err := m.RefreshRole(ctx, h, db); err != nil { //force refresh + if err := m.RefreshRole(ctx, h, db, expectedMasterCount); err != nil { //force refresh return false, err } @@ -1229,13 +1229,13 @@ func (m *Manager) AutoAssignRole(ctx context.Context, h *models.Host, db *gorm.D return false, nil } -// This function recommends a role for a given host based on these criteria: -// 1. if there are not enough masters and the host has enough capabilities to be +// selectRole recommends a role for a given host based on these criteria: +// - if there are not enough masters and the host has enough capabilities to be // a master the function select it to be a master -// 2. if there are enough masters, or it is a day2 host, or it has not enough capabilities -// to be a master the function select it to be a worker -// 3. in case of missing inventory or an internal error the function returns auto-assign -func (m *Manager) selectRole(ctx context.Context, h *models.Host, db *gorm.DB) (models.HostRole, error) { +// - if there are enough masters, or it is a day2 host, or it does not not have enough capabilities +// to be a master the function select it to be a worker +// - in case of missing inventory or an internal error the function returns auto-assign +func (m *Manager) selectRole(ctx context.Context, h *models.Host, db *gorm.DB, expectedMasterCount *int) (models.HostRole, error) { var ( autoSelectedRole = models.HostRoleAutoAssign log = logutil.FromContext(ctx, m.log) @@ -1264,7 +1264,14 @@ func (m *Manager) selectRole(ctx context.Context, h *models.Host, db *gorm.DB) ( return autoSelectedRole, err } - if len(masters) < common.MinMasterHostsNeededForInstallation { + var count int + if expectedMasterCount == nil { + count = common.MinMasterHostsNeededForInstallationInHaMode + } else { + count = *expectedMasterCount + } + + if len(masters) < count { h.Role = models.HostRoleMaster vc, err = newValidationContext(ctx, h, nil, nil, db, make(InventoryCache), m.hwValidator, m.kubeApiEnabled, m.objectHandler, m.softTimeoutsEnabled) if err != nil { @@ -1276,7 +1283,7 @@ func (m *Manager) selectRole(ctx context.Context, h *models.Host, db *gorm.DB) ( log.WithError(err).Errorf("failed to run validations on host %s", h.ID.String()) return autoSelectedRole, err } - if m.canBeMaster(conditions) { + if m.canBeMaster(conditions, true) { return models.HostRoleMaster, nil } } @@ -1284,7 +1291,7 @@ func (m *Manager) selectRole(ctx context.Context, h *models.Host, db *gorm.DB) ( return models.HostRoleWorker, nil } -func (m *Manager) IsValidMasterCandidate(h *models.Host, c *common.Cluster, db *gorm.DB, log logrus.FieldLogger) (bool, error) { +func (m *Manager) IsValidMasterCandidate(h *models.Host, c *common.Cluster, db *gorm.DB, log logrus.FieldLogger, validateAgainstOperators bool) (bool, error) { if h.Role == models.HostRoleWorker { return false, nil } @@ -1305,20 +1312,25 @@ func (m *Manager) IsValidMasterCandidate(h *models.Host, c *common.Cluster, db * return false, err } - if m.canBeMaster(conditions) { + if m.canBeMaster(conditions, validateAgainstOperators) { return true, nil } return false, nil } -func (m *Manager) canBeMaster(conditions map[string]bool) bool { - return conditions[HasCPUCoresForRole.String()] && - conditions[HasMemoryForRole.String()] && - conditions[AreLsoRequirementsSatisfied.String()] && - conditions[AreOdfRequirementsSatisfied.String()] && - conditions[AreCnvRequirementsSatisfied.String()] && - conditions[AreLvmRequirementsSatisfied.String()] +func (m *Manager) canBeMaster(conditions map[string]bool, validateAgainstOperators bool) bool { + can := conditions[HasCPUCoresForRole.String()] && + conditions[HasMemoryForRole.String()] + + if validateAgainstOperators { + return can && conditions[AreLsoRequirementsSatisfied.String()] && + conditions[AreOdfRequirementsSatisfied.String()] && + conditions[AreCnvRequirementsSatisfied.String()] && + conditions[AreLvmRequirementsSatisfied.String()] + } + + return can } func (m *Manager) GetHostValidDisks(host *models.Host) ([]*models.Disk, error) { diff --git a/internal/host/host_test.go b/internal/host/host_test.go index 98aa2495566e..f2365224d2fd 100644 --- a/internal/host/host_test.go +++ b/internal/host/host_test.go @@ -2922,7 +2922,7 @@ var _ = Describe("AutoAssignRole", func() { if isSelected { mockRoleSuggestionEvent(host) } - selected, err := hapi.AutoAssignRole(ctx, host, db) + selected, err := hapi.AutoAssignRole(ctx, host, db, swag.Int(common.MinMasterHostsNeededForInstallationInHaMode)) Expect(selected).To(Equal(isSelected)) if success { Expect(err).ShouldNot(HaveOccurred()) @@ -2993,7 +2993,7 @@ var _ = Describe("AutoAssignRole", func() { }) It("cluster already have enough master nodes", func() { - for i := 0; i < common.MinMasterHostsNeededForInstallation; i++ { + for i := 0; i < common.MinMasterHostsNeededForInstallationInHaMode; i++ { h := hostutil.GenerateTestHost(strfmt.UUID(uuid.New().String()), infraEnvId, clusterId, models.HostStatusKnown) h.Inventory = hostutil.GenerateMasterInventory() h.Role = models.HostRoleAutoAssign @@ -3129,7 +3129,7 @@ var _ = Describe("IsValidMasterCandidate", func() { Expect(db.Create(&h).Error).ShouldNot(HaveOccurred()) var cluster common.Cluster Expect(db.Preload("Hosts").Take(&cluster, "id = ?", clusterId.String()).Error).ToNot(HaveOccurred()) - isValidReply, err := hapi.IsValidMasterCandidate(&h, &cluster, db, common.GetTestLog()) + isValidReply, err := hapi.IsValidMasterCandidate(&h, &cluster, db, common.GetTestLog(), true) Expect(isValidReply).Should(Equal(t.isValid)) Expect(err).ShouldNot(HaveOccurred()) }) diff --git a/internal/host/hostrole_test.go b/internal/host/hostrole_test.go index 23a66bd70563..f74c04b1d54f 100644 --- a/internal/host/hostrole_test.go +++ b/internal/host/hostrole_test.go @@ -131,7 +131,7 @@ var _ = Describe("Suggested-Role on Refresh", func() { )) } - err := hapi.RefreshRole(ctx, &host, db) + err := hapi.RefreshRole(ctx, &host, db, swag.Int(common.MinMasterHostsNeededForInstallationInHaMode)) Expect(err).ToNot(HaveOccurred()) var resultHost models.Host diff --git a/internal/host/mock_host_api.go b/internal/host/mock_host_api.go index e725a9ef91c0..fa677cc01745 100644 --- a/internal/host/mock_host_api.go +++ b/internal/host/mock_host_api.go @@ -41,18 +41,18 @@ func (m *MockAPI) EXPECT() *MockAPIMockRecorder { } // AutoAssignRole mocks base method. -func (m *MockAPI) AutoAssignRole(arg0 context.Context, arg1 *models.Host, arg2 *gorm.DB) (bool, error) { +func (m *MockAPI) AutoAssignRole(arg0 context.Context, arg1 *models.Host, arg2 *gorm.DB, arg3 *int) (bool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AutoAssignRole", arg0, arg1, arg2) + ret := m.ctrl.Call(m, "AutoAssignRole", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(bool) ret1, _ := ret[1].(error) return ret0, ret1 } // AutoAssignRole indicates an expected call of AutoAssignRole. -func (mr *MockAPIMockRecorder) AutoAssignRole(arg0, arg1, arg2 interface{}) *gomock.Call { +func (mr *MockAPIMockRecorder) AutoAssignRole(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AutoAssignRole", reflect.TypeOf((*MockAPI)(nil).AutoAssignRole), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AutoAssignRole", reflect.TypeOf((*MockAPI)(nil).AutoAssignRole), arg0, arg1, arg2, arg3) } // BindHost mocks base method. @@ -327,18 +327,18 @@ func (mr *MockAPIMockRecorder) IsRequireUserActionReset(arg0 interface{}) *gomoc } // IsValidMasterCandidate mocks base method. -func (m *MockAPI) IsValidMasterCandidate(arg0 *models.Host, arg1 *common.Cluster, arg2 *gorm.DB, arg3 logrus.FieldLogger) (bool, error) { +func (m *MockAPI) IsValidMasterCandidate(arg0 *models.Host, arg1 *common.Cluster, arg2 *gorm.DB, arg3 logrus.FieldLogger, arg4 bool) (bool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "IsValidMasterCandidate", arg0, arg1, arg2, arg3) + ret := m.ctrl.Call(m, "IsValidMasterCandidate", arg0, arg1, arg2, arg3, arg4) ret0, _ := ret[0].(bool) ret1, _ := ret[1].(error) return ret0, ret1 } // IsValidMasterCandidate indicates an expected call of IsValidMasterCandidate. -func (mr *MockAPIMockRecorder) IsValidMasterCandidate(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { +func (mr *MockAPIMockRecorder) IsValidMasterCandidate(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsValidMasterCandidate", reflect.TypeOf((*MockAPI)(nil).IsValidMasterCandidate), arg0, arg1, arg2, arg3) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsValidMasterCandidate", reflect.TypeOf((*MockAPI)(nil).IsValidMasterCandidate), arg0, arg1, arg2, arg3, arg4) } // PermanentHostsDeletion mocks base method. @@ -370,17 +370,17 @@ func (mr *MockAPIMockRecorder) RefreshInventory(arg0, arg1, arg2, arg3 interface } // RefreshRole mocks base method. -func (m *MockAPI) RefreshRole(arg0 context.Context, arg1 *models.Host, arg2 *gorm.DB) error { +func (m *MockAPI) RefreshRole(arg0 context.Context, arg1 *models.Host, arg2 *gorm.DB, arg3 *int) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RefreshRole", arg0, arg1, arg2) + ret := m.ctrl.Call(m, "RefreshRole", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(error) return ret0 } // RefreshRole indicates an expected call of RefreshRole. -func (mr *MockAPIMockRecorder) RefreshRole(arg0, arg1, arg2 interface{}) *gomock.Call { +func (mr *MockAPIMockRecorder) RefreshRole(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshRole", reflect.TypeOf((*MockAPI)(nil).RefreshRole), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshRole", reflect.TypeOf((*MockAPI)(nil).RefreshRole), arg0, arg1, arg2, arg3) } // RefreshStatus mocks base method. diff --git a/internal/host/monitor.go b/internal/host/monitor.go index 3be9155cf9be..767a3d0c0a37 100644 --- a/internal/host/monitor.go +++ b/internal/host/monitor.go @@ -152,6 +152,11 @@ func (m *Manager) clusterHostMonitoring() int64 { } for _, c := range clusters { + expectedMasterCount := c.ControlPlaneCount + if c.ControlPlaneCount == 0 { + expectedMasterCount = common.MinMasterHostsNeededForInstallationInHaMode + } + inventoryCache := make(InventoryCache) sortedHosts, canRefreshRoles := SortHosts(c.Hosts) @@ -171,7 +176,7 @@ func (m *Manager) clusterHostMonitoring() int64 { //all the hosts in the cluster has inventory to avoid race condition //with the reset auto-assign mechanism. if canRefreshRoles { - err = m.refreshRoleInternal(ctx, host, m.db, false) + err = m.refreshRoleInternal(ctx, host, m.db, false, swag.Int(int(expectedMasterCount))) if err != nil { log.WithError(err).Errorf("failed to refresh host %s role", *host.ID) } diff --git a/internal/host/transition_test.go b/internal/host/transition_test.go index b13f4717f564..aff1920404fc 100644 --- a/internal/host/transition_test.go +++ b/internal/host/transition_test.go @@ -4419,18 +4419,17 @@ var _ = Describe("Refresh Host", func() { "ODF unsupported Host Role for Compact Mode.")), inventory: hostutil.GenerateInventoryWithResourcesAndMultipleDisk(11, 8, "worker-1"), validationsChecker: makeJsonChecker(map[validationID]validationCheckResult{ - IsConnected: {status: ValidationSuccess, messagePattern: "Host is connected"}, - HasInventory: {status: ValidationSuccess, messagePattern: "Valid inventory exists for the host"}, - HasMinCPUCores: {status: ValidationSuccess, messagePattern: "Sufficient CPU cores"}, - HasMinMemory: {status: ValidationSuccess, messagePattern: "Sufficient minimum RAM"}, - HasMinValidDisks: {status: ValidationSuccess, messagePattern: "Sufficient disk capacity"}, - IsMachineCidrDefined: {status: ValidationSuccess, messagePattern: "Machine Network CIDR is defined"}, - HasCPUCoresForRole: {status: ValidationSuccess, messagePattern: "Sufficient CPU cores for role worker"}, - HasMemoryForRole: {status: ValidationFailure, messagePattern: "Require at least 8.35 GiB RAM for role worker, found only 8.00 GiB"}, - IsHostnameUnique: {status: ValidationSuccess, messagePattern: "Hostname worker-1 is unique in cluster"}, - BelongsToMachineCidr: {status: ValidationSuccess, messagePattern: "Host belongs to all machine network CIDRs"}, - IsNTPSynced: {status: ValidationSuccess, messagePattern: "Host NTP is synced"}, - AreOdfRequirementsSatisfied: {status: ValidationFailure, messagePattern: "ODF unsupported Host Role for Compact Mode."}, + IsConnected: {status: ValidationSuccess, messagePattern: "Host is connected"}, + HasInventory: {status: ValidationSuccess, messagePattern: "Valid inventory exists for the host"}, + HasMinCPUCores: {status: ValidationSuccess, messagePattern: "Sufficient CPU cores"}, + HasMinMemory: {status: ValidationSuccess, messagePattern: "Sufficient minimum RAM"}, + HasMinValidDisks: {status: ValidationSuccess, messagePattern: "Sufficient disk capacity"}, + IsMachineCidrDefined: {status: ValidationSuccess, messagePattern: "Machine Network CIDR is defined"}, + HasCPUCoresForRole: {status: ValidationSuccess, messagePattern: "Sufficient CPU cores for role worker"}, + HasMemoryForRole: {status: ValidationFailure, messagePattern: "Require at least 8.35 GiB RAM for role worker, found only 8.00 GiB"}, + IsHostnameUnique: {status: ValidationSuccess, messagePattern: "Hostname worker-1 is unique in cluster"}, + BelongsToMachineCidr: {status: ValidationSuccess, messagePattern: "Host belongs to all machine network CIDRs"}, + IsNTPSynced: {status: ValidationSuccess, messagePattern: "Host NTP is synced"}, SucessfullOrUnknownContainerImagesAvailability: {status: ValidationSuccess, messagePattern: "All required container images were either pulled successfully or no attempt was made to pull them"}, }), domainResolutions: common.TestDomainNameResolutionsSuccess, diff --git a/internal/host/validations_test.go b/internal/host/validations_test.go index 041dcf0a2b6e..f011bde60047 100644 --- a/internal/host/validations_test.go +++ b/internal/host/validations_test.go @@ -1159,7 +1159,7 @@ var _ = Describe("Validations test", func() { eventstest.WithHostIdMatcher(h.ID.String()), eventstest.WithInfraEnvIdMatcher(h.InfraEnvID.String()), )) - err := m.RefreshRole(ctx, &h, db) + err := m.RefreshRole(ctx, &h, db, swag.Int(common.MinMasterHostsNeededForInstallationInHaMode)) Expect(err).ToNot(HaveOccurred()) mockAndRefreshStatusWithoutEvents(&h) diff --git a/internal/ignition/installmanifests.go b/internal/ignition/installmanifests.go index bcf89d5a7d51..12f8d48ee4ca 100644 --- a/internal/ignition/installmanifests.go +++ b/internal/ignition/installmanifests.go @@ -433,14 +433,20 @@ func (g *installerGenerator) applyManifestPatches(ctx context.Context) error { func (g *installerGenerator) applyInfrastructureCRPatch(ctx context.Context) error { log := logutil.FromContext(ctx, g.log) - // We are only patching the InfrastructureCR if the hosts count is 4 - // and the three masters are schedulable. - if len(g.cluster.Hosts) != 4 { - log.Debugf("number of hosts is different than 4, no need to patch the Infrastructure CR %d", len(g.cluster.Hosts)) + // hosts roles are known at this stage + _, workers, _ := common.GetHostsByEachRole(&g.cluster.Cluster, false) + // Patch the InfrastructureCR only if there is exactly one worker. + // For multiple workers, the OpenShift installer will handle assigning 'infrastructureTopology: HighlyAvailable'. + // When there are no workers, the control plane nodes also act as workers, so 'infrastructureTopology: HighlyAvailable' is set automatically. + // Explicitly set 'infrastructureTopology: HighlyAvailable' for a single-worker setup, as specified in + // https://github.com/openshift/assisted-service/blob/master/docs/enhancements/4-nodes-cluster-deployment.md + numberOfWorkers := len(workers) + if numberOfWorkers != 1 { + log.Debugf("There are '%d' workers, no need to patch the Infrastructure CR", numberOfWorkers) return nil } - log.Infof("Patching Infrastructure CR: Number of hosts: %d", len(g.cluster.Hosts)) + log.Infof("Patching Infrastructure CR: Number of workers: %d", numberOfWorkers) infraManifest := filepath.Join(g.workDir, "manifests", "cluster-infrastructure-02-config.yml") data, err := os.ReadFile(infraManifest) diff --git a/internal/ignition/installmanifests_test.go b/internal/ignition/installmanifests_test.go index 3b8786158ec9..e69f1d14a6aa 100644 --- a/internal/ignition/installmanifests_test.go +++ b/internal/ignition/installmanifests_test.go @@ -1074,6 +1074,11 @@ var _ = Describe("infrastructureCRPatch", func() { { Inventory: hostInventory, RequestedHostname: "example3", + Role: models.HostRoleMaster, + }, + { + Inventory: hostInventory, + RequestedHostname: "example4", Role: models.HostRoleWorker, }, } diff --git a/internal/operators/common/common.go b/internal/operators/common/common.go index d942063dd6fd..aec31f8a94d1 100644 --- a/internal/operators/common/common.go +++ b/internal/operators/common/common.go @@ -20,6 +20,7 @@ func NonInstallationDiskCount(disks []*models.Disk, installationDiskID string, m } } } + return eligibleDisks, availableDisks } diff --git a/internal/operators/lvm/lvm_operator.go b/internal/operators/lvm/lvm_operator.go index ebdc23be6079..59560fcdf7c5 100644 --- a/internal/operators/lvm/lvm_operator.go +++ b/internal/operators/lvm/lvm_operator.go @@ -112,7 +112,7 @@ func (o *operator) ValidateHost(ctx context.Context, cluster *common.Cluster, ho diskCount, _ := operatorscommon.NonInstallationDiskCount(inventory.Disks, host.InstallationDiskID, minDiskSizeGb) role := common.GetEffectiveRole(host) - areSchedulable := common.AreMastersSchedulable(cluster) + areSchedulable := common.ShouldMastersBeSchedulable(&cluster.Cluster) minSizeMessage := "" if minDiskSizeGb > 0 { minSizeMessage = fmt.Sprintf(" of %dGB minimum", minDiskSizeGb) @@ -157,7 +157,7 @@ func (o *operator) GetHostRequirements(ctx context.Context, cluster *common.Clus } role := common.GetEffectiveRole(host) - areSchedulable := common.AreMastersSchedulable(cluster) + areSchedulable := common.ShouldMastersBeSchedulable(&cluster.Cluster) if role == models.HostRoleMaster && !areSchedulable { return &models.ClusterHostRequirementsDetails{ diff --git a/internal/operators/odf/odf_operator.go b/internal/operators/odf/odf_operator.go index 1e3dbc937880..e30114b586dd 100644 --- a/internal/operators/odf/odf_operator.go +++ b/internal/operators/odf/odf_operator.go @@ -129,6 +129,13 @@ func (o *operator) getValidDiskCount(disks []*models.Disk, installationDiskID st // ValidateHost verifies whether this operator is valid for given host func (o *operator) ValidateHost(_ context.Context, cluster *common.Cluster, host *models.Host, additionalOperatorRequirements *models.ClusterHostRequirementsDetails) (api.ValidationResult, error) { + // temporary disabling ODF for stretched clusters until it will be clear how ODF will work in this scenario. + // We pass host validation to avoid false validation messages. The cluster validation will fail + masters, _, _ := common.GetHostsByEachRole(&cluster.Cluster, true) + if masterCount := len(masters); masterCount > 3 { + return api.ValidationResult{Status: api.Success, ValidationId: o.GetHostValidationID(), Reasons: []string{}}, nil + } + numOfHosts := len(cluster.Hosts) if host.Inventory == "" { return api.ValidationResult{Status: api.Pending, ValidationId: o.GetHostValidationID(), Reasons: []string{"Missing Inventory in the host."}}, nil @@ -192,6 +199,13 @@ func (o *operator) GetMonitoredOperator() *models.MonitoredOperator { // GetHostRequirements provides operator's requirements towards the host func (o *operator) GetHostRequirements(_ context.Context, cluster *common.Cluster, host *models.Host) (*models.ClusterHostRequirementsDetails, error) { + // temporary disabling ODF for stretched clusters until it will be clear how ODF will work in this scenario. + // We pass host validation to avoid false validation messages. The cluster validation will fail + masters, _, _ := common.GetHostsByEachRole(&cluster.Cluster, true) + if masterCount := len(masters); masterCount > 3 { + return &models.ClusterHostRequirementsDetails{CPUCores: 0, RAMMib: 0}, nil + } + numOfHosts := len(cluster.Hosts) mode := getODFDeploymentMode(numOfHosts) diff --git a/internal/operators/odf/odf_operator_test.go b/internal/operators/odf/odf_operator_test.go index 0be8e67b553a..7d84583e53ae 100644 --- a/internal/operators/odf/odf_operator_test.go +++ b/internal/operators/odf/odf_operator_test.go @@ -216,6 +216,13 @@ var _ = Describe("Odf Operator", func() { workerWithNoInventory, &models.ClusterHostRequirementsDetails{CPUCores: operator.config.ODFPerHostCPUStandardMode, RAMMib: conversions.GibToMib(operator.config.ODFPerHostMemoryGiBStandardMode)}, ), + table.Entry("there are more than 3 masters", + &common.Cluster{Cluster: models.Cluster{Hosts: []*models.Host{ + masterWithThreeDisk, masterWithNoDisk, masterWithOneDisk, masterWithLessDiskSize, workerWithThreeDisk, workerWithNoInventory, + }}}, + masterWithLessDiskSize, + &models.ClusterHostRequirementsDetails{CPUCores: 0, RAMMib: 0}, + ), ) }) @@ -349,6 +356,35 @@ var _ = Describe("Odf Operator", func() { workerWithThreeDiskSizeOfOneZero, api.ValidationResult{Status: api.Success, ValidationId: operator.GetHostValidationID(), Reasons: []string{}}, ), + table.Entry("more than 3 masters", + &common.Cluster{Cluster: models.Cluster{Hosts: []*models.Host{ + masterWithThreeDisk, masterWithNoDisk, masterWithOneDisk, masterWithLessDiskSize, workerWithThreeDisk, workerWithThreeDiskSizeOfOneZero, + }}}, + masterWithThreeDisk, + api.ValidationResult{Status: api.Success, ValidationId: operator.GetHostValidationID(), Reasons: []string{}}, + ), ) }) + + Context("ValidateCluster", func() { + It("should fail with more than 3 masters", func() { + cluster := &common.Cluster{Cluster: models.Cluster{Hosts: []*models.Host{ + masterWithThreeDisk, masterWithNoDisk, masterWithOneDisk, masterWithLessDiskSize, workerWithThreeDisk, workerWithThreeDiskSizeOfOneZero, + }}} + + res, err := operator.ValidateCluster(ctx, cluster) + Expect(err).ToNot(HaveOccurred()) + Expect(res).Should(Equal(api.ValidationResult{Status: api.Failure, ValidationId: operator.GetHostValidationID(), Reasons: []string{"There are currently more than 3 hosts designated to be control planes. ODF currently supports clusters with exactly three control plane nodes."}})) + }) + + It("should pass with valid cluster", func() { + cluster := &common.Cluster{Cluster: models.Cluster{Hosts: []*models.Host{ + masterWithThreeDisk, masterWithNoDisk, masterWithOneDisk, workerWithThreeDisk, workerWithThreeDisk, workerWithThreeDisk, + }}} + + res, err := operator.ValidateCluster(ctx, cluster) + Expect(err).ToNot(HaveOccurred()) + Expect(res).Should(Equal(api.ValidationResult{Status: api.Success, ValidationId: operator.GetHostValidationID(), Reasons: []string{"ODF Requirements for Standard Deployment are satisfied."}})) + }) + }) }) diff --git a/internal/operators/odf/validation_test.go b/internal/operators/odf/validation_test.go index bec33d83fc8e..d8b63ba001ad 100644 --- a/internal/operators/odf/validation_test.go +++ b/internal/operators/odf/validation_test.go @@ -20,6 +20,7 @@ import ( "github.com/openshift/assisted-service/internal/metrics" "github.com/openshift/assisted-service/internal/operators" "github.com/openshift/assisted-service/internal/operators/odf" + "github.com/openshift/assisted-service/internal/testing" "github.com/openshift/assisted-service/models" "github.com/openshift/assisted-service/pkg/conversions" "gorm.io/gorm" @@ -101,7 +102,7 @@ var _ = Describe("Ocs Operator use-cases", func() { } mockIsValidMasterCandidate := func() { - mockHostAPI.EXPECT().IsValidMasterCandidate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).AnyTimes() + mockHostAPI.EXPECT().IsValidMasterCandidate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).AnyTimes() } BeforeEach(func() { db, dbName = common.PrepareTestDB() @@ -330,7 +331,7 @@ var _ = Describe("Ocs Operator use-cases", func() { clust.AllHostsAreReadyToInstall: {status: clust.ValidationSuccess, messagePattern: "All hosts in the cluster are ready to install"}, clust.IsDNSDomainDefined: {status: clust.ValidationSuccess, messagePattern: "The base domain is defined"}, clust.IsPullSecretSet: {status: clust.ValidationSuccess, messagePattern: "The pull secret is set"}, - clust.SufficientMastersCount: {status: clust.ValidationFailure, messagePattern: fmt.Sprintf("Clusters must have exactly %d dedicated control plane nodes. Add or remove hosts, or change their roles configurations to meet the requirement.", common.MinMasterHostsNeededForInstallation)}, + clust.SufficientMastersCount: {status: clust.ValidationFailure, messagePattern: fmt.Sprintf("The cluster must have exactly %d dedicated control plane nodes. Add or remove hosts, or change their roles configurations to meet the requirement.", common.MinMasterHostsNeededForInstallationInHaMode)}, clust.IsOdfRequirementsSatisfied: {status: clust.ValidationFailure, messagePattern: "A minimum of 3 hosts is required to deploy ODF."}, }), errorExpected: false, @@ -893,6 +894,10 @@ var _ = Describe("Ocs Operator use-cases", func() { }, } + if cluster.Cluster.OpenshiftVersion == "" { + cluster.Cluster.OpenshiftVersion = testing.ValidOCPVersionForNonStretchedClusters + } + Expect(db.Create(&cluster).Error).ShouldNot(HaveOccurred()) mockIsValidMasterCandidate() for i := range t.hosts { diff --git a/internal/operators/odf/validations.go b/internal/operators/odf/validations.go index 634a37bc9efe..3897f96cfba7 100644 --- a/internal/operators/odf/validations.go +++ b/internal/operators/odf/validations.go @@ -33,6 +33,14 @@ type odfClusterResourcesInfo struct { func (o *operator) validateRequirements(cluster *models.Cluster) (api.ValidationStatus, string) { var status string + + // temporary disabling ODF for stretched clusters until it will be clear how ODF will work in this scenario. + masters, _, _ := common.GetHostsByEachRole(cluster, true) + if masterCount := len(masters); masterCount > 3 { + status = "There are currently more than 3 hosts designated to be control planes. ODF currently supports clusters with exactly three control plane nodes." + return api.Failure, status + } + hosts := cluster.Hosts numAvailableHosts := int64(len(hosts)) diff --git a/internal/testing/common.go b/internal/testing/common.go new file mode 100644 index 000000000000..87697fa07339 --- /dev/null +++ b/internal/testing/common.go @@ -0,0 +1,15 @@ +package testing + +import ( + "fmt" + "strconv" + "strings" + + "github.com/openshift/assisted-service/internal/common" +) + +var ValidOCPVersionForNonStretchedClusters = func(majorMinorOCPVersion string) string { + splittedVersion := strings.Split(majorMinorOCPVersion, ".") + intVersion, _ := strconv.Atoi(splittedVersion[1]) + return fmt.Sprintf("%s.%d", splittedVersion[0], intVersion-1) +}(common.MinimumVersionForStretchedControlPlanesCluster) diff --git a/models/cluster_create_params.go b/models/cluster_create_params.go index 3eeade9fcdc5..1351a6d02ee1 100644 --- a/models/cluster_create_params.go +++ b/models/cluster_create_params.go @@ -42,6 +42,9 @@ type ClusterCreateParams struct { // Cluster networks that are associated with this cluster. ClusterNetworks []*ClusterNetwork `json:"cluster_networks"` + // The amount of control planes which should be part of the cluster. + ControlPlaneCount *int64 `json:"control_plane_count,omitempty"` + // The CPU architecture of the image (x86_64/arm64/etc). // Enum: [x86_64 aarch64 arm64 ppc64le s390x multi] CPUArchitecture string `json:"cpu_architecture,omitempty"` diff --git a/models/v2_cluster_update_params.go b/models/v2_cluster_update_params.go index cf3edc1efcff..437f8dbed610 100644 --- a/models/v2_cluster_update_params.go +++ b/models/v2_cluster_update_params.go @@ -45,6 +45,9 @@ type V2ClusterUpdateParams struct { // Cluster networks that are associated with this cluster. ClusterNetworks []*ClusterNetwork `json:"cluster_networks"` + // The amount of control planes which should be part of the cluster. + ControlPlaneCount *int64 `json:"control_plane_count,omitempty"` + // Installation disks encryption mode and host roles to be applied. DiskEncryption *DiskEncryption `json:"disk_encryption,omitempty" gorm:"embedded;embeddedPrefix:disk_encryption_"` diff --git a/restapi/embedded_spec.go b/restapi/embedded_spec.go index 7349675bdeb0..8ae5bebbe83d 100644 --- a/restapi/embedded_spec.go +++ b/restapi/embedded_spec.go @@ -6623,6 +6623,11 @@ func init() { }, "x-nullable": true }, + "control_plane_count": { + "description": "The amount of control planes which should be part of the cluster.", + "type": "integer", + "x-nullable": true + }, "cpu_architecture": { "description": "The CPU architecture of the image (x86_64/arm64/etc).", "type": "string", @@ -10464,6 +10469,11 @@ func init() { }, "x-nullable": true }, + "control_plane_count": { + "description": "The amount of control planes which should be part of the cluster.", + "type": "integer", + "x-nullable": true + }, "disk_encryption": { "description": "Installation disks encryption mode and host roles to be applied.", "$ref": "#/definitions/disk-encryption" @@ -17487,6 +17497,11 @@ func init() { }, "x-nullable": true }, + "control_plane_count": { + "description": "The amount of control planes which should be part of the cluster.", + "type": "integer", + "x-nullable": true + }, "cpu_architecture": { "description": "The CPU architecture of the image (x86_64/arm64/etc).", "type": "string", @@ -21260,6 +21275,11 @@ func init() { }, "x-nullable": true }, + "control_plane_count": { + "description": "The amount of control planes which should be part of the cluster.", + "type": "integer", + "x-nullable": true + }, "disk_encryption": { "description": "Installation disks encryption mode and host roles to be applied.", "$ref": "#/definitions/disk-encryption" diff --git a/subsystem/cluster_test.go b/subsystem/cluster_test.go index 1b1fa53eb287..34c104dd12f5 100644 --- a/subsystem/cluster_test.go +++ b/subsystem/cluster_test.go @@ -5150,3 +5150,46 @@ var _ = Describe("Verify install-config manifest", func() { Entry("Operation: V2GetClusterInstallConfig, Override config: true", "V2GetClusterInstallConfig", getInstallConfigFromFile, models.PlatformTypeBaremetal, true), ) }) + +var _ = Describe("Verify role assignment for stretched control plane cluster", func() { + var ctx = context.TODO() + + It("with 4 masters, 1 worker", func() { + reply, err := userBMClient.Installer.V2RegisterCluster(ctx, &installer.V2RegisterClusterParams{ + Context: ctx, + NewClusterParams: &models.ClusterCreateParams{ + Name: swag.String("test-cluster"), + OpenshiftVersion: swag.String(common.MinimumVersionForStretchedControlPlanesCluster), + PullSecret: swag.String(pullSecret), + ControlPlaneCount: swag.Int64(4), + }, + }) + + Expect(err).To(BeNil()) + Expect(reply).To(BeAssignableToTypeOf(installer.NewV2RegisterClusterCreated())) + + cluster := reply.GetPayload() + Expect(cluster).ToNot(BeNil()) + + infraEnv := registerInfraEnv(cluster.ID, models.ImageTypeMinimalIso) + Expect(infraEnv).ToNot(BeNil()) + + ips := hostutil.GenerateIPv4Addresses(5, defaultCIDRv4) + for k := 0; k < 5; k++ { + registerNodeWithInventory(ctx, *infraEnv.ID, fmt.Sprintf("host-%d", k), ips[0], getDefaultInventory(defaultCIDRv4)) + } + + Eventually(func() bool { + reply, err := userBMClient.Installer.V2GetCluster(ctx, &installer.V2GetClusterParams{ClusterID: *cluster.ID}) + if err != nil { + return false + } + + c := reply.Payload + masters, workers, autoAssign := common.GetHostsByEachRole(c, true) + + return len(masters) == 4 && len(workers) == 1 && len(autoAssign) == 0 + + }, "60s", "2s").Should(BeTrue()) + }) +}) diff --git a/subsystem/kubeapi_test.go b/subsystem/kubeapi_test.go index 7fa13bd3e464..03ec03f20a9b 100644 --- a/subsystem/kubeapi_test.go +++ b/subsystem/kubeapi_test.go @@ -140,6 +140,7 @@ var ( "openshift-v4.11.0": "quay.io/openshift-release-dev/ocp-release:4.11.0-x86_64", "openshift-v4.11.0-multi": "quay.io/openshift-release-dev/ocp-release:4.11.0-multi", "openshift-v4.14.0": "quay.io/openshift-release-dev/ocp-release:4.14.0-ec.4-x86_64", + "openshift-v4.18.0-ec.2": "quay.io/openshift-release-dev/ocp-release:4.18.0-ec.2-x86_64", } ) @@ -5335,6 +5336,115 @@ spec: updatedInfraEnv := getInfraEnvFromDBByKubeKey(ctx, db, infraNsName, waitForReconcileTimeout) Expect(updatedInfraEnv.OpenshiftVersion).To(Equal("4.16")) }) + + It("deploying OCP 4.18 cluster with 4 masters and 1 worker", func() { + By("Deploy 4.18 cluster Image Set") + clusterImageSetReference := hivev1.ClusterImageSetReference{ + Name: "openshift-v4.18.0-ec.2", + } + deployClusterImageSetCRD(ctx, kubeClient, &clusterImageSetReference) + + By("Deploy cluster deployment") + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + + By("Deploy agent cluster install") + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + aciSpec.ProvisionRequirements.ControlPlaneAgents = 4 + aciSpec.ProvisionRequirements.WorkerAgents = 1 + aciSpec.ImageSetRef = &clusterImageSetReference + aciSpec.Networking.NetworkType = models.ClusterCreateParamsNetworkTypeOVNKubernetes + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) + clusterKey := types.NamespacedName{ + Namespace: Options.Namespace, + Name: clusterDeploymentSpec.ClusterName, + } + Eventually(func() *common.Cluster { + cluster := getClusterFromDB(ctx, kubeClient, db, clusterKey, waitForReconcileTimeout) + return cluster + }, "15s", "1s").Should(Not(BeNil())) + + By("Deploy infraenv") + infraEnvKey := types.NamespacedName{ + Namespace: Options.Namespace, + Name: infraNsName.Name, + } + deployInfraEnvCRD(ctx, kubeClient, infraNsName.Name, infraEnvSpec) + Eventually(func() string { + url := getInfraEnvCRD(ctx, kubeClient, infraEnvKey).Status.ISODownloadURL + return url + }, "30s", "3s").Should(Not(BeEmpty())) + + By("Register hosts") + infraEnv := getInfraEnvFromDBByKubeKey(ctx, db, infraEnvKey, waitForReconcileTimeout) + Expect(infraEnv).ToNot(BeNil()) + + configureLocalAgentClient(infraEnv.ID.String()) + ips := hostutil.GenerateIPv4Addresses(5, defaultCIDRv4) + hosts := make([]*models.Host, 0) + + for i := 0; i < 5; i++ { + hostname := fmt.Sprintf("h%d", i) + hosts = append(hosts, registerNode(ctx, *infraEnv.ID, hostname, ips[i])) + } + + for _, host := range hosts { + checkAgentCondition(ctx, host.ID.String(), v1beta1.ValidatedCondition, v1beta1.ValidationsFailingReason) + hostkey := types.NamespacedName{ + Namespace: Options.Namespace, + Name: host.ID.String(), + } + agent := getAgentCRD(ctx, kubeClient, hostkey) + Expect(agent.Status.ValidationsInfo).ToNot(BeNil()) + } + + generateFullMeshConnectivity(ctx, ips[0], hosts...) + + for _, h := range hosts { + generateDomainResolution(ctx, h, clusterDeploymentSpec.ClusterName, "hive.example.com") + } + + // approving the agents so the infraenv could be deregistered successfully + By("Wait for agent roles and approve") + masterCount := 0 + workerCount := 0 + + for _, host := range hosts { + hostkey := types.NamespacedName{ + Namespace: Options.Namespace, + Name: host.ID.String(), + } + + var agent *v1beta1.Agent + Eventually(func() bool { + agent = getAgentCRD(ctx, kubeClient, hostkey) + if agent == nil { + return false + } + + if agent.Status.Role == models.HostRoleAutoAssign { + return false + } + + return true + }, "30s", "10s").Should(BeTrue()) + + if agent.Status.Role == models.HostRoleMaster { + masterCount++ + } + + if agent.Status.Role == models.HostRoleWorker { + workerCount++ + } + + Eventually(func() error { + agent.Spec.Approved = true + return kubeClient.Update(ctx, agent) + }, "30s", "10s").Should(BeNil()) + } + + Expect(masterCount).To(Equal(4)) + Expect(workerCount).To(Equal(1)) + }) }) var _ = Describe("bmac reconcile flow", func() { diff --git a/swagger.yaml b/swagger.yaml index a64f0b86645f..5ee08c7b1c84 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -4911,6 +4911,10 @@ definitions: type: string description: A comma-separated list of tags that are associated to the cluster. x-nullable: true + control_plane_count: + type: integer + description: The amount of control planes which should be part of the cluster. + x-nullable: true host-update-params: type: object @@ -5088,6 +5092,10 @@ definitions: type: string description: A comma-separated list of tags that are associated to the cluster. x-nullable: true + control_plane_count: + type: integer + description: The amount of control planes which should be part of the cluster. + x-nullable: true import-cluster-params: type: object diff --git a/vendor/github.com/openshift/assisted-service/api/hiveextension/v1beta1/agentclusterinstall_types.go b/vendor/github.com/openshift/assisted-service/api/hiveextension/v1beta1/agentclusterinstall_types.go index 8ef92a34ada6..327020fd25bc 100644 --- a/vendor/github.com/openshift/assisted-service/api/hiveextension/v1beta1/agentclusterinstall_types.go +++ b/vendor/github.com/openshift/assisted-service/api/hiveextension/v1beta1/agentclusterinstall_types.go @@ -22,13 +22,11 @@ const ( ClusterInstallationStoppedReason string = "ClusterInstallationStopped" ClusterInstallationStoppedMsg string = "The cluster installation stopped" ClusterInsufficientAgentsReason string = "InsufficientAgents" - ClusterInsufficientAgentsMsg string = "The cluster currently requires %d agents but only %d have registered" + ClusterInsufficientAgentsMsg string = "The cluster currently requires exactly %d master agents and %d worker agents, but currently registered %d master agents and %d worker agents" ClusterUnapprovedAgentsReason string = "UnapprovedAgents" ClusterUnapprovedAgentsMsg string = "The installation is pending on the approval of %d agents" ClusterUnsyncedAgentsReason string = "UnsyncedAgents" ClusterUnsyncedAgentsMsg string = "The cluster currently has %d agents with spec error" - ClusterAdditionalAgentsReason string = "AdditionalAgents" - ClusterAdditionalAgentsMsg string = "The cluster currently requires exactly %d agents but have %d registered" ClusterValidatedCondition hivev1.ClusterInstallConditionType = "Validated" ClusterValidationsOKMsg string = "The cluster's validations are passing" @@ -355,7 +353,7 @@ type ClusterNetworkEntry struct { type ProvisionRequirements struct { // ControlPlaneAgents is the number of matching approved and ready Agents with the control plane role - // required to launch the install. Must be either 1 or 3. + // required to launch the install. Must be either 1 or 3-5. ControlPlaneAgents int `json:"controlPlaneAgents"` // WorkerAgents is the minimum number of matching approved and ready Agents with the worker role diff --git a/vendor/github.com/openshift/assisted-service/models/cluster_create_params.go b/vendor/github.com/openshift/assisted-service/models/cluster_create_params.go index 3eeade9fcdc5..1351a6d02ee1 100644 --- a/vendor/github.com/openshift/assisted-service/models/cluster_create_params.go +++ b/vendor/github.com/openshift/assisted-service/models/cluster_create_params.go @@ -42,6 +42,9 @@ type ClusterCreateParams struct { // Cluster networks that are associated with this cluster. ClusterNetworks []*ClusterNetwork `json:"cluster_networks"` + // The amount of control planes which should be part of the cluster. + ControlPlaneCount *int64 `json:"control_plane_count,omitempty"` + // The CPU architecture of the image (x86_64/arm64/etc). // Enum: [x86_64 aarch64 arm64 ppc64le s390x multi] CPUArchitecture string `json:"cpu_architecture,omitempty"` diff --git a/vendor/github.com/openshift/assisted-service/models/v2_cluster_update_params.go b/vendor/github.com/openshift/assisted-service/models/v2_cluster_update_params.go index cf3edc1efcff..437f8dbed610 100644 --- a/vendor/github.com/openshift/assisted-service/models/v2_cluster_update_params.go +++ b/vendor/github.com/openshift/assisted-service/models/v2_cluster_update_params.go @@ -45,6 +45,9 @@ type V2ClusterUpdateParams struct { // Cluster networks that are associated with this cluster. ClusterNetworks []*ClusterNetwork `json:"cluster_networks"` + // The amount of control planes which should be part of the cluster. + ControlPlaneCount *int64 `json:"control_plane_count,omitempty"` + // Installation disks encryption mode and host roles to be applied. DiskEncryption *DiskEncryption `json:"disk_encryption,omitempty" gorm:"embedded;embeddedPrefix:disk_encryption_"`