MGMT-21810: Support TNF cluster installation using kube-api

api/v1beta1/agent_types.go

@@ -199,6 +199,8 @@ type AgentSpec struct {
 	IgnitionEndpointHTTPHeaders map[string]string `json:"ignitionEndpointHTTPHeaders,omitempty"`
 	// NodeLabels are the labels to be applied on the node associated with this agent
 	NodeLabels map[string]string `json:"nodeLabels,omitempty"`
+	// FencingCredentialsSecretRef is a name of a secret in the Agent's namespace that contains fencing credentials
+	FencingCredentialsSecretRef string `json:"fencingCredentialsSecretRef,omitempty"`
 }
 
 type IgnitionEndpointTokenReference struct {

config/crd/bases/agent-install.openshift.io_agents.yaml

@@ -82,6 +82,10 @@ spec:
                       name must be unique.
                     type: string
                 type: object
+              fencingCredentialsSecretRef:
+                description: FencingCredentialsSecretRef is a name of a secret in
+                  the Agent's namespace that contains fencing credentials
+                type: string
               hostname:
                 type: string
               ignitionConfigOverrides:

config/crd/resources.yaml

@@ -915,6 +915,10 @@ spec:
                       name must be unique.
                     type: string
                 type: object
+              fencingCredentialsSecretRef:
+                description: FencingCredentialsSecretRef is a name of a secret in
+                  the Agent's namespace that contains fencing credentials
+                type: string
               hostname:
                 type: string
               ignitionConfigOverrides:

deploy/olm-catalog/manifests/agent-install.openshift.io_agents.yaml

@@ -92,6 +92,10 @@ spec:
                       name must be unique.
                     type: string
                 type: object
+              fencingCredentialsSecretRef:
+                description: FencingCredentialsSecretRef is a name of a secret in
+                  the Agent's namespace that contains fencing credentials
+                type: string
               hostname:
                 type: string
               ignitionConfigOverrides:

internal/bminventory/inventory.go

@@ -5960,6 +5960,15 @@ func (b *bareMetalInventory) BindHostInternal(ctx context.Context, params instal
 		return nil, common.NewApiError(http.StatusBadRequest, err)
 	}
 
+	fencingClustersSupported, err := common.BaseVersionGreaterOrEqual(common.MinimumVersionForTwoNodesWithFencing, cluster.OpenshiftVersion)
+	if err != nil {
+		return nil, common.NewApiError(http.StatusInternalServerError, err)
+	}
+	if host.FencingCredentials != "" && !fencingClustersSupported {
+		err = errors.Errorf("Host %s has fencing credentials, it must be bound to a cluster with openshift version %s or newer", host.ID, common.MinimumVersionForTwoNodesWithFencing)
+		return nil, common.NewApiError(http.StatusBadRequest, err)
+	}
+
 	if err = b.clusterApi.AcceptRegistration(cluster); err != nil {
 		log.WithError(err).Errorf("failed to bind host <%s> to cluster %s due to: %s",
 			params.HostID.String(), *params.BindHostParams.ClusterID, err.Error())

internal/bminventory/inventory_test.go

@@ -18165,7 +18165,7 @@ var _ = Describe("BindHost", func() {
 		hostID = strfmt.UUID(uuid.New().String())
 		infraEnvID = strfmt.UUID(uuid.New().String())
 		bm = createInventory(db, cfg)
-		err := db.Create(&common.Cluster{Cluster: models.Cluster{ID: &clusterID, Kind: swag.String(models.ClusterKindCluster)}}).Error
+		err := db.Create(&common.Cluster{Cluster: models.Cluster{ID: &clusterID, Kind: swag.String(models.ClusterKindCluster), OpenshiftVersion: common.TestDefaultConfig.OpenShiftVersion}}).Error
 		Expect(err).ShouldNot(HaveOccurred())
 		err = db.Create(&common.InfraEnv{InfraEnv: models.InfraEnv{ID: &infraEnvID}}).Error
 		Expect(err).ShouldNot(HaveOccurred())
@@ -18451,6 +18451,54 @@ var _ = Describe("BindHost", func() {
 		response = bm.V2DeregisterCluster(ctx, deregisterParams)
 		Expect(response).To(BeAssignableToTypeOf(&installer.V2DeregisterClusterNoContent{}))
 	})
+
+	It("successful bind with fencing credentials", func() {
+		var clusterObj models.Cluster
+		Expect(db.First(&clusterObj, "id = ?", clusterID).Error).ShouldNot(HaveOccurred())
+		Expect(db.Model(&clusterObj).Update("openshift_version", common.MinimumVersionForTwoNodesWithFencing).Error).ShouldNot(HaveOccurred())
+
+		var hostObj models.Host
+		Expect(db.First(&hostObj, "id = ?", hostID).Error).ShouldNot(HaveOccurred())
+		Expect(db.Model(&hostObj).Update("fencing_credentials", "credentials").Error).ShouldNot(HaveOccurred())
+
+		params := installer.BindHostParams{
+			HostID:         hostID,
+			InfraEnvID:     infraEnvID,
+			BindHostParams: &models.BindHostParams{ClusterID: &clusterID},
+		}
+		mockEvents.EXPECT().SendHostEvent(gomock.Any(), eventstest.NewEventMatcher(
+			eventstest.WithNameMatcher(eventgen.HostBindSucceededEventName),
+			eventstest.WithHostIdMatcher(params.HostID.String()),
+			eventstest.WithInfraEnvIdMatcher(infraEnvID.String()),
+			eventstest.WithSeverityMatcher(models.EventSeverityInfo)))
+		mockClusterApi.EXPECT().AcceptRegistration(gomock.Any()).Return(nil).Times(1)
+		mockClusterApi.EXPECT().RefreshSchedulableMastersForcedTrue(gomock.Any(), gomock.Any()).Return(nil).Times(1)
+		mockHostApi.EXPECT().BindHost(ctx, gomock.Any(), clusterID, gomock.Any())
+
+		response := bm.BindHost(ctx, params)
+		Expect(response).To(BeAssignableToTypeOf(&installer.BindHostOK{}))
+	})
+
+	It("failed bind because openshift version doesn't support fencing credentials", func() {
+		var hostObj models.Host
+		Expect(db.First(&hostObj, "id = ?", hostID).Error).ShouldNot(HaveOccurred())
+		Expect(db.Model(&hostObj).Update("fencing_credentials", "credentials").Error).ShouldNot(HaveOccurred())
+
+		params := installer.BindHostParams{
+			HostID:         hostID,
+			InfraEnvID:     infraEnvID,
+			BindHostParams: &models.BindHostParams{ClusterID: &clusterID},
+		}
+		mockEvents.EXPECT().SendHostEvent(gomock.Any(), eventstest.NewEventMatcher(
+			eventstest.WithNameMatcher(eventgen.HostBindFailedEventName),
+			eventstest.WithHostIdMatcher(params.HostID.String()),
+			eventstest.WithInfraEnvIdMatcher(infraEnvID.String()),
+			eventstest.WithSeverityMatcher(models.EventSeverityError)))
+		mockHostApi.EXPECT().BindHost(ctx, gomock.Any(), clusterID, gomock.Any()).Times(0)
+
+		response := bm.BindHost(ctx, params)
+		verifyApiErrorString(response, http.StatusBadRequest, "has fencing credentials")
+	})
 })
 
 var _ = Describe("BindHost - with rhsso auth", func() {
@@ -18485,9 +18533,10 @@ var _ = Describe("BindHost - with rhsso auth", func() {
 
 		err := db.Create(&common.Cluster{
 			Cluster: models.Cluster{
-				ID:       &clusterID,
-				Kind:     swag.String(models.ClusterKindCluster),
-				UserName: userName1}}).Error
+				ID:               &clusterID,
+				Kind:             swag.String(models.ClusterKindCluster),
+				OpenshiftVersion: common.TestDefaultConfig.OpenShiftVersion,
+				UserName:         userName1}}).Error
 		Expect(err).ShouldNot(HaveOccurred())
 		err = db.Create(&common.InfraEnv{InfraEnv: models.InfraEnv{ID: &infraEnvID}}).Error
 		Expect(err).ShouldNot(HaveOccurred())

internal/controller/controllers/agent_controller.go

@@ -1661,6 +1661,60 @@ func (r *AgentReconciler) updateNodeLabels(log logrus.FieldLogger, host *common.
 	return false, nil
 }
 
+func (r *AgentReconciler) updateHostFencingCredentials(ctx context.Context, log logrus.FieldLogger, host *common.Host, agent *aiv1beta1.Agent, params *installer.V2UpdateHostParams) (bool, error) {
+	if agent.Spec.FencingCredentialsSecretRef == "" {
+		return false, nil
+	}
+
+	secretRef := types.NamespacedName{Namespace: agent.Namespace, Name: agent.Spec.FencingCredentialsSecretRef}
+	secret, err := getSecret(ctx, r.Client, r.APIReader, secretRef)
+	if err != nil {
+		log.WithError(err).Errorf("failed to get fencing credentials secret for host %s infra-env %s", host.ID.String(), host.InfraEnvID.String())
+		return false, err
+	}
+
+	agentFencingCredentials := &models.FencingCredentialsParams{
+		Address:  swag.String(string(secret.Data["address"])),
+		Password: swag.String(string(secret.Data["password"])),
+		Username: swag.String(string(secret.Data["username"])),
+	}
+	certificateVerification, ok := secret.Data["certificateVerification"]
+	if ok {
+		agentFencingCredentials.CertificateVerification = swag.String(string(certificateVerification))
+	}
+
+	fencingCredentials, err := json.Marshal(agentFencingCredentials)
+	if err != nil {
+		log.WithError(err).Errorf("failed to marshal fencing credentials for host %s infra-env %s", host.ID.String(), host.InfraEnvID.String())
+		return false, err
+	}
+
+	if host.FencingCredentials != string(fencingCredentials) {
+		params.HostUpdateParams.FencingCredentials = agentFencingCredentials
+		return true, nil
+	}
+	return false, nil
+}
+
+func (r *AgentReconciler) updateHostIgnitionEndpointToken(ctx context.Context, log logrus.FieldLogger, host *common.Host, agent *aiv1beta1.Agent, params *installer.V2UpdateHostParams) (bool, error) {
+	if agent.Spec.IgnitionEndpointTokenReference != nil {
+		token, err := r.getIgnitionToken(ctx, agent.Spec.IgnitionEndpointTokenReference)
+		if err != nil {
+			log.WithError(err).Errorf("Failed to get ignition token")
+			return false, err
+		}
+
+		if token != host.IgnitionEndpointToken {
+			params.HostUpdateParams.IgnitionEndpointToken = &token
+			return true, nil
+		}
+	} else if host.IgnitionEndpointToken != "" {
+		params.HostUpdateParams.IgnitionEndpointToken = swag.String("")
+		return true, nil
+	}
+	return false, nil
+}
+
 func (r *AgentReconciler) updateIfNeeded(ctx context.Context, log logrus.FieldLogger, agent *aiv1beta1.Agent, internalHost *common.Host) (*common.Host, error) {
 	spec := agent.Spec
 	var err error
@@ -1696,7 +1750,12 @@ func (r *AgentReconciler) updateIfNeeded(ctx context.Context, log logrus.FieldLo
 		return internalHost, err
 	}
 
-	hostUpdate = hostUpdate || nodesUpdated
+	fencingCredentialsUpdated, err := r.updateHostFencingCredentials(ctx, log, internalHost, agent, params)
+	if err != nil {
+		return internalHost, err
+	}
+
+	hostUpdate = hostUpdate || nodesUpdated || fencingCredentialsUpdated
 
 	if spec.Hostname != "" && spec.Hostname != internalHost.RequestedHostname {
 		hostUpdate = true
@@ -1721,24 +1780,11 @@ func (r *AgentReconciler) updateIfNeeded(ctx context.Context, log logrus.FieldLo
 		}
 	}
 
-	if spec.IgnitionEndpointTokenReference != nil {
-		var token string
-		token, err = r.getIgnitionToken(ctx, agent.Spec.IgnitionEndpointTokenReference)
-		if err != nil {
-			log.WithError(err).Errorf("Failed to get ignition token")
-			return internalHost, err
-		}
-
-		if token != internalHost.IgnitionEndpointToken {
-			hostUpdate = true
-			params.HostUpdateParams.IgnitionEndpointToken = &token
-		}
-	} else {
-		if internalHost.IgnitionEndpointToken != "" {
-			hostUpdate = true
-			params.HostUpdateParams.IgnitionEndpointToken = swag.String("")
-		}
+	IgnitionEndpointTokenUpdated, err := r.updateHostIgnitionEndpointToken(ctx, log, internalHost, agent, params)
+	if err != nil {
+		return internalHost, err
 	}
+	hostUpdate = hostUpdate || IgnitionEndpointTokenUpdated
 
 	if agent.Spec.IgnitionEndpointHTTPHeaders != nil {
 		hostIgnitionEndpointHTTPHeaders := make(map[string]string)

internal/controller/controllers/agent_controller_test.go

@@ -715,6 +715,114 @@ var _ = Describe("agent reconcile", func() {
 		})
 	})
 
+	Context("update host fencing credentials", func() {
+		var (
+			hostId, infraEnvId strfmt.UUID
+			commonHost         *common.Host
+			host               *v1beta1.Agent
+			clusterDeployment  *hivev1.ClusterDeployment
+		)
+		BeforeEach(func() {
+			hostId = strfmt.UUID(uuid.New().String())
+			infraEnvId = strfmt.UUID(uuid.New().String())
+			commonHost = &common.Host{
+				Host: models.Host{
+					ID:         &hostId,
+					ClusterID:  &sId,
+					Inventory:  common.GenerateTestDefaultInventory(),
+					Status:     swag.String(models.HostStatusKnown),
+					StatusInfo: swag.String("Some status info"),
+					InfraEnvID: infraEnvId,
+				},
+			}
+			backEndCluster = &common.Cluster{Cluster: models.Cluster{
+				ID: &sId,
+				Hosts: []*models.Host{
+					&commonHost.Host,
+				}}}
+
+			host = newAgent("host", testNamespace, v1beta1.AgentSpec{ClusterDeploymentName: &v1beta1.ClusterReference{Name: "clusterDeployment", Namespace: testNamespace}})
+			clusterDeployment = newClusterDeployment("clusterDeployment", testNamespace, getDefaultClusterDeploymentSpec("clusterDeployment-test", "test-cluster-aci", "pull-secret"))
+			Expect(c.Create(ctx, clusterDeployment)).To(BeNil())
+
+			mockInstallerInternal.EXPECT().GetHostByKubeKey(gomock.Any()).Return(commonHost, nil).AnyTimes()
+			mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil).Times(1)
+		})
+
+		It("secret doesn't set certificateVerification", func() {
+			fencingCredentials := &models.FencingCredentialsParams{
+				Address:  swag.String("https://bmc.example.com"),
+				Username: swag.String("admin"),
+				Password: swag.String("password123"),
+			}
+			fencingSecret := &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "fencing-secret",
+					Namespace: testNamespace,
+				},
+				Data: map[string][]byte{
+					"address":  []byte(*fencingCredentials.Address),
+					"username": []byte(*fencingCredentials.Username),
+					"password": []byte(*fencingCredentials.Password),
+				},
+			}
+			Expect(c.Create(ctx, fencingSecret)).To(Succeed())
+			host.Spec.FencingCredentialsSecretRef = "fencing-secret"
+
+			mockInstallerInternal.EXPECT().V2UpdateHostInternal(gomock.Any(), gomock.Any(), bminventory.NonInteractive).Do(
+				func(ctx context.Context, params installer.V2UpdateHostParams, interactive bminventory.Interactivity) {
+					Expect(params.HostUpdateParams.FencingCredentials).To(Equal(fencingCredentials))
+				}).Return(commonHost, nil).Times(1)
+			allowGetInfraEnvInternal(mockInstallerInternal, infraEnvId, "infraEnvName")
+			Expect(c.Create(ctx, host)).To(BeNil())
+			result, err := hr.Reconcile(ctx, newHostRequest(host))
+			Expect(err).To(BeNil())
+			Expect(result).To(Equal(ctrl.Result{}))
+		})
+
+		It("secret sets certificateVerification", func() {
+			fencingCredentials := &models.FencingCredentialsParams{
+				Address:                 swag.String("https://bmc.example.com"),
+				Username:                swag.String("admin"),
+				Password:                swag.String("password123"),
+				CertificateVerification: swag.String("Disabled"),
+			}
+			fencingSecret := &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "fencing-secret",
+					Namespace: testNamespace,
+				},
+				Data: map[string][]byte{
+					"address":                 []byte(*fencingCredentials.Address),
+					"username":                []byte(*fencingCredentials.Username),
+					"password":                []byte(*fencingCredentials.Password),
+					"certificateVerification": []byte(*fencingCredentials.CertificateVerification),
+				},
+			}
+			Expect(c.Create(ctx, fencingSecret)).To(Succeed())
+			host.Spec.FencingCredentialsSecretRef = "fencing-secret"
+
+			mockInstallerInternal.EXPECT().V2UpdateHostInternal(gomock.Any(), gomock.Any(), bminventory.NonInteractive).Do(
+				func(ctx context.Context, params installer.V2UpdateHostParams, interactive bminventory.Interactivity) {
+					Expect(params.HostUpdateParams.FencingCredentials).To(Equal(fencingCredentials))
+				}).Return(commonHost, nil).Times(1)
+			allowGetInfraEnvInternal(mockInstallerInternal, infraEnvId, "infraEnvName")
+			Expect(c.Create(ctx, host)).To(BeNil())
+			result, err := hr.Reconcile(ctx, newHostRequest(host))
+			Expect(err).To(BeNil())
+			Expect(result).To(Equal(ctrl.Result{}))
+		})
+
+		It("secret does not exist", func() {
+			host.Spec.FencingCredentialsSecretRef = "fencing-secret"
+			allowGetInfraEnvInternal(mockInstallerInternal, infraEnvId, "infraEnvName")
+			Expect(c.Create(ctx, host)).To(BeNil())
+			result, err := hr.Reconcile(ctx, newHostRequest(host))
+			Expect(err).To(BeNil())
+			Expect(result).To(Equal(ctrl.Result{RequeueAfter: defaultRequeueAfterOnError}))
+		})
+	})
+
 	It("Agent update empty disk path", func() {
 		newInstallDiskPath := ""
 		hostId := strfmt.UUID(uuid.New().String())