From 936018ed59d19215d8cad12d8f76e7e92232dc83 Mon Sep 17 00:00:00 2001 From: enxebre Date: Tue, 12 Dec 2023 20:38:25 +0100 Subject: [PATCH 1/5] Add ROSAControlPlaneReadyCondition, ocmclient and WorkerRoleARN field This commit introduces several improvements to the rosa control plane: - Add ROSAControlPlaneReadyCondition - Add helpers for ocmclient - Add WorkerRoleARN field to the API to satisfy latest ocm API requirements --- ...ne.cluster.x-k8s.io_rosacontrolplanes.yaml | 6 + .../rosa/api/v1beta2/conditions_consts.go | 24 ++ .../api/v1beta2/rosacontrolplane_types.go | 4 + .../rosa/api/v1beta2/zz_generated.deepcopy.go | 10 + .../rosacontrolplane_controller.go | 212 +++++++++++------- 5 files changed, 180 insertions(+), 76 deletions(-) create mode 100644 controlplane/rosa/api/v1beta2/conditions_consts.go diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_rosacontrolplanes.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_rosacontrolplanes.yaml index 3d5c7e6a36..2aa835ef2f 100644 --- a/config/crd/bases/controlplane.cluster.x-k8s.io_rosacontrolplanes.yaml +++ b/config/crd/bases/controlplane.cluster.x-k8s.io_rosacontrolplanes.yaml @@ -249,6 +249,8 @@ spec: version: description: Openshift version, for example "openshift-v4.12.15". type: string + workerRoleARN: + type: string required: - accountID - availabilityZones @@ -261,6 +263,7 @@ spec: - subnets - supportRoleARN - version + - workerRoleARN type: object status: properties: @@ -320,6 +323,9 @@ spec: description: ErrorMessage indicates that there is a terminal problem reconciling the state, and will be set to a descriptive error message. type: string + id: + description: ID is the cluster ID given by ROSA. + type: string initialized: description: Initialized denotes whether or not the control plane has the uploaded kubernetes config-map. diff --git a/controlplane/rosa/api/v1beta2/conditions_consts.go b/controlplane/rosa/api/v1beta2/conditions_consts.go new file mode 100644 index 0000000000..79351f148e --- /dev/null +++ b/controlplane/rosa/api/v1beta2/conditions_consts.go @@ -0,0 +1,24 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta2 + +import clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + +const ( + // ROSAControlPlaneReadyCondition condition reports on the successful reconciliation of ROSAControlPlane. + ROSAControlPlaneReadyCondition clusterv1.ConditionType = "ROSAControlPlaneReady" +) diff --git a/controlplane/rosa/api/v1beta2/rosacontrolplane_types.go b/controlplane/rosa/api/v1beta2/rosacontrolplane_types.go index 528965f0d9..fcbb3ffd93 100644 --- a/controlplane/rosa/api/v1beta2/rosacontrolplane_types.go +++ b/controlplane/rosa/api/v1beta2/rosacontrolplane_types.go @@ -55,6 +55,7 @@ type RosaControlPlaneSpec struct { //nolint: maligned CreatorARN *string `json:"creatorARN"` InstallerRoleARN *string `json:"installerRoleARN"` SupportRoleARN *string `json:"supportRoleARN"` + WorkerRoleARN *string `json:"workerRoleARN"` } // AWSRolesRef contains references to various AWS IAM roles required for operators to make calls against the AWS API. @@ -454,6 +455,9 @@ type RosaControlPlaneStatus struct { FailureMessage *string `json:"failureMessage,omitempty"` // Conditions specifies the cpnditions for the managed control plane Conditions clusterv1.Conditions `json:"conditions,omitempty"` + + // ID is the cluster ID given by ROSA. + ID *string `json:"id,omitempty"` } // +kubebuilder:object:root=true diff --git a/controlplane/rosa/api/v1beta2/zz_generated.deepcopy.go b/controlplane/rosa/api/v1beta2/zz_generated.deepcopy.go index d64e3629cf..40c87b2b27 100644 --- a/controlplane/rosa/api/v1beta2/zz_generated.deepcopy.go +++ b/controlplane/rosa/api/v1beta2/zz_generated.deepcopy.go @@ -155,6 +155,11 @@ func (in *RosaControlPlaneSpec) DeepCopyInto(out *RosaControlPlaneSpec) { *out = new(string) **out = **in } + if in.WorkerRoleARN != nil { + in, out := &in.WorkerRoleARN, &out.WorkerRoleARN + *out = new(string) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RosaControlPlaneSpec. @@ -187,6 +192,11 @@ func (in *RosaControlPlaneStatus) DeepCopyInto(out *RosaControlPlaneStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.ID != nil { + in, out := &in.ID, &out.ID + *out = new(string) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RosaControlPlaneStatus. diff --git a/controlplane/rosa/controllers/rosacontrolplane_controller.go b/controlplane/rosa/controllers/rosacontrolplane_controller.go index cd55018258..73ffa2431a 100644 --- a/controlplane/rosa/controllers/rosacontrolplane_controller.go +++ b/controlplane/rosa/controllers/rosacontrolplane_controller.go @@ -26,6 +26,7 @@ import ( sdk "github.com/openshift-online/ocm-sdk-go" cmv1 "github.com/openshift-online/ocm-sdk-go/clustersmgmt/v1" + ocmerrors "github.com/openshift-online/ocm-sdk-go/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" @@ -42,6 +43,7 @@ import ( clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/util" capiannotations "sigs.k8s.io/cluster-api/util/annotations" + "sigs.k8s.io/cluster-api/util/conditions" "sigs.k8s.io/cluster-api/util/predicates" ) @@ -171,7 +173,7 @@ func (r *ROSAControlPlaneReconciler) reconcileNormal(ctx context.Context, rosaSc // Create the cluster: clusterBuilder := cmv1.NewCluster(). - Name(rosaScope.ControlPlane.Name). + Name(rosaScope.ControlPlane.Name[:15]). MultiAZ(true). Product( cmv1.NewProduct(). @@ -256,17 +258,16 @@ func (r *ROSAControlPlaneReconciler) reconcileNormal(ctx context.Context, rosaSc stsBuilder = stsBuilder.OperatorIAMRoles(roles...) instanceIAMRolesBuilder := cmv1.NewInstanceIAMRoles() - instanceIAMRolesBuilder.MasterRoleARN("TODO") - instanceIAMRolesBuilder.WorkerRoleARN("TODO") + instanceIAMRolesBuilder.WorkerRoleARN(*rosaScope.ControlPlane.Spec.WorkerRoleARN) stsBuilder = stsBuilder.InstanceIAMRoles(instanceIAMRolesBuilder) stsBuilder.OidcConfig(cmv1.NewOidcConfig().ID(*rosaScope.ControlPlane.Spec.OIDCID)) - stsBuilder.AutoMode(true) awsBuilder := cmv1.NewAWS(). - AccountID(*rosaScope.ControlPlane.Spec.AccountID) - awsBuilder = awsBuilder.SubnetIDs(rosaScope.ControlPlane.Spec.Subnets...) - awsBuilder = awsBuilder.STS(stsBuilder) + AccountID(*rosaScope.ControlPlane.Spec.AccountID). + BillingAccountID(*rosaScope.ControlPlane.Spec.AccountID). + SubnetIDs(rosaScope.ControlPlane.Spec.Subnets...). + STS(stsBuilder) clusterBuilder = clusterBuilder.AWS(awsBuilder) clusterNodesBuilder := cmv1.NewClusterNodes() @@ -282,100 +283,80 @@ func (r *ROSAControlPlaneReconciler) reconcileNormal(ctx context.Context, rosaSc return ctrl.Result{}, fmt.Errorf("failed to create description of cluster: %v", err) } - // Create a logger that has the debug level enabled: - ocmLogger, err := sdk.NewGoLoggerBuilder(). - Debug(true). - Build() + // create OCM NodePool + ocmClient, err := newOCMClient() if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to build logger: %w", err) + return ctrl.Result{}, err } + defer func() { + ocmClient.ocm.Close() + }() - // Create the connection, and remember to close it: - token := os.Getenv("OCM_TOKEN") - ocmAPIUrl := os.Getenv("OCM_API_URL") - if ocmAPIUrl == "" { - ocmAPIUrl = "https://api.openshift.com" - } - connection, err := sdk.NewConnectionBuilder(). - Logger(ocmLogger). - Tokens(token). - URL(ocmAPIUrl). - Build() + cluster, err := ocmClient.GetCluster(rosaScope) if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to build connection: %w", err) + return ctrl.Result{}, err } - defer func() { - if err := connection.Close(); err != nil { - reterr = errors.Join(reterr, err) - } - }() log := logger.FromContext(ctx) - cluster, err := connection.ClustersMgmt().V1().Clusters(). - Add(). - // Parameter("dryRun", *config.DryRun). - Body(clusterSpec). - Send() + if cluster.ID() != "" { + clusterID := cluster.ID() + rosaScope.ControlPlane.Status.ID = &clusterID + conditions.MarkFalse(rosaScope.ControlPlane, + rosacontrolplanev1.ROSAControlPlaneReadyCondition, + string(cluster.Status().State()), + clusterv1.ConditionSeverityInfo, + "") + + if cluster.Status().State() == "ready" { + conditions.MarkTrue(rosaScope.ControlPlane, rosacontrolplanev1.ROSAControlPlaneReadyCondition) + rosaScope.ControlPlane.Status.Ready = true + } + + if err := rosaScope.PatchObject(); err != nil { + return ctrl.Result{}, err + } + + log.Info("cluster exists", "state", cluster.Status().State()) + return ctrl.Result{}, nil + } + + newCluster, err := ocmClient.CreateCluster(clusterSpec) if err != nil { log.Info("error", "error", err) return ctrl.Result{RequeueAfter: 10 * time.Second}, nil } - clusterObject := cluster.Body() - log.Info("result", "body", clusterObject) + log.Info("cluster created", "state", newCluster.Status().State()) + clusterID := newCluster.ID() + rosaScope.ControlPlane.Status.ID = &clusterID + if err := rosaScope.PatchObject(); err != nil { + return ctrl.Result{}, err + } return ctrl.Result{}, nil } func (r *ROSAControlPlaneReconciler) reconcileDelete(_ context.Context, rosaScope *scope.ROSAControlPlaneScope) (res ctrl.Result, reterr error) { - // log := logger.FromContext(ctx) - rosaScope.Info("Reconciling ROSAControlPlane delete") - // Create a logger that has the debug level enabled: - ocmLogger, err := sdk.NewGoLoggerBuilder(). - Debug(true). - Build() - if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to build logger: %w", err) - } - - // Create the connection, and remember to close it: - // TODO: token should be read from a secret: https://github.com/kubernetes-sigs/cluster-api-provider-aws/issues/4460 - token := os.Getenv("OCM_TOKEN") - ocmAPIUrl := os.Getenv("OCM_API_URL") - if ocmAPIUrl == "" { - ocmAPIUrl = "https://api.openshift.com" - } - connection, err := sdk.NewConnectionBuilder(). - Logger(ocmLogger). - Tokens(token). - URL(ocmAPIUrl). - Build() + // create OCM NodePool + ocmClient, err := newOCMClient() if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to build connection: %w", err) + return ctrl.Result{}, err } defer func() { - if err := connection.Close(); err != nil { - reterr = errors.Join(reterr, err) - } + ocmClient.ocm.Close() }() - cluster, err := r.getOcmCluster(rosaScope, connection) + cluster, err := ocmClient.GetCluster(rosaScope) if err != nil { return ctrl.Result{}, err } - response, err := connection.ClustersMgmt().V1().Clusters(). - Cluster(cluster.ID()). - Delete(). - Send() - if err != nil { - msg := response.Error().Reason() - if msg == "" { - msg = err.Error() + if cluster != nil { + if _, err := ocmClient.DeleteCluster(cluster.ID()); err != nil { + return ctrl.Result{}, err } - return ctrl.Result{}, fmt.Errorf(msg) } controllerutil.RemoveFinalizer(rosaScope.ControlPlane, ROSAControlPlaneFinalizer) @@ -426,13 +407,66 @@ func (r *ROSAControlPlaneReconciler) rosaClusterToROSAControlPlane(log *logger.L } } -func (r *ROSAControlPlaneReconciler) getOcmCluster(rosaScope *scope.ROSAControlPlaneScope, ocmConnection *sdk.Connection) (*cmv1.Cluster, error) { - clusterKey := rosaScope.ControlPlane.Name +// OCMClient is a temporary helper to talk to OCM API. +// TODO(alberto): vendor this from https://github.com/openshift/rosa/tree/master/pkg/ocm or build its own package here. +type OCMClient struct { + ocm *sdk.Connection +} + +func newOCMClient() (*OCMClient, error) { + // Create the connection, and remember to close it: + token := os.Getenv("OCM_TOKEN") + ocmAPIUrl := os.Getenv("OCM_API_URL") + if ocmAPIUrl == "" { + ocmAPIUrl = "https://api.openshift.com" + } + + // Create a logger that has the debug level enabled: + ocmLogger, err := sdk.NewGoLoggerBuilder(). + Debug(false). + Build() + if err != nil { + return nil, fmt.Errorf("failed to build logger: %w", err) + } + + connection, err := sdk.NewConnectionBuilder(). + Logger(ocmLogger). + Tokens(token). + URL(ocmAPIUrl). + Build() + if err != nil { + return nil, fmt.Errorf("failed to ocm client: %w", err) + } + ocmClient := OCMClient{ocm: connection} + + return &ocmClient, nil +} + +func (client *OCMClient) Close() error { + return client.ocm.Close() +} + +func (client *OCMClient) CreateCluster(clusterSpec *cmv1.Cluster) (*cmv1.Cluster, error) { + cluster, err := client.ocm.ClustersMgmt().V1().Clusters(). + Add(). + Body(clusterSpec). + Send() + if err != nil { + return nil, handleErr(cluster.Error(), err) + } + + clusterObject := cluster.Body() + + return clusterObject, nil +} + +func (client *OCMClient) GetCluster(rosaScope *scope.ROSAControlPlaneScope) (*cmv1.Cluster, error) { + clusterKey := rosaScope.ControlPlane.Name[:15] query := fmt.Sprintf("%s AND (id = '%s' OR name = '%s' OR external_id = '%s')", getClusterFilter(rosaScope), clusterKey, clusterKey, clusterKey, ) - response, err := ocmConnection.ClustersMgmt().V1().Clusters().List(). + response, err := client.ocm.ClustersMgmt().V1().Clusters().List(). Search(query). Page(1). Size(1). @@ -443,7 +477,7 @@ func (r *ROSAControlPlaneReconciler) getOcmCluster(rosaScope *scope.ROSAControlP switch response.Total() { case 0: - return nil, fmt.Errorf("there is no cluster with identifier or name '%s'", clusterKey) + return nil, nil case 1: return response.Items().Slice()[0], nil default: @@ -451,6 +485,18 @@ func (r *ROSAControlPlaneReconciler) getOcmCluster(rosaScope *scope.ROSAControlP } } +func (client *OCMClient) DeleteCluster(clusterID string) (*cmv1.Cluster, error) { + response, err := client.ocm.ClustersMgmt().V1().Clusters(). + Cluster(clusterID). + Delete(). + Send() + if err != nil { + return nil, handleErr(response.Error(), err) + } + + return nil, nil +} + // Generate a query that filters clusters running on the current AWS session account. func getClusterFilter(rosaScope *scope.ROSAControlPlaneScope) string { filter := "product.id = 'rosa'" @@ -462,3 +508,17 @@ func getClusterFilter(rosaScope *scope.ROSAControlPlaneScope) string { } return filter } + +func handleErr(res *ocmerrors.Error, err error) error { + msg := res.Reason() + if msg == "" { + msg = err.Error() + } + // Hack to always display the correct terms and conditions message + if res.Code() == "CLUSTERS-MGMT-451" { + msg = "You must accept the Terms and Conditions in order to continue.\n" + + "Go to https://www.redhat.com/wapps/tnc/ackrequired?site=ocm&event=register\n" + + "Once you accept the terms, you will need to retry the action that was blocked." + } + return fmt.Errorf(msg) +} From e63fee9918500f46a04fd7cf5f4ade40af3459fc Mon Sep 17 00:00:00 2001 From: Mulham Raee Date: Fri, 5 Jan 2024 16:37:47 +0100 Subject: [PATCH 2/5] Add RosaClusterName field to ROSAControlPlane - ensure RosaClusterName is valid using kubebuild validation - moved ocmClient to a seperate package and renamed to rosaClient - updated cluster-template-rosa.yaml - set ControlPlane.Status.Initialized - requeue ROSAControlPlane to poll cluster status until ready --- ...ne.cluster.x-k8s.io_rosacontrolplanes.yaml | 12 + .../api/v1beta2/rosacontrolplane_types.go | 10 + .../rosacontrolplane_controller.go | 223 ++++-------------- .../src/topics/rosa/creating-a-cluster.md | 2 +- pkg/cloud/scope/rosacontrolplane.go | 16 +- pkg/rosa/client.go | 59 +++++ pkg/rosa/clusters.go | 73 ++++++ pkg/rosa/util.go | 21 ++ templates/cluster-template-rosa.yaml | 4 +- 9 files changed, 235 insertions(+), 185 deletions(-) create mode 100644 pkg/rosa/client.go create mode 100644 pkg/rosa/clusters.go create mode 100644 pkg/rosa/util.go diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_rosacontrolplanes.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_rosacontrolplanes.yaml index 2aa835ef2f..a92058625b 100644 --- a/config/crd/bases/controlplane.cluster.x-k8s.io_rosacontrolplanes.yaml +++ b/config/crd/bases/controlplane.cluster.x-k8s.io_rosacontrolplanes.yaml @@ -237,6 +237,17 @@ spec: - nodePoolManagementARN - storageARN type: object + rosaClusterName: + description: Cluster name must be valid DNS-1035 label, so it must + consist of lower case alphanumeric characters or '-', start with + an alphabetic character, end with an alphanumeric character and + have a max length of 15 characters. + maxLength: 15 + pattern: ^[a-z]([-a-z0-9]*[a-z0-9])?$ + type: string + x-kubernetes-validations: + - message: rosaClusterName is immutable + rule: self == oldSelf subnets: description: The Subnet IDs to use when installing the cluster. SubnetIDs should come in pairs; two per availability zone, one private and @@ -260,6 +271,7 @@ spec: - oidcID - region - rolesRef + - rosaClusterName - subnets - supportRoleARN - version diff --git a/controlplane/rosa/api/v1beta2/rosacontrolplane_types.go b/controlplane/rosa/api/v1beta2/rosacontrolplane_types.go index fcbb3ffd93..9f010f9e63 100644 --- a/controlplane/rosa/api/v1beta2/rosacontrolplane_types.go +++ b/controlplane/rosa/api/v1beta2/rosacontrolplane_types.go @@ -23,6 +23,16 @@ import ( ) type RosaControlPlaneSpec struct { //nolint: maligned + // Cluster name must be valid DNS-1035 label, so it must consist of lower case alphanumeric + // characters or '-', start with an alphabetic character, end with an alphanumeric character + // and have a max length of 15 characters. + // + // +immutable + // +kubebuilder:validation:XValidation:rule="self == oldSelf", message="rosaClusterName is immutable" + // +kubebuilder:validation:MaxLength:=15 + // +kubebuilder:validation:Pattern:=`^[a-z]([-a-z0-9]*[a-z0-9])?$` + RosaClusterName string `json:"rosaClusterName"` + // The Subnet IDs to use when installing the cluster. // SubnetIDs should come in pairs; two per availability zone, one private and one public. Subnets []string `json:"subnets"` diff --git a/controlplane/rosa/controllers/rosacontrolplane_controller.go b/controlplane/rosa/controllers/rosacontrolplane_controller.go index 73ffa2431a..8e3bdb0466 100644 --- a/controlplane/rosa/controllers/rosacontrolplane_controller.go +++ b/controlplane/rosa/controllers/rosacontrolplane_controller.go @@ -24,9 +24,7 @@ import ( "strings" "time" - sdk "github.com/openshift-online/ocm-sdk-go" cmv1 "github.com/openshift-online/ocm-sdk-go/clustersmgmt/v1" - ocmerrors "github.com/openshift-online/ocm-sdk-go/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" @@ -40,6 +38,7 @@ import ( expinfrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/exp/api/v1beta2" "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/scope" "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/logger" + "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/rosa" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/util" capiannotations "sigs.k8s.io/cluster-api/util/annotations" @@ -171,9 +170,47 @@ func (r *ROSAControlPlaneReconciler) reconcileNormal(ctx context.Context, rosaSc } } + // TODO: token should be read from a secret: https://github.com/kubernetes-sigs/cluster-api-provider-aws/issues/4460 + token := os.Getenv("OCM_TOKEN") + rosaClient, err := rosa.NewRosaClient(token) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to create a rosa client: %w", err) + } + + defer func() { + rosaClient.Close() + }() + + cluster, err := rosaClient.GetCluster(rosaScope.RosaClusterName(), rosaScope.ControlPlane.Spec.CreatorARN) + if err != nil { + return ctrl.Result{}, err + } + + if clusterID := cluster.ID(); clusterID != "" { + rosaScope.ControlPlane.Status.ID = &clusterID + if cluster.Status().State() == "ready" { + conditions.MarkTrue(rosaScope.ControlPlane, rosacontrolplanev1.ROSAControlPlaneReadyCondition) + rosaScope.ControlPlane.Status.Ready = true + // TODO: distinguish when controlPlane is ready vs initialized + rosaScope.ControlPlane.Status.Initialized = true + + return ctrl.Result{}, nil + } + + conditions.MarkFalse(rosaScope.ControlPlane, + rosacontrolplanev1.ROSAControlPlaneReadyCondition, + string(cluster.Status().State()), + clusterv1.ConditionSeverityInfo, + "") + + rosaScope.Info("waiting for cluster to become ready", "state", cluster.Status().State()) + // Requeue so that status.ready is set to true when the cluster is fully created. + return ctrl.Result{RequeueAfter: time.Second * 60}, nil + } + // Create the cluster: clusterBuilder := cmv1.NewCluster(). - Name(rosaScope.ControlPlane.Name[:15]). + Name(rosaScope.RosaClusterName()). MultiAZ(true). Product( cmv1.NewProduct(). @@ -283,55 +320,15 @@ func (r *ROSAControlPlaneReconciler) reconcileNormal(ctx context.Context, rosaSc return ctrl.Result{}, fmt.Errorf("failed to create description of cluster: %v", err) } - // create OCM NodePool - ocmClient, err := newOCMClient() + newCluster, err := rosaClient.CreateCluster(clusterSpec) if err != nil { - return ctrl.Result{}, err - } - defer func() { - ocmClient.ocm.Close() - }() - - cluster, err := ocmClient.GetCluster(rosaScope) - if err != nil { - return ctrl.Result{}, err - } - - log := logger.FromContext(ctx) - if cluster.ID() != "" { - clusterID := cluster.ID() - rosaScope.ControlPlane.Status.ID = &clusterID - conditions.MarkFalse(rosaScope.ControlPlane, - rosacontrolplanev1.ROSAControlPlaneReadyCondition, - string(cluster.Status().State()), - clusterv1.ConditionSeverityInfo, - "") - - if cluster.Status().State() == "ready" { - conditions.MarkTrue(rosaScope.ControlPlane, rosacontrolplanev1.ROSAControlPlaneReadyCondition) - rosaScope.ControlPlane.Status.Ready = true - } - - if err := rosaScope.PatchObject(); err != nil { - return ctrl.Result{}, err - } - - log.Info("cluster exists", "state", cluster.Status().State()) - return ctrl.Result{}, nil - } - - newCluster, err := ocmClient.CreateCluster(clusterSpec) - if err != nil { - log.Info("error", "error", err) + rosaScope.Info("error", "error", err) return ctrl.Result{RequeueAfter: 10 * time.Second}, nil } - log.Info("cluster created", "state", newCluster.Status().State()) + rosaScope.Info("cluster created", "state", newCluster.Status().State()) clusterID := newCluster.ID() rosaScope.ControlPlane.Status.ID = &clusterID - if err := rosaScope.PatchObject(); err != nil { - return ctrl.Result{}, err - } return ctrl.Result{}, nil } @@ -339,30 +336,30 @@ func (r *ROSAControlPlaneReconciler) reconcileNormal(ctx context.Context, rosaSc func (r *ROSAControlPlaneReconciler) reconcileDelete(_ context.Context, rosaScope *scope.ROSAControlPlaneScope) (res ctrl.Result, reterr error) { rosaScope.Info("Reconciling ROSAControlPlane delete") - // create OCM NodePool - ocmClient, err := newOCMClient() + // Create the connection, and remember to close it: + // TODO: token should be read from a secret: https://github.com/kubernetes-sigs/cluster-api-provider-aws/issues/4460 + token := os.Getenv("OCM_TOKEN") + rosaClient, err := rosa.NewRosaClient(token) if err != nil { - return ctrl.Result{}, err + return ctrl.Result{}, fmt.Errorf("failed to create a rosa client: %w", err) } + defer func() { - ocmClient.ocm.Close() + rosaClient.Close() }() - cluster, err := ocmClient.GetCluster(rosaScope) + cluster, err := rosaClient.GetCluster(rosaScope.RosaClusterName(), rosaScope.ControlPlane.Spec.CreatorARN) if err != nil { return ctrl.Result{}, err } if cluster != nil { - if _, err := ocmClient.DeleteCluster(cluster.ID()); err != nil { + if err := rosaClient.DeleteCluster(cluster.ID()); err != nil { return ctrl.Result{}, err } } controllerutil.RemoveFinalizer(rosaScope.ControlPlane, ROSAControlPlaneFinalizer) - if err := rosaScope.PatchObject(); err != nil { - return ctrl.Result{}, err - } return ctrl.Result{}, nil } @@ -406,119 +403,3 @@ func (r *ROSAControlPlaneReconciler) rosaClusterToROSAControlPlane(log *logger.L } } } - -// OCMClient is a temporary helper to talk to OCM API. -// TODO(alberto): vendor this from https://github.com/openshift/rosa/tree/master/pkg/ocm or build its own package here. -type OCMClient struct { - ocm *sdk.Connection -} - -func newOCMClient() (*OCMClient, error) { - // Create the connection, and remember to close it: - token := os.Getenv("OCM_TOKEN") - ocmAPIUrl := os.Getenv("OCM_API_URL") - if ocmAPIUrl == "" { - ocmAPIUrl = "https://api.openshift.com" - } - - // Create a logger that has the debug level enabled: - ocmLogger, err := sdk.NewGoLoggerBuilder(). - Debug(false). - Build() - if err != nil { - return nil, fmt.Errorf("failed to build logger: %w", err) - } - - connection, err := sdk.NewConnectionBuilder(). - Logger(ocmLogger). - Tokens(token). - URL(ocmAPIUrl). - Build() - if err != nil { - return nil, fmt.Errorf("failed to ocm client: %w", err) - } - ocmClient := OCMClient{ocm: connection} - - return &ocmClient, nil -} - -func (client *OCMClient) Close() error { - return client.ocm.Close() -} - -func (client *OCMClient) CreateCluster(clusterSpec *cmv1.Cluster) (*cmv1.Cluster, error) { - cluster, err := client.ocm.ClustersMgmt().V1().Clusters(). - Add(). - Body(clusterSpec). - Send() - if err != nil { - return nil, handleErr(cluster.Error(), err) - } - - clusterObject := cluster.Body() - - return clusterObject, nil -} - -func (client *OCMClient) GetCluster(rosaScope *scope.ROSAControlPlaneScope) (*cmv1.Cluster, error) { - clusterKey := rosaScope.ControlPlane.Name[:15] - query := fmt.Sprintf("%s AND (id = '%s' OR name = '%s' OR external_id = '%s')", - getClusterFilter(rosaScope), - clusterKey, clusterKey, clusterKey, - ) - response, err := client.ocm.ClustersMgmt().V1().Clusters().List(). - Search(query). - Page(1). - Size(1). - Send() - if err != nil { - return nil, err - } - - switch response.Total() { - case 0: - return nil, nil - case 1: - return response.Items().Slice()[0], nil - default: - return nil, fmt.Errorf("there are %d clusters with identifier or name '%s'", response.Total(), clusterKey) - } -} - -func (client *OCMClient) DeleteCluster(clusterID string) (*cmv1.Cluster, error) { - response, err := client.ocm.ClustersMgmt().V1().Clusters(). - Cluster(clusterID). - Delete(). - Send() - if err != nil { - return nil, handleErr(response.Error(), err) - } - - return nil, nil -} - -// Generate a query that filters clusters running on the current AWS session account. -func getClusterFilter(rosaScope *scope.ROSAControlPlaneScope) string { - filter := "product.id = 'rosa'" - if rosaScope.ControlPlane.Spec.CreatorARN != nil { - filter = fmt.Sprintf("%s AND (properties.%s = '%s')", - filter, - rosaCreatorArnProperty, - *rosaScope.ControlPlane.Spec.CreatorARN) - } - return filter -} - -func handleErr(res *ocmerrors.Error, err error) error { - msg := res.Reason() - if msg == "" { - msg = err.Error() - } - // Hack to always display the correct terms and conditions message - if res.Code() == "CLUSTERS-MGMT-451" { - msg = "You must accept the Terms and Conditions in order to continue.\n" + - "Go to https://www.redhat.com/wapps/tnc/ackrequired?site=ocm&event=register\n" + - "Once you accept the terms, you will need to retry the action that was blocked." - } - return fmt.Errorf(msg) -} diff --git a/docs/book/src/topics/rosa/creating-a-cluster.md b/docs/book/src/topics/rosa/creating-a-cluster.md index 150a46f46a..fa33dd4744 100644 --- a/docs/book/src/topics/rosa/creating-a-cluster.md +++ b/docs/book/src/topics/rosa/creating-a-cluster.md @@ -29,7 +29,7 @@ Once Step 3 is done, you will be ready to proceed with creating a ROSA cluster u 1. Prepare the environment: ```bash - export OPENSHIFT_VERSION="openshift-v4.12.15" + export OPENSHIFT_VERSION="openshift-v4.14.5" export CLUSTER_NAME="capi-rosa-quickstart" export AWS_REGION="us-west-2" export AWS_AVAILABILITY_ZONE="us-west-2a" diff --git a/pkg/cloud/scope/rosacontrolplane.go b/pkg/cloud/scope/rosacontrolplane.go index 18a53b9d1e..2e60829cd7 100644 --- a/pkg/cloud/scope/rosacontrolplane.go +++ b/pkg/cloud/scope/rosacontrolplane.go @@ -19,11 +19,7 @@ package scope import ( "context" - amazoncni "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1" "github.com/pkg/errors" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" @@ -33,13 +29,6 @@ import ( "sigs.k8s.io/cluster-api/util/patch" ) -func init() { - _ = amazoncni.AddToScheme(scheme) - _ = appsv1.AddToScheme(scheme) - _ = corev1.AddToScheme(scheme) - _ = rbacv1.AddToScheme(scheme) -} - type ROSAControlPlaneScopeParams struct { Client client.Client Logger *logger.Logger @@ -93,11 +82,14 @@ func (s *ROSAControlPlaneScope) Name() string { } // InfraClusterName returns the AWS cluster name. - func (s *ROSAControlPlaneScope) InfraClusterName() string { return s.ControlPlane.Name } +func (s *ROSAControlPlaneScope) RosaClusterName() string { + return s.ControlPlane.Spec.RosaClusterName +} + // Namespace returns the cluster namespace. func (s *ROSAControlPlaneScope) Namespace() string { return s.Cluster.Namespace diff --git a/pkg/rosa/client.go b/pkg/rosa/client.go new file mode 100644 index 0000000000..5192e2ffa9 --- /dev/null +++ b/pkg/rosa/client.go @@ -0,0 +1,59 @@ +package rosa + +import ( + "fmt" + "os" + + sdk "github.com/openshift-online/ocm-sdk-go" +) + +type rosaClient struct { + ocm *sdk.Connection +} + +// NewRosaClientWithConnection creates a client with a preexisting connection for testing purpose +func NewRosaClientWithConnection(connection *sdk.Connection) *rosaClient { + return &rosaClient{ + ocm: connection, + } +} + +func NewRosaClient(token string) (*rosaClient, error) { + ocmAPIUrl := os.Getenv("OCM_API_URL") + if ocmAPIUrl == "" { + ocmAPIUrl = "https://api.openshift.com" + } + + // Create a logger that has the debug level enabled: + logger, err := sdk.NewGoLoggerBuilder(). + Debug(true). + Build() + if err != nil { + return nil, fmt.Errorf("failed to build logger: %w", err) + } + + connection, err := sdk.NewConnectionBuilder(). + Logger(logger). + Tokens(token). + URL(ocmAPIUrl). + Build() + if err != nil { + return nil, fmt.Errorf("failed to create ocm connection: %w", err) + } + + return &rosaClient{ + ocm: connection, + }, nil +} + +func (c *rosaClient) Close() error { + return c.ocm.Close() +} + +func (c *rosaClient) GetConnectionURL() string { + return c.ocm.URL() +} + +func (c *rosaClient) GetConnectionTokens() (string, string, error) { + return c.ocm.Tokens() +} diff --git a/pkg/rosa/clusters.go b/pkg/rosa/clusters.go new file mode 100644 index 0000000000..b4f50dcea5 --- /dev/null +++ b/pkg/rosa/clusters.go @@ -0,0 +1,73 @@ +package rosa + +import ( + "fmt" + + cmv1 "github.com/openshift-online/ocm-sdk-go/clustersmgmt/v1" +) + +const ( + rosaCreatorArnProperty = "rosa_creator_arn" +) + +func (c *rosaClient) CreateCluster(spec *cmv1.Cluster) (*cmv1.Cluster, error) { + cluster, err := c.ocm.ClustersMgmt().V1().Clusters(). + Add(). + Body(spec). + Send() + if err != nil { + return nil, handleErr(cluster.Error(), err) + } + + clusterObject := cluster.Body() + return clusterObject, nil +} + +func (c *rosaClient) DeleteCluster(clusterID string) error { + response, err := c.ocm.ClustersMgmt().V1().Clusters(). + Cluster(clusterID). + Delete(). + BestEffort(true). + Send() + if err != nil { + return handleErr(response.Error(), err) + } + + return nil +} + +func (c *rosaClient) GetCluster(clusterKey string, creatorArn *string) (*cmv1.Cluster, error) { + query := fmt.Sprintf("%s AND (id = '%s' OR name = '%s' OR external_id = '%s')", + getClusterFilter(creatorArn), + clusterKey, clusterKey, clusterKey, + ) + response, err := c.ocm.ClustersMgmt().V1().Clusters().List(). + Search(query). + Page(1). + Size(1). + Send() + if err != nil { + return nil, handleErr(response.Error(), err) + } + + switch response.Total() { + case 0: + return nil, nil + case 1: + return response.Items().Slice()[0], nil + default: + return nil, fmt.Errorf("there are %d clusters with identifier or name '%s'", response.Total(), clusterKey) + } +} + +// Generate a query that filters clusters running on the current AWS session account. +func getClusterFilter(creatorArn *string) string { + filter := "product.id = 'rosa'" + if creatorArn != nil { + filter = fmt.Sprintf("%s AND (properties.%s = '%s')", + filter, + rosaCreatorArnProperty, + *creatorArn) + } + return filter +} diff --git a/pkg/rosa/util.go b/pkg/rosa/util.go new file mode 100644 index 0000000000..94505de7b3 --- /dev/null +++ b/pkg/rosa/util.go @@ -0,0 +1,21 @@ +package rosa + +import ( + "fmt" + + ocmerrors "github.com/openshift-online/ocm-sdk-go/errors" +) + +func handleErr(res *ocmerrors.Error, err error) error { + msg := res.Reason() + if msg == "" { + msg = err.Error() + } + // Hack to always display the correct terms and conditions message + if res.Code() == "CLUSTERS-MGMT-451" { + msg = "You must accept the Terms and Conditions in order to continue.\n" + + "Go to https://www.redhat.com/wapps/tnc/ackrequired?site=ocm&event=register\n" + + "Once you accept the terms, you will need to retry the action that was blocked." + } + return fmt.Errorf(msg) +} diff --git a/templates/cluster-template-rosa.yaml b/templates/cluster-template-rosa.yaml index cc15af9505..8fd60b9a5f 100644 --- a/templates/cluster-template-rosa.yaml +++ b/templates/cluster-template-rosa.yaml @@ -27,6 +27,7 @@ kind: ROSAControlPlane metadata: name: "${CLUSTER_NAME}-control-plane" spec: + rosaClusterName: ${CLUSTER_NAME:0:15} version: "${OPENSHIFT_VERSION}" region: "${AWS_REGION}" accountID: "${AWS_ACCOUNT_ID}" @@ -36,7 +37,7 @@ spec: ingressARN: "arn:aws:iam::${AWS_ACCOUNT_ID}:role/${OPERATOR_ROLES_PREFIX}-openshift-ingress-operator-cloud-credentials" imageRegistryARN: "arn:aws:iam::${AWS_ACCOUNT_ID}:role/${OPERATOR_ROLES_PREFIX}-openshift-image-registry-installer-cloud-credentials" storageARN: "arn:aws:iam::${AWS_ACCOUNT_ID}:role/${OPERATOR_ROLES_PREFIX}-openshift-cluster-csi-drivers-ebs-cloud-credentials" - networkARN: "arn:aws:iam::${AWS_ACCOUNT_ID}:role/${OPERATOR_ROLES_PREFIX}-openshift-cloud-network-config-controller-cloud-credent" + networkARN: "arn:aws:iam::${AWS_ACCOUNT_ID}:role/${OPERATOR_ROLES_PREFIX}-openshift-cloud-network-config-controller-cloud-credentials" kubeCloudControllerARN: "arn:aws:iam::${AWS_ACCOUNT_ID}:role/${OPERATOR_ROLES_PREFIX}-kube-system-kube-controller-manager" nodePoolManagementARN: "arn:aws:iam::${AWS_ACCOUNT_ID}:role/${OPERATOR_ROLES_PREFIX}-kube-system-capa-controller-manager" controlPlaneOperatorARN: "arn:aws:iam::${AWS_ACCOUNT_ID}:role/${OPERATOR_ROLES_PREFIX}-kube-system-control-plane-operator" @@ -49,3 +50,4 @@ spec: - "${AWS_AVAILABILITY_ZONE}" installerRoleARN: "arn:aws:iam::${AWS_ACCOUNT_ID}:role/${ACCOUNT_ROLES_PREFIX}-HCP-ROSA-Installer-Role" supportRoleARN: "arn:aws:iam::${AWS_ACCOUNT_ID}:role/${ACCOUNT_ROLES_PREFIX}-HCP-ROSA-Support-Role" + workerRoleARN: "arn:aws:iam::${AWS_ACCOUNT_ID}:role/${ACCOUNT_ROLES_PREFIX}-HCP-ROSA-Worker-Role" From 0f3a162cfe7138b64a04d559a67f1176c0ab924e Mon Sep 17 00:00:00 2001 From: Mulham Raee Date: Fri, 5 Jan 2024 17:41:29 +0100 Subject: [PATCH 3/5] Add ROSAMachinePool CRD & Controller This introduces basic support to create/delete ROSAMachinePools Lifecycle is captured in RosaMchinePoolReady condition - add cluster-template-rosa-machinepool.yaml --- ...ure.cluster.x-k8s.io_rosamachinepools.yaml | 168 ++++++++++ config/crd/kustomization.yaml | 1 + config/rbac/role.yaml | 19 ++ exp/api/v1beta2/conditions_consts.go | 8 + exp/api/v1beta2/finalizers.go | 3 + exp/api/v1beta2/rosamachinepool_types.go | 136 ++++++++ exp/api/v1beta2/zz_generated.deepcopy.go | 128 ++++++++ exp/controllers/rosamachinepool_controller.go | 310 ++++++++++++++++++ main.go | 10 + pkg/cloud/scope/rosamachinepool.go | 183 +++++++++++ pkg/rosa/nodepools.go | 67 ++++ .../cluster-template-rosa-machinepool.yaml | 81 +++++ 12 files changed, 1114 insertions(+) create mode 100644 config/crd/bases/infrastructure.cluster.x-k8s.io_rosamachinepools.yaml create mode 100644 exp/api/v1beta2/rosamachinepool_types.go create mode 100644 exp/controllers/rosamachinepool_controller.go create mode 100644 pkg/cloud/scope/rosamachinepool.go create mode 100644 pkg/rosa/nodepools.go create mode 100644 templates/cluster-template-rosa-machinepool.yaml diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_rosamachinepools.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_rosamachinepools.yaml new file mode 100644 index 0000000000..8bbffa9be7 --- /dev/null +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_rosamachinepools.yaml @@ -0,0 +1,168 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.12.1 + name: rosamachinepools.infrastructure.cluster.x-k8s.io +spec: + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: ROSAMachinePool + listKind: ROSAMachinePoolList + plural: rosamachinepools + shortNames: + - rosamp + singular: rosamachinepool + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: MachinePool ready status + jsonPath: .status.ready + name: Ready + type: string + - description: Number of replicas + jsonPath: .status.replicas + name: Replicas + type: integer + name: v1beta2 + schema: + openAPIV3Schema: + description: ROSAMachinePool is the Schema for the rosamachinepools API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: RosaMachinePoolSpec defines the desired state of RosaMachinePool. + properties: + autoRepair: + default: false + description: AutoRepair specifies whether health checks should be + enabled for machines in the NodePool. The default is false. + type: boolean + autoscaling: + description: Autoscaling specifies auto scaling behaviour for this + MachinePool. required if Replicas is not configured + properties: + maxReplicas: + minimum: 1 + type: integer + minReplicas: + minimum: 1 + type: integer + type: object + availabilityZone: + description: AvailabilityZone is an optinal field specifying the availability + zone where instances of this machine pool should run For Multi-AZ + clusters, you can create a machine pool in a Single-AZ of your choice. + type: string + instanceType: + description: InstanceType specifies the AWS instance type + type: string + labels: + additionalProperties: + type: string + description: Labels specifies labels for the Kubernetes node objects + type: object + nodePoolName: + description: NodePoolName specifies the name of the nodepool in Rosa + must be a valid DNS-1035 label, so it must consist of lower case + alphanumeric and have a max length of 15 characters. + maxLength: 15 + pattern: ^[a-z]([-a-z0-9]*[a-z0-9])?$ + type: string + x-kubernetes-validations: + - message: nodepoolName is immutable + rule: self == oldSelf + providerIDList: + description: ProviderIDList contain a ProviderID for each machine + instance that's currently managed by this machine pool. + items: + type: string + type: array + subnet: + type: string + required: + - nodePoolName + type: object + status: + description: RosaMachinePoolStatus defines the observed state of RosaMachinePool. + properties: + conditions: + description: Conditions defines current service state of the managed + machine pool + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + id: + description: ID is the ID given by ROSA. + type: string + ready: + default: false + description: Ready denotes that the RosaMachinePool nodepool has joined + the cluster + type: boolean + replicas: + description: Replicas is the most recently observed number of replicas. + format: int32 + type: integer + required: + - ready + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 1b8b24762e..b7fbdd0d73 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -23,6 +23,7 @@ resources: - bases/bootstrap.cluster.x-k8s.io_eksconfigtemplates.yaml - bases/controlplane.cluster.x-k8s.io_rosacontrolplanes.yaml - bases/infrastructure.cluster.x-k8s.io_rosaclusters.yaml +- bases/infrastructure.cluster.x-k8s.io_rosamachinepools.yaml # +kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index c7faf0f437..04e081860a 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -378,3 +378,22 @@ rules: - get - patch - update +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - rosamachinenepools + verbs: + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - rosamachinenepools/status + verbs: + - get + - patch + - update diff --git a/exp/api/v1beta2/conditions_consts.go b/exp/api/v1beta2/conditions_consts.go index deff1a8c77..5987af5684 100644 --- a/exp/api/v1beta2/conditions_consts.go +++ b/exp/api/v1beta2/conditions_consts.go @@ -102,3 +102,11 @@ const ( // reconciling EKS nodegroup iam roles. IAMFargateRolesReconciliationFailedReason = "IAMFargateRolesReconciliationFailed" ) + +const ( + // RosaMachinePoolReadyCondition condition reports on the successful reconciliation of rosa control plane. + RosaMachinePoolReadyCondition clusterv1.ConditionType = "RosaMchinePoolReady" + // WaitingForRosaControlPlaneReason used when the machine pool is waiting for + // ROSA control plane infrastructure to be ready before proceeding. + WaitingForRosaControlPlaneReason = "WaitingForRosaControlPlane" +) diff --git a/exp/api/v1beta2/finalizers.go b/exp/api/v1beta2/finalizers.go index 045efb83fb..1125449285 100644 --- a/exp/api/v1beta2/finalizers.go +++ b/exp/api/v1beta2/finalizers.go @@ -25,4 +25,7 @@ const ( // ManagedMachinePoolFinalizer allows the controller to clean up resources on delete. ManagedMachinePoolFinalizer = "awsmanagedmachinepools.infrastructure.cluster.x-k8s.io" + + // RosaMachinePoolFinalizer allows the controller to clean up resources on delete. + RosaMachinePoolFinalizer = "rosamachinepools.infrastructure.cluster.x-k8s.io" ) diff --git a/exp/api/v1beta2/rosamachinepool_types.go b/exp/api/v1beta2/rosamachinepool_types.go new file mode 100644 index 0000000000..b29ace3421 --- /dev/null +++ b/exp/api/v1beta2/rosamachinepool_types.go @@ -0,0 +1,136 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta2 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" +) + +// RosaMachinePoolSpec defines the desired state of RosaMachinePool. +type RosaMachinePoolSpec struct { + // NodePoolName specifies the name of the nodepool in Rosa + // must be a valid DNS-1035 label, so it must consist of lower case alphanumeric and have a max length of 15 characters. + // + // +immutable + // +kubebuilder:validation:XValidation:rule="self == oldSelf", message="nodepoolName is immutable" + // +kubebuilder:validation:MaxLength:=15 + // +kubebuilder:validation:Pattern:=`^[a-z]([-a-z0-9]*[a-z0-9])?$` + NodePoolName string `json:"nodePoolName"` + + // AvailabilityZone is an optinal field specifying the availability zone where instances of this machine pool should run + // For Multi-AZ clusters, you can create a machine pool in a Single-AZ of your choice. + // +optional + AvailabilityZone string `json:"availabilityZone,omitempty"` + + // +optional + Subnet string `json:"subnet,omitempty"` + + // Labels specifies labels for the Kubernetes node objects + // +optional + Labels map[string]string `json:"labels,omitempty"` + + // AutoRepair specifies whether health checks should be enabled for machines + // in the NodePool. The default is false. + // +optional + // +kubebuilder:default=false + AutoRepair bool `json:"autoRepair,omitempty"` + + // InstanceType specifies the AWS instance type + InstanceType string `json:"instanceType,omitempty"` + + // Autoscaling specifies auto scaling behaviour for this MachinePool. + // required if Replicas is not configured + // +optional + Autoscaling *RosaMachinePoolAutoScaling `json:"autoscaling,omitempty"` + + // TODO(alberto): Enable and propagate this API input. + // Taints []*Taint `json:"taints,omitempty"` + // TuningConfigs []string `json:"tuningConfigs,omitempty"` + // Version *Version `json:"version,omitempty"` + + // ProviderIDList contain a ProviderID for each machine instance that's currently managed by this machine pool. + // +optional + ProviderIDList []string `json:"providerIDList,omitempty"` +} + +// RosaMachinePoolAutoScaling specifies scaling options. +type RosaMachinePoolAutoScaling struct { + // +kubebuilder:validation:Minimum=1 + MinReplicas int `json:"minReplicas,omitempty"` + // +kubebuilder:validation:Minimum=1 + MaxReplicas int `json:"maxReplicas,omitempty"` +} + +// RosaMachinePoolStatus defines the observed state of RosaMachinePool. +type RosaMachinePoolStatus struct { + // Ready denotes that the RosaMachinePool nodepool has joined + // the cluster + // +kubebuilder:default=false + Ready bool `json:"ready"` + + // Replicas is the most recently observed number of replicas. + // +optional + Replicas int32 `json:"replicas"` + + // Conditions defines current service state of the managed machine pool + // +optional + Conditions clusterv1.Conditions `json:"conditions,omitempty"` + + // ID is the ID given by ROSA. + ID string `json:"id,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:path=rosamachinepools,scope=Namespaced,categories=cluster-api,shortName=rosamp +// +kubebuilder:storageversion +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.ready",description="MachinePool ready status" +// +kubebuilder:printcolumn:name="Replicas",type="integer",JSONPath=".status.replicas",description="Number of replicas" + +// ROSAMachinePool is the Schema for the rosamachinepools API. +type ROSAMachinePool struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec RosaMachinePoolSpec `json:"spec,omitempty"` + Status RosaMachinePoolStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// ROSAMachinePoolList contains a list of RosaMachinePools. +type ROSAMachinePoolList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ROSAMachinePool `json:"items"` +} + +// GetConditions returns the observations of the operational state of the RosaMachinePool resource. +func (r *ROSAMachinePool) GetConditions() clusterv1.Conditions { + return r.Status.Conditions +} + +// SetConditions sets the underlying service state of the RosaMachinePool to the predescribed clusterv1.Conditions. +func (r *ROSAMachinePool) SetConditions(conditions clusterv1.Conditions) { + r.Status.Conditions = conditions +} + +func init() { + SchemeBuilder.Register(&ROSAMachinePool{}, &ROSAMachinePoolList{}) +} diff --git a/exp/api/v1beta2/zz_generated.deepcopy.go b/exp/api/v1beta2/zz_generated.deepcopy.go index e0b9353b5a..73a3ab00ce 100644 --- a/exp/api/v1beta2/zz_generated.deepcopy.go +++ b/exp/api/v1beta2/zz_generated.deepcopy.go @@ -970,6 +970,65 @@ func (in *ROSAClusterStatus) DeepCopy() *ROSAClusterStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ROSAMachinePool) DeepCopyInto(out *ROSAMachinePool) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ROSAMachinePool. +func (in *ROSAMachinePool) DeepCopy() *ROSAMachinePool { + if in == nil { + return nil + } + out := new(ROSAMachinePool) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ROSAMachinePool) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ROSAMachinePoolList) DeepCopyInto(out *ROSAMachinePoolList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ROSAMachinePool, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ROSAMachinePoolList. +func (in *ROSAMachinePoolList) DeepCopy() *ROSAMachinePoolList { + if in == nil { + return nil + } + out := new(ROSAMachinePoolList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ROSAMachinePoolList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RefreshPreferences) DeepCopyInto(out *RefreshPreferences) { *out = *in @@ -1000,6 +1059,75 @@ func (in *RefreshPreferences) DeepCopy() *RefreshPreferences { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RosaMachinePoolAutoScaling) DeepCopyInto(out *RosaMachinePoolAutoScaling) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RosaMachinePoolAutoScaling. +func (in *RosaMachinePoolAutoScaling) DeepCopy() *RosaMachinePoolAutoScaling { + if in == nil { + return nil + } + out := new(RosaMachinePoolAutoScaling) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RosaMachinePoolSpec) DeepCopyInto(out *RosaMachinePoolSpec) { + *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Autoscaling != nil { + in, out := &in.Autoscaling, &out.Autoscaling + *out = new(RosaMachinePoolAutoScaling) + **out = **in + } + if in.ProviderIDList != nil { + in, out := &in.ProviderIDList, &out.ProviderIDList + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RosaMachinePoolSpec. +func (in *RosaMachinePoolSpec) DeepCopy() *RosaMachinePoolSpec { + if in == nil { + return nil + } + out := new(RosaMachinePoolSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RosaMachinePoolStatus) DeepCopyInto(out *RosaMachinePoolStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(v1beta1.Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RosaMachinePoolStatus. +func (in *RosaMachinePoolStatus) DeepCopy() *RosaMachinePoolStatus { + if in == nil { + return nil + } + out := new(RosaMachinePoolStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SuspendProcessesTypes) DeepCopyInto(out *SuspendProcessesTypes) { *out = *in diff --git a/exp/controllers/rosamachinepool_controller.go b/exp/controllers/rosamachinepool_controller.go new file mode 100644 index 0000000000..183594f7a0 --- /dev/null +++ b/exp/controllers/rosamachinepool_controller.go @@ -0,0 +1,310 @@ +package controllers + +import ( + "context" + "fmt" + "os" + "time" + + cmv1 "github.com/openshift-online/ocm-sdk-go/clustersmgmt/v1" + "github.com/pkg/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/tools/record" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + rosacontrolplanev1 "sigs.k8s.io/cluster-api-provider-aws/v2/controlplane/rosa/api/v1beta2" + expinfrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/exp/api/v1beta2" + "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/scope" + "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/logger" + "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/rosa" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + expclusterv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" + "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/cluster-api/util/annotations" + "sigs.k8s.io/cluster-api/util/conditions" + "sigs.k8s.io/cluster-api/util/predicates" +) + +// ROSAMachinePoolReconciler reconciles a RosaMachinePool object. +type ROSAMachinePoolReconciler struct { + client.Client + Recorder record.EventRecorder + WatchFilterValue string +} + +// SetupWithManager is used to setup the controller. +func (r *ROSAMachinePoolReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error { + log := logger.FromContext(ctx) + + gvk, err := apiutil.GVKForObject(new(expinfrav1.ROSAMachinePool), mgr.GetScheme()) + if err != nil { + return errors.Wrapf(err, "failed to find GVK for RosaMachinePool") + } + rosaControlPlaneToRosaMachinePoolMap := rosaControlPlaneToRosaMachinePoolMapFunc(r.Client, gvk, log) + return ctrl.NewControllerManagedBy(mgr). + For(&expinfrav1.ROSAMachinePool{}). + WithOptions(options). + WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(log.GetLogger(), r.WatchFilterValue)). + Watches( + &expclusterv1.MachinePool{}, + handler.EnqueueRequestsFromMapFunc(machinePoolToInfrastructureMapFunc(gvk)), + ). + Watches( + &rosacontrolplanev1.ROSAControlPlane{}, + handler.EnqueueRequestsFromMapFunc(rosaControlPlaneToRosaMachinePoolMap), + ). + Complete(r) +} + +// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=machinepools;machinepools/status,verbs=get;list;watch;patch +// +kubebuilder:rbac:groups=core,resources=events,verbs=get;list;watch;create;patch +// +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=rosacontrolplanes;rosacontrolplanes/status,verbs=get;list;watch +// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=rosamachinenepools,verbs=get;list;watch;update;patch;delete +// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=rosamachinenepools/status,verbs=get;update;patch + +// Reconcile reconciles RosaMachinePool. +func (r *ROSAMachinePoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, reterr error) { + log := logger.FromContext(ctx) + + rosaMachinePool := &expinfrav1.ROSAMachinePool{} + if err := r.Get(ctx, req.NamespacedName, rosaMachinePool); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{Requeue: true}, nil + } + + machinePool, err := getOwnerMachinePool(ctx, r.Client, rosaMachinePool.ObjectMeta) + if err != nil { + log.Error(err, "Failed to retrieve owner MachinePool from the API Server") + return ctrl.Result{}, err + } + if machinePool == nil { + log.Info("MachinePool Controller has not yet set OwnerRef") + return ctrl.Result{}, nil + } + + log = log.WithValues("MachinePool", klog.KObj(machinePool)) + + cluster, err := util.GetClusterFromMetadata(ctx, r.Client, machinePool.ObjectMeta) + if err != nil { + log.Info("Failed to retrieve Cluster from MachinePool") + return reconcile.Result{}, nil + } + + if annotations.IsPaused(cluster, rosaMachinePool) { + log.Info("Reconciliation is paused for this object") + return ctrl.Result{}, nil + } + + log = log.WithValues("cluster", klog.KObj(cluster)) + + controlPlaneKey := client.ObjectKey{ + Namespace: rosaMachinePool.Namespace, + Name: cluster.Spec.ControlPlaneRef.Name, + } + controlPlane := &rosacontrolplanev1.ROSAControlPlane{} + if err := r.Client.Get(ctx, controlPlaneKey, controlPlane); err != nil { + log.Info("Failed to retrieve ControlPlane from MachinePool") + return reconcile.Result{}, nil + } + + machinePoolScope, err := scope.NewRosaMachinePoolScope(scope.RosaMachinePoolScopeParams{ + Client: r.Client, + ControllerName: "rosamachinepool", + Cluster: cluster, + ControlPlane: controlPlane, + MachinePool: machinePool, + RosaMachinePool: rosaMachinePool, + }) + if err != nil { + return ctrl.Result{}, errors.Wrap(err, "failed to create scope") + } + + if !controlPlane.Status.Ready { + log.Info("Control plane is not ready yet") + err := machinePoolScope.RosaMchinePoolReadyFalse(expinfrav1.WaitingForRosaControlPlaneReason, "") + return ctrl.Result{}, err + } + + defer func() { + conditions.SetSummary(machinePoolScope.RosaMachinePool, conditions.WithConditions(expinfrav1.RosaMachinePoolReadyCondition), conditions.WithStepCounter()) + + if err := machinePoolScope.Close(); err != nil && reterr == nil { + reterr = err + } + }() + + if !rosaMachinePool.ObjectMeta.DeletionTimestamp.IsZero() { + return ctrl.Result{}, r.reconcileDelete(ctx, machinePoolScope) + } + + return r.reconcileNormal(ctx, machinePoolScope) +} + +func (r *ROSAMachinePoolReconciler) reconcileNormal( + ctx context.Context, machinePoolScope *scope.RosaMachinePoolScope) (ctrl.Result, error) { + machinePoolScope.Info("Reconciling RosaMachinePool") + + if controllerutil.AddFinalizer(machinePoolScope.RosaMachinePool, expinfrav1.RosaMachinePoolFinalizer) { + if err := machinePoolScope.PatchObject(); err != nil { + return ctrl.Result{}, err + } + } + + // TODO: token should be read from a secret: https://github.com/kubernetes-sigs/cluster-api-provider-aws/issues/4460 + token := os.Getenv("OCM_TOKEN") + rosaClient, err := rosa.NewRosaClient(token) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to create a rosa client: %w", err) + } + + defer func() { + rosaClient.Close() + }() + + rosaMachinePool := machinePoolScope.RosaMachinePool + machinePool := machinePoolScope.MachinePool + controlPlane := machinePoolScope.ControlPlane + + createdNodePool, found, err := rosaClient.GetNodePool(*controlPlane.Status.ID, rosaMachinePool.Spec.NodePoolName) + if err != nil { + return ctrl.Result{}, err + } + if found { + // TODO (alberto): discover and store providerIDs from aws so the CAPI controller can match then to Nodes and report readiness. + rosaMachinePool.Status.Replicas = int32(createdNodePool.Status().CurrentReplicas()) + if createdNodePool.Replicas() == createdNodePool.Status().CurrentReplicas() && createdNodePool.Status().Message() == "" { + conditions.MarkTrue(rosaMachinePool, expinfrav1.RosaMachinePoolReadyCondition) + rosaMachinePool.Status.Ready = true + + return ctrl.Result{}, nil + } + + conditions.MarkFalse(rosaMachinePool, + expinfrav1.RosaMachinePoolReadyCondition, + createdNodePool.Status().Message(), + clusterv1.ConditionSeverityInfo, + "") + + machinePoolScope.Info("waiting for NodePool to become ready", "state", createdNodePool.Status().Message()) + // Requeue so that status.ready is set to true when the nodepool is fully created. + return ctrl.Result{RequeueAfter: time.Second * 60}, nil + } + + npBuilder := cmv1.NewNodePool() + npBuilder.ID(rosaMachinePool.Spec.NodePoolName). + Labels(rosaMachinePool.Spec.Labels). + AutoRepair(rosaMachinePool.Spec.AutoRepair) + + if rosaMachinePool.Spec.Autoscaling != nil { + npBuilder = npBuilder.Autoscaling( + cmv1.NewNodePoolAutoscaling(). + MinReplica(rosaMachinePool.Spec.Autoscaling.MinReplicas). + MaxReplica(rosaMachinePool.Spec.Autoscaling.MaxReplicas)) + } else { + var replicas int = 1 + if machinePool.Spec.Replicas != nil { + replicas = int(*machinePool.Spec.Replicas) + } + npBuilder = npBuilder.Replicas(replicas) + } + + if rosaMachinePool.Spec.Subnet != "" { + npBuilder.Subnet(rosaMachinePool.Spec.Subnet) + } + + npBuilder.AWSNodePool(cmv1.NewAWSNodePool().InstanceType(rosaMachinePool.Spec.InstanceType)) + + nodePoolSpec, err := npBuilder.Build() + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to build rosa nodepool: %w", err) + } + + createdNodePool, err = rosaClient.CreateNodePool(*controlPlane.Status.ID, nodePoolSpec) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to create nodepool: %w", err) + } + + machinePoolScope.RosaMachinePool.Status.ID = createdNodePool.ID() + + return ctrl.Result{}, nil +} + +func (r *ROSAMachinePoolReconciler) reconcileDelete( + _ context.Context, machinePoolScope *scope.RosaMachinePoolScope) error { + machinePoolScope.Info("Reconciling deletion of RosaMachinePool") + + // TODO: token should be read from a secret: https://github.com/kubernetes-sigs/cluster-api-provider-aws/issues/4460 + token := os.Getenv("OCM_TOKEN") + rosaClient, err := rosa.NewRosaClient(token) + if err != nil { + return fmt.Errorf("failed to create a rosa client: %w", err) + } + + defer func() { + rosaClient.Close() + }() + + nodePool, found, err := rosaClient.GetNodePool(*machinePoolScope.ControlPlane.Status.ID, machinePoolScope.NodePoolName()) + if err != nil { + return err + } + if found { + if err := rosaClient.DeleteNodePool(*machinePoolScope.ControlPlane.Status.ID, nodePool.ID()); err != nil { + return err + } + } + + controllerutil.RemoveFinalizer(machinePoolScope.RosaMachinePool, expinfrav1.RosaMachinePoolFinalizer) + + return nil +} + +func rosaControlPlaneToRosaMachinePoolMapFunc(c client.Client, gvk schema.GroupVersionKind, log logger.Wrapper) handler.MapFunc { + return func(ctx context.Context, o client.Object) []reconcile.Request { + rosaControlPlane, ok := o.(*rosacontrolplanev1.ROSAControlPlane) + if !ok { + klog.Errorf("Expected a RosaControlPlane but got a %T", o) + } + + if !rosaControlPlane.ObjectMeta.DeletionTimestamp.IsZero() { + return nil + } + + clusterKey, err := GetOwnerClusterKey(rosaControlPlane.ObjectMeta) + if err != nil { + log.Error(err, "couldn't get ROSA control plane owner ObjectKey") + return nil + } + if clusterKey == nil { + return nil + } + + managedPoolForClusterList := expclusterv1.MachinePoolList{} + if err := c.List( + ctx, &managedPoolForClusterList, client.InNamespace(clusterKey.Namespace), client.MatchingLabels{clusterv1.ClusterNameLabel: clusterKey.Name}, + ); err != nil { + log.Error(err, "couldn't list pools for cluster") + return nil + } + + mapFunc := machinePoolToInfrastructureMapFunc(gvk) + + var results []ctrl.Request + for i := range managedPoolForClusterList.Items { + rosaMachinePool := mapFunc(ctx, &managedPoolForClusterList.Items[i]) + results = append(results, rosaMachinePool...) + } + + return results + } +} diff --git a/main.go b/main.go index d61f1c37e8..89018ca193 100644 --- a/main.go +++ b/main.go @@ -224,6 +224,16 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "ROSACluster") os.Exit(1) } + + setupLog.Debug("enabling ROSA machinepool controller") + if err := (&expcontrollers.ROSAMachinePoolReconciler{ + Client: mgr.GetClient(), + Recorder: mgr.GetEventRecorderFor("rosamachinepool-controller"), + WatchFilterValue: watchFilterValue, + }).SetupWithManager(ctx, mgr, controller.Options{MaxConcurrentReconciles: awsClusterConcurrency, RecoverPanic: pointer.Bool(true)}); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "ROSAMachinePool") + os.Exit(1) + } } // +kubebuilder:scaffold:builder diff --git a/pkg/cloud/scope/rosamachinepool.go b/pkg/cloud/scope/rosamachinepool.go new file mode 100644 index 0000000000..24c505d1c3 --- /dev/null +++ b/pkg/cloud/scope/rosamachinepool.go @@ -0,0 +1,183 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scope + +import ( + "context" + + "github.com/pkg/errors" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + + rosacontrolplanev1 "sigs.k8s.io/cluster-api-provider-aws/v2/controlplane/rosa/api/v1beta2" + expinfrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/exp/api/v1beta2" + "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud" + "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/logger" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + expclusterv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" + "sigs.k8s.io/cluster-api/util/conditions" + "sigs.k8s.io/cluster-api/util/patch" +) + +// RosaMachinePoolScopeParams defines the input parameters used to create a new Scope. +type RosaMachinePoolScopeParams struct { + Client client.Client + Logger *logger.Logger + Cluster *clusterv1.Cluster + ControlPlane *rosacontrolplanev1.ROSAControlPlane + RosaMachinePool *expinfrav1.ROSAMachinePool + MachinePool *expclusterv1.MachinePool + ControllerName string +} + +// NewRosaMachinePoolScope creates a new Scope from the supplied parameters. +// This is meant to be called for each reconcile iteration. +func NewRosaMachinePoolScope(params RosaMachinePoolScopeParams) (*RosaMachinePoolScope, error) { + if params.ControlPlane == nil { + return nil, errors.New("failed to generate new scope from nil RosaControlPlane") + } + if params.MachinePool == nil { + return nil, errors.New("failed to generate new scope from nil MachinePool") + } + if params.RosaMachinePool == nil { + return nil, errors.New("failed to generate new scope from nil RosaMachinePool") + } + if params.Logger == nil { + log := klog.Background() + params.Logger = logger.NewLogger(log) + } + + ammpHelper, err := patch.NewHelper(params.RosaMachinePool, params.Client) + if err != nil { + return nil, errors.Wrap(err, "failed to init RosaMachinePool patch helper") + } + mpHelper, err := patch.NewHelper(params.MachinePool, params.Client) + if err != nil { + return nil, errors.Wrap(err, "failed to init MachinePool patch helper") + } + + return &RosaMachinePoolScope{ + Logger: *params.Logger, + Client: params.Client, + patchHelper: ammpHelper, + capiMachinePoolPatchHelper: mpHelper, + + Cluster: params.Cluster, + ControlPlane: params.ControlPlane, + RosaMachinePool: params.RosaMachinePool, + MachinePool: params.MachinePool, + controllerName: params.ControllerName, + }, nil +} + +// RosaMachinePoolScope defines the basic context for an actuator to operate upon. +type RosaMachinePoolScope struct { + logger.Logger + client.Client + patchHelper *patch.Helper + capiMachinePoolPatchHelper *patch.Helper + + Cluster *clusterv1.Cluster + ControlPlane *rosacontrolplanev1.ROSAControlPlane + RosaMachinePool *expinfrav1.ROSAMachinePool + MachinePool *expclusterv1.MachinePool + + controllerName string +} + +// RosaMachinePoolName returns the rosa machine pool name. +func (s *RosaMachinePoolScope) RosaMachinePoolName() string { + return s.RosaMachinePool.Name +} + +// NodePoolName returns the nodePool name of this machine pool. +func (s *RosaMachinePoolScope) NodePoolName() string { + return s.RosaMachinePool.Spec.NodePoolName +} + +// ClusterName returns the cluster name. +func (s *RosaMachinePoolScope) RosaClusterName() string { + return s.ControlPlane.Spec.RosaClusterName +} + +// ControlPlaneSubnets returns the control plane subnets. +func (s *RosaMachinePoolScope) ControlPlaneSubnets() []string { + return s.ControlPlane.Spec.Subnets +} + +// InfraCluster returns the AWS infrastructure cluster or control plane object. +func (s *RosaMachinePoolScope) InfraCluster() cloud.ClusterObject { + return s.ControlPlane +} + +// ClusterObj returns the cluster object. +func (s *RosaMachinePoolScope) ClusterObj() cloud.ClusterObject { + return s.Cluster +} + +// ControllerName returns the name of the controller that +// created the RosaMachinePool. +func (s *RosaMachinePoolScope) ControllerName() string { + return s.controllerName +} + +func (s *RosaMachinePoolScope) GetSetter() conditions.Setter { + return s.RosaMachinePool +} + +// RosaMchinePoolReadyFalse marks the ready condition false using warning if error isn't +// empty. +func (s *RosaMachinePoolScope) RosaMchinePoolReadyFalse(reason string, err string) error { + severity := clusterv1.ConditionSeverityWarning + if err == "" { + severity = clusterv1.ConditionSeverityInfo + } + conditions.MarkFalse( + s.RosaMachinePool, + expinfrav1.RosaMachinePoolReadyCondition, + reason, + severity, + err, + ) + if err := s.PatchObject(); err != nil { + return errors.Wrap(err, "failed to mark rosa machinepool not ready") + } + return nil +} + +// PatchObject persists the control plane configuration and status. +func (s *RosaMachinePoolScope) PatchObject() error { + return s.patchHelper.Patch( + context.TODO(), + s.RosaMachinePool, + patch.WithOwnedConditions{Conditions: []clusterv1.ConditionType{ + expinfrav1.RosaMachinePoolReadyCondition, + }}) +} + +// PatchCAPIMachinePoolObject persists the capi machinepool configuration and status. +func (s *RosaMachinePoolScope) PatchCAPIMachinePoolObject(ctx context.Context) error { + return s.capiMachinePoolPatchHelper.Patch( + ctx, + s.MachinePool, + ) +} + +// Close closes the current scope persisting the control plane configuration and status. +func (s *RosaMachinePoolScope) Close() error { + return s.PatchObject() +} diff --git a/pkg/rosa/nodepools.go b/pkg/rosa/nodepools.go new file mode 100644 index 0000000000..0a93bdae18 --- /dev/null +++ b/pkg/rosa/nodepools.go @@ -0,0 +1,67 @@ +package rosa + +import cmv1 "github.com/openshift-online/ocm-sdk-go/clustersmgmt/v1" + +func (c *rosaClient) CreateNodePool(clusterID string, nodePool *cmv1.NodePool) (*cmv1.NodePool, error) { + response, err := c.ocm.ClustersMgmt().V1(). + Clusters().Cluster(clusterID). + NodePools(). + Add().Body(nodePool). + Send() + if err != nil { + return nil, handleErr(response.Error(), err) + } + return response.Body(), nil +} + +func (c *rosaClient) GetNodePools(clusterID string) ([]*cmv1.NodePool, error) { + response, err := c.ocm.ClustersMgmt().V1(). + Clusters().Cluster(clusterID). + NodePools(). + List().Page(1).Size(-1). + Send() + if err != nil { + return nil, handleErr(response.Error(), err) + } + return response.Items().Slice(), nil +} + +func (c *rosaClient) GetNodePool(clusterID string, nodePoolID string) (*cmv1.NodePool, bool, error) { + response, err := c.ocm.ClustersMgmt().V1(). + Clusters().Cluster(clusterID). + NodePools(). + NodePool(nodePoolID). + Get(). + Send() + if response.Status() == 404 { + return nil, false, nil + } + if err != nil { + return nil, false, handleErr(response.Error(), err) + } + return response.Body(), true, nil +} + +func (c *rosaClient) UpdateNodePool(clusterID string, nodePool *cmv1.NodePool) (*cmv1.NodePool, error) { + response, err := c.ocm.ClustersMgmt().V1(). + Clusters().Cluster(clusterID). + NodePools().NodePool(nodePool.ID()). + Update().Body(nodePool). + Send() + if err != nil { + return nil, handleErr(response.Error(), err) + } + return response.Body(), nil +} + +func (c *rosaClient) DeleteNodePool(clusterID string, nodePoolID string) error { + response, err := c.ocm.ClustersMgmt().V1(). + Clusters().Cluster(clusterID). + NodePools().NodePool(nodePoolID). + Delete(). + Send() + if err != nil { + return handleErr(response.Error(), err) + } + return nil +} diff --git a/templates/cluster-template-rosa-machinepool.yaml b/templates/cluster-template-rosa-machinepool.yaml new file mode 100644 index 0000000000..a5568e1673 --- /dev/null +++ b/templates/cluster-template-rosa-machinepool.yaml @@ -0,0 +1,81 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: "${CLUSTER_NAME}" +spec: + clusterNetwork: + pods: + cidrBlocks: ["192.168.0.0/16"] + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: ROSACluster + name: "${CLUSTER_NAME}" + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: ROSAControlPlane + name: "${CLUSTER_NAME}-control-plane" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: ROSACluster +metadata: + name: "${CLUSTER_NAME}" +spec: {} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: ROSAControlPlane +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + rosaClusterName: ${CLUSTER_NAME:0:15} + version: "${OPENSHIFT_VERSION}" + region: "${AWS_REGION}" + accountID: "${AWS_ACCOUNT_ID}" + creatorARN: "${AWS_CREATOR_ARN}" + machineCIDR: "10.0.0.0/16" + rolesRef: + ingressARN: "arn:aws:iam::${AWS_ACCOUNT_ID}:role/${OPERATOR_ROLES_PREFIX}-openshift-ingress-operator-cloud-credentials" + imageRegistryARN: "arn:aws:iam::${AWS_ACCOUNT_ID}:role/${OPERATOR_ROLES_PREFIX}-openshift-image-registry-installer-cloud-credentials" + storageARN: "arn:aws:iam::${AWS_ACCOUNT_ID}:role/${OPERATOR_ROLES_PREFIX}-openshift-cluster-csi-drivers-ebs-cloud-credentials" + networkARN: "arn:aws:iam::${AWS_ACCOUNT_ID}:role/${OPERATOR_ROLES_PREFIX}-openshift-cloud-network-config-controller-cloud-credentials" + kubeCloudControllerARN: "arn:aws:iam::${AWS_ACCOUNT_ID}:role/${OPERATOR_ROLES_PREFIX}-kube-system-kube-controller-manager" + nodePoolManagementARN: "arn:aws:iam::${AWS_ACCOUNT_ID}:role/${OPERATOR_ROLES_PREFIX}-kube-system-capa-controller-manager" + controlPlaneOperatorARN: "arn:aws:iam::${AWS_ACCOUNT_ID}:role/${OPERATOR_ROLES_PREFIX}-kube-system-control-plane-operator" + kmsProviderARN: "arn:aws:iam::${AWS_ACCOUNT_ID}:role/${OPERATOR_ROLES_PREFIX}-kube-system-kms-provider" + oidcID: "${OIDC_CONFIG_ID}" + subnets: + - "${PUBLIC_SUBNET_ID}" + - "${PRIVATE_SUBNET_ID}" + availabilityZones: + - "${AWS_AVAILABILITY_ZONE}" + installerRoleARN: "arn:aws:iam::${AWS_ACCOUNT_ID}:role/${ACCOUNT_ROLES_PREFIX}-HCP-ROSA-Installer-Role" + supportRoleARN: "arn:aws:iam::${AWS_ACCOUNT_ID}:role/${ACCOUNT_ROLES_PREFIX}-HCP-ROSA-Support-Role" + workerRoleARN: "arn:aws:iam::${AWS_ACCOUNT_ID}:role/${ACCOUNT_ROLES_PREFIX}-HCP-ROSA-Worker-Role" +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachinePool +metadata: + name: "${CLUSTER_NAME}-pool-0" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: 1 + template: + spec: + clusterName: "${CLUSTER_NAME}" + bootstrap: + dataSecretName: "" + infrastructureRef: + name: "${CLUSTER_NAME}-pool-0" + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: RosaMachinePool +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: RosaMachinePool +metadata: + name: "${CLUSTER_NAME}-pool-0" +spec: + nodePoolName: "nodepool-0" + instanceType: "m5.xlarge" + subnet: "${PRIVATE_SUBNET_ID}" + + From 4b036342bf0a24960c8935dccbd3b568cca24d1e Mon Sep 17 00:00:00 2001 From: Mulham Raee Date: Wed, 17 Jan 2024 16:13:55 +0100 Subject: [PATCH 4/5] read credentials(token) from a referenced secret --- ...ne.cluster.x-k8s.io_rosacontrolplanes.yaml | 12 +++++ .../api/v1beta2/rosacontrolplane_types.go | 8 ++++ .../rosa/api/v1beta2/zz_generated.deepcopy.go | 6 +++ .../rosacontrolplane_controller.go | 26 +++------- .../src/topics/rosa/creating-a-cluster.md | 41 ++++++++++++---- exp/controllers/rosamachinepool_controller.go | 45 +++++++++-------- pkg/cloud/scope/rosacontrolplane.go | 21 +++++++- pkg/cloud/scope/rosamachinepool.go | 2 +- pkg/rosa/client.go | 48 +++++++++++++++---- pkg/rosa/clusters.go | 5 +- 10 files changed, 152 insertions(+), 62 deletions(-) diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_rosacontrolplanes.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_rosacontrolplanes.yaml index a92058625b..3b5fd569f1 100644 --- a/config/crd/bases/controlplane.cluster.x-k8s.io_rosacontrolplanes.yaml +++ b/config/crd/bases/controlplane.cluster.x-k8s.io_rosacontrolplanes.yaml @@ -72,6 +72,18 @@ spec: type: object creatorARN: type: string + credentialsSecretRef: + description: 'CredentialsSecretRef references a secret with necessary + credentials to connect to the OCM API. The secret should contain + the following data keys: - ocmToken: eyJhbGciOiJIUzI1NiIsI.... - + ocmApiUrl: Optional, defaults to ''https://api.openshift.com''' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic installerRoleARN: type: string machineCIDR: diff --git a/controlplane/rosa/api/v1beta2/rosacontrolplane_types.go b/controlplane/rosa/api/v1beta2/rosacontrolplane_types.go index 9f010f9e63..acc5c8e625 100644 --- a/controlplane/rosa/api/v1beta2/rosacontrolplane_types.go +++ b/controlplane/rosa/api/v1beta2/rosacontrolplane_types.go @@ -17,6 +17,7 @@ limitations under the License. package v1beta2 import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" @@ -66,6 +67,13 @@ type RosaControlPlaneSpec struct { //nolint: maligned InstallerRoleARN *string `json:"installerRoleARN"` SupportRoleARN *string `json:"supportRoleARN"` WorkerRoleARN *string `json:"workerRoleARN"` + + // CredentialsSecretRef references a secret with necessary credentials to connect to the OCM API. + // The secret should contain the following data keys: + // - ocmToken: eyJhbGciOiJIUzI1NiIsI.... + // - ocmApiUrl: Optional, defaults to 'https://api.openshift.com' + // +optional + CredentialsSecretRef *corev1.LocalObjectReference `json:"credentialsSecretRef,omitempty"` } // AWSRolesRef contains references to various AWS IAM roles required for operators to make calls against the AWS API. diff --git a/controlplane/rosa/api/v1beta2/zz_generated.deepcopy.go b/controlplane/rosa/api/v1beta2/zz_generated.deepcopy.go index 40c87b2b27..3d7f3b8b77 100644 --- a/controlplane/rosa/api/v1beta2/zz_generated.deepcopy.go +++ b/controlplane/rosa/api/v1beta2/zz_generated.deepcopy.go @@ -22,6 +22,7 @@ limitations under the License. package v1beta2 import ( + "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/cluster-api/api/v1beta1" ) @@ -160,6 +161,11 @@ func (in *RosaControlPlaneSpec) DeepCopyInto(out *RosaControlPlaneSpec) { *out = new(string) **out = **in } + if in.CredentialsSecretRef != nil { + in, out := &in.CredentialsSecretRef, &out.CredentialsSecretRef + *out = new(v1.LocalObjectReference) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RosaControlPlaneSpec. diff --git a/controlplane/rosa/controllers/rosacontrolplane_controller.go b/controlplane/rosa/controllers/rosacontrolplane_controller.go index 8e3bdb0466..2d6a9c3e5f 100644 --- a/controlplane/rosa/controllers/rosacontrolplane_controller.go +++ b/controlplane/rosa/controllers/rosacontrolplane_controller.go @@ -20,7 +20,6 @@ import ( "context" "errors" "fmt" - "os" "strings" "time" @@ -170,18 +169,13 @@ func (r *ROSAControlPlaneReconciler) reconcileNormal(ctx context.Context, rosaSc } } - // TODO: token should be read from a secret: https://github.com/kubernetes-sigs/cluster-api-provider-aws/issues/4460 - token := os.Getenv("OCM_TOKEN") - rosaClient, err := rosa.NewRosaClient(token) + rosaClient, err := rosa.NewRosaClient(ctx, rosaScope) if err != nil { return ctrl.Result{}, fmt.Errorf("failed to create a rosa client: %w", err) } + defer rosaClient.Close() - defer func() { - rosaClient.Close() - }() - - cluster, err := rosaClient.GetCluster(rosaScope.RosaClusterName(), rosaScope.ControlPlane.Spec.CreatorARN) + cluster, err := rosaClient.GetCluster() if err != nil { return ctrl.Result{}, err } @@ -333,22 +327,16 @@ func (r *ROSAControlPlaneReconciler) reconcileNormal(ctx context.Context, rosaSc return ctrl.Result{}, nil } -func (r *ROSAControlPlaneReconciler) reconcileDelete(_ context.Context, rosaScope *scope.ROSAControlPlaneScope) (res ctrl.Result, reterr error) { +func (r *ROSAControlPlaneReconciler) reconcileDelete(ctx context.Context, rosaScope *scope.ROSAControlPlaneScope) (res ctrl.Result, reterr error) { rosaScope.Info("Reconciling ROSAControlPlane delete") - // Create the connection, and remember to close it: - // TODO: token should be read from a secret: https://github.com/kubernetes-sigs/cluster-api-provider-aws/issues/4460 - token := os.Getenv("OCM_TOKEN") - rosaClient, err := rosa.NewRosaClient(token) + rosaClient, err := rosa.NewRosaClient(ctx, rosaScope) if err != nil { return ctrl.Result{}, fmt.Errorf("failed to create a rosa client: %w", err) } + defer rosaClient.Close() - defer func() { - rosaClient.Close() - }() - - cluster, err := rosaClient.GetCluster(rosaScope.RosaClusterName(), rosaScope.ControlPlane.Spec.CreatorARN) + cluster, err := rosaClient.GetCluster() if err != nil { return ctrl.Result{}, err } diff --git a/docs/book/src/topics/rosa/creating-a-cluster.md b/docs/book/src/topics/rosa/creating-a-cluster.md index fa33dd4744..4472f9489c 100644 --- a/docs/book/src/topics/rosa/creating-a-cluster.md +++ b/docs/book/src/topics/rosa/creating-a-cluster.md @@ -5,18 +5,25 @@ CAPA controller requires an API token in order to be able to provision ROSA clus 1. Visit [https://console.redhat.com/openshift/token](https://console.redhat.com/openshift/token) to retrieve your API authentication token -2. Edit CAPA controller deployment: +1. Create a credentials secret with the token to be referenced later by `ROSAControlePlane` + ```shell + kubectl create secret generic rosa-creds-secret \ + --from-literal=ocmToken='eyJhbGciOiJIUzI1NiIsI....' \ + --from-literal=ocmApiUrl='https://api.openshift.com' + ``` + + Alternatively, you can edit CAPA controller deployment to provide the credentials: ```shell kubectl edit deployment -n capa-system capa-controller-manager ``` - + and add the following environment variables to the manager container: ```yaml - env: - - name: OCM_TOKEN - value: "" - - name: OCM_API_URL - value: "https://api.openshift.com" # or https://api.stage.openshift.com + env: + - name: OCM_TOKEN + value: "" + - name: OCM_API_URL + value: "https://api.openshift.com" # or https://api.stage.openshift.com ``` ## Prerequisites @@ -45,9 +52,25 @@ Once Step 3 is done, you will be ready to proceed with creating a ROSA cluster u export PRIVATE_SUBNET_ID="subnet-05e72222222222222" ``` -1. Create a cluster using the ROSA cluster template: - ```bash +1. Render the cluster manifest using the ROSA cluster template: + ```shell cat templates/cluster-template-rosa.yaml | envsubst > rosa-capi-cluster.yaml + ``` +1. If a credentials secret was created earlier, edit `ROSAControlPlane` to refernce it: + + ```yaml + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: ROSAControlPlane + metadata: + name: "capi-rosa-quickstart-control-plane" + spec: + credentialsSecretRef: + name: rosa-creds-secret + ... + ``` + +1. Finally apply the manifest to create your Rosa cluster: + ```shell kubectl apply -f rosa-capi-cluster.yaml ``` diff --git a/exp/controllers/rosamachinepool_controller.go b/exp/controllers/rosamachinepool_controller.go index 183594f7a0..0166ed953e 100644 --- a/exp/controllers/rosamachinepool_controller.go +++ b/exp/controllers/rosamachinepool_controller.go @@ -3,7 +3,6 @@ package controllers import ( "context" "fmt" - "os" "time" cmv1 "github.com/openshift-online/ocm-sdk-go/clustersmgmt/v1" @@ -129,6 +128,16 @@ func (r *ROSAMachinePoolReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, errors.Wrap(err, "failed to create scope") } + rosaControlPlaneScope, err := scope.NewROSAControlPlaneScope(scope.ROSAControlPlaneScopeParams{ + Client: r.Client, + Cluster: cluster, + ControlPlane: controlPlane, + ControllerName: "rosaControlPlane", + }) + if err != nil { + return ctrl.Result{}, errors.Wrap(err, "failed to create control plane scope") + } + if !controlPlane.Status.Ready { log.Info("Control plane is not ready yet") err := machinePoolScope.RosaMchinePoolReadyFalse(expinfrav1.WaitingForRosaControlPlaneReason, "") @@ -144,14 +153,16 @@ func (r *ROSAMachinePoolReconciler) Reconcile(ctx context.Context, req ctrl.Requ }() if !rosaMachinePool.ObjectMeta.DeletionTimestamp.IsZero() { - return ctrl.Result{}, r.reconcileDelete(ctx, machinePoolScope) + return ctrl.Result{}, r.reconcileDelete(ctx, machinePoolScope, rosaControlPlaneScope) } - return r.reconcileNormal(ctx, machinePoolScope) + return r.reconcileNormal(ctx, machinePoolScope, rosaControlPlaneScope) } -func (r *ROSAMachinePoolReconciler) reconcileNormal( - ctx context.Context, machinePoolScope *scope.RosaMachinePoolScope) (ctrl.Result, error) { +func (r *ROSAMachinePoolReconciler) reconcileNormal(ctx context.Context, + machinePoolScope *scope.RosaMachinePoolScope, + rosaControlPlaneScope *scope.ROSAControlPlaneScope, +) (ctrl.Result, error) { machinePoolScope.Info("Reconciling RosaMachinePool") if controllerutil.AddFinalizer(machinePoolScope.RosaMachinePool, expinfrav1.RosaMachinePoolFinalizer) { @@ -160,16 +171,11 @@ func (r *ROSAMachinePoolReconciler) reconcileNormal( } } - // TODO: token should be read from a secret: https://github.com/kubernetes-sigs/cluster-api-provider-aws/issues/4460 - token := os.Getenv("OCM_TOKEN") - rosaClient, err := rosa.NewRosaClient(token) + rosaClient, err := rosa.NewRosaClient(ctx, rosaControlPlaneScope) if err != nil { return ctrl.Result{}, fmt.Errorf("failed to create a rosa client: %w", err) } - - defer func() { - rosaClient.Close() - }() + defer rosaClient.Close() rosaMachinePool := machinePoolScope.RosaMachinePool machinePool := machinePoolScope.MachinePool @@ -211,7 +217,7 @@ func (r *ROSAMachinePoolReconciler) reconcileNormal( MinReplica(rosaMachinePool.Spec.Autoscaling.MinReplicas). MaxReplica(rosaMachinePool.Spec.Autoscaling.MaxReplicas)) } else { - var replicas int = 1 + replicas := 1 if machinePool.Spec.Replicas != nil { replicas = int(*machinePool.Spec.Replicas) } @@ -240,19 +246,16 @@ func (r *ROSAMachinePoolReconciler) reconcileNormal( } func (r *ROSAMachinePoolReconciler) reconcileDelete( - _ context.Context, machinePoolScope *scope.RosaMachinePoolScope) error { + ctx context.Context, machinePoolScope *scope.RosaMachinePoolScope, + rosaControlPlaneScope *scope.ROSAControlPlaneScope, +) error { machinePoolScope.Info("Reconciling deletion of RosaMachinePool") - // TODO: token should be read from a secret: https://github.com/kubernetes-sigs/cluster-api-provider-aws/issues/4460 - token := os.Getenv("OCM_TOKEN") - rosaClient, err := rosa.NewRosaClient(token) + rosaClient, err := rosa.NewRosaClient(ctx, rosaControlPlaneScope) if err != nil { return fmt.Errorf("failed to create a rosa client: %w", err) } - - defer func() { - rosaClient.Close() - }() + defer rosaClient.Close() nodePool, found, err := rosaClient.GetNodePool(*machinePoolScope.ControlPlane.Status.ID, machinePoolScope.NodePoolName()) if err != nil { diff --git a/pkg/cloud/scope/rosacontrolplane.go b/pkg/cloud/scope/rosacontrolplane.go index 2e60829cd7..88b318e0dd 100644 --- a/pkg/cloud/scope/rosacontrolplane.go +++ b/pkg/cloud/scope/rosacontrolplane.go @@ -20,6 +20,8 @@ import ( "context" "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" @@ -95,12 +97,29 @@ func (s *ROSAControlPlaneScope) Namespace() string { return s.Cluster.Namespace } +// CredentialsSecret returns the CredentialsSecret object. +func (s *ROSAControlPlaneScope) CredentialsSecret() *corev1.Secret { + secretRef := s.ControlPlane.Spec.CredentialsSecretRef + if secretRef == nil { + return nil + } + + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: s.ControlPlane.Spec.CredentialsSecretRef.Name, + Namespace: s.ControlPlane.Namespace, + }, + } +} + // PatchObject persists the control plane configuration and status. func (s *ROSAControlPlaneScope) PatchObject() error { return s.patchHelper.Patch( context.TODO(), s.ControlPlane, - patch.WithOwnedConditions{Conditions: []clusterv1.ConditionType{}}) + patch.WithOwnedConditions{Conditions: []clusterv1.ConditionType{ + rosacontrolplanev1.ROSAControlPlaneReadyCondition, + }}) } // Close closes the current scope persisting the control plane configuration and status. diff --git a/pkg/cloud/scope/rosamachinepool.go b/pkg/cloud/scope/rosamachinepool.go index 24c505d1c3..c39372b6b0 100644 --- a/pkg/cloud/scope/rosamachinepool.go +++ b/pkg/cloud/scope/rosamachinepool.go @@ -109,7 +109,7 @@ func (s *RosaMachinePoolScope) NodePoolName() string { return s.RosaMachinePool.Spec.NodePoolName } -// ClusterName returns the cluster name. +// RosaClusterName returns the cluster name. func (s *RosaMachinePoolScope) RosaClusterName() string { return s.ControlPlane.Spec.RosaClusterName } diff --git a/pkg/rosa/client.go b/pkg/rosa/client.go index 5192e2ffa9..fb30f04a1b 100644 --- a/pkg/rosa/client.go +++ b/pkg/rosa/client.go @@ -1,27 +1,56 @@ package rosa import ( + "context" "fmt" "os" sdk "github.com/openshift-online/ocm-sdk-go" + "sigs.k8s.io/controller-runtime/pkg/client" + + "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/scope" +) + +const ( + ocmTokenKey = "ocmToken" + ocmAPIURLKey = "ocmApiUrl" ) type rosaClient struct { - ocm *sdk.Connection + ocm *sdk.Connection + rosaScope *scope.ROSAControlPlaneScope } -// NewRosaClientWithConnection creates a client with a preexisting connection for testing purpose -func NewRosaClientWithConnection(connection *sdk.Connection) *rosaClient { +// NewRosaClientWithConnection creates a client with a preexisting connection for testing purposes. +func NewRosaClientWithConnection(connection *sdk.Connection, rosaScope *scope.ROSAControlPlaneScope) *rosaClient { return &rosaClient{ - ocm: connection, + ocm: connection, + rosaScope: rosaScope, } } -func NewRosaClient(token string) (*rosaClient, error) { - ocmAPIUrl := os.Getenv("OCM_API_URL") - if ocmAPIUrl == "" { - ocmAPIUrl = "https://api.openshift.com" +func NewRosaClient(ctx context.Context, rosaScope *scope.ROSAControlPlaneScope) (*rosaClient, error) { + var token string + var ocmAPIUrl string + + secret := rosaScope.CredentialsSecret() + if secret != nil { + if err := rosaScope.Client.Get(ctx, client.ObjectKeyFromObject(secret), secret); err != nil { + return nil, fmt.Errorf("failed to get credentials secret: %w", err) + } + + token = string(secret.Data[ocmTokenKey]) + ocmAPIUrl = string(secret.Data[ocmAPIURLKey]) + } else { + // fallback to env variables if secrert is not set + token = os.Getenv("OCM_TOKEN") + if ocmAPIUrl = os.Getenv("OCM_API_URL"); ocmAPIUrl == "" { + ocmAPIUrl = "https://api.openshift.com" + } + } + + if token == "" { + return nil, fmt.Errorf("token is not provided, be sure to set OCM_TOKEN env variable or reference a credentials secret with key %s", ocmTokenKey) } // Create a logger that has the debug level enabled: @@ -42,7 +71,8 @@ func NewRosaClient(token string) (*rosaClient, error) { } return &rosaClient{ - ocm: connection, + ocm: connection, + rosaScope: rosaScope, }, nil } diff --git a/pkg/rosa/clusters.go b/pkg/rosa/clusters.go index b4f50dcea5..1b5cd3bbeb 100644 --- a/pkg/rosa/clusters.go +++ b/pkg/rosa/clusters.go @@ -36,9 +36,10 @@ func (c *rosaClient) DeleteCluster(clusterID string) error { return nil } -func (c *rosaClient) GetCluster(clusterKey string, creatorArn *string) (*cmv1.Cluster, error) { +func (c *rosaClient) GetCluster() (*cmv1.Cluster, error) { + clusterKey := c.rosaScope.RosaClusterName() query := fmt.Sprintf("%s AND (id = '%s' OR name = '%s' OR external_id = '%s')", - getClusterFilter(creatorArn), + getClusterFilter(c.rosaScope.ControlPlane.Spec.CreatorARN), clusterKey, clusterKey, clusterKey, ) response, err := c.ocm.ClustersMgmt().V1().Clusters().List(). From 74a0ce724c761009590670105c93d490b6172eeb Mon Sep 17 00:00:00 2001 From: Mulham Raee Date: Wed, 17 Jan 2024 16:14:08 +0100 Subject: [PATCH 5/5] fixed unit tests - reset CommandLine flagSet before calling klog.InitFlags(nil) to avoid conflicts if an imported package already called it. --- test/helpers/envtest.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/helpers/envtest.go b/test/helpers/envtest.go index 8740d6d8cc..d912b7b7db 100644 --- a/test/helpers/envtest.go +++ b/test/helpers/envtest.go @@ -18,6 +18,7 @@ package helpers import ( "context" + "flag" "fmt" "go/build" "net" @@ -65,6 +66,8 @@ var ( ) func init() { + // reset flags to avoid conflicts if an imported package already called klog.InitFlags() + flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) klog.InitFlags(nil) // additionally force all the controllers to use the Ginkgo logger. ctrl.SetLogger(klog.Background())