diff --git a/api/v1alpha4/kubevirtcluster_types.go b/api/v1alpha4/kubevirtcluster_types.go index 4d8214ef6..2bc8b53c7 100644 --- a/api/v1alpha4/kubevirtcluster_types.go +++ b/api/v1alpha4/kubevirtcluster_types.go @@ -35,10 +35,13 @@ type KubevirtClusterSpec struct { // ControlPlaneEndpoint represents the endpoint used to communicate with the control plane. // +optional - ControlPlaneEndpoint APIEndpoint `json:"controlPlaneEndpoint"` + ControlPlaneEndpoint APIEndpoint `json:"controlPlaneEndpoint,omitempty"` // SSHKeys is a reference to a local struct for SSH keys persistence. - SshKeys SSHKeys `json:"sshKeys"` + SshKeys SSHKeys `json:"sshKeys,omitempty"` + + // InfraClusterSecretRef is a reference to a secret with a kubeconfig for external cluster used for infra. + InfraClusterSecretRef *corev1.ObjectReference `json:"infraClusterSecretRef,omitempty"` } // KubevirtClusterStatus defines the observed state of KubevirtCluster. diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_kubevirtclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_kubevirtclusters.yaml index 97b89746c..97076968e 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_kubevirtclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_kubevirtclusters.yaml @@ -52,6 +52,43 @@ spec: - host - port type: object + infraClusterSecretRef: + description: InfraClusterSecretRef is a reference to a secret with + a kubeconfig for external cluster used for infra. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object sshKeys: description: SSHKeys is a reference to a local struct for SSH keys persistence. @@ -99,8 +136,6 @@ spec: ssh keys. type: string type: object - required: - - sshKeys type: object status: description: KubevirtClusterStatus defines the observed state of KubevirtCluster. diff --git a/controllers/kubevirtcluster_controller.go b/controllers/kubevirtcluster_controller.go index 547b4b102..99bcc7577 100644 --- a/controllers/kubevirtcluster_controller.go +++ b/controllers/kubevirtcluster_controller.go @@ -24,6 +24,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" infrav1 "sigs.k8s.io/cluster-api-provider-kubevirt/api/v1alpha4" "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/context" + "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/infracluster" "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/loadbalancer" "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/ssh" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4" @@ -36,12 +37,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/source" + "time" ) // KubevirtClusterReconciler reconciles a KubevirtCluster object. type KubevirtClusterReconciler struct { client.Client - Log logr.Logger + InfraCluster infracluster.InfraCluster + Log logr.Logger } // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=kubevirtclusters,verbs=get;list;watch;create;update;patch;delete @@ -82,8 +85,17 @@ func (r *KubevirtClusterReconciler) Reconcile(goctx gocontext.Context, req ctrl. Logger: ctrl.LoggerFrom(goctx).WithName(req.Namespace).WithName(req.Name), } + infraClusterClient, infraClusterNamespace, err := r.InfraCluster.GenerateInfraClusterClient(clusterContext) + if err != nil { + return ctrl.Result{RequeueAfter: 10 * time.Second}, errors.Wrap(err, "failed to generate infra cluster client") + } + if infraClusterClient == nil { + clusterContext.Logger.Info("Waiting for infra cluster client...") + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } + // Create a helper for managing a service hosting the load-balancer. - externalLoadBalancer, err := loadbalancer.NewLoadBalancer(clusterContext, r.Client) + externalLoadBalancer, err := loadbalancer.NewLoadBalancer(clusterContext, infraClusterClient, infraClusterNamespace) if err != nil { return ctrl.Result{}, errors.Wrapf(err, "failed to create helper for managing the externalLoadBalancer") } @@ -111,7 +123,7 @@ func (r *KubevirtClusterReconciler) Reconcile(goctx gocontext.Context, req ctrl. // Handle deleted clusters if !kubevirtCluster.DeletionTimestamp.IsZero() { - return r.reconcileDelete(clusterContext) + return r.reconcileDelete(clusterContext, externalLoadBalancer) } // Handle non-deleted clusters @@ -169,7 +181,12 @@ func (r *KubevirtClusterReconciler) reconcileNormal(ctx *context.ClusterContext, return ctrl.Result{}, nil } -func (r *KubevirtClusterReconciler) reconcileDelete(ctx *context.ClusterContext) (ctrl.Result, error) { +func (r *KubevirtClusterReconciler) reconcileDelete(ctx *context.ClusterContext, externalLoadBalancer *loadbalancer.LoadBalancer) (ctrl.Result, error) { + ctx.Logger.Info("Deleting load balancer service...") + if err := externalLoadBalancer.Delete(ctx); err != nil { + ctx.Logger.Error(err, "Failed to delete load balancer service.") + } + // Set the LoadBalancerAvailableCondition reporting delete is started, and issue a patch in order to make // this visible to the users. patchHelper, err := patch.NewHelper(ctx.KubevirtCluster, r.Client) diff --git a/controllers/kubevirtmachine_controller.go b/controllers/kubevirtmachine_controller.go index 7fc30d790..71fdb03b6 100644 --- a/controllers/kubevirtmachine_controller.go +++ b/controllers/kubevirtmachine_controller.go @@ -23,11 +23,11 @@ import ( "regexp" "time" + "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/context" "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/ssh" + "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/infracluster" "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/workloadcluster" - infrav1 "sigs.k8s.io/cluster-api-provider-kubevirt/api/v1alpha4" - "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/context" kubevirthandler "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/kubevirt" "github.com/pkg/errors" @@ -37,7 +37,6 @@ import ( "k8s.io/apimachinery/pkg/types" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4" "sigs.k8s.io/cluster-api/util" - clusterutil "sigs.k8s.io/cluster-api/util" "sigs.k8s.io/cluster-api/util/conditions" "sigs.k8s.io/cluster-api/util/patch" "sigs.k8s.io/cluster-api/util/predicates" @@ -52,6 +51,7 @@ import ( // KubevirtMachineReconciler reconciles a KubevirtMachine object. type KubevirtMachineReconciler struct { client.Client + InfraCluster infracluster.InfraCluster WorkloadCluster workloadcluster.WorkloadCluster } @@ -171,31 +171,25 @@ func (r *KubevirtMachineReconciler) Reconcile(goctx gocontext.Context, req ctrl. func (r *KubevirtMachineReconciler) reconcileNormal(ctx *context.MachineContext) (res ctrl.Result, retErr error) { // If the machine is already provisioned, return if ctx.KubevirtMachine.Status.Ready { + ctx.Logger.Info("KubevirtMachine.Status.Ready is set -- nothing to do!") return ctrl.Result{}, nil } // Make sure bootstrap data is available and populated. if ctx.Machine.Spec.Bootstrap.DataSecretName == nil { if !util.IsControlPlaneMachine(ctx.Machine) && !conditions.IsTrue(ctx.Cluster, clusterv1.ControlPlaneInitializedCondition) { - ctx.Logger.Info("Waiting for the control plane to be initialized") + ctx.Logger.Info("Waiting for the control plane to be initialized...") conditions.MarkFalse(ctx.KubevirtMachine, infrav1.VMProvisionedCondition, clusterv1.WaitingForControlPlaneAvailableReason, clusterv1.ConditionSeverityInfo, "") return ctrl.Result{}, nil } - ctx.Logger.Info("Waiting for the Bootstrap provider controller to set bootstrap data") + ctx.Logger.Info("Waiting for Machine.Spec.Bootstrap.DataSecretName...") conditions.MarkFalse(ctx.KubevirtMachine, infrav1.VMProvisionedCondition, infrav1.WaitingForBootstrapDataReason, clusterv1.ConditionSeverityInfo, "") return ctrl.Result{}, nil } - clusterContext := &context.ClusterContext{ - Context: ctx.Context, - Cluster: ctx.Cluster, - KubevirtCluster: ctx.KubevirtCluster, - Logger: ctx.Logger, - } - // Fetch SSH keys to be used for cluster nodes, and update bootstrap script cloud-init with public key - clusterNodeSshKeys := ssh.NewClusterNodeSshKeys(clusterContext, r.Client) + clusterNodeSshKeys := ssh.NewClusterNodeSshKeys(ctx.ClusterContext(), r.Client) if persisted := clusterNodeSshKeys.IsPersistedToSecret(); !persisted { ctx.Logger.Info("Waiting for ssh keys data secret to be created by KubevirtCluster controller...") return ctrl.Result{RequeueAfter: 10 * time.Second}, nil @@ -204,21 +198,28 @@ func (r *KubevirtMachineReconciler) reconcileNormal(ctx *context.MachineContext) return ctrl.Result{}, errors.Wrap(err, "failed to fetch ssh keys for cluster nodes") } - if err := r.reconcileKubevirtBootstrapSecret(ctx, clusterNodeSshKeys); err != nil { - ctx.Logger.Info("Waiting for the Bootstrap provider controller to set bootstrap data") + infraClusterClient, infraClusterNamespace, err := r.InfraCluster.GenerateInfraClusterClient(ctx.ClusterContext()) + if err != nil { + return ctrl.Result{RequeueAfter: 10 * time.Second}, errors.Wrap(err, "failed to generate infra cluster client") + } + if infraClusterClient == nil { + ctx.Logger.Info("Waiting for infra cluster client...") + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } + + if err := r.reconcileKubevirtBootstrapSecret(ctx, infraClusterClient, infraClusterNamespace, clusterNodeSshKeys); err != nil { conditions.MarkFalse(ctx.KubevirtMachine, infrav1.VMProvisionedCondition, infrav1.WaitingForBootstrapDataReason, clusterv1.ConditionSeverityInfo, "") - return ctrl.Result{}, nil + return ctrl.Result{RequeueAfter: 10 * time.Second}, errors.Wrap(err, "failed to fetch kubevirt bootstrap secret") } // Create a helper for managing the KubeVirt VM hosting the machine. - externalMachine, err := kubevirthandler.NewMachine(ctx, r.Client, clusterNodeSshKeys) + externalMachine, err := kubevirthandler.NewMachine(ctx, infraClusterClient, infraClusterNamespace, clusterNodeSshKeys) if err != nil { return ctrl.Result{}, errors.Wrapf(err, "failed to create helper for managing the externalMachine") } // Provision the underlying VM if not existing if !externalMachine.Exists() { - ctx.Logger.Info("Creating underlying VM instance...") if err := externalMachine.Create(); err != nil { return ctrl.Result{}, errors.Wrap(err, "failed to create VM instance") } @@ -226,14 +227,13 @@ func (r *KubevirtMachineReconciler) reconcileNormal(ctx *context.MachineContext) // Wait for VM to boot if !externalMachine.IsReady() { - ctx.Logger.Info("Waiting for underlying VM instance to boot...") + ctx.Logger.Info("KubeVirt VM is not ready...") return ctrl.Result{RequeueAfter: 20 * time.Second}, nil } conditions.MarkTrue(ctx.KubevirtMachine, infrav1.VMProvisionedCondition) ipAddress := externalMachine.Address() - ctx.Logger.Info(fmt.Sprintf("KubevirtMachine %s: Got ipAddress <%s>", ctx.KubevirtMachine.Name, ipAddress)) if ipAddress == "" { ctx.Logger.Info(fmt.Sprintf("KubevirtMachine %s: Got empty ipAddress, requeue", ctx.KubevirtMachine.Name)) return ctrl.Result{RequeueAfter: 20 * time.Second}, nil @@ -247,7 +247,7 @@ func (r *KubevirtMachineReconciler) reconcileNormal(ctx *context.MachineContext) } // Update the condition BootstrapExecSucceededCondition conditions.MarkTrue(ctx.KubevirtMachine, infrav1.BootstrapExecSucceededCondition) - ctx.Logger.Info("Underlying VM has boostrapped") + ctx.Logger.Info("Underlying VM has boostrapped.") } ctx.KubevirtMachine.Status.Addresses = []clusterv1.MachineAddress{ @@ -271,7 +271,7 @@ func (r *KubevirtMachineReconciler) reconcileNormal(ctx *context.MachineContext) providerID, err := externalMachine.GenerateProviderID() if err != nil { - ctx.Logger.Error(err, "Failed to patch node with provider id...") + ctx.Logger.Error(err, "Failed to patch node with provider id.") return ctrl.Result{RequeueAfter: 5 * time.Second}, nil } @@ -298,7 +298,6 @@ func (r *KubevirtMachineReconciler) updateNodeProviderID(ctx *context.MachineCon if workloadClusterClient == nil { ctx.Logger.Info("Waiting for workload cluster client...") return ctrl.Result{RequeueAfter: 10 * time.Second}, nil - } // using workload cluster client, get the corresponding cluster node @@ -329,6 +328,41 @@ func (r *KubevirtMachineReconciler) updateNodeProviderID(ctx *context.MachineCon } func (r *KubevirtMachineReconciler) reconcileDelete(ctx *context.MachineContext) (ctrl.Result, error) { + infraClusterClient, infraClusterNamespace, err := r.InfraCluster.GenerateInfraClusterClient(ctx.ClusterContext()) + if err != nil { + return ctrl.Result{RequeueAfter: 10 * time.Second}, errors.Wrap(err, "failed to generate infra cluster client") + } + if infraClusterClient == nil { + ctx.Logger.Info("Waiting for infra cluster client...") + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } + + // Fetch SSH keys to be used for cluster nodes, and update bootstrap script cloud-init with public key + clusterNodeSshKeys := ssh.NewClusterNodeSshKeys(ctx.ClusterContext(), r.Client) + if persisted := clusterNodeSshKeys.IsPersistedToSecret(); !persisted { + ctx.Logger.Info("Waiting for ssh keys data secret to be created by KubevirtCluster controller...") + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } + if err := clusterNodeSshKeys.FetchPersistedKeysFromSecret(); err != nil { + return ctrl.Result{}, errors.Wrap(err, "failed to fetch ssh keys for cluster nodes") + } + + ctx.Logger.Info("Deleting VM bootstrap secret...") + if err := r.deleteKubevirtBootstrapSecret(ctx, infraClusterClient, infraClusterNamespace); err != nil { + return ctrl.Result{RequeueAfter: 10 * time.Second}, errors.Wrap(err, "failed to delete bootstrap secret") + } + + ctx.Logger.Info("Deleting VM...") + externalMachine, err := kubevirthandler.NewMachine(ctx, infraClusterClient, infraClusterNamespace, clusterNodeSshKeys) + if err != nil { + return ctrl.Result{RequeueAfter: 10 * time.Second}, errors.Wrap(err, "failed to create helper for externalMachine access") + } + if externalMachine.Exists() { + if err := externalMachine.Delete(); err != nil { + return ctrl.Result{RequeueAfter: 10 * time.Second}, errors.Wrap(err, "failed to delete VM") + } + } + // Set the VMProvisionedCondition reporting delete is started, and issue a patch in order to make // this visible to the users. patchHelper, err := patch.NewHelper(ctx.KubevirtMachine, r.Client) @@ -410,15 +444,15 @@ func (r *KubevirtMachineReconciler) KubevirtClusterToKubevirtMachines(o client.O } // reconcileKubevirtBootstrapSecret creates bootstrap cloud-init secret for KubeVirt virtual machines -func (r *KubevirtMachineReconciler) reconcileKubevirtBootstrapSecret(ctx *context.MachineContext, sshKeys *ssh.ClusterNodeSshKeys) error { +func (r *KubevirtMachineReconciler) reconcileKubevirtBootstrapSecret(ctx *context.MachineContext, infraClusterClient client.Client, infraClusterNamespace string, sshKeys *ssh.ClusterNodeSshKeys) error { if ctx.Machine.Spec.Bootstrap.DataSecretName == nil { return errors.New("error retrieving bootstrap data: linked Machine's bootstrap.dataSecretName is nil") } // Exit early if exists. bootstrapDataSecret := &corev1.Secret{} - bootstrapDataSecretKey := client.ObjectKey{Namespace: ctx.Machine.GetNamespace(), Name: *ctx.Machine.Spec.Bootstrap.DataSecretName + "-userdata"} - if err := r.Client.Get(ctx, bootstrapDataSecretKey, bootstrapDataSecret); err == nil { + bootstrapDataSecretKey := client.ObjectKey{Namespace: infraClusterNamespace, Name: *ctx.Machine.Spec.Bootstrap.DataSecretName + "-userdata"} + if err := infraClusterClient.Get(ctx, bootstrapDataSecretKey, bootstrapDataSecret); err == nil { ctx.BootstrapDataSecret = bootstrapDataSecret return nil } @@ -442,33 +476,17 @@ func (r *KubevirtMachineReconciler) reconcileKubevirtBootstrapSecret(ctx *contex newBootstrapDataSecret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: s.Name + "-userdata", - Namespace: ctx.Machine.GetNamespace(), + Namespace: infraClusterNamespace, }, } ctx.BootstrapDataSecret = newBootstrapDataSecret - _, err := controllerutil.CreateOrUpdate(ctx, r.Client, newBootstrapDataSecret, func() error { + _, err := controllerutil.CreateOrUpdate(ctx, infraClusterClient, newBootstrapDataSecret, func() error { newBootstrapDataSecret.Type = clusterv1.ClusterSecretType newBootstrapDataSecret.Data = map[string][]byte{ "userdata": value, } - // set owner reference for secret - mutateFn := func() (err error) { - newBootstrapDataSecret.SetOwnerReferences(clusterutil.EnsureOwnerRef( - newBootstrapDataSecret.OwnerReferences, - metav1.OwnerReference{ - APIVersion: ctx.KubevirtMachine.APIVersion, - Kind: ctx.KubevirtMachine.Kind, - Name: ctx.KubevirtMachine.Name, - UID: ctx.KubevirtMachine.UID, - })) - return nil - } - if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, newBootstrapDataSecret, mutateFn); err != nil { - return errors.Wrapf(err, "failed to set owner reference for secret") - } - return nil }) @@ -479,6 +497,22 @@ func (r *KubevirtMachineReconciler) reconcileKubevirtBootstrapSecret(ctx *contex return nil } +// deleteKubevirtBootstrapSecret deletes bootstrap cloud-init secret for KubeVirt virtual machines +func (r *KubevirtMachineReconciler) deleteKubevirtBootstrapSecret(ctx *context.MachineContext, infraClusterClient client.Client, infraClusterNamespace string) error { + bootstrapDataSecret := &corev1.Secret{} + bootstrapDataSecretKey := client.ObjectKey{Namespace: infraClusterNamespace, Name: *ctx.Machine.Spec.Bootstrap.DataSecretName + "-userdata"} + if err := infraClusterClient.Get(ctx, bootstrapDataSecretKey, bootstrapDataSecret); err != nil { + // the secret does not exist, exit without error + return nil + } + + if err := infraClusterClient.Delete(ctx, bootstrapDataSecret); err != nil { + return errors.Wrapf(err, "failed to delete kubevirt bootstrap secret for cluster") + } + + return nil +} + func isCloudConfigUserData(userData []byte) bool { return regexp.MustCompile(`(?m)^#cloud-config`).MatchString(string(userData)) } diff --git a/controllers/kubevirtmachine_controller_test.go b/controllers/kubevirtmachine_controller_test.go index db1a5e9ee..569e0b687 100644 --- a/controllers/kubevirtmachine_controller_test.go +++ b/controllers/kubevirtmachine_controller_test.go @@ -27,7 +27,8 @@ import ( "github.com/pkg/errors" "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/context" "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/testing" - "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/workloadcluster/mock" + infraclustermock "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/infracluster/mock" + workloadclustermock "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/workloadcluster/mock" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -131,7 +132,8 @@ var _ = Describe("utility functions", func() { var _ = Describe("reconcile a kubevirt machine", func() { mockCtrl = gomock.NewController(GinkgoT()) - workloadClusterMock := mock.NewMockWorkloadCluster(mockCtrl) + workloadClusterMock := workloadclustermock.NewMockWorkloadCluster(mockCtrl) + infraClusterMock := infraclustermock.NewMockInfraCluster(mockCtrl) testLogger := ctrl.Log.WithName("test") var machineContext *context.MachineContext @@ -220,6 +222,7 @@ var _ = Describe("reconcile a kubevirt machine", func() { kubevirtMachineReconciler = KubevirtMachineReconciler{ Client: fakeClient, WorkloadCluster: workloadClusterMock, + InfraCluster: infraClusterMock, } } @@ -238,6 +241,9 @@ var _ = Describe("reconcile a kubevirt machine", func() { setupClient(objects) + clusterContext := &context.ClusterContext{Context: machineContext.Context, Cluster: machineContext.Cluster, KubevirtCluster: machineContext.KubevirtCluster, Logger: machineContext.Logger} + infraClusterMock.EXPECT().GenerateInfraClusterClient(clusterContext).Return(fakeClient, cluster.Namespace, nil) + out, err := kubevirtMachineReconciler.reconcileNormal(machineContext) Expect(err).ShouldNot(HaveOccurred()) @@ -284,6 +290,9 @@ var _ = Describe("reconcile a kubevirt machine", func() { setupClient(objects) + clusterContext := &context.ClusterContext{Context: machineContext.Context, Cluster: machineContext.Cluster, KubevirtCluster: machineContext.KubevirtCluster, Logger: machineContext.Logger} + infraClusterMock.EXPECT().GenerateInfraClusterClient(clusterContext).Return(fakeClient, cluster.Namespace, nil) + Expect(machineContext.KubevirtMachine.Status.Ready).To(BeFalse()) out, err := kubevirtMachineReconciler.reconcileNormal(machineContext) @@ -306,7 +315,8 @@ var _ = Describe("reconcile a kubevirt machine", func() { var _ = Describe("updateNodeProviderID", func() { mockCtrl = gomock.NewController(GinkgoT()) - workloadClusterMock := mock.NewMockWorkloadCluster(mockCtrl) + workloadClusterMock := workloadclustermock.NewMockWorkloadCluster(mockCtrl) + infraClusterMock := infraclustermock.NewMockInfraCluster(mockCtrl) expectedProviderId := "aa-66@test" testLogger := ctrl.Log.WithName("test") @@ -323,6 +333,7 @@ var _ = Describe("updateNodeProviderID", func() { kubevirtMachineReconciler = KubevirtMachineReconciler{ Client: fakeClient, WorkloadCluster: workloadClusterMock, + InfraCluster: infraClusterMock, } workloadClusterObjects := []client.Object{ @@ -337,7 +348,6 @@ var _ = Describe("updateNodeProviderID", func() { }, } fakeWorkloadClusterClient = fake.NewClientBuilder().WithScheme(setupScheme()).WithObjects(workloadClusterObjects...).Build() - }) AfterEach(func() {}) diff --git a/main.go b/main.go index e69d5102f..3c998d980 100644 --- a/main.go +++ b/main.go @@ -24,6 +24,7 @@ import ( "time" "sigs.k8s.io/cluster-api-provider-kubevirt/controllers" + "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/infracluster" "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/workloadcluster" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -150,6 +151,7 @@ func setupChecks(mgr ctrl.Manager) { func setupReconcilers(ctx context.Context, mgr ctrl.Manager) { if err := (&controllers.KubevirtMachineReconciler{ Client: mgr.GetClient(), + InfraCluster: infracluster.New(mgr.GetClient()), WorkloadCluster: workloadcluster.New(mgr.GetClient()), }).SetupWithManager(ctx, mgr, controller.Options{ MaxConcurrentReconciles: concurrency, @@ -158,8 +160,9 @@ func setupReconcilers(ctx context.Context, mgr ctrl.Manager) { os.Exit(1) } if err := (&controllers.KubevirtClusterReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("KubevirtCluster"), + Client: mgr.GetClient(), + InfraCluster: infracluster.New(mgr.GetClient()), + Log: ctrl.Log.WithName("controllers").WithName("KubevirtCluster"), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "KubevirtCluster") os.Exit(1) diff --git a/pkg/context/machine_context.go b/pkg/context/machine_context.go index e9d2864d0..279178687 100644 --- a/pkg/context/machine_context.go +++ b/pkg/context/machine_context.go @@ -42,6 +42,16 @@ type MachineContext struct { Logger logr.Logger } +// ClusterContext returns cluster context from this machine context +func (c *MachineContext) ClusterContext() *ClusterContext { + return &ClusterContext{ + Context: c.Context, + Cluster: c.Cluster, + KubevirtCluster: c.KubevirtCluster, + Logger: c.Logger, + } +} + // String returns KubeVirt machine GroupVersionKind func (c *MachineContext) String() string { return fmt.Sprintf("%s %s/%s", c.KubevirtMachine.GroupVersionKind(), c.KubevirtMachine.Namespace, c.KubevirtMachine.Name) diff --git a/pkg/infracluster/infracluster.go b/pkg/infracluster/infracluster.go new file mode 100644 index 000000000..32fb89e9a --- /dev/null +++ b/pkg/infracluster/infracluster.go @@ -0,0 +1,67 @@ +package infracluster + +import ( + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/context" + "sigs.k8s.io/controller-runtime/pkg/client" + "strings" +) + +//go:generate mockgen -source=./infracluster.go -destination=./mock/infracluster_generated.go -package=mock +type InfraCluster interface { + GenerateInfraClusterClient(ctx *context.ClusterContext) (client.Client, string, error) +} + +// New creates new InfraCluster instance +func New(client client.Client) InfraCluster { + return &infraCluster{ + Client: client, + } +} + +type infraCluster struct { + client.Client +} + +// GenerateInfraClusterClient creates a client for infra cluster. +func (w *infraCluster) GenerateInfraClusterClient(ctx *context.ClusterContext) (client.Client, string, error) { + infraClusterSecretRef := ctx.KubevirtCluster.Spec.InfraClusterSecretRef + + if infraClusterSecretRef == nil { + return w.Client, ctx.Cluster.Namespace, nil + } + + infraKubeconfigSecret := &corev1.Secret{} + infraKubeconfigSecretKey := client.ObjectKey{Namespace: infraClusterSecretRef.Namespace, Name: infraClusterSecretRef.Name} + if err := w.Client.Get(ctx.Context, infraKubeconfigSecretKey, infraKubeconfigSecret); err != nil { + return nil, "", errors.Wrapf(err, "failed to fetch infra kubeconfig secret %s/%s", infraClusterSecretRef.Namespace, infraClusterSecretRef.Name) + } + + kubeConfig, ok := infraKubeconfigSecret.Data["kubeconfig"] + if !ok { + return nil, "", errors.New("Failed to retrieve infra kubeconfig from secret: 'kubeconfig' key is missing.") + } + + namespace := "default" + namespaceBytes, ok := infraKubeconfigSecret.Data["namespace"] + if ok { + namespace = string(namespaceBytes) + namespace = strings.TrimSpace(namespace) + } + + // generate REST config + restConfig, err := clientcmd.RESTConfigFromKubeConfig(kubeConfig) + if err != nil { + return nil, "", errors.Wrap(err, "failed to create REST config") + } + + // create the client + infraClusterClient, err := client.New(restConfig, client.Options{Scheme: w.Client.Scheme()}) + if err != nil { + return nil, "", errors.Wrap(err, "failed to create infra cluster client") + } + + return infraClusterClient, namespace, nil +} diff --git a/pkg/infracluster/mock/infracluster_generated.go b/pkg/infracluster/mock/infracluster_generated.go new file mode 100644 index 000000000..17af2a14b --- /dev/null +++ b/pkg/infracluster/mock/infracluster_generated.go @@ -0,0 +1,52 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: ./infracluster.go + +// Package mock is a generated GoMock package. +package mock + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + context "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/context" + client "sigs.k8s.io/controller-runtime/pkg/client" +) + +// MockInfraCluster is a mock of InfraCluster interface. +type MockInfraCluster struct { + ctrl *gomock.Controller + recorder *MockInfraClusterMockRecorder +} + +// MockInfraClusterMockRecorder is the mock recorder for MockInfraCluster. +type MockInfraClusterMockRecorder struct { + mock *MockInfraCluster +} + +// NewMockInfraCluster creates a new mock instance. +func NewMockInfraCluster(ctrl *gomock.Controller) *MockInfraCluster { + mock := &MockInfraCluster{ctrl: ctrl} + mock.recorder = &MockInfraClusterMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockInfraCluster) EXPECT() *MockInfraClusterMockRecorder { + return m.recorder +} + +// GenerateInfraClusterClient mocks base method. +func (m *MockInfraCluster) GenerateInfraClusterClient(ctx *context.ClusterContext) (client.Client, string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GenerateInfraClusterClient", ctx) + ret0, _ := ret[0].(client.Client) + ret1, _ := ret[1].(string) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// GenerateInfraClusterClient indicates an expected call of GenerateInfraClusterClient. +func (mr *MockInfraClusterMockRecorder) GenerateInfraClusterClient(ctx interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GenerateInfraClusterClient", reflect.TypeOf((*MockInfraCluster)(nil).GenerateInfraClusterClient), ctx) +} diff --git a/pkg/kubevirt/machine.go b/pkg/kubevirt/machine.go index 2efd401ce..adeb6904c 100644 --- a/pkg/kubevirt/machine.go +++ b/pkg/kubevirt/machine.go @@ -23,13 +23,11 @@ import ( "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" kubevirtv1 "kubevirt.io/api/core/v1" "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/context" "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/ssh" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4" - "sigs.k8s.io/cluster-api/util" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) @@ -37,6 +35,7 @@ import ( // Machine implement a service for managing the KubeVirt VM hosting a kubernetes node. type Machine struct { client client.Client + namespace string machineContext *context.MachineContext vmInstance *kubevirtv1.VirtualMachineInstance @@ -45,16 +44,17 @@ type Machine struct { } // NewMachine returns a new Machine service for the given context. -func NewMachine(ctx *context.MachineContext, client client.Client, sshKeys *ssh.ClusterNodeSshKeys) (*Machine, error) { +func NewMachine(ctx *context.MachineContext, client client.Client, namespace string, sshKeys *ssh.ClusterNodeSshKeys) (*Machine, error) { machine := &Machine{ client: client, + namespace: namespace, machineContext: ctx, vmInstance: nil, sshKeys: sshKeys, getCommandExecutor: ssh.NewVMCommandExecutor, } - namespacedName := types.NamespacedName{Namespace: ctx.KubevirtMachine.Namespace, Name: ctx.KubevirtMachine.Name} + namespacedName := types.NamespacedName{Namespace: namespace, Name: ctx.KubevirtMachine.Name} vmi := &kubevirtv1.VirtualMachineInstance{} err := client.Get(ctx.Context, namespacedName, vmi) @@ -80,24 +80,13 @@ func (m *Machine) Exists() bool { func (m *Machine) Create() error { m.machineContext.Logger.Info(fmt.Sprintf("Creating VM with role '%s'...", nodeRole(m.machineContext))) - virtualMachine := newVirtualMachineFromKubevirtMachine(m.machineContext) + virtualMachine := newVirtualMachineFromKubevirtMachine(m.machineContext, m.namespace) mutateFn := func() (err error) { - // Ensure the KubevirtMachine is marked as an owner of the VirtualMachine. - virtualMachine.SetOwnerReferences(util.EnsureOwnerRef( - virtualMachine.OwnerReferences, - metav1.OwnerReference{ - APIVersion: m.machineContext.KubevirtMachine.APIVersion, - Kind: m.machineContext.KubevirtMachine.Kind, - Name: m.machineContext.KubevirtMachine.Name, - UID: m.machineContext.KubevirtMachine.UID, - })) - - // TODO: to remove those labels if virtualMachine.Labels == nil { virtualMachine.Labels = map[string]string{} } - virtualMachine.Labels[clusterv1.ClusterLabelName] = "capk" + virtualMachine.Labels[clusterv1.ClusterLabelName] = m.machineContext.Cluster.Name return nil } @@ -181,3 +170,21 @@ func (m *Machine) GenerateProviderID() (string, error) { return providerID, nil } + +// Delete deletes VM for this machine. +func (m *Machine) Delete() error { + namespacedName := types.NamespacedName{Namespace: m.machineContext.KubevirtMachine.Namespace, Name: m.machineContext.KubevirtMachine.Name} + vm := &kubevirtv1.VirtualMachine{} + if err := m.client.Get(m.machineContext.Context, namespacedName, vm); err != nil { + if apierrors.IsNotFound(err) { + m.machineContext.Logger.Info(fmt.Sprintf("VM does not exist, nothing to do.")) + return nil + } + } + + if err := m.client.Delete(gocontext.Background(), vm); err != nil { + return errors.Wrapf(err, "failed to delete VM") + } + + return nil +} diff --git a/pkg/kubevirt/machine_test.go b/pkg/kubevirt/machine_test.go index 8547ddb56..cb4c0ab73 100644 --- a/pkg/kubevirt/machine_test.go +++ b/pkg/kubevirt/machine_test.go @@ -240,7 +240,7 @@ func (e FakeVMCommandExecutor) ExecuteCommand(command string) (string, error) { func defaultTestMachine(ctx *context.MachineContext, client client.Client, vmExecutor FakeVMCommandExecutor, sshPubKey []byte) (*Machine, error) { - machine, err := NewMachine(ctx, client, &ssh.ClusterNodeSshKeys{PublicKey: sshPubKey}) + machine, err := NewMachine(ctx, client, ctx.Cluster.Namespace, &ssh.ClusterNodeSshKeys{PublicKey: sshPubKey}) machine.getCommandExecutor = func(fake string, fakeKeys *ssh.ClusterNodeSshKeys) ssh.VMCommandExecutor { return vmExecutor diff --git a/pkg/kubevirt/utils.go b/pkg/kubevirt/utils.go index 9974c5ddb..ff3e929e7 100644 --- a/pkg/kubevirt/utils.go +++ b/pkg/kubevirt/utils.go @@ -35,7 +35,7 @@ type CommandExecutor interface { } // newVirtualMachineFromKubevirtMachine creates VirtualMachine instance. -func newVirtualMachineFromKubevirtMachine(ctx *context.MachineContext) *kubevirtv1.VirtualMachine { +func newVirtualMachineFromKubevirtMachine(ctx *context.MachineContext, namespace string) *kubevirtv1.VirtualMachine { runAlways := kubevirtv1.RunStrategyAlways vmiTemplate := buildVirtualMachineInstanceTemplate(ctx) @@ -51,7 +51,7 @@ func newVirtualMachineFromKubevirtMachine(ctx *context.MachineContext) *kubevirt virtualMachine.ObjectMeta = metav1.ObjectMeta{ Name: ctx.KubevirtMachine.Name, - Namespace: ctx.KubevirtMachine.Namespace, + Namespace: namespace, Labels: map[string]string{ "kubevirt.io/vm": ctx.KubevirtMachine.Name, clusterLabelKey: ctx.KubevirtCluster.Name, diff --git a/pkg/loadbalancer/loadbalancer.go b/pkg/loadbalancer/loadbalancer.go index 567756877..080a408e7 100644 --- a/pkg/loadbalancer/loadbalancer.go +++ b/pkg/loadbalancer/loadbalancer.go @@ -18,6 +18,8 @@ package loadbalancer import ( "fmt" + "github.com/pkg/errors" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -25,38 +27,33 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" infrav1 "sigs.k8s.io/cluster-api-provider-kubevirt/api/v1alpha4" "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/context" - clusterutil "sigs.k8s.io/cluster-api/util" runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" ctrlutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/kind/pkg/cluster/constants" ) -//type lbCreator interface { -// CreateExternalLoadBalancerNode(name, image, clusterLabel, listenAddress string, port int32) (*types.Node, error) -//} - // LoadBalancer manages the load balancer for a specific KubeVirt cluster. type LoadBalancer struct { name string service *corev1.Service - client runtimeclient.Client kubevirtCluster *infrav1.KubevirtCluster + infraClient runtimeclient.Client + infraNamespace string } // NewLoadBalancer returns a new helper for managing a mock load-balancer (using service). -func NewLoadBalancer(ctx *context.ClusterContext, client runtimeclient.Client) (*LoadBalancer, error) { +func NewLoadBalancer(ctx *context.ClusterContext, client runtimeclient.Client, namespace string) (*LoadBalancer, error) { name := ctx.KubevirtCluster.Name + "-lb" // Look for the service that is mocking the load-balancer for the cluster. // Filter based on the label and the roles regardless of whether or not it is running. loadBalancer := &corev1.Service{} loadBalancerKey := runtimeclient.ObjectKey{ - Namespace: ctx.KubevirtCluster.Namespace, + Namespace: namespace, Name: name, } if err := client.Get(ctx.Context, loadBalancerKey, loadBalancer); err != nil { if apierrors.IsNotFound(err) { loadBalancer = nil - ctx.Logger.Info("No load balancer found") } else { return nil, err } @@ -65,21 +62,27 @@ func NewLoadBalancer(ctx *context.ClusterContext, client runtimeclient.Client) ( return &LoadBalancer{ name: name, service: loadBalancer, - client: client, kubevirtCluster: ctx.KubevirtCluster, + infraClient: client, + infraNamespace: namespace, }, nil } +// IsFound checks if load balancer already exists +func (l *LoadBalancer) IsFound() bool { + return l.service != nil +} + // Create creates a service of ClusterIP type to serve as a load-balancer for the cluster. func (l *LoadBalancer) Create(ctx *context.ClusterContext) error { // Skip creation if exists. - if l.service != nil { + if l.IsFound() { return fmt.Errorf("the load balancer service already exists") } lbService := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Namespace: l.kubevirtCluster.Namespace, + Namespace: l.infraNamespace, Name: l.name, }, Spec: corev1.ServiceSpec{ @@ -97,34 +100,28 @@ func (l *LoadBalancer) Create(ctx *context.ClusterContext) error { }, } mutateFn := func() (err error) { - lbService.SetOwnerReferences(clusterutil.EnsureOwnerRef( - lbService.OwnerReferences, - metav1.OwnerReference{ - APIVersion: l.kubevirtCluster.APIVersion, - Kind: l.kubevirtCluster.Kind, - Name: l.kubevirtCluster.Name, - UID: l.kubevirtCluster.UID, - })) + if lbService.Labels == nil { + lbService.Labels = map[string]string{} + } + lbService.Labels[clusterv1.ClusterLabelName] = ctx.Cluster.Name + return nil } - if _, err := ctrlutil.CreateOrUpdate(ctx, l.client, lbService, mutateFn); err != nil { + if _, err := ctrlutil.CreateOrUpdate(ctx.Context, l.infraClient, lbService, mutateFn); err != nil { return corev1.ErrIntOverflowGenerated } return nil } -func (l *LoadBalancer) IsFound() bool { - return l.service != nil -} - +// IP returns ip address of the load balancer func (l *LoadBalancer) IP(ctx *context.ClusterContext) (string, error) { loadBalancer := &corev1.Service{} loadBalancerKey := runtimeclient.ObjectKey{ - Namespace: l.kubevirtCluster.Namespace, + Namespace: l.infraNamespace, Name: l.name, } - if err := l.client.Get(ctx.Context, loadBalancerKey, loadBalancer); err != nil { + if err := l.infraClient.Get(ctx.Context, loadBalancerKey, loadBalancer); err != nil { return "", err } @@ -134,3 +131,16 @@ func (l *LoadBalancer) IP(ctx *context.ClusterContext) (string, error) { return loadBalancer.Spec.ClusterIP, nil } + +// Delete deletes load-balancer service. +func (l *LoadBalancer) Delete(ctx *context.ClusterContext) error { + if !l.IsFound() { + return nil + } + + if err := l.infraClient.Delete(ctx, l.service); err != nil { + return errors.Wrapf(err, "failed to delete load balancer service") + } + + return nil +} diff --git a/pkg/loadbalancer/loadbalancer_test.go b/pkg/loadbalancer/loadbalancer_test.go index b4505634f..2f8715bd8 100644 --- a/pkg/loadbalancer/loadbalancer_test.go +++ b/pkg/loadbalancer/loadbalancer_test.go @@ -67,7 +67,7 @@ var _ = Describe("Load Balancer", func() { }) It("should initialize load balancer without error", func() { - lb, err = loadbalancer.NewLoadBalancer(clusterContext, fakeClient) + lb, err = loadbalancer.NewLoadBalancer(clusterContext, fakeClient, "") Expect(err).NotTo(HaveOccurred()) }) @@ -97,7 +97,7 @@ var _ = Describe("Load Balancer", func() { }) It("should initialize load balancer without error", func() { - lb, err = loadbalancer.NewLoadBalancer(clusterContext, fakeClient) + lb, err = loadbalancer.NewLoadBalancer(clusterContext, fakeClient, "") Expect(err).NotTo(HaveOccurred()) }) diff --git a/pkg/ssh/cluster_node_ssh_keys.go b/pkg/ssh/cluster_node_ssh_keys.go index 2dbe6ca8d..c3ee80298 100644 --- a/pkg/ssh/cluster_node_ssh_keys.go +++ b/pkg/ssh/cluster_node_ssh_keys.go @@ -94,6 +94,12 @@ func (c *ClusterNodeSshKeys) PersistKeysToSecret() (*corev1.Secret, error) { Name: c.ClusterContext.KubevirtCluster.Name, UID: c.ClusterContext.KubevirtCluster.UID, })) + + if newSecret.Labels == nil { + newSecret.Labels = map[string]string{} + } + newSecret.Labels[clusterv1.ClusterLabelName] = c.ClusterContext.Cluster.Name + return nil } if _, err := controllerutil.CreateOrUpdate(c.ClusterContext.Context, c.Client, newSecret, mutateFn); err != nil { diff --git a/pkg/ssh/ssh_command_executor.go b/pkg/ssh/ssh_command_executor.go index 166455a92..7505664b7 100644 --- a/pkg/ssh/ssh_command_executor.go +++ b/pkg/ssh/ssh_command_executor.go @@ -62,24 +62,22 @@ func (e vmCommandExecutor) ExecuteCommand(command string) (string, error) { connection, err := ssh.Dial("tcp", hostAddress, sshConfig) if err != nil { - return "", fmt.Errorf("failed to dial IP %s: %s", hostAddress, err) + return "", fmt.Errorf("ssh: failed to dial IP %s, error: %s", hostAddress, err.Error()) } session, err := connection.NewSession() if err != nil { - return "", fmt.Errorf("failed to create session: %s", err) + return "", fmt.Errorf("ssh: failed to create session, error: %s", err.Error()) } defer session.Close() - // ctx.Logger.Info(fmt.Sprintf("ssh: running command inside VM `%s`...", command)) var b bytes.Buffer session.Stdout = &b if err := session.Run(command); err != nil { - return "", fmt.Errorf("failed to run the command: " + err.Error()) + return "", fmt.Errorf("ssh: failed to run command `%s`, error: %s", command, err.Error()) } output := strings.Trim(b.String(), "\n") - // ctx.Logger.Info(fmt.Sprintf("ssh: command `%s` output is `%s`", command, output)) return output, nil } diff --git a/pkg/testing/common.go b/pkg/testing/common.go index a1451ceeb..eb63e1ec2 100644 --- a/pkg/testing/common.go +++ b/pkg/testing/common.go @@ -101,7 +101,6 @@ func NewVirtualMachineInstance(kubevirtMachine *infrav1.KubevirtMachine) *kubevi } func NewBootstrapDataSecret(userData []byte) *corev1.Secret { - s := &corev1.Secret{} s.Data = make(map[string][]byte) s.Data["userdata"] = userData diff --git a/templates/cluster-template-ext-infra.yaml b/templates/cluster-template-ext-infra.yaml new file mode 100644 index 000000000..2cc134a1d --- /dev/null +++ b/templates/cluster-template-ext-infra.yaml @@ -0,0 +1,143 @@ +--- +apiVersion: cluster.x-k8s.io/v1alpha4 +kind: Cluster +metadata: + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" +spec: + clusterNetwork: + pods: + cidrBlocks: + - 10.244.0.0/16 + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 + kind: KubevirtCluster + name: '${CLUSTER_NAME}' + namespace: "${NAMESPACE}" + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1alpha4 + kind: KubeadmControlPlane + name: '${CLUSTER_NAME}-control-plane' + namespace: "${NAMESPACE}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 +kind: KubevirtCluster +metadata: + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" +spec: + infraClusterSecretRef: + apiVersion: v1 + kind: Secret + name: external-infra-kubeconfig + namespace: capk-system +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 +kind: KubevirtMachineTemplate +metadata: + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" +spec: + template: + spec: + vmSpec: + domain: + cpu: + cores: 2 + memory: + guest: "4Gi" + devices: + disks: + - disk: + bus: virtio + name: containervolume + volumes: + - containerDisk: + image: "${NODE_VM_IMAGE_TEMPLATE}" + name: containervolume +--- +kind: KubeadmControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1alpha4 +metadata: + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" +spec: + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + machineTemplate: + infrastructureRef: + kind: KubevirtMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" + kubeadmConfigSpec: + clusterConfiguration: + imageRepository: ${IMAGE_REPO} + initConfiguration: + nodeRegistration: + criSocket: "${CRI_PATH}" + joinConfiguration: + nodeRegistration: + criSocket: "{CRI_PATH}" + version: "${KUBERNETES_VERSION}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 +kind: KubevirtMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" + namespace: "${NAMESPACE}" +spec: + template: + spec: + vmSpec: + domain: + cpu: + cores: 2 + memory: + guest: "4Gi" + devices: + disks: + - disk: + bus: virtio + name: containervolume + volumes: + - containerDisk: + image: "${NODE_VM_IMAGE_TEMPLATE}" + name: containervolume +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" + namespace: "${NAMESPACE}" +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: {} +--- +apiVersion: cluster.x-k8s.io/v1alpha4 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-0" + namespace: "${NAMESPACE}" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + template: + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + bootstrap: + configRef: + name: "${CLUSTER_NAME}-md-0" + namespace: "${NAMESPACE}" + apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4 + kind: KubeadmConfigTemplate + infrastructureRef: + name: "${CLUSTER_NAME}-md-0" + namespace: "${NAMESPACE}" + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 + kind: KubevirtMachineTemplate