From 779caacca38078ca6c4083d54af1e9e4bbd7e760 Mon Sep 17 00:00:00 2001 From: Mayur Das Date: Fri, 13 May 2022 10:52:57 +0530 Subject: [PATCH] K8sInstallerConfig controller implementation added (#526) K8sInstallerConfig controller will be generation the installation secret for the bundleType to install on byohost test cases added To increase the test coverage, few test cases added for k8sinstallercontroller - should not return reconcile request if ByoMachine InstallerRef doesn't exists - should not return reconcile request if ByoMachine InstallerRef doesn't refer to K8sInstallerConfitTemplate - should return reconcile request if ByoMachine refer to K8sInstallerConfigTemplate installer Signed-off-by: Mayur Das --- Dockerfile | 2 + PROJECT | 1 + agent/installer/bundle_downloader.go | 18 +- agent/installer/installer.go | 22 +- agent/installer/steps.go | 63 --- .../v1beta1/k8sinstallerconfig_types.go | 7 + common/installer/downloader.go | 19 + common/installer/installer.go | 33 ++ .../installer/internal/algo/ubuntu20_4k8s.go | 146 ++++++ common/utils.go | 3 +- config/rbac/role.yaml | 39 ++ .../k8sinstallerconfig_controller.go | 319 ++++++++++++ .../k8sinstallerconfig_controller_test.go | 477 ++++++++++++++++++ controllers/infrastructure/suite_test.go | 48 +- main.go | 8 + test/builder/builders.go | 137 +++++ 16 files changed, 1243 insertions(+), 99 deletions(-) delete mode 100644 agent/installer/steps.go create mode 100644 common/installer/downloader.go create mode 100644 common/installer/installer.go create mode 100644 common/installer/internal/algo/ubuntu20_4k8s.go create mode 100644 controllers/infrastructure/k8sinstallerconfig_controller.go create mode 100644 controllers/infrastructure/k8sinstallerconfig_controller_test.go diff --git a/Dockerfile b/Dockerfile index df509e111..24b73423f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,8 @@ RUN go mod download COPY main.go main.go COPY apis/ apis/ COPY controllers/ controllers/ +COPY common/installer common/installer +COPY agent/installer agent/installer # Build RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go diff --git a/PROJECT b/PROJECT index f17a28515..34512b427 100644 --- a/PROJECT +++ b/PROJECT @@ -47,6 +47,7 @@ resources: - api: crdVersion: v1 namespaced: true + controller: true domain: cluster.x-k8s.io group: infrastructure kind: K8sInstallerConfig diff --git a/agent/installer/bundle_downloader.go b/agent/installer/bundle_downloader.go index d55cbf974..dc3f865f2 100644 --- a/agent/installer/bundle_downloader.go +++ b/agent/installer/bundle_downloader.go @@ -22,12 +22,22 @@ var ( // bundleDownloader for downloading an OCI image. type bundleDownloader struct { - bundleType bundleType + bundleType BundleType repoAddr string downloadPath string logger logr.Logger } +// NewBundleDownloader will return a new bundle downloader instance +func NewBundleDownloader(bundleType BundleType, repoAddr, downloadPath string, logger logr.Logger) *bundleDownloader { + return &bundleDownloader{ + bundleType: bundleType, + repoAddr: repoAddr, + downloadPath: downloadPath, + logger: logger, + } +} + // Download is a method that downloads the bundle from repoAddr to downloadPath. // It automatically downloads and extracts the given version for the current linux // distribution. Creates the folder where the bundle should be saved if it does not exist. @@ -87,7 +97,7 @@ func (bd *bundleDownloader) DownloadFromRepo( if err != nil { return err } - bundleAddr := bd.getBundleAddr(normalizedOsVersion, k8sVersion, tag) + bundleAddr := bd.GetBundleAddr(normalizedOsVersion, k8sVersion, tag) err = convertError(downloadByTool(bundleAddr, dir)) if err != nil { return err @@ -148,8 +158,8 @@ func (bd *bundleDownloader) getBundlePathWithRepo() string { return filepath.Join(bd.downloadPath, strings.ReplaceAll(bd.repoAddr, "/", ".")) } -// getBundleAddr returns the exact address to the bundle in the repo. -func (bd *bundleDownloader) getBundleAddr(normalizedOsVersion, k8sVersion, tag string) string { +// GetBundleAddr returns the exact address to the bundle in the repo. +func (bd *bundleDownloader) GetBundleAddr(normalizedOsVersion, k8sVersion, tag string) string { return fmt.Sprintf("%s/%s:%s", bd.repoAddr, GetBundleName(normalizedOsVersion), tag) } diff --git a/agent/installer/installer.go b/agent/installer/installer.go index ae220d79d..5c53fdb7b 100644 --- a/agent/installer/installer.go +++ b/agent/installer/installer.go @@ -33,11 +33,11 @@ const ( ) // BundleType is used to support various bundles -type bundleType string +type BundleType string const ( // BundleTypeK8s represents a vanilla k8s bundle - BundleTypeK8s bundleType = "k8s" + BundleTypeK8s BundleType = "k8s" ) var preRequisitePackages = []string{"socat", "ebtables", "ethtool", "conntrack"} @@ -49,8 +49,8 @@ type installer struct { logger logr.Logger } -// getSupportedRegistry returns a registry with installers for the supported OS and K8s -func getSupportedRegistry(ob algo.OutputBuilder) registry { +// GetSupportedRegistry returns a registry with installers for the supported OS and K8s +func GetSupportedRegistry(ob algo.OutputBuilder) registry { reg := newRegistry() addBundleInstaller := func(osBundle, k8sBundle string, stepProvider algo.K8sStepProvider) { @@ -116,7 +116,7 @@ func (bd *bundleDownloader) DownloadOrPreview(os, k8s, tag string) error { // New returns an installer that downloads bundles for the current OS from OCI repository with // address bundleRepo and stores them under downloadPath. Download path is created, // if it does not exist. -func New(downloadPath string, bundleType bundleType, logger logr.Logger) (*installer, error) { +func New(downloadPath string, bundleType BundleType, logger logr.Logger) (*installer, error) { if downloadPath == "" { return nil, fmt.Errorf("empty download path") } @@ -139,17 +139,17 @@ func New(downloadPath string, bundleType bundleType, logger logr.Logger) (*insta // newUnchecked returns an installer bypassing os detection and checks of downloadPath. // If it is empty, returned installer will run in preview mode, i.e. // executes everything except the actual commands. -func newUnchecked(currentOs string, bundleType bundleType, downloadPath string, logger logr.Logger, outputBuilder algo.OutputBuilder) (*installer, error) { - bd := bundleDownloader{repoAddr: "", bundleType: bundleType, downloadPath: downloadPath, logger: logger} +func newUnchecked(currentOs string, bundleType BundleType, downloadPath string, logger logr.Logger, outputBuilder algo.OutputBuilder) (*installer, error) { + bd := NewBundleDownloader(bundleType, "", downloadPath, logger) - reg := getSupportedRegistry(outputBuilder) + reg := GetSupportedRegistry(outputBuilder) if len(reg.ListK8s(currentOs)) == 0 { return nil, ErrOsK8sNotSupported } return &installer{ algoRegistry: reg, - bundleDownloader: bd, + bundleDownloader: *bd, detectedOs: currentOs, logger: logger}, nil } @@ -228,7 +228,7 @@ func ListSupportedK8s(os string) []string { // getSupportedRegistryDescription returns a description registry of supported OS and k8s. // It that can only be queried for OS and k8s but cannot be used for install/uninstall. func getSupportedRegistryDescription() registry { - return getSupportedRegistry(nil) + return GetSupportedRegistry(nil) } // PreviewChanges describes the changes to install and uninstall K8s on OS without actually applying them. @@ -236,7 +236,7 @@ func getSupportedRegistryDescription() registry { // Can be invoked on a non-supported OS func PreviewChanges(os, k8sVer string) (install, uninstall string, err error) { stepPreviewer := stringPrinter{msgFmt: "# %s"} - reg := getSupportedRegistry(&stepPreviewer) + reg := GetSupportedRegistry(&stepPreviewer) installer, _ := reg.GetInstaller(os, k8sVer) if installer == nil { diff --git a/agent/installer/steps.go b/agent/installer/steps.go deleted file mode 100644 index 773467bde..000000000 --- a/agent/installer/steps.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2021 VMware, Inc. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package installer - -// contains the installation and uninstallation steps for the supported os and k8s -var ( - DoUbuntu20_4K8s1_22 = ` -set -euo pipefail - -BUNDLE_PATH=${BUNDLE_PATH:-"/var/lib/byoh/bundles"} - -## disable swap -swapoff -a && sed -ri '/\sswap\s/s/^#?/#/' /etc/fstab - -## disable firewall -ufw disable - -## load kernal modules -modprobe overlay && modprobe br_netfilter - -## adding os configuration -tar -C / -xvf "$BUNDLE_PATH/conf.tar" && sysctl --system - -## installing deb packages -for pkg in cri-tools kubernetes-cni kubectl kubeadm kubelet; do - dpkg --install "$BUNDLE_PATH/$pkg.deb" && apt-mark hold $pkg -done - -## intalling containerd -tar -C / -xvf "$BUNDLE_PATH/containerd.tar" - -## starting containerd service -systemctl daemon-reload && systemctl enable containerd && systemctl start containerd` - - UndoUbuntu20_4K8s1_22 = ` -set -euo pipefail - -BUNDLE_PATH=${BUNDLE_PATH:-"/var/lib/byoh/bundles"} - -## enable swap -swapon -a && sed -ri '/\sswap\s/s/^#?//' /etc/fstab - -## enable firewall -ufw enable - -## remove kernal modules -modprobe -r overlay && modprobe -r br_netfilter - -## removing os configuration -tar tf "$BUNDLE_PATH/conf.tar" | xargs -n 1 echo '/' | sed 's/ //g' | xargs rm -f - -## removing deb packages -for pkg in cri-tools kubernetes-cni kubectl kubeadm kubelet; do - dpkg --purge $pkg -done - -## removing containerd configurations and cni plugins -rm -rf /opt/cni/ && rm -rf /opt/containerd/ && tar tf "$BUNDLE_PATH/containerd.tar" | xargs -n 1 echo '/' | sed 's/ //g' | grep -e '[^/]$' | xargs rm -f - -## disabling containerd service -systemctl stop containerd && systemctl disable containerd && systemctl daemon-reload` -) diff --git a/apis/infrastructure/v1beta1/k8sinstallerconfig_types.go b/apis/infrastructure/v1beta1/k8sinstallerconfig_types.go index 3109edf42..456e08e25 100644 --- a/apis/infrastructure/v1beta1/k8sinstallerconfig_types.go +++ b/apis/infrastructure/v1beta1/k8sinstallerconfig_types.go @@ -8,6 +8,13 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +const ( + // K8sInstallerConfigFinalizer allows ReconcileK8sInstallerConfig to clean up secret + // resources associated with K8sInstallerConfig before removing it from the + // API Server. + K8sInstallerConfigFinalizer = "k8sinstallerconfig.infrastructure.cluster.x-k8s.io" +) + // K8sInstallerConfigSpec defines the desired state of K8sInstallerConfig type K8sInstallerConfigSpec struct { // BundleRepo is the OCI registry from which the carvel imgpkg bundle will be downloaded diff --git a/common/installer/downloader.go b/common/installer/downloader.go new file mode 100644 index 000000000..65cf7ba98 --- /dev/null +++ b/common/installer/downloader.go @@ -0,0 +1,19 @@ +// Copyright 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package installer + +import ( + "github.com/go-logr/logr" + "github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/agent/installer" +) + +// BundleDownloader represent a bundle downloader interface +type BundleDownloader interface { + GetBundleAddr(normalizedOsVersion, k8sVersion, tag string) string +} + +// DefaultBundleDownloader implement the downloader interface +func DefaultBundleDownloader(bundleType, repoAddr, downloadPath string, logger logr.Logger) BundleDownloader { + return installer.NewBundleDownloader(installer.BundleType(bundleType), repoAddr, downloadPath, logger) +} diff --git a/common/installer/installer.go b/common/installer/installer.go new file mode 100644 index 000000000..97250ebeb --- /dev/null +++ b/common/installer/installer.go @@ -0,0 +1,33 @@ +// Copyright 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package installer + +import ( + "context" + "strings" + + "github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/agent/installer" + "github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/common/installer/internal/algo" +) + +// K8sInstaller represent k8s installer interface +type K8sInstaller interface { + Install() string + Uninstall() string +} + +// NewInstaller will return a new installer +func NewInstaller(ctx context.Context, osDist, arch, k8sVersion string, downloader BundleDownloader) (K8sInstaller, error) { + // normalizing os image name and adding arch + osArch := strings.ReplaceAll(osDist, " ", "_") + "_" + arch + + reg := installer.GetSupportedRegistry(nil) + if len(reg.ListK8s(osArch)) == 0 { + return nil, installer.ErrOsK8sNotSupported + } + _, osbundle := reg.GetInstaller(osArch, k8sVersion) + addrs := downloader.GetBundleAddr(osbundle, k8sVersion, k8sVersion) + + return algo.NewUbuntu20_04Installer(ctx, arch, addrs) +} diff --git a/common/installer/internal/algo/ubuntu20_4k8s.go b/common/installer/internal/algo/ubuntu20_4k8s.go new file mode 100644 index 000000000..54ae2c593 --- /dev/null +++ b/common/installer/internal/algo/ubuntu20_4k8s.go @@ -0,0 +1,146 @@ +// Copyright 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package algo + +import ( + "bytes" + "context" + "fmt" + "html/template" +) + +const ( + // ImgpkgVersion defines the imgpkg version that will be installed on host if imgpkg is not already installed + ImgpkgVersion = "v0.27.0" +) + +// Ubuntu20_04Installer represent the installer implementation for ubunto20.04.* os distribution +type Ubuntu20_04Installer struct { + install string + uninstall string +} + +// NewUbuntu20_04Installer will return new Ubuntu20_04Installer instance +func NewUbuntu20_04Installer(ctx context.Context, arch, bundleAddrs string) (*Ubuntu20_04Installer, error) { + parseFn := func(script string) (string, error) { + parser, err := template.New("parser").Parse(script) + if err != nil { + return "", fmt.Errorf("unable to parse install script") + } + var tpl bytes.Buffer + if err = parser.Execute(&tpl, map[string]string{ + "BundleAddrs": bundleAddrs, + "Arch": arch, + "ImgpkgVersion": ImgpkgVersion, + "BundleDownloadPath": "{{.BundleDownloadPath}}", + }); err != nil { + return "", fmt.Errorf("unable to apply install parsed template to the data object") + } + return tpl.String(), nil + } + + install, err := parseFn(DoUbuntu20_4K8s1_22) + if err != nil { + return nil, err + } + uninstall, err := parseFn(UndoUbuntu20_4K8s1_22) + if err != nil { + return nil, err + } + return &Ubuntu20_04Installer{ + install: install, + uninstall: uninstall, + }, nil +} + +// Install will return k8s install script +func (s *Ubuntu20_04Installer) Install() string { + return s.install +} + +// Uninstall will return k8s uninstall script +func (s *Ubuntu20_04Installer) Uninstall() string { + return s.uninstall +} + +// contains the installation and uninstallation steps for the supported os and k8s +var ( + DoUbuntu20_4K8s1_22 = ` +set -euo pipefail + +BUNDLE_DOWNLOAD_PATH={{.BundleDownloadPath}} +BUNDLE_ADDR={{.BundleAddrs}} +IMGPKG_VERSION={{.ImgpkgVersion}} +ARCH={{.Arch}} +BUNDLE_PATH=$BUNDLE_DOWNLOAD_PATH/$BUNDLE_ADDR + + +if ! command -v imgpkg >>/dev/null; then + echo "installing imgpkg" + wget -nv -O- github.com/vmware-tanzu/carvel-imgpkg/releases/download/$IMGPKG_VERSION/imgpkg-linux-$ARCH > /tmp/imgpkg + mv /tmp/imgpkg /usr/local/bin/imgpkg + chmod +x /usr/local/bin/imgpkg +fi + +if [ ! -d $BUNDLE_PATH ]; then + echo "downloading bundle" + imgpkg pull -r -i $BUNDLE_ADDR -o $BUNDLE_PATH +fi + + +## disable swap +swapoff -a && sed -ri '/\sswap\s/s/^#?/#/' /etc/fstab + +## disable firewall +ufw disable + +## load kernal modules +modprobe overlay && modprobe br_netfilter + +## adding os configuration +tar -C / -xvf "$BUNDLE_PATH/conf.tar" && sysctl --system + +## installing deb packages +for pkg in cri-tools kubernetes-cni kubectl kubeadm kubelet; do + dpkg --install "$BUNDLE_PATH/$pkg.deb" && apt-mark hold $pkg +done + +## intalling containerd +tar -C / -xvf "$BUNDLE_PATH/containerd.tar" + +## starting containerd service +systemctl daemon-reload && systemctl enable containerd && systemctl start containerd` + + UndoUbuntu20_4K8s1_22 = ` +set -euo pipefail + +BUNDLE_DOWNLOAD_PATH={{.BundleDownloadPath}} +BUNDLE_ADDR={{.BundleAddrs}} +BUNDLE_PATH=$BUNDLE_DOWNLOAD_PATH/$BUNDLE_ADDR + +## enable swap +swapon -a && sed -ri '/\sswap\s/s/^#?//' /etc/fstab + +## enable firewall +ufw enable + +## remove kernal modules +modprobe -r overlay && modprobe -r br_netfilter + +## removing os configuration +tar tf "$BUNDLE_PATH/conf.tar" | xargs -n 1 echo '/' | sed 's/ //g' | xargs rm -f + +## removing deb packages +for pkg in cri-tools kubernetes-cni kubectl kubeadm kubelet; do + dpkg --purge $pkg +done + +## removing containerd configurations and cni plugins +rm -rf /opt/cni/ && rm -rf /opt/containerd/ && tar tf "$BUNDLE_PATH/containerd.tar" | xargs -n 1 echo '/' | sed 's/ //g' | grep -e '[^/]$' | xargs rm -f + +## disabling containerd service +systemctl stop containerd && systemctl disable containerd && systemctl daemon-reload + +rm -rf $BUNDLE_PATH` +) diff --git a/common/utils.go b/common/utils.go index e10aac8d0..40dc92b2f 100644 --- a/common/utils.go +++ b/common/utils.go @@ -6,10 +6,11 @@ package common import ( "bytes" "compress/gzip" - "github.com/pkg/errors" "io" "os" "path/filepath" + + "github.com/pkg/errors" ) // GzipData compresses the data bytes diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 5c16188e8..b006a89b8 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -17,6 +17,19 @@ rules: - patch - update - watch +- apiGroups: + - "" + resources: + - events + - secrets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - "" resources: @@ -165,3 +178,29 @@ rules: - get - patch - update +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - k8sinstallerconfigs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - k8sinstallerconfigs/finalizers + verbs: + - update +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - k8sinstallerconfigs/status + verbs: + - get + - patch + - update diff --git a/controllers/infrastructure/k8sinstallerconfig_controller.go b/controllers/infrastructure/k8sinstallerconfig_controller.go new file mode 100644 index 000000000..837117a77 --- /dev/null +++ b/controllers/infrastructure/k8sinstallerconfig_controller.go @@ -0,0 +1,319 @@ +// Copyright 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package controllers + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + "github.com/pkg/errors" + infrav1 "github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/apis/infrastructure/v1beta1" + "github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/common/installer" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/pointer" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/cluster-api/util/annotations" + "sigs.k8s.io/cluster-api/util/conditions" + "sigs.k8s.io/cluster-api/util/patch" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +// K8sInstallerConfigReconciler reconciles a K8sInstallerConfig object +type K8sInstallerConfigReconciler struct { + client.Client + Scheme *runtime.Scheme +} + +// k8sInstallerConfigScope defines a scope defined around a K8sInstallerConfig and its ByoMachine +type k8sInstallerConfigScope struct { + Client client.Client + Logger logr.Logger + Cluster *clusterv1.Cluster + ByoMachine *infrav1.ByoMachine + Config *infrav1.K8sInstallerConfig +} + +//+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=k8sinstallerconfigs,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=k8sinstallerconfigs/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=k8sinstallerconfigs/finalizers,verbs=update +// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=byomachines,verbs=get;list;watch +// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=byomachines/status,verbs=get +// +kubebuilder:rbac:groups="",resources=secrets;events,verbs=get;list;watch;create;update;patch;delete + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +func (r *K8sInstallerConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, reterr error) { + logger := log.FromContext(ctx) + logger.Info("Reconcile request received") + + // Fetch the K8sInstallerConfig instance + config := &infrav1.K8sInstallerConfig{} + err := r.Client.Get(ctx, req.NamespacedName, config) + if err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + logger.Error(err, "Failed to get K8sInstallerConfig") + return ctrl.Result{}, err + } + + // Fetch the ByoMachine + byoMachine, err := GetOwnerByoMachine(ctx, r.Client, &config.ObjectMeta) + if err != nil { + logger.Error(err, "failed to get Owner ByoMachine") + return ctrl.Result{}, err + } + + if byoMachine == nil { + logger.Info("Waiting for ByoMachine Controller to set OwnerRef on InstallerConfig") + return ctrl.Result{}, nil + } + logger.Info("byoMachine found") + + // Fetch the Cluster + cluster, err := util.GetClusterFromMetadata(ctx, r.Client, byoMachine.ObjectMeta) + if err != nil { + logger.Error(err, "ByoMachine owner Machine is missing cluster label or cluster does not exist") + return ctrl.Result{}, err + } + + logger = logger.WithValues("cluster", cluster.Name) + + if annotations.IsPaused(cluster, config) { + logger.Info("Reconciliation is paused for this object") + return ctrl.Result{}, nil + } + + // Create the K8sInstallerConfig scope + scope := &k8sInstallerConfigScope{ + Client: r.Client, + Logger: logger, + Cluster: cluster, + ByoMachine: byoMachine, + Config: config, + } + + helper, err := patch.NewHelper(config, r.Client) + if err != nil { + logger.Error(err, "unable to create helper") + return ctrl.Result{}, err + } + defer func() { + if err = helper.Patch(ctx, config); err != nil && reterr == nil { + logger.Error(err, "failed to patch K8sInstallerConfig") + reterr = err + } + }() + + // Add finalizer first if not exist + if !controllerutil.ContainsFinalizer(scope.Config, infrav1.K8sInstallerConfigFinalizer) { + controllerutil.AddFinalizer(scope.Config, infrav1.K8sInstallerConfigFinalizer) + } + + // Handle deleted K8sInstallerConfig + if !config.ObjectMeta.DeletionTimestamp.IsZero() { + return r.reconcileDelete(ctx, scope) + } + + switch { + // waiting for ByoMachine to updating it's ByoHostReady condition to false for reason InstallationSecretNotAvailableReason + case conditions.GetReason(byoMachine, infrav1.BYOHostReady) != infrav1.InstallationSecretNotAvailableReason: + logger.Info("ByoMachine is not waiting for InstallationSecret", "reason", conditions.GetReason(byoMachine, infrav1.BYOHostReady)) + return ctrl.Result{}, nil + // Status is ready means a config has been generated. + case config.Status.Ready: + logger.Info("K8sInstallerConfig is ready") + return ctrl.Result{}, nil + } + + return r.reconcileNormal(ctx, scope) +} + +func (r *K8sInstallerConfigReconciler) reconcileNormal(ctx context.Context, scope *k8sInstallerConfigScope) (reconcile.Result, error) { + logger := scope.Logger + logger.Info("Reconciling K8sInstallerConfig") + + k8sVersion := scope.Config.GetAnnotations()[infrav1.K8sVersionAnnotation] + downloader := installer.DefaultBundleDownloader(scope.Config.Spec.BundleType, scope.Config.Spec.BundleRepo, "{{.BUNDLE_DOWNLOAD_PATH}}", logger) + installerObj, err := installer.NewInstaller(ctx, scope.ByoMachine.Status.HostInfo.OSImage, scope.ByoMachine.Status.HostInfo.Architecture, k8sVersion, downloader) + if err != nil { + logger.Error(err, "failed to create installer instance", "osImage", scope.ByoMachine.Status.HostInfo.OSImage, "architecture", scope.ByoMachine.Status.HostInfo.Architecture, "k8sVersion", k8sVersion) + return ctrl.Result{}, err + } + + // creating installation secret + if err := r.storeInstallationData(ctx, scope, installerObj.Install(), installerObj.Uninstall()); err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil +} + +// storeInstallationData creates a new secret with the install and unstall data passed in as input, +// sets the reference in the configuration status and ready to true. +func (r *K8sInstallerConfigReconciler) storeInstallationData(ctx context.Context, scope *k8sInstallerConfigScope, install, uninstall string) error { + logger := scope.Logger + logger.Info("creating installation secret") + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: scope.Config.Name, + Namespace: scope.Config.Namespace, + Labels: map[string]string{ + clusterv1.ClusterLabelName: scope.Cluster.Name, + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: infrav1.GroupVersion.String(), + Kind: scope.Config.Kind, + Name: scope.Config.Name, + UID: scope.Config.UID, + Controller: pointer.BoolPtr(true), + }, + }, + }, + Data: map[string][]byte{ + "install": []byte(install), + "uninstall": []byte(uninstall), + }, + Type: clusterv1.ClusterSecretType, + } + + // as secret creation and scope.Config status patch are not atomic operations + // it is possible that secret creation happens but the config.Status patches are not applied + if err := r.Client.Create(ctx, secret); err != nil { + if !apierrors.IsAlreadyExists(err) { + return errors.Wrapf(err, "failed to create installation secret for K8sInstallerConfig %s/%s", scope.Config.Namespace, scope.Config.Name) + } + logger.Info("installation secret for K8sInstallerConfig already exists, updating", "secret", secret.Name, "K8sInstallerConfig", scope.Config.Name) + if err := r.Client.Update(ctx, secret); err != nil { + return errors.Wrapf(err, "failed to update installation secret for K8sInstallerConfig %s/%s", scope.Config.Namespace, scope.Config.Name) + } + } + scope.Config.Status.InstallationSecret = &corev1.ObjectReference{ + Kind: secret.Kind, + Namespace: secret.Namespace, + Name: secret.Name, + } + scope.Config.Status.Ready = true + logger.Info("created installation secret") + return nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *K8sInstallerConfigReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&infrav1.K8sInstallerConfig{}). + Watches( + &source.Kind{Type: &infrav1.ByoMachine{}}, + handler.EnqueueRequestsFromMapFunc(r.ByoMachineToK8sInstallerConfigMapFunc), + ). + Complete(r) +} + +// ByoMachineToK8sInstallerConfigMapFunc is a handler.ToRequestsFunc to be used to enqeue +// request for reconciliation of K8sInstallerConfig. +func (r *K8sInstallerConfigReconciler) ByoMachineToK8sInstallerConfigMapFunc(o client.Object) []ctrl.Request { + ctx := context.TODO() + logger := log.FromContext(ctx) + + m, ok := o.(*infrav1.ByoMachine) + if !ok { + panic(fmt.Sprintf("Expected a ByoMachine but got a %T", o)) + } + m.GetObjectKind().SetGroupVersionKind(infrav1.GroupVersion.WithKind("ByoMachine")) + + result := []ctrl.Request{} + if m.Spec.InstallerRef != nil && m.Spec.InstallerRef.GroupVersionKind() == infrav1.GroupVersion.WithKind("K8sInstallerConfigTemplate") { + configList := &infrav1.K8sInstallerConfigList{} + if err := r.Client.List(ctx, configList, client.InNamespace(m.Namespace)); err != nil { + logger.Error(err, "failed to list K8sInstallerConfig") + return result + } + for idx := range configList.Items { + config := &configList.Items[idx] + if hasOwnerReferenceFrom(config, m) { + name := client.ObjectKey{Namespace: config.Namespace, Name: config.Name} + result = append(result, ctrl.Request{NamespacedName: name}) + } + } + } + return result +} + +func (r *K8sInstallerConfigReconciler) reconcileDelete(ctx context.Context, scope *k8sInstallerConfigScope) (reconcile.Result, error) { + logger := scope.Logger + logger.Info("Deleting K8sInstallerConfig") + + secretRef := scope.Config.Status.InstallationSecret + if secretRef != nil { + // fetching the secret from reference + obj := &corev1.Secret{} + err := r.Client.Get(ctx, types.NamespacedName{Name: secretRef.Name, Namespace: secretRef.Namespace}, obj) + if err != nil && !apierrors.IsNotFound(err) { + return reconcile.Result{}, errors.Wrapf(err, "failed to get %s %q for K8sInstallerConfig %q in namespace %q", + secretRef.GroupVersionKind(), secretRef.Name, scope.Config.Name, scope.Config.Namespace) + } + + if obj != nil && obj.Name != "" { + // deleting the referred secret + logger.Info("Deleting secret", "secret", secretRef.Name) + if err := r.Client.Delete(ctx, obj); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, errors.Wrapf(err, + "failed to delete %s %q for K8sInstallerConfig %q in namespace %q", + obj.GroupVersionKind(), obj.GetName(), scope.Config.Name, scope.Config.Namespace) + } + } + } + + controllerutil.RemoveFinalizer(scope.Config, infrav1.K8sInstallerConfigFinalizer) + return reconcile.Result{}, nil +} + +// GetOwnerByoMachine returns the ByoMachine object owning the current resource. +func GetOwnerByoMachine(ctx context.Context, c client.Client, obj *metav1.ObjectMeta) (*infrav1.ByoMachine, error) { + for _, ref := range obj.OwnerReferences { + gv, err := schema.ParseGroupVersion(ref.APIVersion) + if err != nil { + return nil, err + } + if ref.Kind == "ByoMachine" && gv.Group == infrav1.GroupVersion.Group { + return GetByoMachineByName(ctx, c, obj.Namespace, ref.Name) + } + } + return nil, nil +} + +// GetByoMachineByName finds and return a ByoMachine object using the specified params. +func GetByoMachineByName(ctx context.Context, c client.Client, namespace, name string) (*infrav1.ByoMachine, error) { + m := &infrav1.ByoMachine{} + key := client.ObjectKey{Name: name, Namespace: namespace} + if err := c.Get(ctx, key, m); err != nil { + return nil, err + } + return m, nil +} + +// hasOwnerReferenceFrom will check if object have owner reference of the given owner +func hasOwnerReferenceFrom(obj, owner client.Object) bool { + for _, o := range obj.GetOwnerReferences() { + if o.Kind == owner.GetObjectKind().GroupVersionKind().Kind && o.Name == owner.GetName() { + return true + } + } + return false +} diff --git a/controllers/infrastructure/k8sinstallerconfig_controller_test.go b/controllers/infrastructure/k8sinstallerconfig_controller_test.go new file mode 100644 index 000000000..800666b6e --- /dev/null +++ b/controllers/infrastructure/k8sinstallerconfig_controller_test.go @@ -0,0 +1,477 @@ +// Copyright 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package controllers_test + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + infrav1 "github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/apis/infrastructure/v1beta1" + "github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/test/builder" + eventutils "github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/test/utils/events" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/utils/pointer" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/cluster-api/util/annotations" + "sigs.k8s.io/cluster-api/util/conditions" + "sigs.k8s.io/cluster-api/util/patch" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +var _ = Describe("Controllers/K8sInstallerConfigController", func() { + var ( + ctx context.Context + byoMachine *infrav1.ByoMachine + k8sinstallerConfig *infrav1.K8sInstallerConfig + k8sinstallerConfigTemplate *infrav1.K8sInstallerConfigTemplate + machine *clusterv1.Machine + k8sClientUncached client.Client + k8sInstallerConfigLookupKey types.NamespacedName + installerSecretLookupKey types.NamespacedName + testClusterVersion = "v1.22.1_xyz" + testBundleRepo = "test-repo" + testBundleType = "k8s" + ) + + BeforeEach(func() { + ctx = context.Background() + + var clientErr error + k8sClientUncached, clientErr = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(clientErr).NotTo(HaveOccurred()) + + machine = builder.Machine(defaultNamespace, defaultMachineName). + WithClusterName(defaultClusterName). + WithClusterVersion(testClusterVersion). + WithBootstrapDataSecret(fakeBootstrapSecret). + Build() + Expect(k8sClientUncached.Create(ctx, machine)).Should(Succeed()) + + byoMachine = builder.ByoMachine(defaultNamespace, defaultByoMachineName). + WithClusterLabel(defaultClusterName). + WithOwnerMachine(machine). + Build() + Expect(k8sClientUncached.Create(ctx, byoMachine)).Should(Succeed()) + + k8sinstallerConfigTemplate = builder.K8sInstallerConfigTemplate(defaultNamespace, defaultK8sInstallerConfigName). + WithBundleRepo(testBundleRepo). + WithBundleType(testBundleType). + Build() + Expect(k8sClientUncached.Create(ctx, k8sinstallerConfigTemplate)).Should(Succeed()) + + k8sinstallerConfig = builder.K8sInstallerConfig(defaultNamespace, defaultK8sInstallerConfigName). + WithClusterLabel(defaultClusterName). + WithOwnerByoMachine(byoMachine). + WithBundleRepo(testBundleRepo). + WithBundleType(testBundleType). + Build() + Expect(k8sClientUncached.Create(ctx, k8sinstallerConfig)).Should(Succeed()) + + WaitForObjectsToBePopulatedInCache(machine, byoMachine, k8sinstallerConfig, k8sinstallerConfigTemplate) + + k8sInstallerConfigLookupKey = types.NamespacedName{Name: k8sinstallerConfig.Name, Namespace: k8sinstallerConfig.Namespace} + installerSecretLookupKey = types.NamespacedName{Name: k8sinstallerConfig.Name, Namespace: k8sinstallerConfig.Namespace} + }) + + AfterEach(func() { + eventutils.DrainEvents(recorder.Events) + }) + + It("should ignore k8sinstallerconfig if it is not found", func() { + _, err := k8sInstallerConfigReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: "non-existent-k8sinstallerconfig", + Namespace: "non-existent-k8sinstallerconfig"}}) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should ignore when owner is not set", func() { + + k8sinstallerconfigWithNoOwner := builder.K8sInstallerConfig(defaultNamespace, defaultK8sInstallerConfigName). + WithClusterLabel(defaultClusterName). + Build() + Expect(k8sClientUncached.Create(ctx, k8sinstallerconfigWithNoOwner)).Should(Succeed()) + + WaitForObjectsToBePopulatedInCache(k8sinstallerconfigWithNoOwner) + + _, err := k8sInstallerConfigReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: k8sinstallerconfigWithNoOwner.Name, + Namespace: k8sinstallerconfigWithNoOwner.Namespace}}) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should return error when byomachine does not contain cluster name", func() { + + byoMachineWithNonExistingCluster := builder.ByoMachine(defaultNamespace, defaultByoMachineName). + WithOwnerMachine(machine). + Build() + Expect(k8sClientUncached.Create(ctx, byoMachineWithNonExistingCluster)).Should(Succeed()) + + k8sinstallerconfigWithNonExistingCluster := builder.K8sInstallerConfig(defaultNamespace, defaultK8sInstallerConfigName). + WithClusterLabel("non-existent-cluster"). + WithOwnerByoMachine(byoMachineWithNonExistingCluster). + Build() + Expect(k8sClientUncached.Create(ctx, k8sinstallerconfigWithNonExistingCluster)).Should(Succeed()) + + WaitForObjectsToBePopulatedInCache(byoMachineWithNonExistingCluster, k8sinstallerconfigWithNonExistingCluster) + + _, err := k8sInstallerConfigReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: k8sinstallerconfigWithNonExistingCluster.Name, + Namespace: k8sinstallerconfigWithNonExistingCluster.Namespace}}) + Expect(err).To(MatchError(util.ErrNoCluster)) + }) + + It("should return error when cluster does not exist", func() { + + byoMachineWithNonExistingCluster := builder.ByoMachine(defaultNamespace, defaultByoMachineName). + WithClusterLabel("non-existent-cluster"). + WithOwnerMachine(machine). + Build() + Expect(k8sClientUncached.Create(ctx, byoMachineWithNonExistingCluster)).Should(Succeed()) + + k8sinstallerconfigWithNonExistingCluster := builder.K8sInstallerConfig(defaultNamespace, defaultK8sInstallerConfigName). + WithClusterLabel("non-existent-cluster"). + WithOwnerByoMachine(byoMachineWithNonExistingCluster). + Build() + Expect(k8sClientUncached.Create(ctx, k8sinstallerconfigWithNonExistingCluster)).Should(Succeed()) + + WaitForObjectsToBePopulatedInCache(byoMachineWithNonExistingCluster, k8sinstallerconfigWithNonExistingCluster) + + _, err := k8sInstallerConfigReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: k8sinstallerconfigWithNonExistingCluster.Name, + Namespace: k8sinstallerconfigWithNonExistingCluster.Namespace}}) + Expect(err).To(MatchError("failed to get Cluster/non-existent-cluster: Cluster.cluster.x-k8s.io \"non-existent-cluster\" not found")) + }) + + It("should ignore when k8sinstallerconfig is paused", func() { + + ph, err := patch.NewHelper(k8sinstallerConfig, k8sClientUncached) + Expect(err).ShouldNot(HaveOccurred()) + pauseAnnotations := map[string]string{ + clusterv1.PausedAnnotation: "paused", + } + annotations.AddAnnotations(k8sinstallerConfig, pauseAnnotations) + Expect(ph.Patch(ctx, k8sinstallerConfig, patch.WithStatusObservedGeneration{})).Should(Succeed()) + WaitForObjectToBeUpdatedInCache(k8sinstallerConfig, func(object client.Object) bool { + return annotations.HasPausedAnnotation(object.(*infrav1.K8sInstallerConfig)) + }) + + _, err = k8sInstallerConfigReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: k8sinstallerConfig.Name, + Namespace: k8sinstallerConfig.Namespace}}) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should ignore when byomachine is not waiting for InstallationSecret", func() { + _, err := k8sInstallerConfigReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: k8sinstallerConfig.Name, + Namespace: k8sinstallerConfig.Namespace}}) + Expect(err).NotTo(HaveOccurred()) + }) + + Context("When ByoMachine wait for InstallerSecret", func() { + + BeforeEach(func() { + ph, err := patch.NewHelper(byoMachine, k8sClientUncached) + Expect(err).ShouldNot(HaveOccurred()) + byoMachine.Status.HostInfo = infrav1.HostInfo{ + Architecture: "x86-64", + OSName: "linux", + OSImage: "Ubuntu 20.04.1", + } + conditions.Set(byoMachine, &clusterv1.Condition{ + Type: infrav1.BYOHostReady, + Reason: infrav1.InstallationSecretNotAvailableReason, + }) + Expect(ph.Patch(ctx, byoMachine, patch.WithStatusObservedGeneration{})).Should(Succeed()) + WaitForObjectToBeUpdatedInCache(byoMachine, func(object client.Object) bool { + return object.(*infrav1.ByoMachine).Status.HostInfo.Architecture == "x86-64" + }) + }) + + It("should add K8sInstallerConfigFinalizer on K8sInstallerConfig", func() { + _, err := k8sInstallerConfigReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: k8sinstallerConfig.Name, + Namespace: k8sinstallerConfig.Namespace}}) + Expect(err).NotTo(HaveOccurred()) + + updatedConfig := &infrav1.K8sInstallerConfig{} + err = k8sClientUncached.Get(ctx, k8sInstallerConfigLookupKey, updatedConfig) + Expect(err).ToNot(HaveOccurred()) + Expect(controllerutil.ContainsFinalizer(updatedConfig, infrav1.K8sInstallerConfigFinalizer)).To(BeTrue()) + }) + + It("should ignore when K8sInstallerConfig status is ready", func() { + + ph, err := patch.NewHelper(k8sinstallerConfig, k8sClientUncached) + Expect(err).ShouldNot(HaveOccurred()) + k8sinstallerConfig.Status.Ready = true + Expect(ph.Patch(ctx, k8sinstallerConfig, patch.WithStatusObservedGeneration{})).Should(Succeed()) + WaitForObjectToBeUpdatedInCache(k8sinstallerConfig, func(object client.Object) bool { + return object.(*infrav1.K8sInstallerConfig).Status.Ready + }) + + _, err = k8sInstallerConfigReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: k8sinstallerConfig.Name, + Namespace: k8sinstallerConfig.Namespace}}) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should throw error if os distribution is not supported", func() { + ph, err := patch.NewHelper(byoMachine, k8sClientUncached) + Expect(err).ShouldNot(HaveOccurred()) + unsupportedOsDist := "unsupportedOsDist" + byoMachine.Status.HostInfo.OSImage = unsupportedOsDist + Expect(ph.Patch(ctx, byoMachine, patch.WithStatusObservedGeneration{})).Should(Succeed()) + WaitForObjectToBeUpdatedInCache(byoMachine, func(object client.Object) bool { + return object.(*infrav1.ByoMachine).Status.HostInfo.OSImage == unsupportedOsDist + }) + + _, err = k8sInstallerConfigReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: k8sinstallerConfig.Name, + Namespace: k8sinstallerConfig.Namespace}}) + Expect(err).Should(MatchError("No k8s support for OS")) + }) + + It("should throw error if architecture is not supported", func() { + ph, err := patch.NewHelper(byoMachine, k8sClientUncached) + Expect(err).ShouldNot(HaveOccurred()) + unsupportedArch := "unsupportedArch" + byoMachine.Status.HostInfo.Architecture = unsupportedArch + Expect(ph.Patch(ctx, byoMachine, patch.WithStatusObservedGeneration{})).Should(Succeed()) + WaitForObjectToBeUpdatedInCache(byoMachine, func(object client.Object) bool { + return object.(*infrav1.ByoMachine).Status.HostInfo.Architecture == unsupportedArch + }) + + _, err = k8sInstallerConfigReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: k8sinstallerConfig.Name, + Namespace: k8sinstallerConfig.Namespace}}) + Expect(err).Should(MatchError("No k8s support for OS")) + }) + + It("should create secret of same name as of K8sInstallerConfig", func() { + _, err := k8sInstallerConfigReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: k8sinstallerConfig.Name, + Namespace: k8sinstallerConfig.Namespace}}) + Expect(err).NotTo(HaveOccurred()) + + createdSecret := &corev1.Secret{} + err = k8sClientUncached.Get(ctx, installerSecretLookupKey, createdSecret) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should create secret with data fields install and uninstall", func() { + _, err := k8sInstallerConfigReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: k8sinstallerConfig.Name, + Namespace: k8sinstallerConfig.Namespace}}) + Expect(err).NotTo(HaveOccurred()) + + createdSecret := &corev1.Secret{} + err = k8sClientUncached.Get(ctx, installerSecretLookupKey, createdSecret) + Expect(err).ToNot(HaveOccurred()) + _, exists := createdSecret.Data["install"] + Expect(exists).To(BeTrue()) + _, exists = createdSecret.Data["uninstall"] + Expect(exists).To(BeTrue()) + }) + + It("should be add secret reference to K8sInstallerConfig", func() { + _, err := k8sInstallerConfigReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: k8sinstallerConfig.Name, + Namespace: k8sinstallerConfig.Namespace}}) + Expect(err).NotTo(HaveOccurred()) + + updatedConfig := &infrav1.K8sInstallerConfig{} + err = k8sClientUncached.Get(ctx, k8sInstallerConfigLookupKey, updatedConfig) + Expect(err).ToNot(HaveOccurred()) + + createdSecret := &corev1.Secret{} + err = k8sClientUncached.Get(ctx, installerSecretLookupKey, createdSecret) + Expect(err).ToNot(HaveOccurred()) + + Expect(updatedConfig.Status.InstallationSecret.Name).Should(Equal(createdSecret.Name)) + Expect(updatedConfig.Status.InstallationSecret.Namespace).Should(Equal(createdSecret.Namespace)) + }) + + It("should be add secret reference to K8sInstallerConfig even if secret already exists", func() { + + createdSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: k8sinstallerConfig.Name, + Namespace: k8sinstallerConfig.Namespace, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: infrav1.GroupVersion.String(), + Kind: "K8sInstallerConfig", + Name: k8sinstallerConfig.Name, + UID: k8sinstallerConfig.UID, + Controller: pointer.BoolPtr(true), + }, + }, + }, + Data: map[string][]byte{ + "install": []byte("dummy install"), + "uninstall": []byte("dummy uninstall"), + }, + Type: clusterv1.ClusterSecretType, + } + err := k8sClientUncached.Create(ctx, createdSecret) + Expect(err).NotTo(HaveOccurred()) + + _, err = k8sInstallerConfigReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: k8sinstallerConfig.Name, + Namespace: k8sinstallerConfig.Namespace}}) + Expect(err).NotTo(HaveOccurred()) + + updatedConfig := &infrav1.K8sInstallerConfig{} + err = k8sClientUncached.Get(ctx, k8sInstallerConfigLookupKey, updatedConfig) + Expect(err).ToNot(HaveOccurred()) + + Expect(updatedConfig.Status.InstallationSecret.Name).Should(Equal(createdSecret.Name)) + Expect(updatedConfig.Status.InstallationSecret.Namespace).Should(Equal(createdSecret.Namespace)) + }) + + It("should be make K8sInstallerConfig ready after secret creation", func() { + _, err := k8sInstallerConfigReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: k8sinstallerConfig.Name, + Namespace: k8sinstallerConfig.Namespace}}) + Expect(err).NotTo(HaveOccurred()) + + updatedConfig := &infrav1.K8sInstallerConfig{} + err = k8sClientUncached.Get(ctx, k8sInstallerConfigLookupKey, updatedConfig) + Expect(err).ToNot(HaveOccurred()) + + Expect(updatedConfig.Status.Ready).To(BeTrue()) + }) + + Context("When K8sInstallerConfig is deleted", func() { + BeforeEach(func() { + _, err := k8sInstallerConfigReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: k8sinstallerConfig.Name, + Namespace: k8sinstallerConfig.Namespace}}) + Expect(err).NotTo(HaveOccurred()) + + Expect(k8sClientUncached.Delete(ctx, k8sinstallerConfig)).Should(Succeed()) + + WaitForObjectToBeUpdatedInCache(k8sinstallerConfig, func(object client.Object) bool { + return !object.(*infrav1.K8sInstallerConfig).ObjectMeta.DeletionTimestamp.IsZero() + }) + }) + + It("should ignore error if k8sInstallerConfig created secret not found and k8sInstallerConfig config should be deleted", func() { + createdSecret := &corev1.Secret{} + Expect(k8sClientUncached.Get(ctx, installerSecretLookupKey, createdSecret)).ToNot(HaveOccurred()) + + deletedConfig := &infrav1.K8sInstallerConfig{} + Expect(k8sClientUncached.Get(ctx, k8sInstallerConfigLookupKey, deletedConfig)).Should(Not(HaveOccurred())) + + Expect(k8sClientUncached.Delete(ctx, createdSecret)).Should(Succeed()) + + _, err := k8sInstallerConfigReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: k8sinstallerConfig.Name, + Namespace: k8sinstallerConfig.Namespace}}) + Expect(err).NotTo(HaveOccurred()) + + err = k8sClientUncached.Get(ctx, k8sInstallerConfigLookupKey, deletedConfig) + Expect(err).To(MatchError(fmt.Sprintf("k8sinstallerconfigs.infrastructure.cluster.x-k8s.io %q not found", k8sInstallerConfigLookupKey.Name))) + }) + + It("should delete the k8sInstallerConfig created secret", func() { + createdSecret := &corev1.Secret{} + Expect(k8sClientUncached.Get(ctx, installerSecretLookupKey, createdSecret)).ToNot(HaveOccurred()) + + _, err := k8sInstallerConfigReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: k8sinstallerConfig.Name, + Namespace: k8sinstallerConfig.Namespace}}) + Expect(err).NotTo(HaveOccurred()) + + err = k8sClientUncached.Get(ctx, k8sInstallerConfigLookupKey, createdSecret) + Expect(err).To(MatchError(fmt.Sprintf("secrets %q not found", createdSecret.Name))) + }) + + It("should delete the k8sInstallerConfig object", func() { + deletedConfig := &infrav1.K8sInstallerConfig{} + // assert K8sInstallerConfig Exists before reconcile + Expect(k8sClientUncached.Get(ctx, k8sInstallerConfigLookupKey, deletedConfig)).Should(Not(HaveOccurred())) + _, err := k8sInstallerConfigReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: k8sinstallerConfig.Name, + Namespace: k8sinstallerConfig.Namespace}}) + Expect(err).NotTo(HaveOccurred()) + + // assert K8sInstallerConfig does not exists + err = k8sClientUncached.Get(ctx, k8sInstallerConfigLookupKey, deletedConfig) + Expect(err).To(MatchError(fmt.Sprintf("k8sinstallerconfigs.infrastructure.cluster.x-k8s.io %q not found", k8sInstallerConfigLookupKey.Name))) + }) + }) + }) + + Context("ByoMachine to K8sInstallerConfig reconcile request", func() { + It("should not return reconcile request if ByoMachine InstallerRef doesn't exists", func() { + result := k8sInstallerConfigReconciler.ByoMachineToK8sInstallerConfigMapFunc(byoMachine) + Expect(len(result)).To(BeZero()) + }) + + It("should not return reconcile request if ByoMachine InstallerRef doesn't refer to K8sInstallerConfitTemplate", func() { + ph, err := patch.NewHelper(byoMachine, k8sClientUncached) + Expect(err).ShouldNot(HaveOccurred()) + byoMachine.Spec.InstallerRef = &corev1.ObjectReference{ + Kind: "RandomInstallerTemplate", + Name: k8sinstallerConfig.Name, + Namespace: k8sinstallerConfig.Namespace, + } + Expect(ph.Patch(ctx, byoMachine, patch.WithStatusObservedGeneration{})).Should(Succeed()) + WaitForObjectToBeUpdatedInCache(byoMachine, func(object client.Object) bool { + return object.(*infrav1.ByoMachine).Spec.InstallerRef != nil + }) + + result := k8sInstallerConfigReconciler.ByoMachineToK8sInstallerConfigMapFunc(byoMachine) + Expect(len(result)).To(BeZero()) + }) + + It("should return reconcile request if ByoMachine refer to K8sInstallerConfigTemplate installer", func() { + ph, err := patch.NewHelper(byoMachine, k8sClientUncached) + Expect(err).ShouldNot(HaveOccurred()) + byoMachine.Spec.InstallerRef = &corev1.ObjectReference{ + Kind: "K8sInstallerConfigTemplate", + Name: k8sinstallerConfigTemplate.Name, + Namespace: k8sinstallerConfigTemplate.Namespace, + APIVersion: infrav1.GroupVersion.String(), + } + Expect(ph.Patch(ctx, byoMachine, patch.WithStatusObservedGeneration{})).Should(Succeed()) + WaitForObjectToBeUpdatedInCache(byoMachine, func(object client.Object) bool { + return object.(*infrav1.ByoMachine).Spec.InstallerRef != nil + }) + + result := k8sInstallerConfigReconciler.ByoMachineToK8sInstallerConfigMapFunc(byoMachine) + Expect(len(result)).NotTo(BeZero()) + }) + }) + +}) diff --git a/controllers/infrastructure/suite_test.go b/controllers/infrastructure/suite_test.go index 5e07e4f94..dbd96ceb8 100644 --- a/controllers/infrastructure/suite_test.go +++ b/controllers/infrastructure/suite_test.go @@ -39,26 +39,28 @@ import ( // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. var ( - testEnv *envtest.Environment - clientFake client.Client - clientSetFake = fakeclientset.NewSimpleClientset() - reconciler *controllers.ByoMachineReconciler - byoClusterReconciler *controllers.ByoClusterReconciler - byoAdmissionReconciler *controllers.ByoAdmissionReconciler - recorder *record.FakeRecorder - byoCluster *infrastructurev1beta1.ByoCluster - capiCluster *clusterv1.Cluster - defaultClusterName = "my-cluster" - defaultNodeName = "my-host" - defaultByoHostName = "my-host" - defaultMachineName = "my-machine" - defaultByoMachineName = "my-byomachine" - defaultNamespace = "default" - fakeBootstrapSecret = "fakeBootstrapSecret" - k8sManager ctrl.Manager - cfg *rest.Config - ctx context.Context - cancel context.CancelFunc + testEnv *envtest.Environment + clientFake client.Client + clientSetFake = fakeclientset.NewSimpleClientset() + reconciler *controllers.ByoMachineReconciler + byoClusterReconciler *controllers.ByoClusterReconciler + byoAdmissionReconciler *controllers.ByoAdmissionReconciler + k8sInstallerConfigReconciler *controllers.K8sInstallerConfigReconciler + recorder *record.FakeRecorder + byoCluster *infrastructurev1beta1.ByoCluster + capiCluster *clusterv1.Cluster + defaultClusterName = "my-cluster" + defaultNodeName = "my-host" + defaultByoHostName = "my-host" + defaultMachineName = "my-machine" + defaultByoMachineName = "my-byomachine" + defaultK8sInstallerConfigName = "my-k8sinstallerconfig" + defaultNamespace = "default" + fakeBootstrapSecret = "fakeBootstrapSecret" + k8sManager ctrl.Manager + cfg *rest.Config + ctx context.Context + cancel context.CancelFunc ) func TestAPIs(t *testing.T) { @@ -141,6 +143,12 @@ var _ = BeforeSuite(func() { err = byoAdmissionReconciler.SetupWithManager(k8sManager) Expect(err).NotTo(HaveOccurred()) + k8sInstallerConfigReconciler = &controllers.K8sInstallerConfigReconciler{ + Client: k8sManager.GetClient(), + } + err = k8sInstallerConfigReconciler.SetupWithManager(k8sManager) + Expect(err).NotTo(HaveOccurred()) + go func() { err = k8sManager.GetCache().Start(ctx) Expect(err).NotTo(HaveOccurred()) diff --git a/main.go b/main.go index bd63d9fbe..d7cd1bee9 100644 --- a/main.go +++ b/main.go @@ -26,6 +26,7 @@ import ( byohcontrollers "github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/controllers/infrastructure" infrastructurev1beta1 "github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/apis/infrastructure/v1beta1" + //+kubebuilder:scaffold:imports clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/controllers/remote" @@ -42,6 +43,8 @@ var ( func init() { klog.InitFlags(nil) + // clear any discard loggers set by dependecies + klog.ClearLogger() utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(infrastructurev1beta1.AddToScheme(scheme)) @@ -134,6 +137,11 @@ func main() { os.Exit(1) } + if err = (&byohcontrollers.K8sInstallerConfigReconciler{Client: mgr.GetClient(), Scheme: mgr.GetScheme()}).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "K8sInstallerConfig") + os.Exit(1) + } + mgr.GetWebhookServer().Register("/validate-infrastructure-cluster-x-k8s-io-v1beta1-byohost", &webhook.Admission{Handler: &infrastructurev1beta1.ByoHostValidator{}}) //+kubebuilder:scaffold:builder diff --git a/test/builder/builders.go b/test/builder/builders.go index 54cba4cfd..cd3e79a0c 100644 --- a/test/builder/builders.go +++ b/test/builder/builders.go @@ -469,3 +469,140 @@ func (csrb *CertificateSigningRequestBuilder) Build() (*certv1.CertificateSignin } return csr, nil } + +// K8sInstallerConfigBuilder holds the variables and objects required to build an infrastructurev1beta1.K8sInstallerConfig +type K8sInstallerConfigBuilder struct { + namespace string + name string + generatedName string + clusterLabel string + byomachine *infrastructurev1beta1.ByoMachine + bundleType string + bundleRepo string +} + +// K8sInstallerConfig returns a K8sInstallerConfigBuilder with the given generated name and namespace +func K8sInstallerConfig(namespace, generatedName string) *K8sInstallerConfigBuilder { + return &K8sInstallerConfigBuilder{ + namespace: namespace, + generatedName: generatedName, + } +} + +// WithName adds the name to K8sInstallerConfigBuilder +func (b *K8sInstallerConfigBuilder) WithName(name string) *K8sInstallerConfigBuilder { + b.name = name + return b +} + +// WithClusterLabel adds the passed cluster label to the K8sInstallerConfigBuilder +func (b *K8sInstallerConfigBuilder) WithClusterLabel(clusterName string) *K8sInstallerConfigBuilder { + b.clusterLabel = clusterName + return b +} + +// WithOwnerByoMachine adds the passed Owner ByoMachine to the K8sInstallerConfigBuilder +func (b *K8sInstallerConfigBuilder) WithOwnerByoMachine(byomachine *infrastructurev1beta1.ByoMachine) *K8sInstallerConfigBuilder { + b.byomachine = byomachine + return b +} + +// WithBundleRepo adds the passed bundleRepo to the K8sInstallerConfigBuilder +func (b *K8sInstallerConfigBuilder) WithBundleRepo(bundleRepo string) *K8sInstallerConfigBuilder { + b.bundleRepo = bundleRepo + return b +} + +// WithBundleType adds the passed bundleType to the K8sInstallerConfigBuilder +func (b *K8sInstallerConfigBuilder) WithBundleType(bundleType string) *K8sInstallerConfigBuilder { + b.bundleType = bundleType + return b +} + +// Build returns a K8sInstallerConfig with the attributes added to the K8sInstallerConfigBuilder +func (b *K8sInstallerConfigBuilder) Build() *infrastructurev1beta1.K8sInstallerConfig { + k8sinstallerconfig := &infrastructurev1beta1.K8sInstallerConfig{ + TypeMeta: metav1.TypeMeta{ + Kind: "K8sInstallerConfig", + APIVersion: infrastructurev1beta1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: b.name, + GenerateName: b.generatedName, + Namespace: b.namespace, + }, + Spec: infrastructurev1beta1.K8sInstallerConfigSpec{}, + } + if b.byomachine != nil { + k8sinstallerconfig.ObjectMeta.OwnerReferences = []metav1.OwnerReference{ + { + Kind: "ByoMachine", + Name: b.byomachine.Name, + APIVersion: infrastructurev1beta1.GroupVersion.String(), + UID: b.byomachine.UID, + }, + } + } + if b.clusterLabel != "" { + k8sinstallerconfig.ObjectMeta.Labels = map[string]string{ + clusterv1.ClusterLabelName: b.clusterLabel, + } + } + if b.bundleRepo != "" { + k8sinstallerconfig.Spec.BundleRepo = b.bundleRepo + } + if b.bundleType != "" { + k8sinstallerconfig.Spec.BundleType = b.bundleType + } + return k8sinstallerconfig +} + +// K8sInstallerConfigTemplateBuilder holds the variables and objects required to build an infrastructurev1beta1.K8sInstallerConfigTemplate +type K8sInstallerConfigTemplateBuilder struct { + namespace string + name string + bundleType string + bundleRepo string +} + +// K8sInstallerConfigTemplate returns a K8sInstallerConfigTemplateBuilder with the given name and namespace +func K8sInstallerConfigTemplate(namespace, name string) *K8sInstallerConfigTemplateBuilder { + return &K8sInstallerConfigTemplateBuilder{ + namespace: namespace, + name: name, + } +} + +// WithBundleRepo adds the passed bundleRepo to the K8sInstallerConfigTemplateBuilder +func (b *K8sInstallerConfigTemplateBuilder) WithBundleRepo(bundleRepo string) *K8sInstallerConfigTemplateBuilder { + b.bundleRepo = bundleRepo + return b +} + +// WithBundleType adds the passed bundleType to the K8sInstallerConfigTemplateBuilder +func (b *K8sInstallerConfigTemplateBuilder) WithBundleType(bundleType string) *K8sInstallerConfigTemplateBuilder { + b.bundleType = bundleType + return b +} + +// Build returns a K8sInstallerConfigTemplate with the attributes added to the K8sInstallerConfigTemplateBuilder +func (b *K8sInstallerConfigTemplateBuilder) Build() *infrastructurev1beta1.K8sInstallerConfigTemplate { + k8sinstallerconfigtemplate := &infrastructurev1beta1.K8sInstallerConfigTemplate{ + TypeMeta: metav1.TypeMeta{ + Kind: "K8sInstallerConfigTemplate", + APIVersion: infrastructurev1beta1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + GenerateName: b.name, + Namespace: b.namespace, + }, + Spec: infrastructurev1beta1.K8sInstallerConfigTemplateSpec{}, + } + if b.bundleRepo != "" { + k8sinstallerconfigtemplate.Spec.Template.Spec.BundleRepo = b.bundleRepo + } + if b.bundleType != "" { + k8sinstallerconfigtemplate.Spec.Template.Spec.BundleType = b.bundleType + } + return k8sinstallerconfigtemplate +}