MCO-1131: Merge implementation machineOSBuild and machineOSConfig in MCO

cmd/machine-config-controller/start.go

@@ -210,6 +210,7 @@ func createControllers(ctx *ctrlcommon.ControllerContext) []ctrlcommon.Controlle
 			ctx.InformerFactory.Machineconfiguration().V1().MachineConfigPools(),
 			ctx.KubeInformerFactory.Core().V1().Nodes(),
 			ctx.KubeInformerFactory.Core().V1().Pods(),
+			ctx.InformerFactory.Machineconfiguration().V1alpha1().MachineOSBuilds(),
 			ctx.ConfigInformerFactory.Config().V1().Schedulers(),
 			ctx.ClientBuilder.KubeClientOrDie("node-update-controller"),
 			ctx.ClientBuilder.MachineConfigClientOrDie("node-update-controller"),

cmd/machine-os-builder/start.go

@@ -4,15 +4,16 @@ import (
 	"context"
 	"errors"
 	"flag"
-	"fmt"
 	"os"
 
+	mcfgv1alpha1 "github.com/openshift/api/machineconfiguration/v1alpha1"
+
+	"github.com/openshift/machine-config-operator/cmd/common"
 	"github.com/openshift/machine-config-operator/internal/clients"
 	"github.com/openshift/machine-config-operator/pkg/controller/build"
 	ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
-	corev1 "k8s.io/api/core/v1"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/tools/leaderelection"
 
 	"github.com/openshift/machine-config-operator/pkg/version"
 	"github.com/spf13/cobra"
@@ -37,33 +38,16 @@ func init() {
 	startCmd.PersistentFlags().StringVar(&startOpts.kubeconfig, "kubeconfig", "", "Kubeconfig file to access a remote cluster (testing only)")
 }
 
-// Checks if the on-cluster-build-config ConfigMap exists. If it exists, return the ConfigMap.
-// If not, return an error.
-func getBuildControllerConfigMap(ctx context.Context, cb *clients.Builder) (*corev1.ConfigMap, error) {
-	kubeclient := cb.KubeClientOrDie(componentName)
-	cmName := build.OnClusterBuildConfigMapName
-	cm, err := kubeclient.CoreV1().ConfigMaps(ctrlcommon.MCONamespace).Get(ctx, cmName, metav1.GetOptions{})
-
-	if err != nil && apierrors.IsNotFound(err) {
-		return nil, fmt.Errorf("configmap %s does not exist. Please create it before opting into on-cluster builds", cmName)
-	}
-
-	if err != nil {
-		return nil, err
-	}
+func getMachineOSConfigs(ctx context.Context, cb *clients.Builder) (*mcfgv1alpha1.MachineOSConfigList, error) {
+	mcfgClient := cb.MachineConfigClientOrDie(componentName)
+	return mcfgClient.MachineconfigurationV1alpha1().MachineOSConfigs().List(ctx, metav1.ListOptions{})
 
-	return cm, nil
 }
 
 // Creates a new BuildController configured for a certain image builder based
-// upon the imageBuilderType key in the on-cluster-build-config ConfigMap.
-func getBuildController(ctx context.Context, cb *clients.Builder) (*build.Controller, error) {
-	onClusterBuildConfigMap, err := getBuildControllerConfigMap(ctx, cb)
-	if err != nil {
-		return nil, err
-	}
-
-	imageBuilderType, err := build.GetImageBuilderType(onClusterBuildConfigMap)
+// upon the imageBuilderType key in the MOSC.
+func getBuildControllers(ctx context.Context, cb *clients.Builder) ([]*build.Controller, error) {
+	machineOSConfigs, err := getMachineOSConfigs(ctx, cb)
 	if err != nil {
 		return nil, err
 	}
@@ -72,11 +56,16 @@ func getBuildController(ctx context.Context, cb *clients.Builder) (*build.Contro
 	buildClients := build.NewClientsFromControllerContext(ctrlCtx)
 	cfg := build.DefaultBuildControllerConfig()
 
-	if imageBuilderType == build.OpenshiftImageBuilder {
-		return build.NewWithImageBuilder(cfg, buildClients), nil
-	}
+	controllersToStart := []*build.Controller{}
 
-	return build.NewWithCustomPodBuilder(cfg, buildClients), nil
+	podRequestExisted := 0
+	for _, mosc := range machineOSConfigs.Items {
+		if mosc.Spec.BuildInputs.ImageBuilder.ImageBuilderType == mcfgv1alpha1.MachineOSImageBuilderType("PodImageBuilder") && podRequestExisted == 0 {
+			controllersToStart = append(controllersToStart, build.NewWithCustomPodBuilder(cfg, buildClients))
+			podRequestExisted++
+		}
+	}
+	return controllersToStart, nil
 }
 
 func runStartCmd(_ *cobra.Command, _ []string) {
@@ -89,25 +78,49 @@ func runStartCmd(_ *cobra.Command, _ []string) {
 	// To help debugging, immediately log version
 	klog.Infof("Version: %+v (%s)", version.Raw, version.Hash)
 
+	// This is the 'main' context that we thread through the build controller context and
+	// the leader elections. Cancelling this is "stop everything, we are shutting down".
 	ctx, cancel := context.WithCancel(context.Background())
-
 	cb, err := clients.NewBuilder("")
 	if err != nil {
 		klog.Fatalln(err)
 	}
 
-	ctrl, err := getBuildController(ctx, cb)
-	if err != nil {
-		klog.Fatalln(err)
-		var invalidImageBuiler *build.ErrInvalidImageBuilder
-		if errors.As(err, &invalidImageBuiler) {
-			klog.Errorf("The user passed an invalid imageBuilderType of %s", invalidImageBuiler.InvalidType)
-			cancel()
-			os.Exit(255)
+	run := func(ctx context.Context) {
+		go common.SignalHandler(cancel)
+
+		controllers, err := getBuildControllers(ctx, cb)
+		if err != nil {
+			klog.Fatalln(err)
+			var invalidImageBuiler *build.ErrInvalidImageBuilder
+			if errors.As(err, &invalidImageBuiler) {
+				klog.Errorf("The user passed an invalid imageBuilderType of %s", invalidImageBuiler.InvalidType)
+				cancel()
+				os.Exit(255)
+			}
 		}
+		for _, ctrl := range controllers {
+			go ctrl.Run(ctx, 3)
+		}
+		<-ctx.Done()
+		cancel()
 	}
 
-	go ctrl.Run(ctx, 5)
-	<-ctx.Done()
-	cancel()
+	leaderElectionCfg := common.GetLeaderElectionConfig(cb.GetBuilderConfig())
+
+	leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
+		Lock:            common.CreateResourceLock(cb, ctrlcommon.MCONamespace, componentName),
+		ReleaseOnCancel: true,
+		LeaseDuration:   leaderElectionCfg.LeaseDuration.Duration,
+		RenewDeadline:   leaderElectionCfg.RenewDeadline.Duration,
+		RetryPeriod:     leaderElectionCfg.RetryPeriod.Duration,
+		Callbacks: leaderelection.LeaderCallbacks{
+			OnStartedLeading: run,
+			OnStoppedLeading: func() {
+				klog.Infof("Stopped leading. MOB terminating.")
+				os.Exit(0)
+			},
+		},
+	})
+
 }

install/0000_80_machine-config_00_clusterreader_clusterrole.yaml

@@ -56,3 +56,23 @@ rules:
       - get
       - list
       - watch
+  - apiGroups: 
+      - machineconfiguration.openshift.io
+    resources: 
+      - machineosconfigs 
+      - machineosconfigs/status
+    verbs: 
+      - create
+      - update 
+      - patch 
+      - get
+  - apiGroups: 
+      - machineconfiguration.openshift.io
+    resources: 
+      - machineosbuilds
+      - machineosbuilds/status
+    verbs: 
+      - create
+      - update 
+      - patch
+      - get

manifests/machineconfigcontroller/clusterrole.yaml

@@ -42,6 +42,12 @@ rules:
 - apiGroups: ["operator.openshift.io"]
   resources: ["machineconfigurations"]
   verbs: ["get","list","watch"]
+- apiGroups: ["machineconfiguration.openshift.io"]
+  resources: ["machineosconfigs", "machineosconfigs/status"]
+  verbs: ["create", "update", "patch", "get"]
+- apiGroups: ["machineconfiguration.openshift.io"]
+  resources: ["machineosbuilds", "machineosbuilds/status"]
+  verbs: ["create", "update", "patch", "get"]
 - apiGroups:
   - authentication.k8s.io
   resources:

pkg/apihelpers/apihelpers.go

@@ -9,6 +9,7 @@ import (
 
 	mcfgv1 "github.com/openshift/api/machineconfiguration/v1"
 	opv1 "github.com/openshift/api/operator/v1"
+
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 

pkg/apihelpers/machineosbuild_apihelpers.go

@@ -0,0 +1,79 @@
+package apihelpers
+
+import (
+	mcfgv1alpha1 "github.com/openshift/api/machineconfiguration/v1alpha1"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// NewMachineOSBuildCondition creates a new MachineOSBuild condition.
+func NewMachineOSBuildCondition(condType string, status metav1.ConditionStatus, reason, message string) *metav1.Condition {
+	return &metav1.Condition{
+		Type:               condType,
+		Status:             status,
+		LastTransitionTime: metav1.Now(),
+		Reason:             reason,
+		Message:            message,
+	}
+}
+
+func GetMachineOSBuildCondition(status mcfgv1alpha1.MachineOSBuildStatus, condType mcfgv1alpha1.BuildProgress) *metav1.Condition {
+	// in case of sync errors, return the last condition that matches, not the first
+	// this exists for redundancy and potential race conditions.
+	var LatestState *metav1.Condition
+	for i := range status.Conditions {
+		c := status.Conditions[i]
+		if mcfgv1alpha1.BuildProgress(c.Type) == condType {
+			LatestState = &c
+		}
+	}
+	return LatestState
+}
+
+// SetMachineOSBuildCondition updates the MachineOSBuild to include the provided condition. If the condition that
+// we are about to add already exists and has the same status and reason then we are not going to update.
+func SetMachineOSBuildCondition(status *mcfgv1alpha1.MachineOSBuildStatus, condition metav1.Condition) {
+	currentCond := GetMachineOSBuildCondition(*status, mcfgv1alpha1.BuildProgress(condition.Type))
+	if currentCond != nil && currentCond.Status == condition.Status && currentCond.Reason == condition.Reason && currentCond.Message == condition.Message {
+		return
+	}
+	// Do not update lastTransitionTime if the status of the condition doesn't change.
+	if currentCond != nil && currentCond.Status == condition.Status {
+		condition.LastTransitionTime = currentCond.LastTransitionTime
+	}
+
+	// this may not be necessary
+	newConditions := filterOutMachineOSBuildCondition(status.Conditions, mcfgv1alpha1.BuildProgress(condition.Type))
+	status.Conditions = append(newConditions, condition)
+}
+
+// RemoveMachineOSBuildCondition removes the MachineOSBuild condition with the provided type.
+func RemoveMachineOSBuildCondition(status *mcfgv1alpha1.MachineOSBuildStatus, condType mcfgv1alpha1.BuildProgress) {
+	status.Conditions = filterOutMachineOSBuildCondition(status.Conditions, condType)
+}
+
+// filterOutMachineOSBuildCondition returns a new slice of MachineOSBuild conditions without conditions with the provided type.
+func filterOutMachineOSBuildCondition(conditions []metav1.Condition, condType mcfgv1alpha1.BuildProgress) []metav1.Condition {
+	var newConditions []metav1.Condition
+	for _, c := range conditions {
+		if mcfgv1alpha1.BuildProgress(c.Type) == condType {
+			continue
+		}
+		newConditions = append(newConditions, c)
+	}
+	return newConditions
+}
+
+func IsMachineOSBuildConditionTrue(conditions []metav1.Condition, conditionType mcfgv1alpha1.BuildProgress) bool {
+	return IsMachineOSBuildConditionPresentAndEqual(conditions, conditionType, metav1.ConditionTrue)
+}
+
+// IsMachineOSBuildConditionPresentAndEqual returns true when conditionType is present and equal to status.
+func IsMachineOSBuildConditionPresentAndEqual(conditions []metav1.Condition, conditionType mcfgv1alpha1.BuildProgress, status metav1.ConditionStatus) bool {
+	for _, condition := range conditions {
+		if mcfgv1alpha1.BuildProgress(condition.Type) == conditionType {
+			return condition.Status == status
+		}
+	}
+	return false
+}

pkg/controller/build/assets/Dockerfile.on-cluster-build-template → pkg/controller/build/assets/Containerfile.on-cluster-build-template

@@ -5,22 +5,22 @@
 # Decode and extract the MachineConfig from the gzipped ConfigMap and move it
 # into position. We do this in a separate stage so that we don't have the
 # gzipped MachineConfig laying around.
-FROM {{.BaseImage.Pullspec}} AS extract
+FROM {{.MachineOSConfig.Spec.BuildInputs.BaseOSImagePullspec}} AS extract
 COPY ./machineconfig/machineconfig.json.gz /tmp/machineconfig.json.gz
 RUN mkdir -p /etc/machine-config-daemon && \
 	cat /tmp/machineconfig.json.gz | base64 -d | gunzip - > /etc/machine-config-daemon/currentconfig
 
-{{if .ExtensionsImage.Pullspec}}
+{{if .MachineOSConfig.Spec.BuildInputs.BaseOSExtensionsImagePullspec}}
 # Pull our extensions image. Not sure yet what / how this should be wired up
 # though. Ideally, I'd like to use some Buildah tricks to have the extensions
 # directory mounted into the container at build-time so that I don't have to
 # copy the RPMs into the container, configure the repo, and do the
 # installation. Alternatively, I'd have to start a pod with an HTTP server.
-FROM {{.ExtensionsImage.Pullspec}} AS extensions
+FROM {{.MachineOSConfig.Spec.BuildInputs.BaseOSExtensionsImagePullspec}} AS extensions
 {{end}}
 
 
-FROM {{.BaseImage.Pullspec}} AS configs
+FROM {{.MachineOSConfig.Spec.BuildInputs.BaseOSImagePullspec}} AS configs
 # Copy the extracted MachineConfig into the expected place in the image.
 COPY --from=extract /etc/machine-config-daemon/currentconfig /etc/machine-config-daemon/currentconfig
 # Do the ignition live-apply, extracting the Ignition config from the MachineConfig.
@@ -29,11 +29,11 @@ COPY --from=extract /etc/machine-config-daemon/currentconfig /etc/machine-config
 RUN container="oci" exec -a ignition-apply /usr/lib/dracut/modules.d/30ignition/ignition --ignore-unsupported <(cat /etc/machine-config-daemon/currentconfig | jq '.spec.config') && \
 	ostree container commit
 
-LABEL machineconfig={{.Pool.Spec.Configuration.Name}}
-LABEL machineconfigpool={{.Pool.Name}}
-LABEL releaseversion={{.ReleaseVersion}}
-LABEL baseOSContainerImage={{.BaseImage.Pullspec}}
+LABEL machineconfig={{.MachineOSBuild.Spec.DesiredConfig.Name}}
+LABEL machineconfigpool={{.MachineOSConfig.Spec.MachineConfigPool.Name}}
+LABEL releaseversion={{.MachineOSConfig.Spec.BuildInputs.ReleaseVersion}}
+LABEL baseOSContainerImage={{.MachineOSConfig.Spec.BuildInputs.BaseOSImagePullspec}}
 
-{{if .CustomDockerfile}}
-{{.CustomDockerfile}}
+{{if .Containerfile}}
+{{.Containerfile}}
 {{end}}

pkg/controller/build/assets/buildah-build.sh

@@ -5,6 +5,10 @@
 # custom build pod.
 set -xeuo
 
+ETC_PKI_ENTITLEMENT_MOUNTPOINT="${ETC_PKI_ENTITLEMENT_MOUNTPOINT:-}"
+ETC_PKI_RPM_GPG_MOUNTPOINT="${ETC_PKI_RPM_GPG_MOUNTPOINT:-}"
+ETC_YUM_REPOS_D_MOUNTPOINT="${ETC_YUM_REPOS_D_MOUNTPOINT:-}"
+
 build_context="$HOME/context"
 
 # Create a directory to hold our build context.
@@ -14,12 +18,58 @@ mkdir -p "$build_context/machineconfig"
 cp /tmp/dockerfile/Dockerfile "$build_context"
 cp /tmp/machineconfig/machineconfig.json.gz "$build_context/machineconfig/"
 
-# Build our image using Buildah.
-buildah bud \
-	--storage-driver vfs \
-	--authfile="$BASE_IMAGE_PULL_CREDS" \
-	--tag "$TAG" \
-	--file="$build_context/Dockerfile" "$build_context"
+build_args=(
+	--log-level=DEBUG
+	--storage-driver vfs
+	--authfile="$BASE_IMAGE_PULL_CREDS"
+	--tag "$TAG"
+	--file="$build_context/Dockerfile"
+)
+
+mount_opts="z,rw"
+
+# If we have RHSM certs, copy them into a tempdir to avoid SELinux issues, and
+# tell Buildah about them.
+rhsm_path="/var/run/secrets/rhsm"
+if [[ -d "$rhsm_path" ]]; then
+	rhsm_certs="$(mktemp -d)"
+	cp -r -v "$rhsm_path/." "$rhsm_certs"
+	chmod -R 0755 "$rhsm_certs"
+	build_args+=("--volume=$rhsm_certs:/run/secrets/rhsm:$mount_opts")
+fi
+
+# If we have /etc/pki/entitlement certificates, commonly used with RHEL
+# entitlements, copy them into a tempdir to avoid SELinux issues, and tell
+# Buildah about them.
+if [[ -n "$ETC_PKI_ENTITLEMENT_MOUNTPOINT" ]] && [[ -d "$ETC_PKI_ENTITLEMENT_MOUNTPOINT" ]]; then
+	configs="$(mktemp -d)"
+	cp -r -v "$ETC_PKI_ENTITLEMENT_MOUNTPOINT/." "$configs"
+	chmod -R 0755 "$configs"
+	build_args+=("--volume=$configs:$ETC_PKI_ENTITLEMENT_MOUNTPOINT:$mount_opts")
+fi
+
+# If we have /etc/yum.repos.d configs, commonly used with Red Hat Satellite
+# subscriptions, copy them into a tempdir to avoid SELinux issues, and tell
+# Buildah about them.
+if [[ -n "$ETC_YUM_REPOS_D_MOUNTPOINT" ]] && [[ -d "$ETC_YUM_REPOS_D_MOUNTPOINT" ]]; then
+	configs="$(mktemp -d)"
+	cp -r -v "$ETC_YUM_REPOS_D_MOUNTPOINT/." "$configs"
+	chmod -R 0755 "$configs"
+	build_args+=("--volume=$configs:$ETC_YUM_REPOS_D_MOUNTPOINT:$mount_opts")
+fi
+
+# If we have /etc/pki/rpm-gpg configs, commonly used with Red Hat Satellite
+# subscriptions, copy them into a tempdir to avoid SELinux issues, and tell
+# Buildah about them.
+if [[ -n "$ETC_PKI_RPM_GPG_MOUNTPOINT" ]] && [[ -d "$ETC_PKI_RPM_GPG_MOUNTPOINT" ]]; then
+	configs="$(mktemp -d)"
+	cp -r -v "$ETC_PKI_RPM_GPG_MOUNTPOINT/." "$configs"
+	chmod -R 0755 "$configs"
+	build_args+=("--volume=$configs:$ETC_PKI_RPM_GPG_MOUNTPOINT:$mount_opts")
+fi
+
+# Build our image.
+buildah bud "${build_args[@]}" "$build_context"
 
 # Push our built image.
 buildah push \