MCO-703: Lifecycle Buildah with MCO

Dockerfile

@@ -38,7 +38,13 @@ RUN --mount=type=cache,target=/var/cache/dnf,z \
     # rewrite image names for scos
     sed -i 's/rhel-coreos/stream-coreos/g' /manifests/*; fi && \
     dnf --setopt=keepcache=true -y install 'nmstate >= 2.2.10' && \
-    if ! rpm -q util-linux; then dnf install --setopt=keepcache=true -y util-linux; fi
+    if ! rpm -q util-linux; then dnf install --setopt=keepcache=true -y util-linux; fi && \
+    # We also need to install fuse-overlayfs and cpp for Buildah to work correctly.
+    if ! rpm -q buildah; then dnf install --setopt=keepcache=true -y buildah fuse-overlayfs cpp --exclude container-selinux; fi && \
+    # Create the build user which will be used for doing OS image builds. We
+    # use the username "build" and the uid 1000 since this matches what is in
+    # the official Buildah image.
+    useradd --uid 1000 build
 # Copy the binaries *after* we install nmstate so we don't invalidate our cache for local builds.
 COPY --from=rhel9-builder /go/src/github.com/openshift/machine-config-operator/instroot-rhel9.tar /tmp/instroot-rhel9.tar
 RUN cd / && tar xf /tmp/instroot-rhel9.tar && rm -f /tmp/instroot-rhel9.tar

Dockerfile.rhel7

@@ -39,7 +39,13 @@ RUN --mount=type=cache,target=/var/cache/dnf,z \
     # rewrite image names for scos
     sed -i 's/rhel-coreos/stream-coreos/g' /manifests/*; fi && \
     dnf --setopt=keepcache=true -y install 'nmstate >= 2.2.10' && \
-    if ! rpm -q util-linux; then dnf install --setopt=keepcache=true -y util-linux; fi
+    if ! rpm -q util-linux; then dnf install --setopt=keepcache=true -y util-linux; fi && \
+    # We also need to install fuse-overlayfs and cpp for Buildah to work correctly.
+    if ! rpm -q buildah; then dnf install --setopt=keepcache=true -y buildah fuse-overlayfs cpp --exclude container-selinux; fi && \
+    # Create the build user which will be used for doing OS image builds. We
+    # use the username "build" and the uid 1000 since this matches what is in
+    # the official Buildah image.
+    useradd --uid 1000 build
 # Copy the binaries *after* we install nmstate so we don't invalidate our cache for local builds.
 COPY --from=rhel9-builder /go/src/github.com/openshift/machine-config-operator/instroot-rhel9.tar /tmp/instroot-rhel9.tar
 RUN cd / && tar xf /tmp/instroot-rhel9.tar && rm -f /tmp/instroot-rhel9.tar

cmd/machine-config-operator/bootstrap.go

@@ -10,6 +10,7 @@ import (
 
 	imagev1 "github.com/openshift/api/image/v1"
 	"github.com/openshift/library-go/pkg/operator/resource/resourceread"
+	ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
 	"github.com/openshift/machine-config-operator/pkg/operator"
 	"github.com/openshift/machine-config-operator/pkg/version"
 )
@@ -151,8 +152,8 @@ func runBootstrapCmd(_ *cobra.Command, _ []string) {
 		}
 	}
 
-	imgs := operator.Images{
-		RenderConfigImages: operator.RenderConfigImages{
+	imgs := ctrlcommon.Images{
+		RenderConfigImages: ctrlcommon.RenderConfigImages{
 			MachineConfigOperator:          bootstrapOpts.mcoImage,
 			KeepalivedBootstrap:            bootstrapOpts.keepalivedImage,
 			CorednsBootstrap:               bootstrapOpts.corednsImage,
@@ -162,7 +163,7 @@ func runBootstrapCmd(_ *cobra.Command, _ []string) {
 			BaseOSContainerImage:           bootstrapOpts.baseOSContainerImage,
 			BaseOSExtensionsContainerImage: bootstrapOpts.baseOSExtensionsContainerImage,
 		},
-		ControllerConfigImages: operator.ControllerConfigImages{
+		ControllerConfigImages: ctrlcommon.ControllerConfigImages{
 			InfraImage:          bootstrapOpts.infraImage,
 			Keepalived:          bootstrapOpts.keepalivedImage,
 			Coredns:             bootstrapOpts.corednsImage,

pkg/controller/build/assets/wait.sh

@@ -6,13 +6,27 @@
 
 # Wait until the digestfile file appears. The presence of this file indicates
 # that the build operation is complete.
+set -euo
+
+# Wait until the done file appears.
 while [ ! -f "/tmp/done/digestfile" ]
 do
 	sleep 1
 done
 
 # Inject the contents of the digestfile into a ConfigMap.
+set -x
+
+# Create the digestfile ConfigMap
 oc create configmap \
 	"$DIGEST_CONFIGMAP_NAME" \
 	--namespace openshift-machine-config-operator \
 	--from-file=digest=/tmp/done/digestfile
+
+# Label the digestfile ConfigMap
+# shellcheck disable=SC2086
+oc label configmap \
+	"$DIGEST_CONFIGMAP_NAME" \
+	--namespace openshift-machine-config-operator \
+	--overwrite=true \
+	$DIGEST_CONFIGMAP_LABELS

pkg/controller/build/build_controller.go

@@ -16,7 +16,6 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	aggerrors "k8s.io/apimachinery/pkg/util/errors"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
-	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
 	coreclientsetv1 "k8s.io/client-go/kubernetes/typed/core/v1"
@@ -55,62 +54,6 @@ import (
 	"github.com/openshift/machine-config-operator/internal/clients"
 )
 
-const (
-	targetMachineConfigPoolLabel = "machineconfiguration.openshift.io/targetMachineConfigPool"
-	// TODO(zzlotnik): Is there a constant for this someplace else?
-	desiredConfigLabel = "machineconfiguration.openshift.io/desiredConfig"
-)
-
-// on-cluster-build-custom-dockerfile ConfigMap name.
-const (
-	customDockerfileConfigMapName = "on-cluster-build-custom-dockerfile"
-)
-
-// on-cluster-build-config ConfigMap keys.
-const (
-	// Name of ConfigMap which contains knobs for configuring the build controller.
-	OnClusterBuildConfigMapName = "on-cluster-build-config"
-
-	// The on-cluster-build-config ConfigMap key which contains a K8s secret capable of pulling of the base OS image.
-	BaseImagePullSecretNameConfigKey = "baseImagePullSecretName"
-
-	// The on-cluster-build-config ConfigMap key which contains a K8s secret capable of pushing the final OS image.
-	FinalImagePushSecretNameConfigKey = "finalImagePushSecretName"
-
-	// The on-cluster-build-config ConfigMap key which contains the pullspec of where to push the final OS image (e.g., registry.hostname.com/org/repo:tag).
-	FinalImagePullspecConfigKey = "finalImagePullspec"
-)
-
-// machine-config-osimageurl ConfigMap keys.
-const (
-	// TODO: Is this a constant someplace else?
-	machineConfigOSImageURLConfigMapName = "machine-config-osimageurl"
-
-	// The machine-config-osimageurl ConfigMap key which contains the pullspec of the base OS image (e.g., registry.hostname.com/org/repo:tag).
-	baseOSContainerImageConfigKey = "baseOSContainerImage"
-
-	// The machine-config-osimageurl ConfigMap key which contains the pullspec of the base OS image (e.g., registry.hostname.com/org/repo:tag).
-	baseOSExtensionsContainerImageConfigKey = "baseOSExtensionsContainerImage"
-
-	// The machine-config-osimageurl ConfigMap key which contains the current OpenShift release version.
-	releaseVersionConfigKey = "releaseVersion"
-
-	// The machine-config-osimageurl ConfigMap key which contains the osImageURL
-	// value. I don't think we actually use this anywhere though.
-	osImageURLConfigKey = "osImageURL"
-)
-
-const (
-	// Name of the etc-pki-entitlement secret from the openshift-config-managed namespace.
-	etcPkiEntitlementSecretName = "etc-pki-entitlement"
-
-	// Name of the etc-pki-rpm-gpg secret.
-	etcPkiRpmGpgSecretName = "etc-pki-rpm-gpg"
-
-	// Name of the etc-yum-repos-d ConfigMap.
-	etcYumReposDConfigMapName = "etc-yum-repos-d"
-)
-
 type ErrInvalidImageBuilder struct {
 	Message     string
 	InvalidType string
@@ -120,20 +63,10 @@ func (e *ErrInvalidImageBuilder) Error() string {
 	return e.Message
 }
 
-// Image builder constants.
-type ImageBuilderType string
-
-const (
-
-	// CustomPodImageBuilder is the constant indicating use of the custom pod image builder.
-	CustomPodImageBuilder ImageBuilderType = "CustomPodBuilder"
-)
-
 var (
 	// controllerKind contains the schema.GroupVersionKind for this controller type.
 	//nolint:varcheck,deadcode // This will be used eventually
-	controllerKind         = mcfgv1.SchemeGroupVersion.WithKind("MachineConfigPool")
-	validImageBuilderTypes = sets.New[ImageBuilderType](CustomPodImageBuilder)
+	controllerKind = mcfgv1.SchemeGroupVersion.WithKind("MachineConfigPool")
 )
 
 //nolint:revive // If I name this ControllerConfig, that name will be overloaded :P
@@ -184,7 +117,7 @@ type Controller struct {
 
 	config           BuildControllerConfig
 	imageBuilder     ImageBuilder
-	imageBuilderType ImageBuilderType
+	imageBuilderType mcfgv1alpha1.MachineOSImageBuilderType
 }
 
 // Creates a BuildControllerConfig with sensible production defaults.
@@ -391,7 +324,7 @@ func (ctrl *Controller) processNextMosWorkItem() bool {
 
 // Reconciles the MachineConfigPool state with the state of a custom pod object.
 func (ctrl *Controller) customBuildPodUpdater(pod *corev1.Pod) error {
-	pool, err := ctrl.mcfgclient.MachineconfigurationV1().MachineConfigPools().Get(context.TODO(), pod.Labels[targetMachineConfigPoolLabel], metav1.GetOptions{})
+	pool, err := ctrl.mcfgclient.MachineconfigurationV1().MachineConfigPools().Get(context.TODO(), pod.Labels[TargetMachineConfigPoolLabelKey], metav1.GetOptions{})
 	if err != nil {
 		return err
 	}
@@ -967,42 +900,67 @@ func (ctrl *Controller) updateConfigAndBuild(mosc *mcfgv1alpha1.MachineOSConfig,
 	return ctrl.syncAvailableStatus(newMosb)
 }
 
+func (ctrl *Controller) getOSImageURLConfig() (*ctrlcommon.OSImageURLConfig, error) {
+	cm, err := ctrl.kubeclient.CoreV1().ConfigMaps(ctrlcommon.MCONamespace).Get(context.TODO(), ctrlcommon.MachineConfigOSImageURLConfigMapName, metav1.GetOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("could not get ConfigMap %s: %w", ctrlcommon.MachineConfigOSImageURLConfigMapName, err)
+	}
+
+	return ctrlcommon.ParseOSImageURLConfigMap(cm)
+}
+
+func (ctrl *Controller) getImagesConfig() (*ctrlcommon.Images, error) {
+	cm, err := ctrl.kubeclient.CoreV1().ConfigMaps(ctrlcommon.MCONamespace).Get(context.TODO(), ctrlcommon.MachineConfigOperatorImagesConfigMapName, metav1.GetOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("could not get configmap %s: %w", ctrlcommon.MachineConfigOperatorImagesConfigMapName, err)
+	}
+
+	return ctrlcommon.ParseImagesFromConfigMap(cm)
+}
+
 // Prepares all of the objects needed to perform an image build.
 func (ctrl *Controller) prepareForBuild(mosb *mcfgv1alpha1.MachineOSBuild, mosc *mcfgv1alpha1.MachineOSConfig) (ImageBuildRequest, error) {
 	ibr := newImageBuildRequestFromBuildInputs(mosb, mosc)
 
+	imagesConfig, err := ctrl.getImagesConfig()
+	if err != nil {
+		return ibr, fmt.Errorf("could not get images.json config: %w", err)
+	}
+
+	ibr.MCOImagePullspec = imagesConfig.MachineConfigOperator
+
 	// populate the "optional" fields, if the user did not specify them
-	osImageURL, err := ctrl.kubeclient.CoreV1().ConfigMaps(ctrlcommon.MCONamespace).Get(context.TODO(), machineConfigOSImageURLConfigMapName, metav1.GetOptions{})
+	osImageURLConfig, err := ctrl.getOSImageURLConfig()
 	if err != nil {
-		return ibr, fmt.Errorf("could not get OS image URL: %w", err)
+		return ibr, fmt.Errorf("could not get osImageURL config: %w", err)
 	}
+
 	moscNew := mosc.DeepCopy()
 
-	url := newExtensionsImageInfo(osImageURL, mosc)
 	if moscNew.Spec.BuildInputs.BaseOSExtensionsImagePullspec == "" {
-		moscNew.Spec.BuildInputs.BaseOSExtensionsImagePullspec = url.Pullspec
+		moscNew.Spec.BuildInputs.BaseOSExtensionsImagePullspec = osImageURLConfig.BaseOSExtensionsContainerImage
 	}
-	url = newBaseImageInfo(osImageURL, mosc)
+
 	if moscNew.Spec.BuildInputs.BaseOSImagePullspec == "" {
-		moscNew.Spec.BuildInputs.BaseOSImagePullspec = url.Pullspec
-		moscNew.Spec.BuildInputs.ReleaseVersion = osImageURL.Data[releaseVersionConfigKey]
+		moscNew.Spec.BuildInputs.BaseOSImagePullspec = osImageURLConfig.BaseOSContainerImage
+		moscNew.Spec.BuildInputs.ReleaseVersion = osImageURLConfig.ReleaseVersion
 	}
 
-	etcPkiEntitlements, err := ctrl.getOptionalSecret(etcPkiEntitlementSecretName)
+	etcPkiEntitlements, err := ctrl.getOptionalSecret(EtcPkiEntitlementSecretName)
 	if err != nil {
 		return ImageBuildRequest{}, err
 	}
 
 	ibr.HasEtcPkiEntitlementKeys = etcPkiEntitlements != nil
 
-	etcPkiRpmGpgKeys, err := ctrl.getOptionalSecret(etcPkiRpmGpgSecretName)
+	etcPkiRpmGpgKeys, err := ctrl.getOptionalSecret(EtcPkiRpmGpgSecretName)
 	if err != nil {
 		return ImageBuildRequest{}, err
 	}
 
 	ibr.HasEtcPkiRpmGpgKeys = etcPkiRpmGpgKeys != nil
 
-	etcYumReposDConfigs, err := ctrl.getOptionalConfigMap(etcYumReposDConfigMapName)
+	etcYumReposDConfigs, err := ctrl.getOptionalConfigMap(EtcYumReposDConfigMapName)
 	if err != nil {
 		return ImageBuildRequest{}, err
 	}
@@ -1192,7 +1150,7 @@ func (ctrl *Controller) validatePullSecret(name string) (*corev1.Secret, error)
 	// If we have a canonicalized secret, get the original secret name from the
 	// label, then retry validation with the original secret. This will cause the
 	// canonicalized secret to be updated if the original secret has changed.
-	return ctrl.validatePullSecret(canonicalized.Labels[originalSecretNameLabel])
+	return ctrl.validatePullSecret(canonicalized.Labels[OriginalSecretNameLabelKey])
 }
 
 // Attempt to create a canonicalized pull secret. If the secret already exsits, we should update it.
@@ -1249,7 +1207,7 @@ func (ctrl *Controller) createCanonicalizedSecret(secret *corev1.Secret) (*corev
 	// have been updated and we should handle that more gracefully.
 	if k8serrors.IsAlreadyExists(err) {
 		klog.Infof("Canonicalized secret %s already exists", secret.Name)
-		return ctrl.validatePullSecret(secret.Labels[originalSecretNameLabel])
+		return ctrl.validatePullSecret(secret.Labels[OriginalSecretNameLabelKey])
 	}
 
 	return nil, fmt.Errorf("could not create canonicalized secret %q: %w", secret.Name, err)
@@ -1439,21 +1397,14 @@ func shouldWeDoABuild(builder interface {
 	return false, nil
 }
 
-// Determines if a pod or build is managed by this controller by examining its labels.
-func hasAllRequiredOSBuildLabels(labels map[string]string) bool {
-	requiredLabels := []string{
-		ctrlcommon.OSImageBuildPodLabel,
-		targetMachineConfigPoolLabel,
-		desiredConfigLabel,
-	}
-
-	for _, label := range requiredLabels {
-		if _, ok := labels[label]; !ok {
-			return false
-		}
-	}
+// Determines if an object is an ephemeral build object by examining its labels.
+func isEphemeralBuildObject(obj metav1.Object) bool {
+	return EphemeralBuildObjectSelector().Matches(labels.Set(obj.GetLabels()))
+}
 
-	return true
+// Determines if an object is managed by this controller by examining its labels.
+func hasAllRequiredOSBuildLabels(inLabels map[string]string) bool {
+	return OSBuildSelector().Matches(labels.Set(inLabels))
 }
 
 func (ctrl *Controller) doesMOSBExist(mosc *mcfgv1alpha1.MachineOSConfig) (*mcfgv1alpha1.MachineOSBuild, bool) {

pkg/controller/build/build_controller_test.go

@@ -173,7 +173,8 @@ func TestCanonicalizedSecrets(t *testing.T) {
 		assert.Contains(t, s.Name, canonicalSecretSuffix)
 		assert.True(t, isCanonicalizedSecret(s))
 		assert.True(t, hasCanonicalizedSecretLabels(s))
-		assert.Equal(t, s.Labels[originalSecretNameLabel], legacyPullSecretName)
+		assert.Equal(t, s.Labels[OriginalSecretNameLabelKey], legacyPullSecretName)
+		assert.True(t, IsObjectCreatedByBuildController(s))
 	}
 
 	testFunc := func(t *testing.T) {
@@ -288,6 +289,8 @@ func getClientsForTest() *Clients {
 		},
 	})
 
+	imagesConfigMap := getImagesConfigMap()
+
 	osImageURLConfigMap := getOSImageURLConfigMap()
 
 	legacyPullSecret := `{"registry.hostname.com": {"username": "user", "password": "s3kr1t", "auth": "s00pers3kr1t", "email": "[email protected]"}}`
@@ -297,6 +300,7 @@ func getClientsForTest() *Clients {
 	return &Clients{
 		mcfgclient: fakeclientmachineconfigv1.NewSimpleClientset(objects...),
 		kubeclient: fakecorev1client.NewSimpleClientset(
+			imagesConfigMap,
 			osImageURLConfigMap,
 			&corev1.Secret{
 				ObjectMeta: metav1.ObjectMeta{