Start image customization controller

controllers/provisioning_controller.go

@@ -242,6 +242,8 @@ func (r *ProvisioningReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		provisioning.EnsureMetal3StateService,
 		provisioning.EnsureImageCache,
 		provisioning.EnsureBaremetalOperatorWebhook,
+		provisioning.EnsureImageCustomizationService,
+		provisioning.EnsureImageCustomizationDeployment,
 	} {
 		updated, err := ensureResource(info)
 		if err != nil {
@@ -381,6 +383,12 @@ func (r *ProvisioningReconciler) deleteMetal3Resources(info *provisioning.Provis
 	if err := provisioning.DeleteImageCache(info); err != nil {
 		return errors.Wrap(err, "failed to delete metal3 image cache")
 	}
+	if err := provisioning.DeleteImageCustomizationService(info); err != nil {
+		return errors.Wrap(err, "failed to delete metal3 image customization service")
+	}
+	if err := provisioning.DeleteImageCustomizationDeployment(info); err != nil {
+		return errors.Wrap(err, "failed to delete metal3 image customization deployment")
+	}
 	return nil
 }
 

manifests/0000_31_cluster-baremetal-operator_01_images.configmap.yaml

@@ -18,6 +18,7 @@ data:
       "baremetalMachineOsDownloader": "registry.ci.openshift.org/openshift:ironic-machine-os-downloader",
       "baremetalStaticIpManager": "registry.ci.openshift.org/openshift:ironic-static-ip-manager",
       "baremetalIronicAgent": "registry.ci.openshift.org/openshift:ironic-agent",
-      "imageCustomizationController": "registry.ci.openshift.org/openshift:image-customization-controller"
+      "imageCustomizationController": "registry.ci.openshift.org/openshift:image-customization-controller",
+      "machineOSImages": "registry.ci.openshift.org/openshift:machine-os-images"
     }
 

manifests/image-references

@@ -39,3 +39,7 @@ spec:
     from:
       kind: DockerImage
       name: registry.ci.openshift.org/openshift:image-customization-controller
+  - name: machine-os-images
+    from:
+      kind: DockerImage
+      name: registry.ci.openshift.org/openshift:machine-os-images

provisioning/baremetal_config.go

@@ -32,13 +32,11 @@ var (
 	baremetalIronicPort            = "6385"
 	baremetalIronicInspectorPort   = "5050"
 	baremetalKernelUrlSubPath      = "images/ironic-python-agent.kernel"
-	baremetalRamdiskUrlSubPath     = "images/ironic-python-agent.initramfs"
 	baremetalIronicEndpointSubpath = "v1/"
 	provisioningIP                 = "PROVISIONING_IP"
 	provisioningInterface          = "PROVISIONING_INTERFACE"
 	provisioningMacAddresses       = "PROVISIONING_MACS"
 	deployKernelUrl                = "DEPLOY_KERNEL_URL"
-	deployRamdiskUrl               = "DEPLOY_RAMDISK_URL"
 	ironicEndpoint                 = "IRONIC_ENDPOINT"
 	ironicInspectorEndpoint        = "IRONIC_INSPECTOR_ENDPOINT"
 	httpPort                       = "HTTP_PORT"
@@ -78,11 +76,6 @@ func getDeployKernelUrl() *string {
 	return &deployKernelUrl
 }
 
-func getDeployRamdiskUrl() *string {
-	deployRamdiskUrl := fmt.Sprintf("http://localhost:%d/%s", imageCachePort, baremetalRamdiskUrlSubPath)
-	return &deployRamdiskUrl
-}
-
 func getIronicEndpoint() *string {
 	ironicEndpoint := fmt.Sprintf("https://localhost:%s/%s", baremetalIronicPort, baremetalIronicEndpointSubpath)
 	return &ironicEndpoint
@@ -100,33 +93,6 @@ func getProvisioningOSDownloadURL(config *metal3iov1alpha1.ProvisioningSpec) *st
 	return nil
 }
 
-// Check whether the PreProvisionOSDownloadURLs are set. If yes, we
-// construct a comma-separated list of RHCOS live images and return it
-func getPreProvisioningOSDownloadURLs(config *metal3iov1alpha1.ProvisioningSpec) []string {
-	var liveURLs []string
-	if config.PreProvisioningOSDownloadURLs.IsoURL != "" {
-		liveURLs = append(liveURLs, config.PreProvisioningOSDownloadURLs.IsoURL)
-	}
-	if isCoreOSIPAAvailable(config) {
-		liveURLs = append(liveURLs, config.PreProvisioningOSDownloadURLs.InitramfsURL)
-		liveURLs = append(liveURLs, config.PreProvisioningOSDownloadURLs.KernelURL)
-		liveURLs = append(liveURLs, config.PreProvisioningOSDownloadURLs.RootfsURL)
-	}
-
-	return liveURLs
-}
-
-// isCoreOSIPAAvailable is a helper to check whether the CoreOS based IPA URLs are available.
-// Only return true when kernel, rootfs and initramfs URLs are present
-func isCoreOSIPAAvailable(config *metal3iov1alpha1.ProvisioningSpec) bool {
-	if config.PreProvisioningOSDownloadURLs.KernelURL != "" &&
-		config.PreProvisioningOSDownloadURLs.RootfsURL != "" &&
-		config.PreProvisioningOSDownloadURLs.InitramfsURL != "" {
-		return true
-	}
-	return false
-}
-
 func getBootIsoSource(config *metal3iov1alpha1.ProvisioningSpec) *string {
 	if config.BootIsoSource != "" {
 		return (*string)(&config.BootIsoSource)
@@ -144,8 +110,6 @@ func getMetal3DeploymentConfig(name string, baremetalConfig *metal3iov1alpha1.Pr
 		return pointer.StringPtr(strings.Join(baremetalConfig.ProvisioningMacAddresses, ","))
 	case deployKernelUrl:
 		return getDeployKernelUrl()
-	case deployRamdiskUrl:
-		return getDeployRamdiskUrl()
 	case ironicEndpoint:
 		return getIronicEndpoint()
 	case ironicInspectorEndpoint:

provisioning/baremetal_config_test.go

@@ -56,18 +56,6 @@ func TestGetMetal3DeploymentConfig(t *testing.T) {
 			spec:          disabledProvisioning().build(),
 			expectedValue: "http://localhost:6181/images/ironic-python-agent.kernel",
 		},
-		{
-			name:          "Unmanaged DeployRamdiskUrl",
-			configName:    deployRamdiskUrl,
-			spec:          unmanagedProvisioning().build(),
-			expectedValue: "http://localhost:6181/images/ironic-python-agent.initramfs",
-		},
-		{
-			name:          "Disabled DeployRamdiskUrl",
-			configName:    deployRamdiskUrl,
-			spec:          disabledProvisioning().build(),
-			expectedValue: "http://localhost:6181/images/ironic-python-agent.initramfs",
-		},
 		{
 			name:          "Disabled IronicEndpoint",
 			configName:    ironicEndpoint,

provisioning/baremetal_pod.go

@@ -19,7 +19,6 @@ import (
 	"context"
 	"fmt"
 	"strconv"
-	"strings"
 	"time"
 
 	appsv1 "k8s.io/api/apps/v1"
@@ -28,6 +27,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	appsclientv1 "k8s.io/client-go/kubernetes/typed/apps/v1"
+	coreclientv1 "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -54,6 +54,7 @@ const (
 	mariadbPwdEnvVar                 = "MARIADB_PASSWORD"    // #nosec
 	ironicInsecureEnvVar             = "IRONIC_INSECURE"
 	inspectorInsecureEnvVar          = "IRONIC_INSPECTOR_INSECURE"
+	ironicKernelParamsEnvVar         = "IRONIC_KERNEL_PARAMS"
 	ironicCertEnvVar                 = "IRONIC_CACERT_FILE"
 	sshKeyEnvVar                     = "IRONIC_RAMDISK_SSH_KEY"
 	externalIpEnvVar                 = "IRONIC_EXTERNAL_IP"
@@ -303,89 +304,17 @@ func newMetal3InitContainers(info *ProvisioningInfo) []corev1.Container {
 		initContainers = append(initContainers, createInitContainerStaticIpSet(info.Images, &info.ProvConfig.Spec))
 	}
 
-	// If the PreProvisioningOSDownloadURLs are set, we fetch the URLs of either CoreOS ISO and IPA assets or in some
-	// cases only the IPA assets
-	liveURLs := getPreProvisioningOSDownloadURLs(&info.ProvConfig.Spec)
-	if len(liveURLs) > 0 {
-		initContainers = append(initContainers, createInitContainerMachineOsDownloader(info, strings.Join(liveURLs, ","), true, true))
+	// Extract the pre-provisioning images from a container in the payload
+	initContainers = append(initContainers, createInitContainerMachineOSImages(info, "--all", imageVolumeMount, "/shared/html/images"))
 
-		// If the ISO URL is also specified, start the createInitContainerConfigureCoreOSIPA init container
-		if info.ProvConfig.Spec.PreProvisioningOSDownloadURLs.IsoURL != "" {
-			// Configure the LiveISO by embedding ignition and other startup files
-			initContainers = append(initContainers, createInitContainerConfigureCoreOSIPA(info))
-		}
-	}
 	// If the ProvisioningOSDownloadURL is set, we download the URL specified in it
 	if info.ProvConfig.Spec.ProvisioningOSDownloadURL != "" {
 		initContainers = append(initContainers, createInitContainerMachineOsDownloader(info, info.ProvConfig.Spec.ProvisioningOSDownloadURL, false, true))
 	}
 
-	// If the CoreOS IPA assets are not available we will use the IPA downloader
-	if !isCoreOSIPAAvailable(&info.ProvConfig.Spec) {
-		initContainers = append(initContainers, createInitContainerIpaDownloader(info.Images))
-	}
-
 	return injectProxyAndCA(initContainers, info.Proxy)
 }
 
-func createInitContainerIpaDownloader(images *Images) corev1.Container {
-	initContainer := corev1.Container{
-		Name:            "metal3-ipa-downloader",
-		Image:           images.IpaDownloader,
-		Command:         []string{"/usr/local/bin/get-resource.sh"},
-		ImagePullPolicy: "IfNotPresent",
-		SecurityContext: &corev1.SecurityContext{
-			Privileged: pointer.BoolPtr(true),
-		},
-		VolumeMounts: []corev1.VolumeMount{imageVolumeMount},
-		Env:          []corev1.EnvVar{},
-		Resources: corev1.ResourceRequirements{
-			Requests: corev1.ResourceList{
-				corev1.ResourceCPU:    resource.MustParse("10m"),
-				corev1.ResourceMemory: resource.MustParse("50Mi"),
-			},
-		},
-	}
-	return initContainer
-}
-
-// This initContainer configures RHCOS Live ISO images by embedding the IPA
-// agent ignition. See:
-// https://github.com/openshift/ironic-image/blob/master/scripts/configure-coreos-ipa
-func createInitContainerConfigureCoreOSIPA(info *ProvisioningInfo) corev1.Container {
-	config := &info.ProvConfig.Spec
-	initContainer := corev1.Container{
-		Name:            "metal3-configure-coreos-ipa",
-		Image:           info.Images.Ironic,
-		Command:         []string{"/bin/configure-coreos-ipa"},
-		ImagePullPolicy: "IfNotPresent",
-		SecurityContext: &corev1.SecurityContext{
-			Privileged: pointer.BoolPtr(true),
-		},
-		VolumeMounts: []corev1.VolumeMount{
-			sharedVolumeMount,
-			imageVolumeMount,
-			ironicCredentialsMount,
-			ironicTlsMount,
-		},
-		Env: []corev1.EnvVar{
-			buildEnvVar(provisioningIP, config),
-			buildEnvVar(provisioningInterface, config),
-			buildSSHKeyEnvVar(info.SSHKey),
-			pullSecret,
-			buildEnvVar(provisioningMacAddresses, config),
-			{Name: "IRONIC_AGENT_IMAGE", Value: info.Images.IronicAgent},
-		},
-		Resources: corev1.ResourceRequirements{
-			Requests: corev1.ResourceList{
-				corev1.ResourceCPU:    resource.MustParse("10m"),
-				corev1.ResourceMemory: resource.MustParse("50Mi"),
-			},
-		},
-	}
-	return initContainer
-}
-
 func ipOptionForMachineOsDownloader(info *ProvisioningInfo) string {
 	var optionValue string
 	switch info.NetworkStack {
@@ -472,10 +401,10 @@ func newMetal3Containers(info *ProvisioningInfo) []corev1.Container {
 		createContainerMetal3BaremetalOperator(info.Images, &info.ProvConfig.Spec, info.BaremetalWebhookEnabled),
 		createContainerMetal3Mariadb(info.Images),
 		createContainerMetal3Httpd(info.Images, &info.ProvConfig.Spec, info.SSHKey),
-		createContainerMetal3IronicConductor(info.Images, &info.ProvConfig.Spec, info.SSHKey),
+		createContainerMetal3IronicConductor(info.Images, info, &info.ProvConfig.Spec, info.SSHKey),
 		createContainerMetal3IronicApi(info.Images, &info.ProvConfig.Spec),
 		createContainerMetal3RamdiskLogs(info.Images),
-		createContainerMetal3IronicInspector(info.Images, &info.ProvConfig.Spec),
+		createContainerMetal3IronicInspector(info.Images, info, &info.ProvConfig.Spec),
 	}
 
 	// If the provisioning network is disabled, and the user hasn't requested a
@@ -532,7 +461,7 @@ func createContainerMetal3BaremetalOperator(images *Images, config *metal3iov1al
 			},
 		},
 		Command:         []string{"/baremetal-operator"},
-		Args:            []string{"--health-addr", ":9446"},
+		Args:            []string{"--health-addr", ":9446", "-build-preprov-image"},
 		ImagePullPolicy: "IfNotPresent",
 		VolumeMounts: []corev1.VolumeMount{
 			ironicCredentialsMount,
@@ -571,7 +500,6 @@ func createContainerMetal3BaremetalOperator(images *Images, config *metal3iov1al
 				Value: "true",
 			},
 			buildEnvVar(deployKernelUrl, config),
-			buildEnvVar(deployRamdiskUrl, config),
 			buildEnvVar(ironicEndpoint, config),
 			buildEnvVar(ironicInspectorEndpoint, config),
 			{
@@ -718,7 +646,7 @@ func createContainerMetal3Httpd(images *Images, config *metal3iov1alpha1.Provisi
 	return container
 }
 
-func createContainerMetal3IronicConductor(images *Images, config *metal3iov1alpha1.ProvisioningSpec, sshKey string) corev1.Container {
+func createContainerMetal3IronicConductor(images *Images, info *ProvisioningInfo, config *metal3iov1alpha1.ProvisioningSpec, sshKey string) corev1.Container {
 	volumes := []corev1.VolumeMount{
 		sharedVolumeMount,
 		imageVolumeMount,
@@ -750,6 +678,10 @@ func createContainerMetal3IronicConductor(images *Images, config *metal3iov1alph
 				Name:  inspectorInsecureEnvVar,
 				Value: "true",
 			},
+			{
+				Name:  ironicKernelParamsEnvVar,
+				Value: ipOptionForMachineOsDownloader(info),
+			},
 			buildEnvVar(httpPort, config),
 			buildEnvVar(provisioningIP, config),
 			buildEnvVar(provisioningInterface, config),
@@ -848,7 +780,7 @@ func createContainerMetal3RamdiskLogs(images *Images) corev1.Container {
 	return container
 }
 
-func createContainerMetal3IronicInspector(images *Images, config *metal3iov1alpha1.ProvisioningSpec) corev1.Container {
+func createContainerMetal3IronicInspector(images *Images, info *ProvisioningInfo, config *metal3iov1alpha1.ProvisioningSpec) corev1.Container {
 	container := corev1.Container{
 		Name:            "metal3-ironic-inspector",
 		Image:           images.Ironic,
@@ -868,6 +800,10 @@ func createContainerMetal3IronicInspector(images *Images, config *metal3iov1alph
 				Name:  ironicInsecureEnvVar,
 				Value: "true",
 			},
+			{
+				Name:  ironicKernelParamsEnvVar,
+				Value: ipOptionForMachineOsDownloader(info),
+			},
 			buildEnvVar(provisioningIP, config),
 			buildEnvVar(provisioningInterface, config),
 			setIronicHtpasswdHash(htpasswdEnvVar, inspectorSecretName),
@@ -1125,3 +1061,18 @@ func GetDeploymentState(client appsclientv1.DeploymentsGetter, targetNamespace s
 func DeleteMetal3Deployment(info *ProvisioningInfo) error {
 	return client.IgnoreNotFound(info.Client.AppsV1().Deployments(info.Namespace).Delete(context.Background(), baremetalDeploymentName, metav1.DeleteOptions{}))
 }
+
+func getPodHostIP(podClient coreclientv1.PodsGetter, targetNamespace string) (string, error) {
+	listOptions := metav1.ListOptions{
+		LabelSelector: metal3AppName,
+		FieldSelector: "status.hostIP",
+	}
+
+	podList, err := podClient.Pods(targetNamespace).List(context.Background(), listOptions)
+	if err == nil && len(podList.Items) > 0 {
+		// We expect only one pod with the above LabelSelector
+		hostIP := podList.Items[0].Status.HostIP
+		return hostIP, err
+	}
+	return "", err
+}

provisioning/baremetal_pod_test.go

@@ -144,10 +144,6 @@ func TestNewMetal3InitContainers(t *testing.T) {
 			name:   "valid config with pre provisioning os download urls set",
 			config: configWithPreProvisioningOSDownloadURLs().build(),
 			expectedContainers: []corev1.Container{
-				{
-					Name:  "metal3-configure-coreos-ipa",
-					Image: images.Ironic,
-				},
 				{
 					Name:  "metal3-machine-os-downloader-live-images",
 					Image: images.MachineOsDownloader,
@@ -213,7 +209,6 @@ func TestNewMetal3Containers(t *testing.T) {
 				{Name: "IRONIC_CACERT_FILE", Value: "/certs/ironic/tls.crt"},
 				{Name: "IRONIC_INSECURE", Value: "true"},
 				{Name: "DEPLOY_KERNEL_URL", Value: "http://localhost:6181/images/ironic-python-agent.kernel"},
-				{Name: "DEPLOY_RAMDISK_URL", Value: "http://localhost:6181/images/ironic-python-agent.initramfs"},
 				{Name: "IRONIC_ENDPOINT", Value: "https://localhost:6385/v1/"},
 				{Name: "IRONIC_INSPECTOR_ENDPOINT", Value: "https://localhost:5050/v1/"},
 				{Name: "LIVE_ISO_FORCE_PERSISTENT_BOOT_DEVICE", Value: "Never"},
@@ -243,6 +238,7 @@ func TestNewMetal3Containers(t *testing.T) {
 				envWithSecret("MARIADB_PASSWORD", "metal3-mariadb-password", "password"),
 				{Name: "IRONIC_INSECURE", Value: "true"},
 				{Name: "IRONIC_INSPECTOR_INSECURE", Value: "true"},
+				{Name: "IRONIC_KERNEL_PARAMS", Value: "ip=dhcp6"},
 				{Name: "HTTP_PORT", Value: "6180"},
 				{Name: "PROVISIONING_IP", Value: "172.30.20.3/24"},
 				{Name: "PROVISIONING_INTERFACE", Value: "eth0"},
@@ -275,6 +271,7 @@ func TestNewMetal3Containers(t *testing.T) {
 			Name: "metal3-ironic-inspector",
 			Env: []corev1.EnvVar{
 				{Name: "IRONIC_INSECURE", Value: "true"},
+				{Name: "IRONIC_KERNEL_PARAMS", Value: "ip=dhcp6"},
 				{Name: "PROVISIONING_IP", Value: "172.30.20.3/24"},
 				{Name: "PROVISIONING_INTERFACE", Value: "eth0"},
 				envWithSecret("HTTP_BASIC_HTPASSWD", "metal3-ironic-inspector-password", "htpasswd"),

provisioning/container_images.go

@@ -30,6 +30,7 @@ type Images struct {
 	StaticIpManager              string `json:"baremetalStaticIpManager"`
 	IronicAgent                  string `json:"baremetalIronicAgent"`
 	ImageCustomizationController string `json:"imageCustomizationController"`
+	MachineOSImages              string `json:"machineOSImages"`
 }
 
 func GetContainerImages(containerImages *Images, imagesFilePath string) error {

provisioning/container_images_test.go

@@ -13,6 +13,7 @@ var (
 	expectedIronicStaticIpManager        = "registry.ci.openshift.org/openshift:ironic-static-ip-manager"
 	expectedIronicAgent                  = "registry.ci.openshift.org/openshift:ironic-agent"
 	expectedImageCustomizationController = "registry.ci.openshift.org/openshift:image-customization-controller"
+	expectedMachineOSImages              = "registry.ci.openshift.org/openshift:machine-os-images"
 )
 
 func TestGetContainerImages(t *testing.T) {
@@ -50,7 +51,8 @@ func TestGetContainerImages(t *testing.T) {
 					containerImages.MachineOsDownloader != expectedMachineOsDownloader ||
 					containerImages.StaticIpManager != expectedIronicStaticIpManager ||
 					containerImages.IronicAgent != expectedIronicAgent ||
-					containerImages.ImageCustomizationController != expectedImageCustomizationController {
+					containerImages.ImageCustomizationController != expectedImageCustomizationController ||
+					containerImages.MachineOSImages != expectedMachineOSImages {
 					t.Errorf("failed GetContainerImages. One or more Baremetal container images do not match the expected images.")
 				}
 			}