diff --git a/controllers/provisioning_controller.go b/controllers/provisioning_controller.go index 6aad8f5a8..be18f2d3b 100644 --- a/controllers/provisioning_controller.go +++ b/controllers/provisioning_controller.go @@ -242,6 +242,8 @@ func (r *ProvisioningReconciler) Reconcile(ctx context.Context, req ctrl.Request provisioning.EnsureMetal3StateService, provisioning.EnsureImageCache, provisioning.EnsureBaremetalOperatorWebhook, + provisioning.EnsureImageCustomizationService, + provisioning.EnsureImageCustomizationDeployment, } { updated, err := ensureResource(info) if err != nil { @@ -381,6 +383,12 @@ func (r *ProvisioningReconciler) deleteMetal3Resources(info *provisioning.Provis if err := provisioning.DeleteImageCache(info); err != nil { return errors.Wrap(err, "failed to delete metal3 image cache") } + if err := provisioning.DeleteImageCustomizationService(info); err != nil { + return errors.Wrap(err, "failed to delete metal3 image customization service") + } + if err := provisioning.DeleteImageCustomizationDeployment(info); err != nil { + return errors.Wrap(err, "failed to delete metal3 image customization deployment") + } return nil } diff --git a/manifests/0000_31_cluster-baremetal-operator_01_images.configmap.yaml b/manifests/0000_31_cluster-baremetal-operator_01_images.configmap.yaml index e4b6c7be7..b43c78d01 100644 --- a/manifests/0000_31_cluster-baremetal-operator_01_images.configmap.yaml +++ b/manifests/0000_31_cluster-baremetal-operator_01_images.configmap.yaml @@ -18,6 +18,7 @@ data: "baremetalMachineOsDownloader": "registry.ci.openshift.org/openshift:ironic-machine-os-downloader", "baremetalStaticIpManager": "registry.ci.openshift.org/openshift:ironic-static-ip-manager", "baremetalIronicAgent": "registry.ci.openshift.org/openshift:ironic-agent", - "imageCustomizationController": "registry.ci.openshift.org/openshift:image-customization-controller" + "imageCustomizationController": "registry.ci.openshift.org/openshift:image-customization-controller", + "machineOSImages": "registry.ci.openshift.org/openshift:machine-os-images" } diff --git a/manifests/image-references b/manifests/image-references index 24b211829..6b22c051a 100644 --- a/manifests/image-references +++ b/manifests/image-references @@ -39,3 +39,7 @@ spec: from: kind: DockerImage name: registry.ci.openshift.org/openshift:image-customization-controller + - name: machine-os-images + from: + kind: DockerImage + name: registry.ci.openshift.org/openshift:machine-os-images diff --git a/provisioning/baremetal_config.go b/provisioning/baremetal_config.go index d710508ad..434fce73b 100644 --- a/provisioning/baremetal_config.go +++ b/provisioning/baremetal_config.go @@ -32,13 +32,11 @@ var ( baremetalIronicPort = "6385" baremetalIronicInspectorPort = "5050" baremetalKernelUrlSubPath = "images/ironic-python-agent.kernel" - baremetalRamdiskUrlSubPath = "images/ironic-python-agent.initramfs" baremetalIronicEndpointSubpath = "v1/" provisioningIP = "PROVISIONING_IP" provisioningInterface = "PROVISIONING_INTERFACE" provisioningMacAddresses = "PROVISIONING_MACS" deployKernelUrl = "DEPLOY_KERNEL_URL" - deployRamdiskUrl = "DEPLOY_RAMDISK_URL" ironicEndpoint = "IRONIC_ENDPOINT" ironicInspectorEndpoint = "IRONIC_INSPECTOR_ENDPOINT" httpPort = "HTTP_PORT" @@ -78,11 +76,6 @@ func getDeployKernelUrl() *string { return &deployKernelUrl } -func getDeployRamdiskUrl() *string { - deployRamdiskUrl := fmt.Sprintf("http://localhost:%d/%s", imageCachePort, baremetalRamdiskUrlSubPath) - return &deployRamdiskUrl -} - func getIronicEndpoint() *string { ironicEndpoint := fmt.Sprintf("https://localhost:%s/%s", baremetalIronicPort, baremetalIronicEndpointSubpath) return &ironicEndpoint @@ -100,33 +93,6 @@ func getProvisioningOSDownloadURL(config *metal3iov1alpha1.ProvisioningSpec) *st return nil } -// Check whether the PreProvisionOSDownloadURLs are set. If yes, we -// construct a comma-separated list of RHCOS live images and return it -func getPreProvisioningOSDownloadURLs(config *metal3iov1alpha1.ProvisioningSpec) []string { - var liveURLs []string - if config.PreProvisioningOSDownloadURLs.IsoURL != "" { - liveURLs = append(liveURLs, config.PreProvisioningOSDownloadURLs.IsoURL) - } - if isCoreOSIPAAvailable(config) { - liveURLs = append(liveURLs, config.PreProvisioningOSDownloadURLs.InitramfsURL) - liveURLs = append(liveURLs, config.PreProvisioningOSDownloadURLs.KernelURL) - liveURLs = append(liveURLs, config.PreProvisioningOSDownloadURLs.RootfsURL) - } - - return liveURLs -} - -// isCoreOSIPAAvailable is a helper to check whether the CoreOS based IPA URLs are available. -// Only return true when kernel, rootfs and initramfs URLs are present -func isCoreOSIPAAvailable(config *metal3iov1alpha1.ProvisioningSpec) bool { - if config.PreProvisioningOSDownloadURLs.KernelURL != "" && - config.PreProvisioningOSDownloadURLs.RootfsURL != "" && - config.PreProvisioningOSDownloadURLs.InitramfsURL != "" { - return true - } - return false -} - func getBootIsoSource(config *metal3iov1alpha1.ProvisioningSpec) *string { if config.BootIsoSource != "" { return (*string)(&config.BootIsoSource) @@ -144,8 +110,6 @@ func getMetal3DeploymentConfig(name string, baremetalConfig *metal3iov1alpha1.Pr return pointer.StringPtr(strings.Join(baremetalConfig.ProvisioningMacAddresses, ",")) case deployKernelUrl: return getDeployKernelUrl() - case deployRamdiskUrl: - return getDeployRamdiskUrl() case ironicEndpoint: return getIronicEndpoint() case ironicInspectorEndpoint: diff --git a/provisioning/baremetal_config_test.go b/provisioning/baremetal_config_test.go index 153259764..1960c5816 100644 --- a/provisioning/baremetal_config_test.go +++ b/provisioning/baremetal_config_test.go @@ -56,18 +56,6 @@ func TestGetMetal3DeploymentConfig(t *testing.T) { spec: disabledProvisioning().build(), expectedValue: "http://localhost:6181/images/ironic-python-agent.kernel", }, - { - name: "Unmanaged DeployRamdiskUrl", - configName: deployRamdiskUrl, - spec: unmanagedProvisioning().build(), - expectedValue: "http://localhost:6181/images/ironic-python-agent.initramfs", - }, - { - name: "Disabled DeployRamdiskUrl", - configName: deployRamdiskUrl, - spec: disabledProvisioning().build(), - expectedValue: "http://localhost:6181/images/ironic-python-agent.initramfs", - }, { name: "Disabled IronicEndpoint", configName: ironicEndpoint, diff --git a/provisioning/baremetal_pod.go b/provisioning/baremetal_pod.go index e1a0ae195..7508edb59 100644 --- a/provisioning/baremetal_pod.go +++ b/provisioning/baremetal_pod.go @@ -19,7 +19,6 @@ import ( "context" "fmt" "strconv" - "strings" "time" appsv1 "k8s.io/api/apps/v1" @@ -28,6 +27,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" appsclientv1 "k8s.io/client-go/kubernetes/typed/apps/v1" + coreclientv1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -54,6 +54,7 @@ const ( mariadbPwdEnvVar = "MARIADB_PASSWORD" // #nosec ironicInsecureEnvVar = "IRONIC_INSECURE" inspectorInsecureEnvVar = "IRONIC_INSPECTOR_INSECURE" + ironicKernelParamsEnvVar = "IRONIC_KERNEL_PARAMS" ironicCertEnvVar = "IRONIC_CACERT_FILE" sshKeyEnvVar = "IRONIC_RAMDISK_SSH_KEY" externalIpEnvVar = "IRONIC_EXTERNAL_IP" @@ -303,89 +304,17 @@ func newMetal3InitContainers(info *ProvisioningInfo) []corev1.Container { initContainers = append(initContainers, createInitContainerStaticIpSet(info.Images, &info.ProvConfig.Spec)) } - // If the PreProvisioningOSDownloadURLs are set, we fetch the URLs of either CoreOS ISO and IPA assets or in some - // cases only the IPA assets - liveURLs := getPreProvisioningOSDownloadURLs(&info.ProvConfig.Spec) - if len(liveURLs) > 0 { - initContainers = append(initContainers, createInitContainerMachineOsDownloader(info, strings.Join(liveURLs, ","), true, true)) + // Extract the pre-provisioning images from a container in the payload + initContainers = append(initContainers, createInitContainerMachineOSImages(info, "--all", imageVolumeMount, "/shared/html/images")) - // If the ISO URL is also specified, start the createInitContainerConfigureCoreOSIPA init container - if info.ProvConfig.Spec.PreProvisioningOSDownloadURLs.IsoURL != "" { - // Configure the LiveISO by embedding ignition and other startup files - initContainers = append(initContainers, createInitContainerConfigureCoreOSIPA(info)) - } - } // If the ProvisioningOSDownloadURL is set, we download the URL specified in it if info.ProvConfig.Spec.ProvisioningOSDownloadURL != "" { initContainers = append(initContainers, createInitContainerMachineOsDownloader(info, info.ProvConfig.Spec.ProvisioningOSDownloadURL, false, true)) } - // If the CoreOS IPA assets are not available we will use the IPA downloader - if !isCoreOSIPAAvailable(&info.ProvConfig.Spec) { - initContainers = append(initContainers, createInitContainerIpaDownloader(info.Images)) - } - return injectProxyAndCA(initContainers, info.Proxy) } -func createInitContainerIpaDownloader(images *Images) corev1.Container { - initContainer := corev1.Container{ - Name: "metal3-ipa-downloader", - Image: images.IpaDownloader, - Command: []string{"/usr/local/bin/get-resource.sh"}, - ImagePullPolicy: "IfNotPresent", - SecurityContext: &corev1.SecurityContext{ - Privileged: pointer.BoolPtr(true), - }, - VolumeMounts: []corev1.VolumeMount{imageVolumeMount}, - Env: []corev1.EnvVar{}, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("10m"), - corev1.ResourceMemory: resource.MustParse("50Mi"), - }, - }, - } - return initContainer -} - -// This initContainer configures RHCOS Live ISO images by embedding the IPA -// agent ignition. See: -// https://github.com/openshift/ironic-image/blob/master/scripts/configure-coreos-ipa -func createInitContainerConfigureCoreOSIPA(info *ProvisioningInfo) corev1.Container { - config := &info.ProvConfig.Spec - initContainer := corev1.Container{ - Name: "metal3-configure-coreos-ipa", - Image: info.Images.Ironic, - Command: []string{"/bin/configure-coreos-ipa"}, - ImagePullPolicy: "IfNotPresent", - SecurityContext: &corev1.SecurityContext{ - Privileged: pointer.BoolPtr(true), - }, - VolumeMounts: []corev1.VolumeMount{ - sharedVolumeMount, - imageVolumeMount, - ironicCredentialsMount, - ironicTlsMount, - }, - Env: []corev1.EnvVar{ - buildEnvVar(provisioningIP, config), - buildEnvVar(provisioningInterface, config), - buildSSHKeyEnvVar(info.SSHKey), - pullSecret, - buildEnvVar(provisioningMacAddresses, config), - {Name: "IRONIC_AGENT_IMAGE", Value: info.Images.IronicAgent}, - }, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("10m"), - corev1.ResourceMemory: resource.MustParse("50Mi"), - }, - }, - } - return initContainer -} - func ipOptionForMachineOsDownloader(info *ProvisioningInfo) string { var optionValue string switch info.NetworkStack { @@ -472,10 +401,10 @@ func newMetal3Containers(info *ProvisioningInfo) []corev1.Container { createContainerMetal3BaremetalOperator(info.Images, &info.ProvConfig.Spec, info.BaremetalWebhookEnabled), createContainerMetal3Mariadb(info.Images), createContainerMetal3Httpd(info.Images, &info.ProvConfig.Spec, info.SSHKey), - createContainerMetal3IronicConductor(info.Images, &info.ProvConfig.Spec, info.SSHKey), + createContainerMetal3IronicConductor(info.Images, info, &info.ProvConfig.Spec, info.SSHKey), createContainerMetal3IronicApi(info.Images, &info.ProvConfig.Spec), createContainerMetal3RamdiskLogs(info.Images), - createContainerMetal3IronicInspector(info.Images, &info.ProvConfig.Spec), + createContainerMetal3IronicInspector(info.Images, info, &info.ProvConfig.Spec), } // If the provisioning network is disabled, and the user hasn't requested a @@ -532,7 +461,7 @@ func createContainerMetal3BaremetalOperator(images *Images, config *metal3iov1al }, }, Command: []string{"/baremetal-operator"}, - Args: []string{"--health-addr", ":9446"}, + Args: []string{"--health-addr", ":9446", "-build-preprov-image"}, ImagePullPolicy: "IfNotPresent", VolumeMounts: []corev1.VolumeMount{ ironicCredentialsMount, @@ -571,7 +500,6 @@ func createContainerMetal3BaremetalOperator(images *Images, config *metal3iov1al Value: "true", }, buildEnvVar(deployKernelUrl, config), - buildEnvVar(deployRamdiskUrl, config), buildEnvVar(ironicEndpoint, config), buildEnvVar(ironicInspectorEndpoint, config), { @@ -718,7 +646,7 @@ func createContainerMetal3Httpd(images *Images, config *metal3iov1alpha1.Provisi return container } -func createContainerMetal3IronicConductor(images *Images, config *metal3iov1alpha1.ProvisioningSpec, sshKey string) corev1.Container { +func createContainerMetal3IronicConductor(images *Images, info *ProvisioningInfo, config *metal3iov1alpha1.ProvisioningSpec, sshKey string) corev1.Container { volumes := []corev1.VolumeMount{ sharedVolumeMount, imageVolumeMount, @@ -750,6 +678,10 @@ func createContainerMetal3IronicConductor(images *Images, config *metal3iov1alph Name: inspectorInsecureEnvVar, Value: "true", }, + { + Name: ironicKernelParamsEnvVar, + Value: ipOptionForMachineOsDownloader(info), + }, buildEnvVar(httpPort, config), buildEnvVar(provisioningIP, config), buildEnvVar(provisioningInterface, config), @@ -848,7 +780,7 @@ func createContainerMetal3RamdiskLogs(images *Images) corev1.Container { return container } -func createContainerMetal3IronicInspector(images *Images, config *metal3iov1alpha1.ProvisioningSpec) corev1.Container { +func createContainerMetal3IronicInspector(images *Images, info *ProvisioningInfo, config *metal3iov1alpha1.ProvisioningSpec) corev1.Container { container := corev1.Container{ Name: "metal3-ironic-inspector", Image: images.Ironic, @@ -868,6 +800,10 @@ func createContainerMetal3IronicInspector(images *Images, config *metal3iov1alph Name: ironicInsecureEnvVar, Value: "true", }, + { + Name: ironicKernelParamsEnvVar, + Value: ipOptionForMachineOsDownloader(info), + }, buildEnvVar(provisioningIP, config), buildEnvVar(provisioningInterface, config), setIronicHtpasswdHash(htpasswdEnvVar, inspectorSecretName), @@ -1125,3 +1061,18 @@ func GetDeploymentState(client appsclientv1.DeploymentsGetter, targetNamespace s func DeleteMetal3Deployment(info *ProvisioningInfo) error { return client.IgnoreNotFound(info.Client.AppsV1().Deployments(info.Namespace).Delete(context.Background(), baremetalDeploymentName, metav1.DeleteOptions{})) } + +func getPodHostIP(podClient coreclientv1.PodsGetter, targetNamespace string) (string, error) { + listOptions := metav1.ListOptions{ + LabelSelector: metal3AppName, + FieldSelector: "status.hostIP", + } + + podList, err := podClient.Pods(targetNamespace).List(context.Background(), listOptions) + if err == nil && len(podList.Items) > 0 { + // We expect only one pod with the above LabelSelector + hostIP := podList.Items[0].Status.HostIP + return hostIP, err + } + return "", err +} diff --git a/provisioning/baremetal_pod_test.go b/provisioning/baremetal_pod_test.go index f2aa257d5..aa2a76535 100644 --- a/provisioning/baremetal_pod_test.go +++ b/provisioning/baremetal_pod_test.go @@ -144,10 +144,6 @@ func TestNewMetal3InitContainers(t *testing.T) { name: "valid config with pre provisioning os download urls set", config: configWithPreProvisioningOSDownloadURLs().build(), expectedContainers: []corev1.Container{ - { - Name: "metal3-configure-coreos-ipa", - Image: images.Ironic, - }, { Name: "metal3-machine-os-downloader-live-images", Image: images.MachineOsDownloader, @@ -213,7 +209,6 @@ func TestNewMetal3Containers(t *testing.T) { {Name: "IRONIC_CACERT_FILE", Value: "/certs/ironic/tls.crt"}, {Name: "IRONIC_INSECURE", Value: "true"}, {Name: "DEPLOY_KERNEL_URL", Value: "http://localhost:6181/images/ironic-python-agent.kernel"}, - {Name: "DEPLOY_RAMDISK_URL", Value: "http://localhost:6181/images/ironic-python-agent.initramfs"}, {Name: "IRONIC_ENDPOINT", Value: "https://localhost:6385/v1/"}, {Name: "IRONIC_INSPECTOR_ENDPOINT", Value: "https://localhost:5050/v1/"}, {Name: "LIVE_ISO_FORCE_PERSISTENT_BOOT_DEVICE", Value: "Never"}, @@ -243,6 +238,7 @@ func TestNewMetal3Containers(t *testing.T) { envWithSecret("MARIADB_PASSWORD", "metal3-mariadb-password", "password"), {Name: "IRONIC_INSECURE", Value: "true"}, {Name: "IRONIC_INSPECTOR_INSECURE", Value: "true"}, + {Name: "IRONIC_KERNEL_PARAMS", Value: "ip=dhcp6"}, {Name: "HTTP_PORT", Value: "6180"}, {Name: "PROVISIONING_IP", Value: "172.30.20.3/24"}, {Name: "PROVISIONING_INTERFACE", Value: "eth0"}, @@ -275,6 +271,7 @@ func TestNewMetal3Containers(t *testing.T) { Name: "metal3-ironic-inspector", Env: []corev1.EnvVar{ {Name: "IRONIC_INSECURE", Value: "true"}, + {Name: "IRONIC_KERNEL_PARAMS", Value: "ip=dhcp6"}, {Name: "PROVISIONING_IP", Value: "172.30.20.3/24"}, {Name: "PROVISIONING_INTERFACE", Value: "eth0"}, envWithSecret("HTTP_BASIC_HTPASSWD", "metal3-ironic-inspector-password", "htpasswd"), diff --git a/provisioning/container_images.go b/provisioning/container_images.go index 3a5cfd51f..aca3920af 100644 --- a/provisioning/container_images.go +++ b/provisioning/container_images.go @@ -30,6 +30,7 @@ type Images struct { StaticIpManager string `json:"baremetalStaticIpManager"` IronicAgent string `json:"baremetalIronicAgent"` ImageCustomizationController string `json:"imageCustomizationController"` + MachineOSImages string `json:"machineOSImages"` } func GetContainerImages(containerImages *Images, imagesFilePath string) error { diff --git a/provisioning/container_images_test.go b/provisioning/container_images_test.go index eb48e6358..4f56129d2 100644 --- a/provisioning/container_images_test.go +++ b/provisioning/container_images_test.go @@ -13,6 +13,7 @@ var ( expectedIronicStaticIpManager = "registry.ci.openshift.org/openshift:ironic-static-ip-manager" expectedIronicAgent = "registry.ci.openshift.org/openshift:ironic-agent" expectedImageCustomizationController = "registry.ci.openshift.org/openshift:image-customization-controller" + expectedMachineOSImages = "registry.ci.openshift.org/openshift:machine-os-images" ) func TestGetContainerImages(t *testing.T) { @@ -50,7 +51,8 @@ func TestGetContainerImages(t *testing.T) { containerImages.MachineOsDownloader != expectedMachineOsDownloader || containerImages.StaticIpManager != expectedIronicStaticIpManager || containerImages.IronicAgent != expectedIronicAgent || - containerImages.ImageCustomizationController != expectedImageCustomizationController { + containerImages.ImageCustomizationController != expectedImageCustomizationController || + containerImages.MachineOSImages != expectedMachineOSImages { t.Errorf("failed GetContainerImages. One or more Baremetal container images do not match the expected images.") } } diff --git a/provisioning/image_cache.go b/provisioning/image_cache.go index 3c1bacfdd..810c9d043 100644 --- a/provisioning/image_cache.go +++ b/provisioning/image_cache.go @@ -8,7 +8,6 @@ import ( "path" "regexp" "strconv" - "strings" "time" appsv1 "k8s.io/api/apps/v1" @@ -135,22 +134,10 @@ func createContainerImageCache(images *Images) corev1.Container { func newImageCacheInitContainers(info *ProvisioningInfo) ([]corev1.Container, error) { initContainers := []corev1.Container{} - // If the PreProvisioningOSDownloadURLs are set, we fetch the URLs of either CoreOS ISO and IPA URLs or in some - // cases only the IPA URLs - liveURLs := getPreProvisioningOSDownloadURLs(&info.ProvConfig.Spec) - var newURLs []string - if len(liveURLs) > 0 { - for _, URL := range liveURLs { - if URL != "" { - newURL, err := transformURL(info.Namespace, URL) - if err != nil { - return nil, err - } - newURLs = append(newURLs, newURL) - } - } - initContainers = append(initContainers, createInitContainerMachineOsDownloader(info, strings.Join(newURLs, ","), true, false)) - } + + // Extract the pre-provisioning images from a container in the payload + initContainers = append(initContainers, createInitContainerMachineOSImages(info, "--all", imageVolumeMount, "/shared/html/images")) + // Download the transformed URL containing qcow2 if info.ProvConfig.Spec.ProvisioningOSDownloadURL != "" { newURL, err := transformURL(info.Namespace, info.ProvConfig.Spec.ProvisioningOSDownloadURL) @@ -160,10 +147,6 @@ func newImageCacheInitContainers(info *ProvisioningInfo) ([]corev1.Container, er initContainers = append(initContainers, createInitContainerMachineOsDownloader(info, newURL, false, false)) } - // If the CoreOS IPA URLs are not available we will use the IPA downloader - if !isCoreOSIPAAvailable(&info.ProvConfig.Spec) { - initContainers = append(initContainers, createInitContainerIpaDownloader(info.Images)) - } return initContainers, nil } diff --git a/provisioning/image_customization.go b/provisioning/image_customization.go new file mode 100644 index 000000000..abdd34f8b --- /dev/null +++ b/provisioning/image_customization.go @@ -0,0 +1,266 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package provisioning + +import ( + "context" + "fmt" + "strings" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + metal3iov1alpha1 "github.com/openshift/cluster-baremetal-operator/api/v1alpha1" + "github.com/openshift/library-go/pkg/operator/resource/resourceapply" + "github.com/openshift/library-go/pkg/operator/resource/resourcemerge" +) + +const ( + ironicBaseUrl = "IRONIC_BASE_URL" + ironicAgentImage = "IRONIC_AGENT_IMAGE" + imageCustomizationDeploymentName = "metal3-image-customization" + imageCustomizationVolume = "metal3-image-customization-volume" + imageCustomizationPort = 8084 + containerRegistriesConfPath = "/etc/containers/registries.conf" + containerRegistriesEnvVar = "REGISTRIES_CONF_PATH" + deployISOEnvVar = "DEPLOY_ISO" + deployISOFile = "/shared/html/images/ironic-python-agent.iso" + deployInitrdEnvVar = "DEPLOY_INITRD" + deployInitrdFile = "/shared/html/images/ironic-python-agent.initramfs" +) + +var ( + imageRegistriesVolumeMount = corev1.VolumeMount{ + Name: imageCustomizationVolume, + MountPath: containerRegistriesConfPath, + } +) + +func imageRegistriesVolume() corev1.Volume { + // TODO: Should this be corev1.HostPathFile? + volType := corev1.HostPathFileOrCreate + + return corev1.Volume{ + Name: imageCustomizationVolume, + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: containerRegistriesConfPath, + Type: &volType, + }, + }, + } +} + +func getUrlFromIP(ipAddr string) string { + if strings.Contains(ipAddr, ":") { + // This is an IPv6 addr + return "https://" + fmt.Sprintf("[%s]", ipAddr) + } + if ipAddr != "" { + // This is an IPv4 addr + return "https://" + ipAddr + } else { + return "" + } +} + +func createImageCustomizationContainer(images *Images, info *ProvisioningInfo, ironicIP string) corev1.Container { + container := corev1.Container{ + Name: "image-customization-controller", + Image: images.ImageCustomizationController, + Command: []string{"/image-customization-controller", + "-images-bind-addr", fmt.Sprintf(":%d", imageCustomizationPort), + "-images-publish-addr", + fmt.Sprintf("http://%s.%s.svc.cluster.local/", + imageCustomizationService, info.Namespace)}, + + // TODO: This container does not have to run in privileged mode when the i-c-c has + // its own volume and does not have to use the imageCacheSharedVolume + SecurityContext: &corev1.SecurityContext{ + Privileged: pointer.BoolPtr(true), + }, + VolumeMounts: []corev1.VolumeMount{ + imageRegistriesVolumeMount, + imageVolumeMount, + }, + ImagePullPolicy: "IfNotPresent", + Env: []corev1.EnvVar{ + { + Name: deployISOEnvVar, + Value: deployISOFile, + }, + { + Name: deployInitrdEnvVar, + Value: deployInitrdFile, + }, + { + Name: ironicBaseUrl, + Value: getUrlFromIP(ironicIP), + }, + { + Name: ironicAgentImage, + Value: images.IronicAgent, + }, + { + Name: containerRegistriesEnvVar, + Value: containerRegistriesConfPath, + }, + buildSSHKeyEnvVar(info.SSHKey), + pullSecret, + }, + Ports: []corev1.ContainerPort{ + { + Name: "http", + ContainerPort: imageCustomizationPort, + }, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("5m"), + corev1.ResourceMemory: resource.MustParse("50Mi"), + }, + }, + } + return container +} + +func newImageCustomizationPodTemplateSpec(info *ProvisioningInfo, labels *map[string]string, ironicIP string) *corev1.PodTemplateSpec { + containers := []corev1.Container{ + createImageCustomizationContainer(info.Images, info, ironicIP), + } + + tolerations := []corev1.Toleration{ + { + Key: "node-role.kubernetes.io/master", + Effect: corev1.TaintEffectNoSchedule, + Operator: corev1.TolerationOpExists, + }, + { + Key: "CriticalAddonsOnly", + Operator: corev1.TolerationOpExists, + }, + { + Key: "node.kubernetes.io/not-ready", + Effect: corev1.TaintEffectNoExecute, + Operator: corev1.TolerationOpExists, + TolerationSeconds: pointer.Int64Ptr(120), + }, + { + Key: "node.kubernetes.io/unreachable", + Effect: corev1.TaintEffectNoExecute, + Operator: corev1.TolerationOpExists, + TolerationSeconds: pointer.Int64Ptr(120), + }, + } + + return &corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: podTemplateAnnotations, + Labels: *labels, + }, + Spec: corev1.PodSpec{ + Containers: containers, + HostNetwork: false, + DNSPolicy: corev1.DNSClusterFirstWithHostNet, + PriorityClassName: "system-node-critical", + NodeSelector: map[string]string{"node-role.kubernetes.io/master": ""}, + ServiceAccountName: "cluster-baremetal-operator", + Tolerations: tolerations, + Volumes: []corev1.Volume{ + imageRegistriesVolume(), + imageVolume(), + }, + }, + } +} + +func newImageCustomizationDeployment(info *ProvisioningInfo, ironicIP string) *appsv1.Deployment { + selector := &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "k8s-app": metal3AppName, + cboLabelName: imageCustomizationService, + }, + } + podSpecLabels := map[string]string{ + "k8s-app": metal3AppName, + cboLabelName: imageCustomizationService, + } + template := newImageCustomizationPodTemplateSpec(info, &podSpecLabels, ironicIP) + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: imageCustomizationDeploymentName, + Namespace: info.Namespace, + Annotations: map[string]string{}, + Labels: map[string]string{ + "k8s-app": metal3AppName, + cboLabelName: imageCustomizationService, + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: pointer.Int32Ptr(1), + Selector: selector, + Template: *template, + }, + } +} + +func getIronicIP(info *ProvisioningInfo) (string, error) { + config := info.ProvConfig.Spec + if config.ProvisioningNetwork != metal3iov1alpha1.ProvisioningNetworkDisabled && !config.VirtualMediaViaExternalNetwork { + return config.ProvisioningIP, nil + } else { + // Couple of things can go wrong here: + // 1. err != nil could be caused when Pod.List() and hence getPodHostIP() fails due to a temporary + // glitch (like a dropped network connection). So, report the error and try again. + // 2. hostIP == "" can happen when the metal3 pod is still coming up. The image-customization-controller + // has been updated to accept hostIP as "" + + return getPodHostIP(info.Client.CoreV1(), info.Namespace) + } +} + +func EnsureImageCustomizationDeployment(info *ProvisioningInfo) (updated bool, err error) { + ironicIP, err := getIronicIP(info) + if err != nil { + return false, fmt.Errorf("unable to determine Ironic's IP to pass to the image-customization-controller: %w", err) + } + + imageCustomizationDeployment := newImageCustomizationDeployment(info, ironicIP) + expectedGeneration := resourcemerge.ExpectedDeploymentGeneration(imageCustomizationDeployment, info.ProvConfig.Status.Generations) + err = controllerutil.SetControllerReference(info.ProvConfig, imageCustomizationDeployment, info.Scheme) + if err != nil { + err = fmt.Errorf("unable to set controllerReference on image-customization deployment: %w", err) + return + } + deployment, updated, err := resourceapply.ApplyDeployment(info.Client.AppsV1(), + info.EventRecorder, imageCustomizationDeployment, expectedGeneration) + if err != nil { + return updated, err + } + if updated { + resourcemerge.SetDeploymentGeneration(&info.ProvConfig.Status.Generations, deployment) + } + return updated, nil +} + +func DeleteImageCustomizationDeployment(info *ProvisioningInfo) error { + return client.IgnoreNotFound(info.Client.AppsV1().Deployments(info.Namespace).Delete(context.Background(), imageCustomizationDeploymentName, metav1.DeleteOptions{})) +} diff --git a/provisioning/image_customization_service.go b/provisioning/image_customization_service.go new file mode 100644 index 000000000..bc4a897fa --- /dev/null +++ b/provisioning/image_customization_service.go @@ -0,0 +1,62 @@ +package provisioning + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + "github.com/openshift/library-go/pkg/operator/resource/resourceapply" +) + +const ( + imageCustomizationService = "metal3-image-customization-service" +) + +func newImageCustomizationService(targetNamespace string) *corev1.Service { + ports := []corev1.ServicePort{ + { + Name: "http", + Port: 80, + TargetPort: intstr.FromInt(imageCustomizationPort), + }, + } + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: imageCustomizationService, + Namespace: targetNamespace, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + Selector: map[string]string{ + cboLabelName: imageCustomizationService, + }, + Ports: ports, + }, + } +} + +func EnsureImageCustomizationService(info *ProvisioningInfo) (updated bool, err error) { + imageCustomizationService := newImageCustomizationService(info.Namespace) + + err = controllerutil.SetControllerReference(info.ProvConfig, imageCustomizationService, info.Scheme) + if err != nil { + err = fmt.Errorf("unable to set controllerReference on %s service: %w", imageCustomizationService, err) + return + } + + _, updated, err = resourceapply.ApplyService(info.Client.CoreV1(), + info.EventRecorder, imageCustomizationService) + if err != nil { + err = fmt.Errorf("unable to apply %s service: %w", imageCustomizationService, err) + } + return +} + +func DeleteImageCustomizationService(info *ProvisioningInfo) error { + return client.IgnoreNotFound(info.Client.CoreV1().Services(info.Namespace).Delete(context.Background(), imageCustomizationService, metav1.DeleteOptions{})) +} diff --git a/provisioning/machine_os_images.go b/provisioning/machine_os_images.go new file mode 100644 index 000000000..7d70b8650 --- /dev/null +++ b/provisioning/machine_os_images.go @@ -0,0 +1,31 @@ +package provisioning + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +func createInitContainerMachineOSImages(info *ProvisioningInfo, whichImages string, dest corev1.VolumeMount, destPath string) corev1.Container { + container := corev1.Container{ + Name: "machine-os-images", + Image: info.Images.MachineOSImages, + Command: []string{"/bin/copy-metal", whichImages, destPath}, + VolumeMounts: []corev1.VolumeMount{ + dest, + }, + ImagePullPolicy: "IfNotPresent", + Env: []corev1.EnvVar{ + { + Name: ipOptions, + Value: ipOptionForMachineOsDownloader(info), + }, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("5m"), + corev1.ResourceMemory: resource.MustParse("50Mi"), + }, + }, + } + return container +} diff --git a/provisioning/sample_images.json b/provisioning/sample_images.json index de67c60dc..84a18c37a 100644 --- a/provisioning/sample_images.json +++ b/provisioning/sample_images.json @@ -6,5 +6,6 @@ "baremetalMachineOsDownloader": "registry.ci.openshift.org/openshift:ironic-machine-os-downloader", "baremetalStaticIpManager": "registry.ci.openshift.org/openshift:ironic-static-ip-manager", "baremetalIronicAgent": "registry.ci.openshift.org/openshift:ironic-agent", - "imageCustomizationController": "registry.ci.openshift.org/openshift:image-customization-controller" + "imageCustomizationController": "registry.ci.openshift.org/openshift:image-customization-controller", + "machineOSImages": "registry.ci.openshift.org/openshift:machine-os-images" }