[OCPNODE-553] Add CRD ImageDigestMirrorSet,ImageTagMirrorSet

cmd/machine-config-controller/start.go

@@ -146,6 +146,8 @@ func createControllers(ctx *ctrlcommon.ControllerContext) []ctrlcommon.Controlle
 			ctx.InformerFactory.Machineconfiguration().V1().ControllerConfigs(),
 			ctx.InformerFactory.Machineconfiguration().V1().ContainerRuntimeConfigs(),
 			ctx.ConfigInformerFactory.Config().V1().Images(),
+			ctx.ConfigInformerFactory.Config().V1().ImageDigestMirrorSets(),
+			ctx.ConfigInformerFactory.Config().V1().ImageTagMirrorSets(),
 			ctx.OperatorInformerFactory.Operator().V1alpha1().ImageContentSourcePolicies(),
 			ctx.ConfigInformerFactory.Config().V1().ClusterVersions(),
 			ctx.ClientBuilder.KubeClientOrDie("container-runtime-config-controller"),

docs/MachineConfigDaemon.md

@@ -168,8 +168,8 @@ The "Reload Crio" action performs the file write and runs a `systemctl reload cr
 
 1. Container signing GPG keys: these can be changed by pointing `/etc/containers/policy.json` to `/etc/machine-config-daemon/no-reboot/containers-gpg.pub` and storing keys in the latter file. Changes to either file trigger the "Reload Crio" action
 2. **Selected** `/etc/containers/registries.conf` changes: this file is generally changed via ICSP object changes. Only the following changes will avoid a drain:
-   - addition of a registry with `mirror-by-digest-only=true`
-   - addition of a mirror in a registry with `mirror-by-digest-only=true`
+   - addition of a registry with `pull-from-mirror=digest-only` for each mirror
+   - addition of a mirror with `pull-from-mirror=digest-only` in a registry
    - appending items in the `unqualified-search-registries` list
 
 ### With Drain

go.mod

@@ -30,7 +30,7 @@ require (
 	github.com/openshift/api v0.0.0-20221220162201-efeef9d83325
 	github.com/openshift/client-go v0.0.0-20220831193253-4950ae70c8ea
 	github.com/openshift/library-go v0.0.0-20220915130036-73d5a4a82865
-	github.com/openshift/runtime-utils v0.0.0-20220906151503-3beb0b584526
+	github.com/openshift/runtime-utils v0.0.0-20220926190846-5c488b20a19f
 	github.com/prometheus/client_golang v1.13.0
 	github.com/spf13/cobra v1.5.0
 	github.com/spf13/pflag v1.0.5

go.sum

@@ -1014,8 +1014,8 @@ github.com/openshift/client-go v0.0.0-20220831193253-4950ae70c8ea h1:7JbjIzWt3Q7
 github.com/openshift/client-go v0.0.0-20220831193253-4950ae70c8ea/go.mod h1:+J8DqZC60acCdpYkwVy/KH4cudgWiFZRNOBeghCzdGA=
 github.com/openshift/library-go v0.0.0-20220915130036-73d5a4a82865 h1:x7KWaYzkD2KQ3rha9u7OVAfjZpSFTmJRSHb4CHc+CwM=
 github.com/openshift/library-go v0.0.0-20220915130036-73d5a4a82865/go.mod h1:KPBAXGaq7pPmA+1wUVtKr5Axg3R68IomWDkzaOxIhxM=
-github.com/openshift/runtime-utils v0.0.0-20220906151503-3beb0b584526 h1:VZQXj1MoaqmmnP0lZvwNOK/c22I4+ZQufs0+dvYVUCg=
-github.com/openshift/runtime-utils v0.0.0-20220906151503-3beb0b584526/go.mod h1:Zc9dB7MrREj9MwD4znL6jSHLHyeOpG823a92IkLV3d4=
+github.com/openshift/runtime-utils v0.0.0-20220926190846-5c488b20a19f h1:ubRzazPtplWWNWWX07v4ww74S9QL+B2RAxHJ8O00m7o=
+github.com/openshift/runtime-utils v0.0.0-20220926190846-5c488b20a19f/go.mod h1:l9/qeKZuAmYUMl0yicJlbkPGDsIycGhwxOvOAWyaP0E=
 github.com/otiai10/copy v1.2.0 h1:HvG945u96iNadPoG2/Ja2+AUJeW5YuFQMixq9yirC+k=
 github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw=
 github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=

manifests/machineconfigcontroller/clusterrole.yaml

@@ -16,7 +16,7 @@ rules:
   resources: ["images", "clusterversions", "featuregates", "nodes", "nodes/status"]
   verbs: ["*"]
 - apiGroups: ["config.openshift.io"]
-  resources: ["schedulers", "apiservers", "infrastructures"]
+  resources: ["schedulers", "apiservers", "infrastructures", "imagedigestmirrorsets", "imagetagmirrorsets"]
   verbs: ["get", "list", "watch"]
 - apiGroups: ["operator.openshift.io"]
   resources: ["imagecontentsourcepolicies"]

pkg/controller/bootstrap/bootstrap.go

@@ -80,6 +80,8 @@ func (b *Bootstrap) Run(destDir string) error {
 	var configs []*mcfgv1.MachineConfig
 	var crconfigs []*mcfgv1.ContainerRuntimeConfig
 	var icspRules []*apioperatorsv1alpha1.ImageContentSourcePolicy
+	var idmsRules []*apicfgv1.ImageDigestMirrorSet
+	var itmsRules []*apicfgv1.ImageTagMirrorSet
 	var imgCfg *apicfgv1.Image
 	for _, info := range infos {
 		if info.IsDir() {
@@ -121,6 +123,10 @@ func (b *Bootstrap) Run(destDir string) error {
 				kconfigs = append(kconfigs, obj)
 			case *apioperatorsv1alpha1.ImageContentSourcePolicy:
 				icspRules = append(icspRules, obj)
+			case *apicfgv1.ImageDigestMirrorSet:
+				idmsRules = append(idmsRules, obj)
+			case *apicfgv1.ImageTagMirrorSet:
+				itmsRules = append(itmsRules, obj)
 			case *apicfgv1.Image:
 				imgCfg = obj
 			case *apicfgv1.FeatureGate:
@@ -146,7 +152,7 @@ func (b *Bootstrap) Run(destDir string) error {
 	}
 	configs = append(configs, iconfigs...)
 
-	rconfigs, err := containerruntimeconfig.RunImageBootstrap(b.templatesDir, cconfig, pools, icspRules, imgCfg)
+	rconfigs, err := containerruntimeconfig.RunImageBootstrap(b.templatesDir, cconfig, pools, icspRules, idmsRules, itmsRules, imgCfg)
 	if err != nil {
 		return err
 	}

pkg/controller/bootstrap/testdata/bootstrap/image-content-source-policy-0.yaml

@@ -7,4 +7,4 @@ spec:
   repositoryDigestMirrors:
   - mirrors:
     - registry.mirror.example.com/ocp
-    source: registry.product.example.org/ocp/4.2-DATE-VERSION
+    source: registry.product.example.org/ocp/4.2-date-version

pkg/controller/container-runtime-config/container_runtime_config_controller.go

@@ -18,6 +18,7 @@ import (
 	cligolistersv1 "github.com/openshift/client-go/config/listers/config/v1"
 	operatorinformersv1alpha1 "github.com/openshift/client-go/operator/informers/externalversions/operator/v1alpha1"
 	operatorlistersv1alpha1 "github.com/openshift/client-go/operator/listers/operator/v1alpha1"
+	runtimeutils "github.com/openshift/runtime-utils/pkg/registries"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/equality"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -91,6 +92,12 @@ type Controller struct {
 	icspLister       operatorlistersv1alpha1.ImageContentSourcePolicyLister
 	icspListerSynced cache.InformerSynced
 
+	idmsLister       cligolistersv1.ImageDigestMirrorSetLister
+	idmsListerSynced cache.InformerSynced
+
+	itmsLister       cligolistersv1.ImageTagMirrorSetLister
+	itmsListerSynced cache.InformerSynced
+
 	mcpLister       mcfglistersv1.MachineConfigPoolLister
 	mcpListerSynced cache.InformerSynced
 
@@ -108,6 +115,8 @@ func New(
 	ccInformer mcfginformersv1.ControllerConfigInformer,
 	mcrInformer mcfginformersv1.ContainerRuntimeConfigInformer,
 	imgInformer cligoinformersv1.ImageInformer,
+	idmsInformer cligoinformersv1.ImageDigestMirrorSetInformer,
+	itmsInformer cligoinformersv1.ImageTagMirrorSetInformer,
 	icspInformer operatorinformersv1alpha1.ImageContentSourcePolicyInformer,
 	clusterVersionInformer cligoinformersv1.ClusterVersionInformer,
 	kubeClient clientset.Interface,
@@ -145,6 +154,18 @@ func New(
 		DeleteFunc: ctrl.icspConfDeleted,
 	})
 
+	idmsInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc:    ctrl.idmsConfAdded,
+		UpdateFunc: ctrl.idmsConfUpdated,
+		DeleteFunc: ctrl.idmsConfDeleted,
+	})
+
+	itmsInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc:    ctrl.itmsConfAdded,
+		UpdateFunc: ctrl.itmsConfUpdated,
+		DeleteFunc: ctrl.itmsConfDeleted,
+	})
+
 	ctrl.syncHandler = ctrl.syncContainerRuntimeConfig
 	ctrl.syncImgHandler = ctrl.syncImageConfig
 	ctrl.enqueueContainerRuntimeConfig = ctrl.enqueue
@@ -164,6 +185,12 @@ func New(
 	ctrl.icspLister = icspInformer.Lister()
 	ctrl.icspListerSynced = icspInformer.Informer().HasSynced
 
+	ctrl.idmsLister = idmsInformer.Lister()
+	ctrl.idmsListerSynced = idmsInformer.Informer().HasSynced
+
+	ctrl.itmsLister = itmsInformer.Lister()
+	ctrl.itmsListerSynced = itmsInformer.Informer().HasSynced
+
 	ctrl.clusterVersionLister = clusterVersionInformer.Lister()
 	ctrl.clusterVersionListerSynced = clusterVersionInformer.Informer().HasSynced
 
@@ -177,7 +204,7 @@ func (ctrl *Controller) Run(workers int, stopCh <-chan struct{}) {
 	defer ctrl.imgQueue.ShutDown()
 
 	if !cache.WaitForCacheSync(stopCh, ctrl.mcpListerSynced, ctrl.mccrListerSynced, ctrl.ccListerSynced,
-		ctrl.imgListerSynced, ctrl.icspListerSynced, ctrl.clusterVersionListerSynced) {
+		ctrl.imgListerSynced, ctrl.icspListerSynced, ctrl.idmsListerSynced, ctrl.itmsListerSynced, ctrl.clusterVersionListerSynced) {
 		return
 	}
 
@@ -228,6 +255,30 @@ func (ctrl *Controller) icspConfDeleted(obj interface{}) {
 	ctrl.imgQueue.Add("openshift-config")
 }
 
+func (ctrl *Controller) idmsConfAdded(obj interface{}) {
+	ctrl.imgQueue.Add("openshift-config")
+}
+
+func (ctrl *Controller) idmsConfUpdated(oldObj, newObj interface{}) {
+	ctrl.imgQueue.Add("openshift-config")
+}
+
+func (ctrl *Controller) idmsConfDeleted(obj interface{}) {
+	ctrl.imgQueue.Add("openshift-config")
+}
+
+func (ctrl *Controller) itmsConfAdded(obj interface{}) {
+	ctrl.imgQueue.Add("openshift-config")
+}
+
+func (ctrl *Controller) itmsConfUpdated(oldObj, newObj interface{}) {
+	ctrl.imgQueue.Add("openshift-config")
+}
+
+func (ctrl *Controller) itmsConfDeleted(obj interface{}) {
+	ctrl.imgQueue.Add("openshift-config")
+}
+
 func (ctrl *Controller) updateContainerRuntimeConfig(oldObj, newObj interface{}) {
 	oldCtrCfg := oldObj.(*mcfgv1.ContainerRuntimeConfig)
 	newCtrCfg := newObj.(*mcfgv1.ContainerRuntimeConfig)
@@ -650,6 +701,7 @@ func mergeConfigChanges(origFile *ign3types.File, cfg *mcfgv1.ContainerRuntimeCo
 	return cfgTOML, nil
 }
 
+// nolint: gocyclo
 func (ctrl *Controller) syncImageConfig(key string) error {
 	startTime := time.Now()
 	glog.V(4).Infof("Started syncing ImageConfig %q (%v)", key, startTime)
@@ -687,6 +739,25 @@ func (ctrl *Controller) syncImageConfig(key string) error {
 	} else if err != nil {
 		return err
 	}
+	// Find all ImageDigestMirrorSet objects
+	idmsRules, err := ctrl.idmsLister.List(labels.Everything())
+	if err != nil && errors.IsNotFound(err) {
+		idmsRules = []*apicfgv1.ImageDigestMirrorSet{}
+	} else if err != nil {
+		return err
+	}
+
+	// Find all ImageTagMirrorSet objects
+	itmsRules, err := ctrl.itmsLister.List(labels.Everything())
+	if err != nil && errors.IsNotFound(err) {
+		itmsRules = []*apicfgv1.ImageTagMirrorSet{}
+	} else if err != nil {
+		return err
+	}
+
+	if err := runtimeutils.RejectMultiUpdateMirrorSetObjs(icspRules, idmsRules, itmsRules); err != nil {
+		return err
+	}
 
 	var (
 		registriesBlocked, policyBlocked, allowedRegs []string
@@ -698,7 +769,7 @@ func (ctrl *Controller) syncImageConfig(key string) error {
 		// has been recovered
 		releaseImage = clusterVersionCfg.Status.Desired.Image
 		// Go through the registries in the image spec to get and validate the blocked registries
-		registriesBlocked, policyBlocked, allowedRegs, err = getValidBlockedAndAllowedRegistries(releaseImage, &imgcfg.Spec, icspRules)
+		registriesBlocked, policyBlocked, allowedRegs, err = getValidBlockedAndAllowedRegistries(releaseImage, &imgcfg.Spec, icspRules, idmsRules)
 		if err != nil && err != errParsingReference {
 			glog.V(2).Infof("%v, skipping....", err)
 		} else if err == errParsingReference {
@@ -733,7 +804,7 @@ func (ctrl *Controller) syncImageConfig(key string) error {
 		if err := retry.RetryOnConflict(updateBackoff, func() error {
 			registriesIgn, err := registriesConfigIgnition(ctrl.templatesDir, controllerConfig, role, releaseImage,
 				imgcfg.Spec.RegistrySources.InsecureRegistries, registriesBlocked, policyBlocked, allowedRegs,
-				imgcfg.Spec.RegistrySources.ContainerRuntimeSearchRegistries, icspRules)
+				imgcfg.Spec.RegistrySources.ContainerRuntimeSearchRegistries, icspRules, idmsRules, itmsRules)
 			if err != nil {
 				return err
 			}
@@ -794,7 +865,8 @@ func (ctrl *Controller) syncImageConfig(key string) error {
 }
 
 func registriesConfigIgnition(templateDir string, controllerConfig *mcfgv1.ControllerConfig, role, releaseImage string,
-	insecureRegs, registriesBlocked, policyBlocked, allowedRegs, searchRegs []string, icspRules []*apioperatorsv1alpha1.ImageContentSourcePolicy) (*ign3types.Config, error) {
+	insecureRegs, registriesBlocked, policyBlocked, allowedRegs, searchRegs []string,
+	icspRules []*apioperatorsv1alpha1.ImageContentSourcePolicy, idmsRules []*apicfgv1.ImageDigestMirrorSet, itmsRules []*apicfgv1.ImageTagMirrorSet) (*ign3types.Config, error) {
 
 	var (
 		registriesTOML []byte
@@ -807,15 +879,15 @@ func registriesConfigIgnition(templateDir string, controllerConfig *mcfgv1.Contr
 		return nil, fmt.Errorf("could not generate original ContainerRuntime Configs: %w", err)
 	}
 
-	if insecureRegs != nil || registriesBlocked != nil || len(icspRules) != 0 {
+	if insecureRegs != nil || registriesBlocked != nil || len(icspRules) != 0 || len(idmsRules) != 0 || len(itmsRules) != 0 {
 		if originalRegistriesIgn.Contents.Source == nil {
 			return nil, fmt.Errorf("original registries config is empty")
 		}
 		contents, err := ctrlcommon.DecodeIgnitionFileContents(originalRegistriesIgn.Contents.Source, originalRegistriesIgn.Contents.Compression)
 		if err != nil {
 			return nil, fmt.Errorf("could not decode original registries config: %w", err)
 		}
-		registriesTOML, err = updateRegistriesConfig(contents, insecureRegs, registriesBlocked, icspRules)
+		registriesTOML, err = updateRegistriesConfig(contents, insecureRegs, registriesBlocked, icspRules, idmsRules, itmsRules)
 		if err != nil {
 			return nil, fmt.Errorf("could not update registries config with new changes: %w", err)
 		}
@@ -847,17 +919,23 @@ func registriesConfigIgnition(templateDir string, controllerConfig *mcfgv1.Contr
 
 // RunImageBootstrap generates MachineConfig objects for mcpPools that would have been generated by syncImageConfig,
 // except that mcfgv1.Image is not available.
-func RunImageBootstrap(templateDir string, controllerConfig *mcfgv1.ControllerConfig, mcpPools []*mcfgv1.MachineConfigPool, icspRules []*apioperatorsv1alpha1.ImageContentSourcePolicy, imgCfg *apicfgv1.Image) ([]*mcfgv1.MachineConfig, error) {
+func RunImageBootstrap(templateDir string, controllerConfig *mcfgv1.ControllerConfig, mcpPools []*mcfgv1.MachineConfigPool, icspRules []*apioperatorsv1alpha1.ImageContentSourcePolicy,
+	idmsRules []*apicfgv1.ImageDigestMirrorSet, itmsRules []*apicfgv1.ImageTagMirrorSet, imgCfg *apicfgv1.Image) ([]*mcfgv1.MachineConfig, error) {
+
 	var (
 		insecureRegs, registriesBlocked, policyBlocked, allowedRegs, searchRegs []string
 		err                                                                     error
 	)
 
+	if err := runtimeutils.RejectMultiUpdateMirrorSetObjs(icspRules, idmsRules, itmsRules); err != nil {
+		return nil, err
+	}
+
 	// Read the search, insecure, blocked, and allowed registries from the cluster-wide Image CR if it is not nil
 	if imgCfg != nil {
 		insecureRegs = imgCfg.Spec.RegistrySources.InsecureRegistries
 		searchRegs = imgCfg.Spec.RegistrySources.ContainerRuntimeSearchRegistries
-		registriesBlocked, policyBlocked, allowedRegs, err = getValidBlockedAndAllowedRegistries(controllerConfig.Spec.ReleaseImage, &imgCfg.Spec, icspRules)
+		registriesBlocked, policyBlocked, allowedRegs, err = getValidBlockedAndAllowedRegistries(controllerConfig.Spec.ReleaseImage, &imgCfg.Spec, icspRules, idmsRules)
 		if err != nil && err != errParsingReference {
 			glog.V(2).Infof("%v, skipping....", err)
 		} else if err == errParsingReference {
@@ -874,7 +952,7 @@ func RunImageBootstrap(templateDir string, controllerConfig *mcfgv1.ControllerCo
 			return nil, err
 		}
 		registriesIgn, err := registriesConfigIgnition(templateDir, controllerConfig, role, controllerConfig.Spec.ReleaseImage,
-			insecureRegs, registriesBlocked, policyBlocked, allowedRegs, searchRegs, icspRules)
+			insecureRegs, registriesBlocked, policyBlocked, allowedRegs, searchRegs, icspRules, idmsRules, itmsRules)
 		if err != nil {
 			return nil, err
 		}