From 908c6e713cac2de80b7830d27bcc3706be61a242 Mon Sep 17 00:00:00 2001 From: Andrea Fasano Date: Thu, 25 Sep 2025 13:15:36 -0400 Subject: [PATCH 1/2] fix registries.conf generation for multiple yaml mirror files --- pkg/asset/registry/registriesconf.go | 51 ++++++---- pkg/asset/registry/registriesconf_test.go | 110 ++++++++++++++++++++++ 2 files changed, 144 insertions(+), 17 deletions(-) create mode 100644 pkg/asset/registry/registriesconf_test.go diff --git a/pkg/asset/registry/registriesconf.go b/pkg/asset/registry/registriesconf.go index 74c07d0c..a7845a0f 100644 --- a/pkg/asset/registry/registriesconf.go +++ b/pkg/asset/registry/registriesconf.go @@ -1,8 +1,11 @@ package registry import ( + "bytes" "context" "fmt" + "io" + "io/fs" "os" "path/filepath" "strings" @@ -50,6 +53,8 @@ type ImageDigestMirror struct { type RegistriesConf struct { File *asset.File Config *sysregistriesv2.V2RegistriesConf + + fSys fs.FS } var _ asset.Asset = (*RegistriesConf)(nil) @@ -74,7 +79,11 @@ func (i *RegistriesConf) Generate(_ context.Context, dependencies asset.Parents) applianceConfig := &config.ApplianceConfig{} dependencies.Get(envConfig, applianceConfig) - releaseImagesLocation, releaseLocation := i.getEndpointLocations(envConfig.CacheDir) + if i.fSys == nil { + i.fSys = os.DirFS(envConfig.CacheDir) + } + + releaseImagesLocation, releaseLocation := i.getEndpointLocations() registries := &sysregistriesv2.V2RegistriesConf{ Registries: []sysregistriesv2.Registry{ { @@ -121,33 +130,41 @@ func (i *RegistriesConf) Generate(_ context.Context, dependencies asset.Parents) return nil } -func (i *RegistriesConf) getEndpointLocations(cacheDir string) (string, string) { +func (i *RegistriesConf) getEndpointLocations() (string, string) { releaseImagesLocation := "quay.io/openshift-release-dev/ocp-release" releaseLocation := "quay.io/openshift-release-dev/ocp-v4.0-art-dev" - idmsFile, err := os.ReadFile(filepath.Join(cacheDir, idmsFileName)) + idmsFile, err := fs.ReadFile(i.fSys, idmsFileName) if err != nil { logrus.Debugf("missing IDMS yaml (%v), fallback to defaults.", err) return releaseImagesLocation, releaseLocation } - var idms *ImageDigestMirrorSet - if err := yaml.UnmarshalStrict(idmsFile, &idms); err != nil { - logrus.Debugf("failed to unmarshal IDMS yaml (%v), fallback to defaults.", err) - return releaseImagesLocation, releaseLocation - } - for _, digestMirrors := range idms.Spec.ImageDigestMirrors { - if len(digestMirrors.Mirrors) == 0 { - continue + dec := yaml.NewDecoder(bytes.NewReader(idmsFile)) + dec.SetStrict(true) + + for { + var idms ImageDigestMirrorSet + if err := dec.Decode(&idms); err != nil { + if errors.Is(err, io.EOF) { + break + } + logrus.Debugf("failed to unmarshal IDMS yaml (%v)", err) } - location := digestMirrors.Mirrors[0] - if strings.HasSuffix(location, "release-images") { - releaseImagesLocation = digestMirrors.Source - } else if strings.HasSuffix(location, "release") { - releaseLocation = digestMirrors.Source + + for _, digestMirrors := range idms.Spec.ImageDigestMirrors { + if len(digestMirrors.Mirrors) == 0 { + continue + } + location := digestMirrors.Mirrors[0] + if strings.HasSuffix(location, "release-images") { + releaseImagesLocation = digestMirrors.Source + } else if strings.HasSuffix(location, "release") { + releaseLocation = digestMirrors.Source + } } } - + logrus.Debugf("endpoints locations: %s, %s", releaseImagesLocation, releaseLocation) return releaseImagesLocation, releaseLocation } diff --git a/pkg/asset/registry/registriesconf_test.go b/pkg/asset/registry/registriesconf_test.go new file mode 100644 index 00000000..4a7f689e --- /dev/null +++ b/pkg/asset/registry/registriesconf_test.go @@ -0,0 +1,110 @@ +package registry + +import ( + "context" + "testing" + "testing/fstest" + + . "github.com/onsi/ginkgo/v2/dsl/core" + . "github.com/onsi/gomega" + "github.com/openshift/appliance/pkg/asset/config" + "github.com/openshift/installer/pkg/asset" +) + +var _ = Describe("Test RegistriesConf", func() { + var ( + fakeFileSystem fstest.MapFS + deps asset.Parents + r RegistriesConf + ) + + BeforeEach(func() { + fakeFileSystem = fstest.MapFS{} + deps = asset.Parents{} + + deps.Add(&config.EnvConfig{}, &config.ApplianceConfig{}) + r = RegistriesConf{ + fSys: fakeFileSystem, + } + }) + + It("Single Yaml", func() { + fakeFileSystem[idmsFileName] = createSingleYamlIDMSMirrorFile() + + err := r.Generate(context.Background(), deps) + Expect(err).NotTo(HaveOccurred()) + Expect(string(r.File.Data)).To(Equal("unqualified-search-registries = []\n\n[[registry]]\n location = \"registry.ci.openshift.org/ocp/release\"\n prefix = \"\"\n\n [[registry.mirror]]\n location = \"registry.appliance.openshift.com:5000/openshift/release-images\"\n\n [[registry.mirror]]\n location = \"registry.appliance.openshift.com:5001/openshift/release-images\"\n\n[[registry]]\n location = \"quay.io/openshift-release-dev/ocp-v4.0-art-dev\"\n prefix = \"\"\n\n [[registry.mirror]]\n location = \"registry.appliance.openshift.com:5000/openshift/release\"\n\n [[registry.mirror]]\n location = \"registry.appliance.openshift.com:5001/openshift/release\"\n")) + }) + + It("Multiple Yaml", func() { + fakeFileSystem[idmsFileName] = createMultipleYamlIDMSMirrorFile() + + err := r.Generate(context.Background(), deps) + Expect(err).NotTo(HaveOccurred()) + Expect(string(r.File.Data)).Should(ContainSubstring("registry.ci.openshift.org")) + }) + +}) + +func TestRegistriesConf(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "registriesconf_test") +} + +func createSingleYamlIDMSMirrorFile() *fstest.MapFile { + return &fstest.MapFile{ + Data: []byte( + `apiVersion: config.openshift.io/v1 +kind: ImageDigestMirrorSet +metadata: + name: idms-release-0 +spec: + imageDigestMirrors: + - mirrors: + - registry.appliance.openshift.com:5000/openshift/release-images + source: registry.ci.openshift.org/ocp/release + - mirrors: + - registry.appliance.openshift.com:5000/openshift/release + source: quay.io/openshift-release-dev/ocp-v4.0-art-dev +status: {}`)} +} + +func createMultipleYamlIDMSMirrorFile() *fstest.MapFile { + return &fstest.MapFile{ + Data: []byte( + `kind: ImageDigestMirrorSet +metadata: + name: idms-operator-0 +spec: + imageDigestMirrors: + - mirrors: + - registry.appliance.openshift.com:5000/container-native-virtualization + source: registry.redhat.io/container-native-virtualization + - mirrors: + - registry.appliance.openshift.com:5000/openshift4 + source: registry.redhat.io/openshift4 + - mirrors: + - registry.appliance.openshift.com:5000/workload-availability + source: registry.redhat.io/workload-availability + - mirrors: + - registry.appliance.openshift.com:5000/migration-toolkit-virtualization + source: registry.redhat.io/migration-toolkit-virtualization + - mirrors: + - registry.appliance.openshift.com:5000/kube-descheduler-operator + source: registry.redhat.io/kube-descheduler-operator +status: {} +--- +apiVersion: config.openshift.io/v1 +kind: ImageDigestMirrorSet +metadata: + name: idms-release-0 +spec: + imageDigestMirrors: + - mirrors: + - registry.appliance.openshift.com:5000/openshift/release-images + source: registry.ci.openshift.org/ocp/release + - mirrors: + - registry.appliance.openshift.com:5000/openshift/release + source: quay.io/openshift-release-dev/ocp-v4.0-art-dev +status: {}`)} +} From de611acb35f4a07b1b23d488225f900efcbcb709 Mon Sep 17 00:00:00 2001 From: Andrea Fasano Date: Mon, 29 Sep 2025 03:15:06 -0400 Subject: [PATCH 2/2] use GetMultipleYamls for manifests parsing --- pkg/asset/registry/registriesconf.go | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/pkg/asset/registry/registriesconf.go b/pkg/asset/registry/registriesconf.go index a7845a0f..f7322590 100644 --- a/pkg/asset/registry/registriesconf.go +++ b/pkg/asset/registry/registriesconf.go @@ -1,10 +1,8 @@ package registry import ( - "bytes" "context" "fmt" - "io" "io/fs" "os" "path/filepath" @@ -14,10 +12,10 @@ import ( "github.com/openshift/appliance/pkg/asset/config" "github.com/openshift/appliance/pkg/consts" "github.com/openshift/installer/pkg/asset" + agentManifests "github.com/openshift/installer/pkg/asset/agent/manifests" "github.com/pelletier/go-toml" "github.com/pkg/errors" "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" ) const ( @@ -140,18 +138,13 @@ func (i *RegistriesConf) getEndpointLocations() (string, string) { return releaseImagesLocation, releaseLocation } - dec := yaml.NewDecoder(bytes.NewReader(idmsFile)) - dec.SetStrict(true) - - for { - var idms ImageDigestMirrorSet - if err := dec.Decode(&idms); err != nil { - if errors.Is(err, io.EOF) { - break - } - logrus.Debugf("failed to unmarshal IDMS yaml (%v)", err) - } + idmsManifests, err := agentManifests.GetMultipleYamls[ImageDigestMirrorSet](idmsFile) + if err != nil { + logrus.Debugf("could not decode YAML for %s (%v), fallback to defaults.", idmsFileName, err) + return releaseImagesLocation, releaseLocation + } + for _, idms := range idmsManifests { for _, digestMirrors := range idms.Spec.ImageDigestMirrors { if len(digestMirrors.Mirrors) == 0 { continue