rpmmd/reporegistry: allow on-disk repository format to also be yaml

pkg/reporegistry/repository.go

@@ -50,8 +50,9 @@ func loadAllRepositoriesFromFS(confPaths []fs.FS) (rpmmd.DistrosRepoConfigs, err
 			}
 
 			// distro repositories definition is expected to be named "<distro_name>.json"
-			if strings.HasSuffix(fileEntry.Name(), ".json") {
+			if strings.HasSuffix(fileEntry.Name(), ".json") || strings.HasSuffix(fileEntry.Name(), ".yaml") {
 				distroIDStr := strings.TrimSuffix(fileEntry.Name(), ".json")
+				distroIDStr = strings.TrimSuffix(distroIDStr, ".yaml")
 
 				// compatibility layer to support old repository definition filenames
 				// without a dot to separate major and minor release versions
@@ -92,21 +93,27 @@ func loadAllRepositoriesFromFS(confPaths []fs.FS) (rpmmd.DistrosRepoConfigs, err
 // LoadRepositories loads distribution repositories from the given list of paths.
 // If there are duplicate distro repositories definitions found in multiple paths, the first
 // encounter is preferred. For this reason, the order of paths in the passed list should
-// reflect the desired preference.
+// reflect the desired preference. Both json and yaml repository files can be used to load
+// from. When a json file is encountered it takes precedence over a yaml file under the
+// same distro name.
 //
 // Note that the confPaths must point directly to the directory with
-// the json repo files.
+// the json and yaml repo files.
 func LoadRepositories(confPaths []string, distro string) (map[string][]rpmmd.RepoConfig, error) {
 	var repoConfigs map[string][]rpmmd.RepoConfig
-	path := distro + ".json"
 
 	for _, confPath := range confPaths {
 		var err error
-		repoConfigs, err = rpmmd.LoadRepositoriesFromFile(filepath.Join(confPath, path))
-		if os.IsNotExist(err) {
-			continue
-		} else if err != nil {
-			return nil, err
+		paths := []string{distro + ".json", distro + ".yaml"}
+		for _, path := range paths {
+			repoConfigs, err = rpmmd.LoadRepositoriesFromFile(filepath.Join(confPath, path))
+			if os.IsNotExist(err) {
+				continue
+			} else if err != nil {
+				return nil, err
+			} else {
+				break
+			}
 		}
 
 		// Found the distro repository configs in the current path

pkg/reporegistry/repository_test.go

@@ -22,6 +22,7 @@ func getConfPaths(t *testing.T) []string {
 	confPaths := []string{
 		"./test/confpaths/priority1/repositories",
 		"./test/confpaths/priority2/repositories",
+		"./test/confpaths/priority3/repositories",
 	}
 	var absConfPaths []string
 
@@ -64,6 +65,16 @@ func TestLoadRepositoriesExisting(t *testing.T) {
 				test_distro.TestArch2Name: {"fedora-34-p2", "updates-34-p2"},
 			},
 		},
+		{
+			name: "single distro definition but its yaml",
+			args: args{
+				distro: "fedora-35",
+			},
+			want: map[string][]string{
+				test_distro.TestArchName:  {"fedora-35-p3", "updates-35-p3"},
+				test_distro.TestArch2Name: {"fedora-35-p3", "updates-35-p3"},
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -157,6 +168,36 @@ func Test_LoadAllRepositories(t *testing.T) {
 				},
 			},
 		},
+		"fedora-35": {
+			test_distro.TestArchName: {
+				{
+					Name:     "fedora-35-p3",
+					BaseURLs: []string{"https://example.com/fedora-35-p3/test_arch"},
+					GPGKeys:  []string{"FAKE-GPG-KEY"},
+					CheckGPG: common.ToPtr(true),
+				},
+				{
+					Name:     "updates-35-p3",
+					BaseURLs: []string{"https://example.com/updates-35-p3/test_arch"},
+					GPGKeys:  []string{"FAKE-GPG-KEY"},
+					CheckGPG: common.ToPtr(true),
+				},
+			},
+			test_distro.TestArch2Name: {
+				{
+					Name:     "fedora-35-p3",
+					BaseURLs: []string{"https://example.com/fedora-35-p3/test_arch2"},
+					GPGKeys:  []string{"FAKE-GPG-KEY"},
+					CheckGPG: common.ToPtr(true),
+				},
+				{
+					Name:     "updates-35-p3",
+					BaseURLs: []string{"https://example.com/updates-35-p3/test_arch2"},
+					GPGKeys:  []string{"FAKE-GPG-KEY"},
+					CheckGPG: common.ToPtr(true),
+				},
+			},
+		},
 		"rhel-8.7": {
 			test_distro.TestArchName: {
 				{
@@ -354,6 +395,6 @@ func TestLoadRepositoriesError(t *testing.T) {
 		assert.NoError(reposFile.Close())
 
 		_, err = LoadAllRepositories([]string{reposDir}, nil)
-		assert.ErrorContains(err, "failed to load repositories: invalid character '<' looking for beginning of value")
+		assert.ErrorContains(err, "cannot unmarshal !!str")
 	})
 }

pkg/reporegistry/test/confpaths/priority3/repositories/fedora-35.yaml

@@ -0,0 +1,20 @@
+---
+test_arch:
+  - name: "fedora-35-p3"
+    baseurl: "https://example.com/fedora-35-p3/test_arch"
+    gpgkey: "FAKE-GPG-KEY"
+    check_gpg: true
+  - name: "updates-35-p3"
+    baseurl: "https://example.com/updates-35-p3/test_arch"
+    gpgkey: "FAKE-GPG-KEY"
+    check_gpg: true
+
+test_arch2:
+  - name: "fedora-35-p3"
+    baseurl: "https://example.com/fedora-35-p3/test_arch2"
+    gpgkey: "FAKE-GPG-KEY"
+    check_gpg: true
+  - name: "updates-35-p3"
+    baseurl: "https://example.com/updates-35-p3/test_arch2"
+    gpgkey: "FAKE-GPG-KEY"
+    check_gpg: true

pkg/rpmmd/repository.go

@@ -7,6 +7,10 @@ import (
 	"io"
 	"os"
 	"strings"
+
+	"go.yaml.in/yaml/v3"
+
+	"github.com/osbuild/images/internal/common"
 )
 
 // repository is the presentation of a on-disk repository json file
@@ -58,6 +62,10 @@ func (r *repository) UnmarshalJSON(data []byte) (err error) {
 	return nil
 }
 
+func (r *repository) UnmarshalYAML(unmarshal func(any) error) error {
+	return common.UnmarshalYAMLviaJSON(r, unmarshal)
+}
+
 type RepoConfig struct {
 	// the repo id is not always required and is ignored in some cases.
 	// For example, it is not required in osbuild-depsolve-dnf, but it is a required
@@ -133,7 +141,7 @@ func LoadRepositoriesFromReader(r io.Reader) (map[string][]RepoConfig, error) {
 	var reposMap map[string][]repository
 	repoConfigs := make(map[string][]RepoConfig)
 
-	err := json.NewDecoder(r).Decode(&reposMap)
+	err := yaml.NewDecoder(r).Decode(&reposMap)
 	if err != nil {
 		return nil, err
 	}

pkg/rpmmd/repository_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"go.yaml.in/yaml/v3"
 
 	"github.com/osbuild/images/pkg/rpmmd"
 )
@@ -41,7 +42,7 @@ func TestRepoConfigUnmarshalHappy(t *testing.T) {
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			var repos rpmmd.Repository
-			err := json.Unmarshal([]byte(tc.json), &repos)
+			err := yaml.Unmarshal([]byte(tc.json), &repos)
 			assert.NoError(t, err)
 			assert.Equal(t, tc.repo, repos)
 		})
@@ -67,14 +68,14 @@ func TestRepoConfigUnmarshalSad(t *testing.T) {
 		{
 			name:        "wrong json",
 			json:        `all-wrong`,
-			expectedErr: `invalid character 'a' looking for beginning of value`,
+			expectedErr: `cannot unmarshal string into Go value of type struct`,
 		},
 	}
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			var repos rpmmd.Repository
-			err := json.Unmarshal([]byte(tc.json), &repos)
-			assert.EqualError(t, err, tc.expectedErr)
+			err := yaml.Unmarshal([]byte(tc.json), &repos)
+			assert.ErrorContains(t, err, tc.expectedErr)
 		})
 	}
 }