diff --git a/pkg/reporegistry/repository.go b/pkg/reporegistry/repository.go index edf3a94333..7e88939b38 100644 --- a/pkg/reporegistry/repository.go +++ b/pkg/reporegistry/repository.go @@ -50,8 +50,9 @@ func loadAllRepositoriesFromFS(confPaths []fs.FS) (rpmmd.DistrosRepoConfigs, err } // distro repositories definition is expected to be named ".json" - if strings.HasSuffix(fileEntry.Name(), ".json") { + if strings.HasSuffix(fileEntry.Name(), ".json") || strings.HasSuffix(fileEntry.Name(), ".yaml") { distroIDStr := strings.TrimSuffix(fileEntry.Name(), ".json") + distroIDStr = strings.TrimSuffix(distroIDStr, ".yaml") // compatibility layer to support old repository definition filenames // without a dot to separate major and minor release versions @@ -92,21 +93,27 @@ func loadAllRepositoriesFromFS(confPaths []fs.FS) (rpmmd.DistrosRepoConfigs, err // LoadRepositories loads distribution repositories from the given list of paths. // If there are duplicate distro repositories definitions found in multiple paths, the first // encounter is preferred. For this reason, the order of paths in the passed list should -// reflect the desired preference. +// reflect the desired preference. Both json and yaml repository files can be used to load +// from. When a json file is encountered it takes precedence over a yaml file under the +// same distro name. // // Note that the confPaths must point directly to the directory with -// the json repo files. +// the json and yaml repo files. func LoadRepositories(confPaths []string, distro string) (map[string][]rpmmd.RepoConfig, error) { var repoConfigs map[string][]rpmmd.RepoConfig - path := distro + ".json" for _, confPath := range confPaths { var err error - repoConfigs, err = rpmmd.LoadRepositoriesFromFile(filepath.Join(confPath, path)) - if os.IsNotExist(err) { - continue - } else if err != nil { - return nil, err + paths := []string{distro + ".json", distro + ".yaml"} + for _, path := range paths { + repoConfigs, err = rpmmd.LoadRepositoriesFromFile(filepath.Join(confPath, path)) + if os.IsNotExist(err) { + continue + } else if err != nil { + return nil, err + } else { + break + } } // Found the distro repository configs in the current path diff --git a/pkg/reporegistry/repository_test.go b/pkg/reporegistry/repository_test.go index f9cad76fa0..24582b2369 100644 --- a/pkg/reporegistry/repository_test.go +++ b/pkg/reporegistry/repository_test.go @@ -22,6 +22,7 @@ func getConfPaths(t *testing.T) []string { confPaths := []string{ "./test/confpaths/priority1/repositories", "./test/confpaths/priority2/repositories", + "./test/confpaths/priority3/repositories", } var absConfPaths []string @@ -64,6 +65,16 @@ func TestLoadRepositoriesExisting(t *testing.T) { test_distro.TestArch2Name: {"fedora-34-p2", "updates-34-p2"}, }, }, + { + name: "single distro definition but its yaml", + args: args{ + distro: "fedora-35", + }, + want: map[string][]string{ + test_distro.TestArchName: {"fedora-35-p3", "updates-35-p3"}, + test_distro.TestArch2Name: {"fedora-35-p3", "updates-35-p3"}, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -157,6 +168,36 @@ func Test_LoadAllRepositories(t *testing.T) { }, }, }, + "fedora-35": { + test_distro.TestArchName: { + { + Name: "fedora-35-p3", + BaseURLs: []string{"https://example.com/fedora-35-p3/test_arch"}, + GPGKeys: []string{"FAKE-GPG-KEY"}, + CheckGPG: common.ToPtr(true), + }, + { + Name: "updates-35-p3", + BaseURLs: []string{"https://example.com/updates-35-p3/test_arch"}, + GPGKeys: []string{"FAKE-GPG-KEY"}, + CheckGPG: common.ToPtr(true), + }, + }, + test_distro.TestArch2Name: { + { + Name: "fedora-35-p3", + BaseURLs: []string{"https://example.com/fedora-35-p3/test_arch2"}, + GPGKeys: []string{"FAKE-GPG-KEY"}, + CheckGPG: common.ToPtr(true), + }, + { + Name: "updates-35-p3", + BaseURLs: []string{"https://example.com/updates-35-p3/test_arch2"}, + GPGKeys: []string{"FAKE-GPG-KEY"}, + CheckGPG: common.ToPtr(true), + }, + }, + }, "rhel-8.7": { test_distro.TestArchName: { { @@ -354,6 +395,6 @@ func TestLoadRepositoriesError(t *testing.T) { assert.NoError(reposFile.Close()) _, err = LoadAllRepositories([]string{reposDir}, nil) - assert.ErrorContains(err, "failed to load repositories: invalid character '<' looking for beginning of value") + assert.ErrorContains(err, "cannot unmarshal !!str") }) } diff --git a/pkg/reporegistry/test/confpaths/priority3/repositories/fedora-35.yaml b/pkg/reporegistry/test/confpaths/priority3/repositories/fedora-35.yaml new file mode 100644 index 0000000000..d1f982f6dc --- /dev/null +++ b/pkg/reporegistry/test/confpaths/priority3/repositories/fedora-35.yaml @@ -0,0 +1,20 @@ +--- +test_arch: + - name: "fedora-35-p3" + baseurl: "https://example.com/fedora-35-p3/test_arch" + gpgkey: "FAKE-GPG-KEY" + check_gpg: true + - name: "updates-35-p3" + baseurl: "https://example.com/updates-35-p3/test_arch" + gpgkey: "FAKE-GPG-KEY" + check_gpg: true + +test_arch2: + - name: "fedora-35-p3" + baseurl: "https://example.com/fedora-35-p3/test_arch2" + gpgkey: "FAKE-GPG-KEY" + check_gpg: true + - name: "updates-35-p3" + baseurl: "https://example.com/updates-35-p3/test_arch2" + gpgkey: "FAKE-GPG-KEY" + check_gpg: true diff --git a/pkg/rpmmd/repository.go b/pkg/rpmmd/repository.go index 95df3658d2..f1df82eab6 100644 --- a/pkg/rpmmd/repository.go +++ b/pkg/rpmmd/repository.go @@ -7,6 +7,10 @@ import ( "io" "os" "strings" + + "go.yaml.in/yaml/v3" + + "github.com/osbuild/images/internal/common" ) // repository is the presentation of a on-disk repository json file @@ -58,6 +62,10 @@ func (r *repository) UnmarshalJSON(data []byte) (err error) { return nil } +func (r *repository) UnmarshalYAML(unmarshal func(any) error) error { + return common.UnmarshalYAMLviaJSON(r, unmarshal) +} + type RepoConfig struct { // the repo id is not always required and is ignored in some cases. // For example, it is not required in osbuild-depsolve-dnf, but it is a required @@ -133,7 +141,7 @@ func LoadRepositoriesFromReader(r io.Reader) (map[string][]RepoConfig, error) { var reposMap map[string][]repository repoConfigs := make(map[string][]RepoConfig) - err := json.NewDecoder(r).Decode(&reposMap) + err := yaml.NewDecoder(r).Decode(&reposMap) if err != nil { return nil, err } diff --git a/pkg/rpmmd/repository_test.go b/pkg/rpmmd/repository_test.go index b3fe3fd64e..4c81381906 100644 --- a/pkg/rpmmd/repository_test.go +++ b/pkg/rpmmd/repository_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "go.yaml.in/yaml/v3" "github.com/osbuild/images/pkg/rpmmd" ) @@ -41,7 +42,7 @@ func TestRepoConfigUnmarshalHappy(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { var repos rpmmd.Repository - err := json.Unmarshal([]byte(tc.json), &repos) + err := yaml.Unmarshal([]byte(tc.json), &repos) assert.NoError(t, err) assert.Equal(t, tc.repo, repos) }) @@ -67,14 +68,14 @@ func TestRepoConfigUnmarshalSad(t *testing.T) { { name: "wrong json", json: `all-wrong`, - expectedErr: `invalid character 'a' looking for beginning of value`, + expectedErr: `cannot unmarshal string into Go value of type struct`, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { var repos rpmmd.Repository - err := json.Unmarshal([]byte(tc.json), &repos) - assert.EqualError(t, err, tc.expectedErr) + err := yaml.Unmarshal([]byte(tc.json), &repos) + assert.ErrorContains(t, err, tc.expectedErr) }) } }