diff --git a/pkg/agent/containerd/config.go b/pkg/agent/containerd/config.go index ac2c5254eed1..760b5d5a28b7 100644 --- a/pkg/agent/containerd/config.go +++ b/pkg/agent/containerd/config.go @@ -1,6 +1,7 @@ package containerd import ( + "fmt" "net" "net/url" "os" @@ -70,65 +71,60 @@ func writeContainerdHosts(cfg *config.Node, containerdConfig templates.Container func getHostConfigs(registry *registries.Registry, noDefaultEndpoint bool, mirrorAddr string) HostConfigs { hosts := map[string]templates.HostConfig{} + // create config for default endpoints + for host, config := range registry.Configs { + if c, err := defaultHostConfig(host, mirrorAddr, config); err != nil { + logrus.Errorf("Failed to generate config for registry %s: %v", host, err) + } else { + if host == "*" { + host = "_default" + } + hosts[host] = *c + } + } + // create endpoints for mirrors for host, mirror := range registry.Mirrors { - config := templates.HostConfig{ - Program: version.Program, - } - if uri, _, err := normalizeEndpointAddress(host, mirrorAddr); err == nil { - config.DefaultEndpoint = uri.String() + // create the default config, if it wasn't explicitly mentioned in the config section + config, ok := hosts[host] + if !ok { + if c, err := defaultHostConfig(host, mirrorAddr, configForHost(registry.Configs, host)); err != nil { + logrus.Errorf("Failed to generate config for registry %s: %v", host, err) + continue + } else { + if host == "*" || noDefaultEndpoint { + c.Default = nil + } + config = *c + } } // TODO: rewrites are currently copied from the mirror settings into each endpoint. // In the future, we should allow for per-endpoint rewrites, instead of expecting // all mirrors to have the same structure. This will require changes to the registries.yaml // structure, which is defined in rancher/wharfie. - for _, endpoint := range mirror.Endpoints { - uri, override, err := normalizeEndpointAddress(endpoint, mirrorAddr) + for i, endpoint := range mirror.Endpoints { + registryName, url, override, err := normalizeEndpointAddress(endpoint, mirrorAddr) if err != nil { - logrus.Warnf("Ignoring invalid endpoint URL %s for %s: %v", endpoint, host, err) + logrus.Warnf("Ignoring invalid endpoint URL %d=%s for %s: %v", i, endpoint, host, err) } else { var rewrites map[string]string // Do not apply rewrites to the embedded registry endpoint - if uri.Host != mirrorAddr { + if url.Host != mirrorAddr { rewrites = mirror.Rewrites } - config.Endpoints = append(config.Endpoints, templates.RegistryEndpoint{ - Config: registry.Configs[uri.Host], + ep := templates.RegistryEndpoint{ + Config: configForHost(registry.Configs, registryName), Rewrites: rewrites, OverridePath: override, - URI: uri.String(), - }) - } - } - - if host == "*" { - host = "_default" - } - hosts[host] = config - } - - // create endpoints for registries using default endpoints - for host, registry := range registry.Configs { - config, ok := hosts[host] - if !ok { - config = templates.HostConfig{ - Program: version.Program, - } - if uri, _, err := normalizeEndpointAddress(host, mirrorAddr); err == nil { - config.DefaultEndpoint = uri.String() - } - } - // If there is config for this host but no endpoints, inject the config for the default endpoint. - if len(config.Endpoints) == 0 { - uri, _, err := normalizeEndpointAddress(host, mirrorAddr) - if err != nil { - logrus.Warnf("Ignoring invalid endpoint URL %s for %s: %v", host, host, err) - } else { - config.Endpoints = append(config.Endpoints, templates.RegistryEndpoint{ - Config: registry, - URI: uri.String(), - }) + URL: url, + } + if i+1 == len(mirror.Endpoints) && endpointURLEqual(config.Default, &ep) { + // if the last endpoint is the default endpoint, move it there + config.Default = &ep + } else { + config.Endpoints = append(config.Endpoints, ep) + } } } @@ -140,25 +136,8 @@ func getHostConfigs(registry *registries.Registry, noDefaultEndpoint bool, mirro // Clean up hosts and default endpoints where resulting config leaves only defaults for host, config := range hosts { - // if default endpoint is disabled, or this is the wildcard host, delete the default endpoint - if noDefaultEndpoint || host == "_default" { - config.DefaultEndpoint = "" - hosts[host] = config - } - if l := len(config.Endpoints); l > 0 { - if ep := config.Endpoints[l-1]; ep.URI == config.DefaultEndpoint { - // if the last endpoint is the default endpoint - if ep.Config.Auth == nil && ep.Config.TLS == nil && len(ep.Rewrites) == 0 { - // if has no config, delete this host to use the default config - delete(hosts, host) - } else { - // if it has config, delete the default endpoint - config.DefaultEndpoint = "" - hosts[host] = config - } - } - } else { - // if this host has no endpoints, delete this host to use the default config + // if this host has no endpoints and the default has no config, delete this host + if len(config.Endpoints) == 0 && !endpointHasConfig(config.Default) { delete(hosts, host) } } @@ -167,18 +146,18 @@ func getHostConfigs(registry *registries.Registry, noDefaultEndpoint bool, mirro } // normalizeEndpointAddress normalizes the endpoint address. -// If successful, it returns the URL, and a bool indicating if the endpoint path should be overridden. +// If successful, it returns the registry name, URL, and a bool indicating if the endpoint path should be overridden. // If unsuccessful, an error is returned. // Scheme and hostname logic should match containerd: // https://github.com/containerd/containerd/blob/v1.7.13/remotes/docker/config/hosts.go#L99-L131 -func normalizeEndpointAddress(endpoint, mirrorAddr string) (*url.URL, bool, error) { +func normalizeEndpointAddress(endpoint, mirrorAddr string) (string, *url.URL, bool, error) { // Ensure that the endpoint address has a scheme so that the URL is parsed properly if !strings.Contains(endpoint, "://") { endpoint = "//" + endpoint } endpointURL, err := url.Parse(endpoint) if err != nil { - return nil, false, err + return "", nil, false, err } port := endpointURL.Port() @@ -191,14 +170,66 @@ func normalizeEndpointAddress(endpoint, mirrorAddr string) (*url.URL, bool, erro endpointURL.Scheme = "https" } } - endpointURL.Host, _ = docker.DefaultHost(endpointURL.Host) + registry := endpointURL.Host + endpointURL.Host, _ = docker.DefaultHost(registry) + // This is the reverse of the DefaultHost normalization + if endpointURL.Host == "registry-1.docker.io" { + registry = "docker.io" + } switch endpointURL.Path { case "", "/", "/v2": // If the path is empty, /, or /v2, use the default path. endpointURL.Path = "/v2" - return endpointURL, false, nil + return registry, endpointURL, false, nil + } + + return registry, endpointURL, true, nil +} + +func defaultHostConfig(host, mirrorAddr string, config registries.RegistryConfig) (*templates.HostConfig, error) { + _, url, _, err := normalizeEndpointAddress(host, mirrorAddr) + if err != nil { + return nil, fmt.Errorf("invalid endpoint URL %s for %s: %v", host, host, err) } + if host == "*" { + url = nil + } + return &templates.HostConfig{ + Program: version.Program, + Default: &templates.RegistryEndpoint{ + URL: url, + Config: config, + }, + }, nil +} - return endpointURL, true, nil +func configForHost(configs map[string]registries.RegistryConfig, host string) registries.RegistryConfig { + // check for config under modified hostname. If the hostname is unmodified, or there is no config for + // the modified hostname, return the config for the default hostname. + if h, _ := docker.DefaultHost(host); h != host { + if c, ok := configs[h]; ok { + return c + } + } + return configs[host] +} + +// endpointURLEqual compares endpoint URL strings +func endpointURLEqual(a, b *templates.RegistryEndpoint) bool { + var au, bu string + if a != nil && a.URL != nil { + au = a.URL.String() + } + if b != nil && b.URL != nil { + bu = b.URL.String() + } + return au == bu +} + +func endpointHasConfig(ep *templates.RegistryEndpoint) bool { + if ep != nil { + return ep.OverridePath || ep.Config.Auth != nil || ep.Config.TLS != nil || len(ep.Rewrites) > 0 + } + return false } diff --git a/pkg/agent/containerd/config_test.go b/pkg/agent/containerd/config_test.go index 428215343575..dc7b55f160ca 100644 --- a/pkg/agent/containerd/config_test.go +++ b/pkg/agent/containerd/config_test.go @@ -2,6 +2,7 @@ package containerd import ( "net" + "net/url" "os" "path/filepath" "strings" @@ -11,9 +12,22 @@ import ( "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/spegel" "github.com/rancher/wharfie/pkg/registries" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" ) +func init() { + logrus.SetLevel(logrus.DebugLevel) +} + +func u(s string) *url.URL { + u, err := url.Parse(s) + if err != nil { + panic(err) + } + return u +} + func Test_UnitGetHostConfigs(t *testing.T) { type args struct { registryContent string @@ -39,6 +53,18 @@ func Test_UnitGetHostConfigs(t *testing.T) { }, want: HostConfigs{}, }, + { + name: "registry with default endpoint explicitly listed", + args: args{ + registryContent: ` + mirrors: + docker.io: + endpoint: + - docker.io + `, + }, + want: HostConfigs{}, + }, { name: "registry with default endpoint - embedded registry", args: args{ @@ -50,11 +76,13 @@ func Test_UnitGetHostConfigs(t *testing.T) { }, want: HostConfigs{ "docker.io": templates.HostConfig{ - DefaultEndpoint: "https://registry-1.docker.io/v2", - Program: "k3s", + Program: "k3s", + Default: &templates.RegistryEndpoint{ + URL: u("https://registry-1.docker.io/v2"), + }, Endpoints: []templates.RegistryEndpoint{ { - URI: "https://127.0.0.1:6443/v2", + URL: u("https://127.0.0.1:6443/v2"), Config: registries.RegistryConfig{ TLS: ®istries.TLSConfig{ CAFile: "server-ca", @@ -67,15 +95,13 @@ func Test_UnitGetHostConfigs(t *testing.T) { }, "127.0.0.1:6443": templates.HostConfig{ Program: "k3s", - Endpoints: []templates.RegistryEndpoint{ - { - URI: "https://127.0.0.1:6443/v2", - Config: registries.RegistryConfig{ - TLS: ®istries.TLSConfig{ - CAFile: "server-ca", - KeyFile: "client-key", - CertFile: "client-cert", - }, + Default: &templates.RegistryEndpoint{ + URL: u("https://127.0.0.1:6443/v2"), + Config: registries.RegistryConfig{ + TLS: ®istries.TLSConfig{ + CAFile: "server-ca", + KeyFile: "client-key", + CertFile: "client-cert", }, }, }, @@ -98,20 +124,49 @@ func Test_UnitGetHostConfigs(t *testing.T) { want: HostConfigs{ "docker.io": templates.HostConfig{ Program: "k3s", - Endpoints: []templates.RegistryEndpoint{ - { - URI: "https://registry-1.docker.io/v2", - Config: registries.RegistryConfig{ - Auth: ®istries.AuthConfig{ - Username: "user", - Password: "pass", - }, + Default: &templates.RegistryEndpoint{ + URL: u("https://registry-1.docker.io/v2"), + Config: registries.RegistryConfig{ + Auth: ®istries.AuthConfig{ + Username: "user", + Password: "pass", + }, + }, + }, + }, + }, + }, + { + name: "registry with default endpoint explicitly listed and creds", + args: args{ + registryContent: ` + mirrors: + docker.io: + endpoint: + - docker.io + configs: + docker.io: + auth: + username: user + password: pass + `, + }, + want: HostConfigs{ + "docker.io": templates.HostConfig{ + Program: "k3s", + Default: &templates.RegistryEndpoint{ + URL: u("https://registry-1.docker.io/v2"), + Config: registries.RegistryConfig{ + Auth: ®istries.AuthConfig{ + Username: "user", + Password: "pass", }, }, }, }, }, }, + { name: "registry with only creds", args: args{ @@ -126,14 +181,12 @@ func Test_UnitGetHostConfigs(t *testing.T) { want: HostConfigs{ "docker.io": templates.HostConfig{ Program: "k3s", - Endpoints: []templates.RegistryEndpoint{ - { - URI: "https://registry-1.docker.io/v2", - Config: registries.RegistryConfig{ - Auth: ®istries.AuthConfig{ - Username: "user", - Password: "pass", - }, + Default: &templates.RegistryEndpoint{ + URL: u("https://registry-1.docker.io/v2"), + Config: registries.RegistryConfig{ + Auth: ®istries.AuthConfig{ + Username: "user", + Password: "pass", }, }, }, @@ -166,14 +219,12 @@ func Test_UnitGetHostConfigs(t *testing.T) { want: HostConfigs{ "registry.example.com": templates.HostConfig{ Program: "k3s", - Endpoints: []templates.RegistryEndpoint{ - { - URI: "https://registry.example.com/v2", - Config: registries.RegistryConfig{ - Auth: ®istries.AuthConfig{ - Username: "user", - Password: "pass", - }, + Default: &templates.RegistryEndpoint{ + URL: u("https://registry.example.com/v2"), + Config: registries.RegistryConfig{ + Auth: ®istries.AuthConfig{ + Username: "user", + Password: "pass", }, }, }, @@ -194,14 +245,12 @@ func Test_UnitGetHostConfigs(t *testing.T) { want: HostConfigs{ "registry.example.com": templates.HostConfig{ Program: "k3s", - Endpoints: []templates.RegistryEndpoint{ - { - URI: "https://registry.example.com/v2", - Config: registries.RegistryConfig{ - Auth: ®istries.AuthConfig{ - Username: "user", - Password: "pass", - }, + Default: &templates.RegistryEndpoint{ + URL: u("https://registry.example.com/v2"), + Config: registries.RegistryConfig{ + Auth: ®istries.AuthConfig{ + Username: "user", + Password: "pass", }, }, }, @@ -220,12 +269,14 @@ func Test_UnitGetHostConfigs(t *testing.T) { }, want: HostConfigs{ "docker.io": templates.HostConfig{ - DefaultEndpoint: "https://registry-1.docker.io/v2", - Program: "k3s", + Program: "k3s", + Default: &templates.RegistryEndpoint{ + URL: u("https://registry-1.docker.io/v2"), + }, Endpoints: []templates.RegistryEndpoint{ { OverridePath: true, - URI: "https://registry.example.com/prefix/v2", + URL: u("https://registry.example.com/prefix/v2"), }, }, }, @@ -243,12 +294,14 @@ func Test_UnitGetHostConfigs(t *testing.T) { }, want: HostConfigs{ "docker.io": templates.HostConfig{ - DefaultEndpoint: "https://registry-1.docker.io/v2", - Program: "k3s", + Program: "k3s", + Default: &templates.RegistryEndpoint{ + URL: u("https://registry-1.docker.io/v2"), + }, Endpoints: []templates.RegistryEndpoint{ { OverridePath: true, - URI: "https://registry.example.com/prefix/v2", + URL: u("https://registry.example.com/prefix/v2"), }, }, }, @@ -266,11 +319,13 @@ func Test_UnitGetHostConfigs(t *testing.T) { }, want: HostConfigs{ "docker.io": templates.HostConfig{ - DefaultEndpoint: "https://registry-1.docker.io/v2", - Program: "k3s", + Program: "k3s", + Default: &templates.RegistryEndpoint{ + URL: u("https://registry-1.docker.io/v2"), + }, Endpoints: []templates.RegistryEndpoint{ { - URI: "https://registry.example.com/v2", + URL: u("https://registry.example.com/v2"), }, }, }, @@ -288,11 +343,13 @@ func Test_UnitGetHostConfigs(t *testing.T) { }, want: HostConfigs{ "docker.io": templates.HostConfig{ - DefaultEndpoint: "https://registry-1.docker.io/v2", - Program: "k3s", + Program: "k3s", + Default: &templates.RegistryEndpoint{ + URL: u("https://registry-1.docker.io/v2"), + }, Endpoints: []templates.RegistryEndpoint{ { - URI: "https://registry.example.com/v2", + URL: u("https://registry.example.com/v2"), }, }, }, @@ -310,11 +367,13 @@ func Test_UnitGetHostConfigs(t *testing.T) { }, want: HostConfigs{ "docker.io": templates.HostConfig{ - DefaultEndpoint: "https://registry-1.docker.io/v2", - Program: "k3s", + Program: "k3s", + Default: &templates.RegistryEndpoint{ + URL: u("https://registry-1.docker.io/v2"), + }, Endpoints: []templates.RegistryEndpoint{ { - URI: "https://registry.example.com/v2", + URL: u("https://registry.example.com/v2"), }, }, }, @@ -332,11 +391,13 @@ func Test_UnitGetHostConfigs(t *testing.T) { }, want: HostConfigs{ "docker.io": templates.HostConfig{ - DefaultEndpoint: "https://registry-1.docker.io/v2", - Program: "k3s", + Program: "k3s", + Default: &templates.RegistryEndpoint{ + URL: u("https://registry-1.docker.io/v2"), + }, Endpoints: []templates.RegistryEndpoint{ { - URI: "https://registry.example.com/v2", + URL: u("https://registry.example.com/v2"), }, }, }, @@ -354,11 +415,13 @@ func Test_UnitGetHostConfigs(t *testing.T) { }, want: HostConfigs{ "docker.io": templates.HostConfig{ - DefaultEndpoint: "https://registry-1.docker.io/v2", - Program: "k3s", + Program: "k3s", + Default: &templates.RegistryEndpoint{ + URL: u("https://registry-1.docker.io/v2"), + }, Endpoints: []templates.RegistryEndpoint{ { - URI: "https://registry.example.com:443/v2", + URL: u("https://registry.example.com:443/v2"), }, }, }, @@ -376,11 +439,13 @@ func Test_UnitGetHostConfigs(t *testing.T) { }, want: HostConfigs{ "docker.io": templates.HostConfig{ - DefaultEndpoint: "https://registry-1.docker.io/v2", - Program: "k3s", + Program: "k3s", + Default: &templates.RegistryEndpoint{ + URL: u("https://registry-1.docker.io/v2"), + }, Endpoints: []templates.RegistryEndpoint{ { - URI: "https://1.2.3.4/v2", + URL: u("https://1.2.3.4/v2"), }, }, }, @@ -398,11 +463,13 @@ func Test_UnitGetHostConfigs(t *testing.T) { }, want: HostConfigs{ "docker.io": templates.HostConfig{ - DefaultEndpoint: "https://registry-1.docker.io/v2", - Program: "k3s", + Program: "k3s", + Default: &templates.RegistryEndpoint{ + URL: u("https://registry-1.docker.io/v2"), + }, Endpoints: []templates.RegistryEndpoint{ { - URI: "https://1.2.3.4:443/v2", + URL: u("https://1.2.3.4:443/v2"), }, }, }, @@ -420,11 +487,13 @@ func Test_UnitGetHostConfigs(t *testing.T) { }, want: HostConfigs{ "docker.io": templates.HostConfig{ - DefaultEndpoint: "https://registry-1.docker.io/v2", - Program: "k3s", + Program: "k3s", + Default: &templates.RegistryEndpoint{ + URL: u("https://registry-1.docker.io/v2"), + }, Endpoints: []templates.RegistryEndpoint{ { - URI: "http://localhost:5000/v2", + URL: u("http://localhost:5000/v2"), }, }, }, @@ -442,11 +511,13 @@ func Test_UnitGetHostConfigs(t *testing.T) { }, want: HostConfigs{ "docker.io": templates.HostConfig{ - DefaultEndpoint: "https://registry-1.docker.io/v2", - Program: "k3s", + Program: "k3s", + Default: &templates.RegistryEndpoint{ + URL: u("https://registry-1.docker.io/v2"), + }, Endpoints: []templates.RegistryEndpoint{ { - URI: "https://localhost:5000/v2", + URL: u("https://localhost:5000/v2"), }, }, }, @@ -464,11 +535,13 @@ func Test_UnitGetHostConfigs(t *testing.T) { }, want: HostConfigs{ "docker.io": templates.HostConfig{ - DefaultEndpoint: "https://registry-1.docker.io/v2", - Program: "k3s", + Program: "k3s", + Default: &templates.RegistryEndpoint{ + URL: u("https://registry-1.docker.io/v2"), + }, Endpoints: []templates.RegistryEndpoint{ { - URI: "http://127.0.0.1:5000/v2", + URL: u("http://127.0.0.1:5000/v2"), }, }, }, @@ -486,11 +559,13 @@ func Test_UnitGetHostConfigs(t *testing.T) { }, want: HostConfigs{ "docker.io": templates.HostConfig{ - DefaultEndpoint: "https://registry-1.docker.io/v2", - Program: "k3s", + Program: "k3s", + Default: &templates.RegistryEndpoint{ + URL: u("https://registry-1.docker.io/v2"), + }, Endpoints: []templates.RegistryEndpoint{ { - URI: "https://127.0.0.1:5000/v2", + URL: u("https://127.0.0.1:5000/v2"), }, }, }, @@ -513,11 +588,13 @@ func Test_UnitGetHostConfigs(t *testing.T) { }, want: HostConfigs{ "docker.io": templates.HostConfig{ - DefaultEndpoint: "https://registry-1.docker.io/v2", - Program: "k3s", + Program: "k3s", + Default: &templates.RegistryEndpoint{ + URL: u("https://registry-1.docker.io/v2"), + }, Endpoints: []templates.RegistryEndpoint{ { - URI: "https://registry.example.com/v2", + URL: u("https://registry.example.com/v2"), Config: registries.RegistryConfig{ Auth: ®istries.AuthConfig{ Username: "user", @@ -529,14 +606,12 @@ func Test_UnitGetHostConfigs(t *testing.T) { }, "registry.example.com": templates.HostConfig{ Program: "k3s", - Endpoints: []templates.RegistryEndpoint{ - { - URI: "https://registry.example.com/v2", - Config: registries.RegistryConfig{ - Auth: ®istries.AuthConfig{ - Username: "user", - Password: "pass", - }, + Default: &templates.RegistryEndpoint{ + URL: u("https://registry.example.com/v2"), + Config: registries.RegistryConfig{ + Auth: ®istries.AuthConfig{ + Username: "user", + Password: "pass", }, }, }, @@ -563,12 +638,14 @@ func Test_UnitGetHostConfigs(t *testing.T) { }, want: HostConfigs{ "docker.io": templates.HostConfig{ - DefaultEndpoint: "https://registry-1.docker.io/v2", - Program: "k3s", + Program: "k3s", + Default: &templates.RegistryEndpoint{ + URL: u("https://registry-1.docker.io/v2"), + }, Endpoints: []templates.RegistryEndpoint{ { OverridePath: true, - URI: "https://registry.example.com/prefix/v2", + URL: u("https://registry.example.com/prefix/v2"), Config: registries.RegistryConfig{ Auth: ®istries.AuthConfig{ Username: "user", @@ -579,12 +656,20 @@ func Test_UnitGetHostConfigs(t *testing.T) { }, }, "registry.example.com": templates.HostConfig{ - DefaultEndpoint: "https://registry.example.com/v2", - Program: "k3s", + Program: "k3s", + Default: &templates.RegistryEndpoint{ + URL: u("https://registry.example.com/v2"), + Config: registries.RegistryConfig{ + Auth: ®istries.AuthConfig{ + Username: "user", + Password: "pass", + }, + }, + }, Endpoints: []templates.RegistryEndpoint{ { OverridePath: true, - URI: "https://registry.example.com/prefix/v2", + URL: u("https://registry.example.com/prefix/v2"), Config: registries.RegistryConfig{ Auth: ®istries.AuthConfig{ Username: "user", @@ -616,12 +701,14 @@ func Test_UnitGetHostConfigs(t *testing.T) { }, want: HostConfigs{ "docker.io": templates.HostConfig{ - DefaultEndpoint: "https://registry-1.docker.io/v2", - Program: "k3s", + Program: "k3s", + Default: &templates.RegistryEndpoint{ + URL: u("https://registry-1.docker.io/v2"), + }, Endpoints: []templates.RegistryEndpoint{ { OverridePath: true, - URI: "https://registry.example.com/project/registry", + URL: u("https://registry.example.com/project/registry"), Config: registries.RegistryConfig{ Auth: ®istries.AuthConfig{ Username: "user", @@ -632,12 +719,20 @@ func Test_UnitGetHostConfigs(t *testing.T) { }, }, "registry.example.com": templates.HostConfig{ - DefaultEndpoint: "https://registry.example.com/v2", - Program: "k3s", + Program: "k3s", + Default: &templates.RegistryEndpoint{ + URL: u("https://registry.example.com/v2"), + Config: registries.RegistryConfig{ + Auth: ®istries.AuthConfig{ + Username: "user", + Password: "pass", + }, + }, + }, Endpoints: []templates.RegistryEndpoint{ { OverridePath: true, - URI: "https://registry.example.com/project/registry", + URL: u("https://registry.example.com/project/registry"), Config: registries.RegistryConfig{ Auth: ®istries.AuthConfig{ Username: "user", @@ -670,7 +765,7 @@ func Test_UnitGetHostConfigs(t *testing.T) { Program: "k3s", Endpoints: []templates.RegistryEndpoint{ { - URI: "https://registry.example.com/v2", + URL: u("https://registry.example.com/v2"), Config: registries.RegistryConfig{ Auth: ®istries.AuthConfig{ Username: "user", @@ -682,14 +777,12 @@ func Test_UnitGetHostConfigs(t *testing.T) { }, "registry.example.com": templates.HostConfig{ Program: "k3s", - Endpoints: []templates.RegistryEndpoint{ - { - URI: "https://registry.example.com/v2", - Config: registries.RegistryConfig{ - Auth: ®istries.AuthConfig{ - Username: "user", - Password: "pass", - }, + Default: &templates.RegistryEndpoint{ + URL: u("https://registry.example.com/v2"), + Config: registries.RegistryConfig{ + Auth: ®istries.AuthConfig{ + Username: "user", + Password: "pass", }, }, }, @@ -714,11 +807,13 @@ func Test_UnitGetHostConfigs(t *testing.T) { }, want: HostConfigs{ "docker.io": templates.HostConfig{ - DefaultEndpoint: "https://registry-1.docker.io/v2", - Program: "k3s", + Program: "k3s", + Default: &templates.RegistryEndpoint{ + URL: u("https://registry-1.docker.io/v2"), + }, Endpoints: []templates.RegistryEndpoint{ { - URI: "https://127.0.0.1:6443/v2", + URL: u("https://127.0.0.1:6443/v2"), Config: registries.RegistryConfig{ TLS: ®istries.TLSConfig{ CAFile: "server-ca", @@ -728,7 +823,7 @@ func Test_UnitGetHostConfigs(t *testing.T) { }, }, { - URI: "https://registry.example.com/v2", + URL: u("https://registry.example.com/v2"), Config: registries.RegistryConfig{ Auth: ®istries.AuthConfig{ Username: "user", @@ -740,29 +835,25 @@ func Test_UnitGetHostConfigs(t *testing.T) { }, "registry.example.com": templates.HostConfig{ Program: "k3s", - Endpoints: []templates.RegistryEndpoint{ - { - URI: "https://registry.example.com/v2", - Config: registries.RegistryConfig{ - Auth: ®istries.AuthConfig{ - Username: "user", - Password: "pass", - }, + Default: &templates.RegistryEndpoint{ + URL: u("https://registry.example.com/v2"), + Config: registries.RegistryConfig{ + Auth: ®istries.AuthConfig{ + Username: "user", + Password: "pass", }, }, }, }, "127.0.0.1:6443": templates.HostConfig{ Program: "k3s", - Endpoints: []templates.RegistryEndpoint{ - { - URI: "https://127.0.0.1:6443/v2", - Config: registries.RegistryConfig{ - TLS: ®istries.TLSConfig{ - CAFile: "server-ca", - KeyFile: "client-key", - CertFile: "client-cert", - }, + Default: &templates.RegistryEndpoint{ + URL: u("https://127.0.0.1:6443/v2"), + Config: registries.RegistryConfig{ + TLS: ®istries.TLSConfig{ + CAFile: "server-ca", + KeyFile: "client-key", + CertFile: "client-cert", }, }, }, @@ -789,11 +880,13 @@ func Test_UnitGetHostConfigs(t *testing.T) { }, want: HostConfigs{ "docker.io": templates.HostConfig{ - DefaultEndpoint: "https://registry-1.docker.io/v2", - Program: "k3s", + Program: "k3s", + Default: &templates.RegistryEndpoint{ + URL: u("https://registry-1.docker.io/v2"), + }, Endpoints: []templates.RegistryEndpoint{ { - URI: "https://127.0.0.1:6443/v2", + URL: u("https://127.0.0.1:6443/v2"), Config: registries.RegistryConfig{ TLS: ®istries.TLSConfig{ CAFile: "server-ca", @@ -803,7 +896,7 @@ func Test_UnitGetHostConfigs(t *testing.T) { }, }, { - URI: "https://registry.example.com/v2", + URL: u("https://registry.example.com/v2"), Config: registries.RegistryConfig{ Auth: ®istries.AuthConfig{ Username: "user", @@ -818,29 +911,25 @@ func Test_UnitGetHostConfigs(t *testing.T) { }, "registry.example.com": templates.HostConfig{ Program: "k3s", - Endpoints: []templates.RegistryEndpoint{ - { - URI: "https://registry.example.com/v2", - Config: registries.RegistryConfig{ - Auth: ®istries.AuthConfig{ - Username: "user", - Password: "pass", - }, + Default: &templates.RegistryEndpoint{ + URL: u("https://registry.example.com/v2"), + Config: registries.RegistryConfig{ + Auth: ®istries.AuthConfig{ + Username: "user", + Password: "pass", }, }, }, }, "127.0.0.1:6443": templates.HostConfig{ Program: "k3s", - Endpoints: []templates.RegistryEndpoint{ - { - URI: "https://127.0.0.1:6443/v2", - Config: registries.RegistryConfig{ - TLS: ®istries.TLSConfig{ - CAFile: "server-ca", - KeyFile: "client-key", - CertFile: "client-cert", - }, + Default: &templates.RegistryEndpoint{ + URL: u("https://127.0.0.1:6443/v2"), + Config: registries.RegistryConfig{ + TLS: ®istries.TLSConfig{ + CAFile: "server-ca", + KeyFile: "client-key", + CertFile: "client-cert", }, }, }, @@ -869,7 +958,7 @@ func Test_UnitGetHostConfigs(t *testing.T) { Program: "k3s", Endpoints: []templates.RegistryEndpoint{ { - URI: "https://127.0.0.1:6443/v2", + URL: u("https://127.0.0.1:6443/v2"), Config: registries.RegistryConfig{ TLS: ®istries.TLSConfig{ CAFile: "server-ca", @@ -879,7 +968,7 @@ func Test_UnitGetHostConfigs(t *testing.T) { }, }, { - URI: "https://registry.example.com/v2", + URL: u("https://registry.example.com/v2"), Config: registries.RegistryConfig{ Auth: ®istries.AuthConfig{ Username: "user", @@ -891,23 +980,104 @@ func Test_UnitGetHostConfigs(t *testing.T) { }, "registry.example.com": templates.HostConfig{ Program: "k3s", + Default: &templates.RegistryEndpoint{ + URL: u("https://registry.example.com/v2"), + Config: registries.RegistryConfig{ + Auth: ®istries.AuthConfig{ + Username: "user", + Password: "pass", + }, + }, + }, + }, + "127.0.0.1:6443": templates.HostConfig{ + Program: "k3s", + Default: &templates.RegistryEndpoint{ + URL: u("https://127.0.0.1:6443/v2"), + Config: registries.RegistryConfig{ + TLS: ®istries.TLSConfig{ + CAFile: "server-ca", + KeyFile: "client-key", + CertFile: "client-cert", + }, + }, + }, + }, + }, + }, + { + name: "registry with mirror endpoint - embedded registry, default endpoint explicitly listed", + args: args{ + mirrorAddr: "127.0.0.1:6443", + registryContent: ` + mirrors: + docker.io: + endpoint: + - registry.example.com + - registry.example.org + - docker.io + `, + }, + want: HostConfigs{ + "docker.io": templates.HostConfig{ + Program: "k3s", + Default: &templates.RegistryEndpoint{ + URL: u("https://registry-1.docker.io/v2"), + }, Endpoints: []templates.RegistryEndpoint{ { - URI: "https://registry.example.com/v2", + URL: u("https://127.0.0.1:6443/v2"), Config: registries.RegistryConfig{ - Auth: ®istries.AuthConfig{ - Username: "user", - Password: "pass", + TLS: ®istries.TLSConfig{ + CAFile: "server-ca", + KeyFile: "client-key", + CertFile: "client-cert", }, }, }, + { + URL: u("https://registry.example.com/v2"), + }, + { + URL: u("https://registry.example.org/v2"), + }, }, }, "127.0.0.1:6443": templates.HostConfig{ + Program: "k3s", + Default: &templates.RegistryEndpoint{ + URL: u("https://127.0.0.1:6443/v2"), + Config: registries.RegistryConfig{ + TLS: ®istries.TLSConfig{ + CAFile: "server-ca", + KeyFile: "client-key", + CertFile: "client-cert", + }, + }, + }, + }, + }, + }, + { + name: "registry with mirror endpoint - embedded registry and no default endpoint, default endpoint explicitly listed", + args: args{ + mirrorAddr: "127.0.0.1:6443", + noDefaultEndpoint: true, + registryContent: ` + mirrors: + docker.io: + endpoint: + - registry.example.com + - registry.example.org + - docker.io + `, + }, + want: HostConfigs{ + "docker.io": templates.HostConfig{ Program: "k3s", Endpoints: []templates.RegistryEndpoint{ { - URI: "https://127.0.0.1:6443/v2", + URL: u("https://127.0.0.1:6443/v2"), Config: registries.RegistryConfig{ TLS: ®istries.TLSConfig{ CAFile: "server-ca", @@ -916,6 +1086,28 @@ func Test_UnitGetHostConfigs(t *testing.T) { }, }, }, + { + URL: u("https://registry.example.com/v2"), + }, + { + URL: u("https://registry.example.org/v2"), + }, + { + URL: u("https://registry-1.docker.io/v2"), + }, + }, + }, + "127.0.0.1:6443": templates.HostConfig{ + Program: "k3s", + Default: &templates.RegistryEndpoint{ + URL: u("https://127.0.0.1:6443/v2"), + Config: registries.RegistryConfig{ + TLS: ®istries.TLSConfig{ + CAFile: "server-ca", + KeyFile: "client-key", + CertFile: "client-cert", + }, + }, }, }, }, @@ -935,7 +1127,7 @@ func Test_UnitGetHostConfigs(t *testing.T) { Program: "k3s", Endpoints: []templates.RegistryEndpoint{ { - URI: "https://registry.example.com/v2", + URL: u("https://registry.example.com/v2"), }, }, }, @@ -959,21 +1151,49 @@ func Test_UnitGetHostConfigs(t *testing.T) { // note that the embedded registry mirror is NOT listed as an endpoint. // individual registries must be enabled for mirroring by name. { - URI: "https://registry.example.com/v2", + URL: u("https://registry.example.com/v2"), }, }, }, "127.0.0.1:6443": templates.HostConfig{ Program: "k3s", - Endpoints: []templates.RegistryEndpoint{ - { - URI: "https://127.0.0.1:6443/v2", - Config: registries.RegistryConfig{ - TLS: ®istries.TLSConfig{ - CAFile: "server-ca", - KeyFile: "client-key", - CertFile: "client-cert", - }, + Default: &templates.RegistryEndpoint{ + URL: u("https://127.0.0.1:6443/v2"), + Config: registries.RegistryConfig{ + TLS: ®istries.TLSConfig{ + CAFile: "server-ca", + KeyFile: "client-key", + CertFile: "client-cert", + }, + }, + }, + }, + }, + }, + { + name: "wildcard config", + args: args{ + registryContent: ` + configs: + "*": + auth: + username: user + password: pass + tls: + insecure_skip_verify: true + `, + }, + want: HostConfigs{ + "_default": { + Program: "k3s", + Default: &templates.RegistryEndpoint{ + Config: registries.RegistryConfig{ + Auth: ®istries.AuthConfig{ + Username: "user", + Password: "pass", + }, + TLS: ®istries.TLSConfig{ + InsecureSkipVerify: true, }, }, }, @@ -1033,15 +1253,13 @@ func Test_UnitGetHostConfigs(t *testing.T) { // localhost registries are not handled by the embedded registry mirror. "127.0.0.1:6443": templates.HostConfig{ Program: "k3s", - Endpoints: []templates.RegistryEndpoint{ - { - URI: "https://127.0.0.1:6443/v2", - Config: registries.RegistryConfig{ - TLS: ®istries.TLSConfig{ - CAFile: "server-ca", - KeyFile: "client-key", - CertFile: "client-cert", - }, + Default: &templates.RegistryEndpoint{ + URL: u("https://127.0.0.1:6443/v2"), + Config: registries.RegistryConfig{ + TLS: ®istries.TLSConfig{ + CAFile: "server-ca", + KeyFile: "client-key", + CertFile: "client-cert", }, }, }, @@ -1064,11 +1282,18 @@ func Test_UnitGetHostConfigs(t *testing.T) { }, want: HostConfigs{ "localhost:5000": templates.HostConfig{ - DefaultEndpoint: "http://localhost:5000/v2", - Program: "k3s", + Default: &templates.RegistryEndpoint{ + URL: u("http://localhost:5000/v2"), + Config: registries.RegistryConfig{ + TLS: ®istries.TLSConfig{ + InsecureSkipVerify: true, + }, + }, + }, + Program: "k3s", Endpoints: []templates.RegistryEndpoint{ { - URI: "https://localhost:5000/v2", + URL: u("https://localhost:5000/v2"), Config: registries.RegistryConfig{ TLS: ®istries.TLSConfig{ InsecureSkipVerify: true, @@ -1085,8 +1310,10 @@ func Test_UnitGetHostConfigs(t *testing.T) { t.Run(tt.name, func(t *testing.T) { // replace tabs from the inline yaml with spaces; yaml doesn't support tabs for indentation. tt.args.registryContent = strings.ReplaceAll(tt.args.registryContent, "\t", " ") - registriesFile := filepath.Join(t.TempDir(), "registries.yaml") + tempDir := t.TempDir() + registriesFile := filepath.Join(tempDir, "registries.yaml") os.WriteFile(registriesFile, []byte(tt.args.registryContent), 0644) + t.Logf("%s:\n%s", registriesFile, tt.args.registryContent) registry, err := registries.GetPrivateRegistries(registriesFile) if err != nil { @@ -1103,8 +1330,30 @@ func Test_UnitGetHostConfigs(t *testing.T) { conf.InjectMirror(&config.Node{AgentConfig: config.Agent{Registry: registry.Registry}}) } + // Generate config template struct for all hosts got := getHostConfigs(registry.Registry, tt.args.noDefaultEndpoint, tt.args.mirrorAddr) assert.Equal(t, tt.want, got, "getHostConfigs()") + + // Confirm that hosts.toml renders properly for all registries + for host, config := range got { + hostsTemplate, err := templates.ParseHostsTemplateFromConfig(templates.HostsTomlTemplate, config) + assert.NoError(t, err, "ParseHostTemplateFromConfig for %s", host) + t.Logf("%s/hosts.d/%s/hosts.toml\n%s", tempDir, host, hostsTemplate) + } + + // Confirm that the main containerd config.toml renders properly + containerdConfig := templates.ContainerdConfig{ + NodeConfig: &config.Node{ + Containerd: config.Containerd{ + Registry: tempDir + "/hosts.d", + }, + }, + PrivateRegistryConfig: registry.Registry, + Program: "k3s", + } + configTemplate, err := templates.ParseTemplateFromConfig(templates.ContainerdConfigTemplate, containerdConfig) + assert.NoError(t, err, "ParseTemplateFromConfig") + t.Logf("%s/config.toml\n%s", tempDir, configTemplate) }) } } diff --git a/pkg/agent/templates/templates.go b/pkg/agent/templates/templates.go index de273f26c144..82eb1ab4acf0 100644 --- a/pkg/agent/templates/templates.go +++ b/pkg/agent/templates/templates.go @@ -2,6 +2,7 @@ package templates import ( "bytes" + "net/url" "text/template" "github.com/rancher/wharfie/pkg/registries" @@ -28,24 +29,40 @@ type ContainerdConfig struct { type RegistryEndpoint struct { OverridePath bool - URI string + URL *url.URL Rewrites map[string]string Config registries.RegistryConfig } type HostConfig struct { - DefaultEndpoint string - Program string - Endpoints []RegistryEndpoint + Default *RegistryEndpoint + Program string + Endpoints []RegistryEndpoint } const HostsTomlTemplate = ` {{- /* */ -}} # File generated by {{ .Program }}. DO NOT EDIT. -{{ if .DefaultEndpoint }}server = "{{ .DefaultEndpoint }}"{{ end }} +{{ with $e := .Default }} +{{- if $e.URL }} +server = "{{ $e.URL }}" +capabilities = ["pull", "resolve", "push"] +{{ end }} +{{- if $e.Config.TLS }} +{{- if $e.Config.TLS.CAFile }} +ca = [{{ printf "%q" $e.Config.TLS.CAFile }}] +{{- end }} +{{- if or $e.Config.TLS.CertFile $e.Config.TLS.KeyFile }} +client = [[{{ printf "%q" $e.Config.TLS.CertFile }}, {{ printf "%q" $e.Config.TLS.KeyFile }}]] +{{- end }} +{{- if $e.Config.TLS.InsecureSkipVerify }} +skip_verify = true +{{- end }} +{{ end }} +{{ end }} {{ range $e := .Endpoints -}} -[host."{{ $e.URI }}"] +[host."{{ $e.URL }}"] capabilities = ["pull", "resolve"] {{- if $e.OverridePath }} override_path = true @@ -62,7 +79,7 @@ const HostsTomlTemplate = ` {{- end }} {{ end }} {{- if $e.Rewrites }} - [host."{{ $e.URI }}".rewrite] + [host."{{ $e.URL }}".rewrite] {{- range $pattern, $replace := $e.Rewrites }} "{{ $pattern }}" = "{{ $replace }}" {{- end }}