diff --git a/docs/configmap-and-annotations.md b/docs/configmap-and-annotations.md index 92e9971ff8..74ddd7eb3e 100644 --- a/docs/configmap-and-annotations.md +++ b/docs/configmap-and-annotations.md @@ -101,6 +101,11 @@ spec: | N/A | `worker-shutdown-timeout` | Sets the value of the [worker_shutdown_timeout](http://nginx.org/en/docs/ngx_core_module.html#worker_shutdown_timeout) directive. | N/A | | | N/A | `server-names-hash-bucket-size` | Sets the value of the [server_names_hash_bucket_size](http://nginx.org/en/docs/http/ngx_http_core_module.html#server_names_hash_bucket_size) directive. | Depends on the size of the processor’s cache line. | | | N/A | `server-names-hash-max-size` | Sets the value of the [server_names_hash_max_size](http://nginx.org/en/docs/http/ngx_http_core_module.html#server_names_hash_max_size) directive. | `512` | | +| N/A | `resolver-addresses` | Sets the value of the [resolver](http://nginx.org/en/docs/http/ngx_http_core_module.html#resolver) addresses. Note: If you use a DNS name (ex., `kube-dns.kube-system.svc.cluster.local`) as a resolver address, NGINX Plus will resolve it using the system resolver during the start and on every configuration reload. As a consequence, If the name cannot be resolved or the DNS server doesn't respond, NGINX Plus will fail to start or reload. To avoid this, consider using only IP addresses as resolver addresses. Supported in NGINX Plus only. | N/A | [Support for Type ExternalName Services](../examples/externalname-services). | +| N/A | `resolver-ipv6` | Enables IPv6 resolution in the resolver. Supported in NGINX Plus only. | `True` | [Support for Type ExternalName Services](../examples/externalname-services). | +| N/A | `resolver-valid` | Sets the time NGINX caches the resolved DNS records. Supported in NGINX Plus only. | TTL value of a DNS record | [Support for Type ExternalName Services](../examples/externalname-services). | +| N/A | `resolver-timeout` | Sets the [resolver_timeout](http://nginx.org/en/docs/http/ngx_http_core_module.html#resolver_timeout) for name resolution. Supported in NGINX Plus only. | `30s` | [Support for Type ExternalName Services](../examples/externalname-services). | + ### Logging diff --git a/examples/externalname-services/README.md b/examples/externalname-services/README.md new file mode 100644 index 0000000000..30daa693a9 --- /dev/null +++ b/examples/externalname-services/README.md @@ -0,0 +1,61 @@ +# Support for Type ExternalName Services +The Ingress Controller supports routing requests to services of the type [ExternalName](https://kubernetes.io/docs/concepts/services-networking/service/#externalname). + +An ExternalName service is defined by an external DNS name that is resolved into the IP addresses, typically external to the cluster. This enables to use the Ingress Controller to route requests to the destinations outside of the cluster. + +**Note:** This feature is only available in NGINX Plus. + + +## Prerequisites +To use ExternalName services, first you need to configure one or more resolvers using the ConfigMap. NGINX Plus will use those resolvers to resolve DNS names of the services. + +For example, the following ConfigMap configures one resolver: + +```yaml +kind: ConfigMap +apiVersion: v1 +metadata: + name: nginx-config + namespace: nginx-ingress +data: + resolver-addresses: "10.0.0.10" +``` + +Additional resolver parameters, including the caching of DNS records, are available. Check the corresponding [ConfigMap and Annotations](../../docs/configmap-and-annotations.md) section. + + +## Example +In the following yaml file we define an ExternalName service with the name my-service: + +```yaml +kind: Service +apiVersion: v1 +metadata: + name: my-service +spec: + type: ExternalName + externalName: my.service.example.com +``` + +In the following Ingress resource we use my-service: + +```yaml +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: example-ingress + annotations: + kubernetes.io/ingress.class: "nginx" +spec: + rules: + - host: example.com + http: + paths: + - path: / + backend: + serviceName: my-service + servicePort: 80 + +``` + +As a result, NGINX Plus will route requests for “example.com” to the IP addresses behind the DNS name my.service.example.com. \ No newline at end of file diff --git a/internal/controller/controller.go b/internal/controller/controller.go index d481ccc52b..0b6344f178 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -960,15 +960,27 @@ func (lbc *LoadBalancerController) createIngress(ing *extensions.Ingress) (*ngin ingEx.Endpoints = make(map[string][]string) ingEx.HealthChecks = make(map[string]*api_v1.Probe) + ingEx.ExternalNameSvcs = make(map[string]bool) if ing.Spec.Backend != nil { - endps, err := lbc.getEndpointsForIngressBackend(ing.Spec.Backend, ing.Namespace) + endps := []string{} + var external bool + svc, err := lbc.getServiceForIngressBackend(ing.Spec.Backend, ing.Namespace) if err != nil { - glog.Warningf("Error retrieving endpoints for the service %v: %v", ing.Spec.Backend.ServiceName, err) - ingEx.Endpoints[ing.Spec.Backend.ServiceName+ing.Spec.Backend.ServicePort.String()] = []string{} + glog.V(3).Infof("Error getting service %v: %v", ing.Spec.Backend.ServiceName, err) } else { - ingEx.Endpoints[ing.Spec.Backend.ServiceName+ing.Spec.Backend.ServicePort.String()] = endps + endps, external, err = lbc.getEndpointsForIngressBackend(ing.Spec.Backend, ing.Namespace, svc) + if err == nil && external && lbc.isNginxPlus { + ingEx.ExternalNameSvcs[svc.Name] = true + } } + + if err != nil { + glog.Warningf("Error retrieving endpoints for the service %v: %v", ing.Spec.Backend.ServiceName, err) + } + // endps is empty if there was any error before this point + ingEx.Endpoints[ing.Spec.Backend.ServiceName+ing.Spec.Backend.ServicePort.String()] = endps + if lbc.isNginxPlus && lbc.isHealthCheckEnabled(ing) { healthCheck := lbc.getHealthChecksForIngressBackend(ing.Spec.Backend, ing.Namespace) if healthCheck != nil { @@ -988,21 +1000,33 @@ func (lbc *LoadBalancerController) createIngress(ing *extensions.Ingress) (*ngin } for _, path := range rule.HTTP.Paths { - endps, err := lbc.getEndpointsForIngressBackend(&path.Backend, ing.Namespace) + endps := []string{} + var external bool + svc, err := lbc.getServiceForIngressBackend(&path.Backend, ing.Namespace) if err != nil { - glog.Warningf("Error retrieving endpoints for the service %v: %v", path.Backend.ServiceName, err) - ingEx.Endpoints[path.Backend.ServiceName+path.Backend.ServicePort.String()] = []string{} + glog.V(3).Infof("Error getting service %v: %v", &path.Backend.ServiceName, err) } else { - ingEx.Endpoints[path.Backend.ServiceName+path.Backend.ServicePort.String()] = endps + endps, external, err = lbc.getEndpointsForIngressBackend(&path.Backend, ing.Namespace, svc) + if err == nil && external && lbc.isNginxPlus { + ingEx.ExternalNameSvcs[svc.Name] = true + } + } + + if err != nil { + glog.Warningf("Error retrieving endpoints for the service %v: %v", path.Backend.ServiceName, err) } + // endps is empty if there was any error before this point + ingEx.Endpoints[path.Backend.ServiceName+path.Backend.ServicePort.String()] = endps + + // Pull active health checks from k8 api if lbc.isNginxPlus && lbc.isHealthCheckEnabled(ing) { - // Pull active health checks from k8 api healthCheck := lbc.getHealthChecksForIngressBackend(&path.Backend, ing.Namespace) if healthCheck != nil { ingEx.HealthChecks[path.Backend.ServiceName+path.Backend.ServicePort.String()] = healthCheck } } } + validRules++ } @@ -1070,25 +1094,32 @@ func compareContainerPortAndServicePort(containerPort api_v1.ContainerPort, svcP return false } -func (lbc *LoadBalancerController) getEndpointsForIngressBackend(backend *extensions.IngressBackend, namespace string) ([]string, error) { - svc, err := lbc.getServiceForIngressBackend(backend, namespace) - if err != nil { - glog.V(3).Infof("Error getting service %v: %v", backend.ServiceName, err) - return nil, err - } +func (lbc *LoadBalancerController) getExternalEndpointsForIngressBackend(backend *extensions.IngressBackend, namespace string, svc *api_v1.Service) []string { + endpoint := fmt.Sprintf("%s:%d", svc.Spec.ExternalName, int32(backend.ServicePort.IntValue())) + endpoints := []string{endpoint} + return endpoints +} +func (lbc *LoadBalancerController) getEndpointsForIngressBackend(backend *extensions.IngressBackend, namespace string, svc *api_v1.Service) (result []string, isExternal bool, err error) { endps, err := lbc.endpointLister.GetServiceEndpoints(svc) if err != nil { + if svc.Spec.Type == api_v1.ServiceTypeExternalName { + if !lbc.isNginxPlus { + return nil, false, fmt.Errorf("Type ExternalName Services feature is only available in NGINX Plus") + } + result = lbc.getExternalEndpointsForIngressBackend(backend, namespace, svc) + return result, true, nil + } glog.V(3).Infof("Error getting endpoints for service %s from the cache: %v", svc.Name, err) - return nil, err + return nil, false, err } - result, err := lbc.getEndpointsForPort(endps, backend.ServicePort, svc) + result, err = lbc.getEndpointsForPort(endps, backend.ServicePort, svc) if err != nil { glog.V(3).Infof("Error getting endpoints for service %s port %v: %v", svc.Name, backend.ServicePort, err) - return nil, err + return nil, false, err } - return result, nil + return result, false, nil } func (lbc *LoadBalancerController) getEndpointsForPort(endps api_v1.Endpoints, ingSvcPort intstr.IntOrString, svc *api_v1.Service) ([]string, error) { diff --git a/internal/controller/controller_test.go b/internal/controller/controller_test.go index c62f12d38e..f4cc6ff440 100644 --- a/internal/controller/controller_test.go +++ b/internal/controller/controller_test.go @@ -10,7 +10,7 @@ import ( "github.com/nginxinc/kubernetes-ingress/internal/nginx" "github.com/nginxinc/kubernetes-ingress/internal/nginx/plus" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" extensions "k8s.io/api/extensions/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" diff --git a/internal/handlers/service.go b/internal/handlers/service.go index 152e6af519..ef0ad4194f 100644 --- a/internal/handlers/service.go +++ b/internal/handlers/service.go @@ -54,7 +54,7 @@ func CreateServiceHandlers(lbc *controller.LoadBalancerController) cache.Resourc return } oldSvc := old.(*api_v1.Service) - if hasServicePortChanges(oldSvc.Spec.Ports, curSvc.Spec.Ports) { + if hasServiceChanges(oldSvc, curSvc) { glog.V(3).Infof("Service %v changed, syncing", curSvc.Name) lbc.EnqueueIngressForService(curSvc) } @@ -80,6 +80,22 @@ func (a portSort) Less(i, j int) bool { return a[i].Name < a[j].Name } +// hasServicedChanged checks if the service has changed based on custom rules we define (eg. port). +func hasServiceChanges(oldSvc, curSvc *api_v1.Service) bool { + if hasServicePortChanges(oldSvc.Spec.Ports, curSvc.Spec.Ports) { + return true + } + if hasServiceExternalNameChanges(oldSvc, curSvc) { + return true + } + return false +} + +// hasServiceExternalNameChanges only compares Service.Spec.Externalname for Type ExternalName services. +func hasServiceExternalNameChanges(oldSvc, curSvc *api_v1.Service) bool { + return curSvc.Spec.Type == api_v1.ServiceTypeExternalName && oldSvc.Spec.ExternalName != curSvc.Spec.ExternalName +} + // hasServicePortChanges only compares ServicePort.Name and .Port. func hasServicePortChanges(oldServicePorts []api_v1.ServicePort, curServicePorts []api_v1.ServicePort) bool { if len(oldServicePorts) != len(curServicePorts) { diff --git a/internal/nginx/config.go b/internal/nginx/config.go index de3a1b1fa1..14bb5602cd 100644 --- a/internal/nginx/config.go +++ b/internal/nginx/config.go @@ -50,6 +50,10 @@ type Config struct { HealthCheckMandatory bool HealthCheckMandatoryQueue int64 SlowStart string + ResolverAddresses []string + ResolverIPV6 bool + ResolverValid string + ResolverTimeout string // http://nginx.org/en/docs/http/ngx_http_realip_module.html RealIPHeader string @@ -94,6 +98,7 @@ func NewDefaultConfig() *Config { FailTimeout: "10s", LBMethod: "random two least_conn", MainErrorLogLevel: "notice", + ResolverIPV6: true, } } @@ -370,5 +375,46 @@ func ParseConfigMap(cfgm *api_v1.ConfigMap, nginxPlus bool) *Config { cfg.MainStreamSnippets = mainStreamSnippets } } + + if resolverAddresses, exists, err := GetMapKeyAsStringSlice(cfgm.Data, "resolver-addresses", cfgm, ","); exists { + if err != nil { + glog.Error(err) + } else { + if nginxPlus { + cfg.ResolverAddresses = resolverAddresses + } else { + glog.Warning("ConfigMap key 'resolver-addresses' requires NGINX Plus") + } + } + } + + if resolverIpv6, exists, err := GetMapKeyAsBool(cfgm.Data, "resolver-ipv6", cfgm); exists { + if err != nil { + glog.Error(err) + } else { + if nginxPlus { + cfg.ResolverIPV6 = resolverIpv6 + } else { + glog.Warning("ConfigMap key 'resolver-ipv6' requires NGINX Plus") + } + } + } + + if resolverValid, exists := cfgm.Data["resolver-valid"]; exists { + if nginxPlus { + cfg.ResolverValid = resolverValid + } else { + glog.Warning("ConfigMap key 'resolver-valid' requires NGINX Plus") + } + } + + if resolverTimeout, exists := cfgm.Data["resolver-timeout"]; exists { + if nginxPlus { + cfg.ResolverTimeout = resolverTimeout + } else { + glog.Warning("ConfigMap key 'resolver-timeout' requires NGINX Plus") + } + } + return cfg } diff --git a/internal/nginx/configurator.go b/internal/nginx/configurator.go index 0e5337e85a..68ebf9dd22 100644 --- a/internal/nginx/configurator.go +++ b/internal/nginx/configurator.go @@ -830,6 +830,13 @@ func (cnf *Configurator) createUpstream(ingEx *IngressEx, name string, backend * endps, exists := ingEx.Endpoints[backend.ServiceName+backend.ServicePort.String()] if exists { var upsServers []UpstreamServer + // Always false for NGINX OSS + _, isExternalNameSvc := ingEx.ExternalNameSvcs[backend.ServiceName] + if isExternalNameSvc && !cnf.IsResolverConfigured() { + glog.Warningf("A resolver must be configured for Type ExternalName service %s, no upstream servers will be created", backend.ServiceName) + endps = []string{} + } + for _, endp := range endps { addressport := strings.Split(endp, ":") upsServers = append(upsServers, UpstreamServer{ @@ -838,6 +845,7 @@ func (cnf *Configurator) createUpstream(ingEx *IngressEx, name string, backend * MaxFails: cfg.MaxFails, FailTimeout: cfg.FailTimeout, SlowStart: cfg.SlowStart, + Resolve: isExternalNameSvc, }) } if len(upsServers) > 0 { @@ -1087,9 +1095,13 @@ func (cnf *Configurator) updatePlusEndpoints(ingEx *IngressEx) error { name := getNameForUpstream(ingEx.Ingress, emptyHost, ingEx.Ingress.Spec.Backend) endps, exists := ingEx.Endpoints[ingEx.Ingress.Spec.Backend.ServiceName+ingEx.Ingress.Spec.Backend.ServicePort.String()] if exists { - err := cnf.nginxAPI.UpdateServers(name, endps, cfg, cnf.nginx.configVersion) - if err != nil { - return fmt.Errorf("Couldn't update the endpoints for %v: %v", name, err) + if _, isExternalName := ingEx.ExternalNameSvcs[ingEx.Ingress.Spec.Backend.ServiceName]; isExternalName { + glog.V(3).Infof("Service %s is Type ExternalName, skipping NGINX Plus endpoints update via API", ingEx.Ingress.Spec.Backend.ServiceName) + } else { + err := cnf.nginxAPI.UpdateServers(name, endps, cfg, cnf.nginx.configVersion) + if err != nil { + return fmt.Errorf("Couldn't update the endpoints for %v: %v", name, err) + } } } } @@ -1101,6 +1113,10 @@ func (cnf *Configurator) updatePlusEndpoints(ingEx *IngressEx) error { name := getNameForUpstream(ingEx.Ingress, rule.Host, &path.Backend) endps, exists := ingEx.Endpoints[path.Backend.ServiceName+path.Backend.ServicePort.String()] if exists { + if _, isExternalName := ingEx.ExternalNameSvcs[path.Backend.ServiceName]; isExternalName { + glog.V(3).Infof("Service %s is Type ExternalName, skipping NGINX Plus endpoints update via API", path.Backend.ServiceName) + continue + } err := cnf.nginxAPI.UpdateServers(name, endps, cfg, cnf.nginx.configVersion) if err != nil { return fmt.Errorf("Couldn't update the endpoints for %v: %v", name, err) @@ -1171,6 +1187,10 @@ func GenerateNginxMainConfig(config *Config) *MainConfig { WorkerShutdownTimeout: config.MainWorkerShutdownTimeout, WorkerConnections: config.MainWorkerConnections, WorkerRlimitNofile: config.MainWorkerRlimitNofile, + ResolverAddresses: config.ResolverAddresses, + ResolverIPV6: config.ResolverIPV6, + ResolverValid: config.ResolverValid, + ResolverTimeout: config.ResolverTimeout, } return nginxCfg } @@ -1252,3 +1272,8 @@ func (cnf *Configurator) HasMinion(master *extensions.Ingress, minion *extension } return cnf.minions[masterName][objectMetaToFileName(&minion.ObjectMeta)] } + +// IsResolverConfigured checks if a DNS resolver is present in NGINX configuration +func (cnf *Configurator) IsResolverConfigured() bool { + return len(cnf.config.ResolverAddresses) != 0 +} diff --git a/internal/nginx/configurator_test.go b/internal/nginx/configurator_test.go index 400c71a7d5..39388d3254 100644 --- a/internal/nginx/configurator_test.go +++ b/internal/nginx/configurator_test.go @@ -226,6 +226,7 @@ func createCafeIngressEx() IngressEx { "coffee-svc80": {"10.0.0.1:80"}, "tea-svc80": {"10.0.0.2:80"}, }, + ExternalNameSvcs: map[string]bool{}, } return cafeIngressEx } diff --git a/internal/nginx/ingress.go b/internal/nginx/ingress.go index 72ca569c94..ef61d3c4e6 100644 --- a/internal/nginx/ingress.go +++ b/internal/nginx/ingress.go @@ -10,11 +10,12 @@ import ( // IngressEx holds an Ingress along with Secrets and Endpoints of the services // that are referenced in this Ingress type IngressEx struct { - Ingress *extensions.Ingress - TLSSecrets map[string]*api_v1.Secret - JWTKey JWTKey - Endpoints map[string][]string - HealthChecks map[string]*api_v1.Probe + Ingress *extensions.Ingress + TLSSecrets map[string]*api_v1.Secret + JWTKey JWTKey + Endpoints map[string][]string + HealthChecks map[string]*api_v1.Probe + ExternalNameSvcs map[string]bool } // MergeableIngresses is a mergeable ingress of a master and minions diff --git a/internal/nginx/nginx.go b/internal/nginx/nginx.go index b4001555a0..a48ba14eee 100644 --- a/internal/nginx/nginx.go +++ b/internal/nginx/nginx.go @@ -61,6 +61,7 @@ type UpstreamServer struct { MaxFails int FailTimeout string SlowStart string + Resolve bool } // HealthCheck describes an active HTTP health check @@ -175,6 +176,10 @@ type MainConfig struct { WorkerShutdownTimeout string WorkerConnections string WorkerRlimitNofile string + ResolverAddresses []string + ResolverIPV6 bool + ResolverValid string + ResolverTimeout string } // NewUpstreamWithDefaultServer creates an upstream with the default server. diff --git a/internal/nginx/templates/nginx-plus.ingress.tmpl b/internal/nginx/templates/nginx-plus.ingress.tmpl index f1ba59bbeb..4f17c6142c 100644 --- a/internal/nginx/templates/nginx-plus.ingress.tmpl +++ b/internal/nginx/templates/nginx-plus.ingress.tmpl @@ -5,7 +5,7 @@ upstream {{$upstream.Name}} { {{if $upstream.LBMethod }}{{$upstream.LBMethod}};{{end}} {{range $server := $upstream.UpstreamServers}} server {{$server.Address}}:{{$server.Port}} max_fails={{$server.MaxFails}} fail_timeout={{$server.FailTimeout}} - {{- if $server.SlowStart}} slow_start={{$server.SlowStart}}{{end}};{{end}} + {{- if $server.SlowStart}} slow_start={{$server.SlowStart}}{{end}}{{if $server.Resolve}} resolve{{end}};{{end}} {{if $upstream.StickyCookie}} sticky cookie {{$upstream.StickyCookie}}; {{end}} diff --git a/internal/nginx/templates/nginx-plus.tmpl b/internal/nginx/templates/nginx-plus.tmpl index 903a8a6719..915ffd0a65 100644 --- a/internal/nginx/templates/nginx-plus.tmpl +++ b/internal/nginx/templates/nginx-plus.tmpl @@ -60,6 +60,11 @@ http { {{if .SSLPreferServerCiphers}}ssl_prefer_server_ciphers on;{{end}} {{if .SSLDHParam}}ssl_dhparam {{.SSLDHParam}};{{end}} + {{if .ResolverAddresses}} + resolver {{range $resolver := .ResolverAddresses}}{{$resolver}}{{end}}{{if .ResolverValid}} valid={{.ResolverValid}}{{end}}{{if not .ResolverIPV6}} ipv6=off{{end}}; + {{if .ResolverTimeout}}resolver_timeout {{.ResolverTimeout}};{{end}} + {{end}} + server { listen 80 default_server{{if .ProxyProtocol}} proxy_protocol{{end}}; listen 443 ssl default_server{{if .HTTP2}} http2{{end}}{{if .ProxyProtocol}} proxy_protocol{{end}}; diff --git a/internal/nginx/templates/templates_test.go b/internal/nginx/templates/templates_test.go index 98471c0262..bd427ca6d0 100644 --- a/internal/nginx/templates/templates_test.go +++ b/internal/nginx/templates/templates_test.go @@ -99,6 +99,10 @@ var mainCfg = nginx.MainConfig{ WorkerRlimitNofile: "65536", StreamSnippets: []string{"# comment"}, StreamLogFormat: "$remote_addr", + ResolverAddresses: []string{"example.com", "127.0.0.1"}, + ResolverIPV6: false, + ResolverValid: "10s", + ResolverTimeout: "15s", } func TestIngressForNGINXPlus(t *testing.T) {