diff --git a/apis/v1alpha1/nginxproxy_types.go b/apis/v1alpha1/nginxproxy_types.go index 018911da7e..acb42e4f5e 100644 --- a/apis/v1alpha1/nginxproxy_types.go +++ b/apis/v1alpha1/nginxproxy_types.go @@ -53,6 +53,12 @@ type NginxProxySpec struct { // // +optional Telemetry *Telemetry `json:"telemetry,omitempty"` + // RewriteClientIP defines configuration for rewriting the client IP to the original client's IP. + // +kubebuilder:validation:XValidation:message="if mode is set, trustedAddresses is a required field",rule="!(has(self.mode) && (!has(self.trustedAddresses) || size(self.trustedAddresses) == 0))" + // + // +optional + //nolint:lll + RewriteClientIP *RewriteClientIP `json:"rewriteClientIP,omitempty"` // DisableHTTP2 defines if http2 should be disabled for all servers. // Default is false, meaning http2 will be enabled for all servers. // @@ -114,3 +120,86 @@ type TelemetryExporter struct { // +kubebuilder:validation:Pattern=`^(?:http?:\/\/)?[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)*(?::\d{1,5})?$` Endpoint string `json:"endpoint"` } + +// RewriteClientIP specifies the configuration for rewriting the client's IP address. +type RewriteClientIP struct { + // Mode defines how NGINX will rewrite the client's IP address. + // There are two possible modes: + // - ProxyProtocol: NGINX will rewrite the client's IP using the PROXY protocol header. + // - XForwardedFor: NGINX will rewrite the client's IP using the X-Forwarded-For header. + // Sets NGINX directive real_ip_header: https://nginx.org/en/docs/http/ngx_http_realip_module.html#real_ip_header + // + // +optional + Mode *RewriteClientIPModeType `json:"mode,omitempty"` + + // SetIPRecursively configures whether recursive search is used when selecting the client's address from + // the X-Forwarded-For header. It is used in conjunction with TrustedAddresses. + // If enabled, NGINX will recurse on the values in X-Forwarded-Header from the end of array + // to start of array and select the first untrusted IP. + // For example, if X-Forwarded-For is [11.11.11.11, 22.22.22.22, 55.55.55.1], + // and TrustedAddresses is set to 55.55.55.1/32, NGINX will rewrite the client IP to 22.22.22.22. + // If disabled, NGINX will select the IP at the end of the array. + // In the previous example, 55.55.55.1 would be selected. + // Sets NGINX directive real_ip_recursive: https://nginx.org/en/docs/http/ngx_http_realip_module.html#real_ip_recursive + // + // +optional + SetIPRecursively *bool `json:"setIPRecursively,omitempty"` + + // TrustedAddresses specifies the addresses that are trusted to send correct client IP information. + // If a request comes from a trusted address, NGINX will rewrite the client IP information, + // and forward it to the backend in the X-Forwarded-For* and X-Real-IP headers. + // If the request does not come from a trusted address, NGINX will not rewrite the client IP information. + // TrustedAddresses only supports CIDR blocks: 192.33.21.1/24, fe80::1/64. + // To trust all addresses (not recommended for production), set to 0.0.0.0/0. + // If no addresses are provided, NGINX will not rewrite the client IP information. + // Sets NGINX directive set_real_ip_from: https://nginx.org/en/docs/http/ngx_http_realip_module.html#set_real_ip_from + // This field is required if mode is set. + // +kubebuilder:validation:MaxItems=16 + // +listType=map + // +listMapKey=type + // + // +optional + TrustedAddresses []Address `json:"trustedAddresses,omitempty"` +} + +// RewriteClientIPModeType defines how NGINX Gateway Fabric will determine the client's original IP address. +// +kubebuilder:validation:Enum=ProxyProtocol;XForwardedFor +type RewriteClientIPModeType string + +const ( + // RewriteClientIPModeProxyProtocol configures NGINX to accept PROXY protocol and + // set the client's IP address to the IP address in the PROXY protocol header. + // Sets the proxy_protocol parameter on the listen directive of all servers and sets real_ip_header + // to proxy_protocol: https://nginx.org/en/docs/http/ngx_http_realip_module.html#real_ip_header. + RewriteClientIPModeProxyProtocol RewriteClientIPModeType = "ProxyProtocol" + + // RewriteClientIPModeXForwardedFor configures NGINX to set the client's IP address to the + // IP address in the X-Forwarded-For HTTP header. + // https://nginx.org/en/docs/http/ngx_http_realip_module.html#real_ip_header. + RewriteClientIPModeXForwardedFor RewriteClientIPModeType = "XForwardedFor" +) + +// Address is a struct that specifies address type and value. +type Address struct { + // Type specifies the type of address. + // Default is "cidr" which specifies that the address is a CIDR block. + // + // +optional + // +kubebuilder:default:=cidr + Type AddressType `json:"type,omitempty"` + + // Value specifies the address value. + // + // +optional + Value string `json:"value,omitempty"` +} + +// AddressType specifies the type of address. +// +kubebuilder:validation:Enum=cidr +type AddressType string + +const ( + // AddressTypeCIDR specifies that the address is a CIDR block. + // kubebuilder:validation:Pattern=`^[\.a-zA-Z0-9:]*(\/([0-9]?[0-9]?[0-9]))$` + AddressTypeCIDR AddressType = "cidr" +) diff --git a/apis/v1alpha1/zz_generated.deepcopy.go b/apis/v1alpha1/zz_generated.deepcopy.go index 90cdca7a41..bffbb7dfdb 100644 --- a/apis/v1alpha1/zz_generated.deepcopy.go +++ b/apis/v1alpha1/zz_generated.deepcopy.go @@ -10,6 +10,21 @@ import ( "sigs.k8s.io/gateway-api/apis/v1alpha2" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Address) DeepCopyInto(out *Address) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Address. +func (in *Address) DeepCopy() *Address { + if in == nil { + return nil + } + out := new(Address) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClientBody) DeepCopyInto(out *ClientBody) { *out = *in @@ -367,6 +382,11 @@ func (in *NginxProxySpec) DeepCopyInto(out *NginxProxySpec) { *out = new(Telemetry) (*in).DeepCopyInto(*out) } + if in.RewriteClientIP != nil { + in, out := &in.RewriteClientIP, &out.RewriteClientIP + *out = new(RewriteClientIP) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NginxProxySpec. @@ -463,6 +483,36 @@ func (in *ObservabilityPolicySpec) DeepCopy() *ObservabilityPolicySpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RewriteClientIP) DeepCopyInto(out *RewriteClientIP) { + *out = *in + if in.Mode != nil { + in, out := &in.Mode, &out.Mode + *out = new(RewriteClientIPModeType) + **out = **in + } + if in.SetIPRecursively != nil { + in, out := &in.SetIPRecursively, &out.SetIPRecursively + *out = new(bool) + **out = **in + } + if in.TrustedAddresses != nil { + in, out := &in.TrustedAddresses, &out.TrustedAddresses + *out = make([]Address, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RewriteClientIP. +func (in *RewriteClientIP) DeepCopy() *RewriteClientIP { + if in == nil { + return nil + } + out := new(RewriteClientIP) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SpanAttribute) DeepCopyInto(out *SpanAttribute) { *out = *in diff --git a/charts/nginx-gateway-fabric/values.yaml b/charts/nginx-gateway-fabric/values.yaml index 9cfc2064b2..b81fb9063d 100644 --- a/charts/nginx-gateway-fabric/values.yaml +++ b/charts/nginx-gateway-fabric/values.yaml @@ -93,6 +93,17 @@ nginx: {} # disableHTTP2: false # ipFamily: dual + # rewriteClientIP: + # mode: "ProxyProtocol" + # # -- The trusted addresses field needs to be replaced with the load balancer's address and type. + # trustedAddresses: [ + # { + # # -- The CIDR block of the load balancer(s). + # value: "", + # type: "cidr", + # } + # ] + # setIPRecursively: true # telemetry: # exporter: # endpoint: otel-collector.default.svc:4317 diff --git a/config/crd/bases/gateway.nginx.org_nginxproxies.yaml b/config/crd/bases/gateway.nginx.org_nginxproxies.yaml index d0c8ba3330..19ed93c64b 100644 --- a/config/crd/bases/gateway.nginx.org_nginxproxies.yaml +++ b/config/crd/bases/gateway.nginx.org_nginxproxies.yaml @@ -62,6 +62,70 @@ spec: - ipv4 - ipv6 type: string + rewriteClientIP: + description: RewriteClientIP defines configuration for rewriting the + client IP to the original client's IP. + properties: + mode: + description: |- + Mode defines how NGINX will rewrite the client's IP address. + There are two possible modes: + - ProxyProtocol: NGINX will rewrite the client's IP using the PROXY protocol header. + - XForwardedFor: NGINX will rewrite the client's IP using the X-Forwarded-For header. + Sets NGINX directive real_ip_header: https://nginx.org/en/docs/http/ngx_http_realip_module.html#real_ip_header + enum: + - ProxyProtocol + - XForwardedFor + type: string + setIPRecursively: + description: |- + SetIPRecursively configures whether recursive search is used when selecting the client's address from + the X-Forwarded-For header. It is used in conjunction with TrustedAddresses. + If enabled, NGINX will recurse on the values in X-Forwarded-Header from the end of array + to start of array and select the first untrusted IP. + For example, if X-Forwarded-For is [11.11.11.11, 22.22.22.22, 55.55.55.1], + and TrustedAddresses is set to 55.55.55.1/32, NGINX will rewrite the client IP to 22.22.22.22. + If disabled, NGINX will select the IP at the end of the array. + In the previous example, 55.55.55.1 would be selected. + Sets NGINX directive real_ip_recursive: https://nginx.org/en/docs/http/ngx_http_realip_module.html#real_ip_recursive + type: boolean + trustedAddresses: + description: |- + TrustedAddresses specifies the addresses that are trusted to send correct client IP information. + If a request comes from a trusted address, NGINX will rewrite the client IP information, + and forward it to the backend in the X-Forwarded-For* and X-Real-IP headers. + If the request does not come from a trusted address, NGINX will not rewrite the client IP information. + TrustedAddresses only supports CIDR blocks: 192.33.21.1/24, fe80::1/64. + To trust all addresses (not recommended for production), set to 0.0.0.0/0. + If no addresses are provided, NGINX will not rewrite the client IP information. + Sets NGINX directive set_real_ip_from: https://nginx.org/en/docs/http/ngx_http_realip_module.html#set_real_ip_from + This field is required if mode is set. + items: + description: Address is a struct that specifies address type + and value. + properties: + type: + default: cidr + description: |- + Type specifies the type of address. + Default is "cidr" which specifies that the address is a CIDR block. + enum: + - cidr + type: string + value: + description: Value specifies the address value. + type: string + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + x-kubernetes-validations: + - message: if mode is set, trustedAddresses is a required field + rule: '!(has(self.mode) && (!has(self.trustedAddresses) || size(self.trustedAddresses) + == 0))' telemetry: description: Telemetry specifies the OpenTelemetry configuration. properties: diff --git a/deploy/crds.yaml b/deploy/crds.yaml index 90a4c5d113..ef8ffea772 100644 --- a/deploy/crds.yaml +++ b/deploy/crds.yaml @@ -647,6 +647,70 @@ spec: - ipv4 - ipv6 type: string + rewriteClientIP: + description: RewriteClientIP defines configuration for rewriting the + client IP to the original client's IP. + properties: + mode: + description: |- + Mode defines how NGINX will rewrite the client's IP address. + There are two possible modes: + - ProxyProtocol: NGINX will rewrite the client's IP using the PROXY protocol header. + - XForwardedFor: NGINX will rewrite the client's IP using the X-Forwarded-For header. + Sets NGINX directive real_ip_header: https://nginx.org/en/docs/http/ngx_http_realip_module.html#real_ip_header + enum: + - ProxyProtocol + - XForwardedFor + type: string + setIPRecursively: + description: |- + SetIPRecursively configures whether recursive search is used when selecting the client's address from + the X-Forwarded-For header. It is used in conjunction with TrustedAddresses. + If enabled, NGINX will recurse on the values in X-Forwarded-Header from the end of array + to start of array and select the first untrusted IP. + For example, if X-Forwarded-For is [11.11.11.11, 22.22.22.22, 55.55.55.1], + and TrustedAddresses is set to 55.55.55.1/32, NGINX will rewrite the client IP to 22.22.22.22. + If disabled, NGINX will select the IP at the end of the array. + In the previous example, 55.55.55.1 would be selected. + Sets NGINX directive real_ip_recursive: https://nginx.org/en/docs/http/ngx_http_realip_module.html#real_ip_recursive + type: boolean + trustedAddresses: + description: |- + TrustedAddresses specifies the addresses that are trusted to send correct client IP information. + If a request comes from a trusted address, NGINX will rewrite the client IP information, + and forward it to the backend in the X-Forwarded-For* and X-Real-IP headers. + If the request does not come from a trusted address, NGINX will not rewrite the client IP information. + TrustedAddresses only supports CIDR blocks: 192.33.21.1/24, fe80::1/64. + To trust all addresses (not recommended for production), set to 0.0.0.0/0. + If no addresses are provided, NGINX will not rewrite the client IP information. + Sets NGINX directive set_real_ip_from: https://nginx.org/en/docs/http/ngx_http_realip_module.html#set_real_ip_from + This field is required if mode is set. + items: + description: Address is a struct that specifies address type + and value. + properties: + type: + default: cidr + description: |- + Type specifies the type of address. + Default is "cidr" which specifies that the address is a CIDR block. + enum: + - cidr + type: string + value: + description: Value specifies the address value. + type: string + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + x-kubernetes-validations: + - message: if mode is set, trustedAddresses is a required field + rule: '!(has(self.mode) && (!has(self.trustedAddresses) || size(self.trustedAddresses) + == 0))' telemetry: description: Telemetry specifies the OpenTelemetry configuration. properties: diff --git a/internal/mode/static/nginx/config/http/config.go b/internal/mode/static/nginx/config/http/config.go index 4e26604196..f17d51e5d5 100644 --- a/internal/mode/static/nginx/config/http/config.go +++ b/internal/mode/static/nginx/config/http/config.go @@ -2,7 +2,10 @@ package http import "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/shared" -const InternalRoutePathPrefix = "/_ngf-internal" +const ( + InternalRoutePathPrefix = "/_ngf-internal" + HTTPSScheme = "https" +) // Server holds all configuration for an HTTP server. type Server struct { @@ -109,9 +112,10 @@ type ProxySSLVerify struct { // ServerConfig holds configuration for an HTTP server and IP family to be used by NGINX. type ServerConfig struct { - Servers []Server - IPFamily shared.IPFamily - Plus bool + Servers []Server + RewriteClientIP shared.RewriteClientIPSettings + IPFamily shared.IPFamily + Plus bool } // Include defines a file that's included via the include directive. diff --git a/internal/mode/static/nginx/config/servers.go b/internal/mode/static/nginx/config/servers.go index 4ced814bd0..0fd1930f5e 100644 --- a/internal/mode/static/nginx/config/servers.go +++ b/internal/mode/static/nginx/config/servers.go @@ -41,6 +41,22 @@ var httpBaseHeaders = []http.Header{ Name: "Connection", Value: "$connection_upgrade", }, + { + Name: "X-Real-IP", + Value: "$remote_addr", + }, + { + Name: "X-Forwarded-Proto", + Value: "$scheme", + }, + { + Name: "X-Forwarded-Host", + Value: "$host", + }, + { + Name: "X-Forwarded-Port", + Value: "$server_port", + }, } // grpcBaseHeaders contains the constant headers set in each gRPC server block. @@ -57,6 +73,22 @@ var grpcBaseHeaders = []http.Header{ Name: "Authority", Value: "$gw_api_compliant_host", }, + { + Name: "X-Real-IP", + Value: "$remote_addr", + }, + { + Name: "X-Forwarded-Proto", + Value: "$scheme", + }, + { + Name: "X-Forwarded-Host", + Value: "$host", + }, + { + Name: "X-Forwarded-Port", + Value: "$server_port", + }, } func (g GeneratorImpl) newExecuteServersFunc(generator policies.Generator) executeFunc { @@ -66,12 +98,13 @@ func (g GeneratorImpl) newExecuteServersFunc(generator policies.Generator) execu } func (g GeneratorImpl) executeServers(conf dataplane.Configuration, generator policies.Generator) []executeResult { - servers, httpMatchPairs := createServers(conf.HTTPServers, conf.SSLServers, conf.TLSPassthroughServers, generator) + servers, httpMatchPairs := createServers(conf, generator) serverConfig := http.ServerConfig{ - Servers: servers, - IPFamily: getIPFamily(conf.BaseHTTPConfig), - Plus: g.plus, + Servers: servers, + IPFamily: getIPFamily(conf.BaseHTTPConfig), + Plus: g.plus, + RewriteClientIP: getRewriteClientIPSettings(conf.BaseHTTPConfig.RewriteClientIPSettings), } serverResult := executeResult{ @@ -139,28 +172,23 @@ func createIncludeFileResults(servers []http.Server) []executeResult { return results } -func createServers( - httpServers, - sslServers []dataplane.VirtualServer, - tlsPassthroughServers []dataplane.Layer4VirtualServer, - generator policies.Generator, -) ([]http.Server, httpMatchPairs) { - servers := make([]http.Server, 0, len(httpServers)+len(sslServers)) +func createServers(conf dataplane.Configuration, generator policies.Generator) ([]http.Server, httpMatchPairs) { + servers := make([]http.Server, 0, len(conf.HTTPServers)+len(conf.SSLServers)) finalMatchPairs := make(httpMatchPairs) sharedTLSPorts := make(map[int32]struct{}) - for _, passthroughServer := range tlsPassthroughServers { + for _, passthroughServer := range conf.TLSPassthroughServers { sharedTLSPorts[passthroughServer.Port] = struct{}{} } - for idx, s := range httpServers { + for idx, s := range conf.HTTPServers { serverID := fmt.Sprintf("%d", idx) httpServer, matchPairs := createServer(s, serverID, generator) servers = append(servers, httpServer) maps.Copy(finalMatchPairs, matchPairs) } - for idx, s := range sslServers { + for idx, s := range conf.SSLServers { serverID := fmt.Sprintf("SSL_%d", idx) sslServer, matchPairs := createSSLServer(s, serverID, generator) @@ -874,3 +902,18 @@ func createDefaultRootLocation() http.Location { func isNonSlashedPrefixPath(pathType dataplane.PathType, path string) bool { return pathType == dataplane.PathTypePrefix && !strings.HasSuffix(path, "/") } + +// getRewriteClientIPSettings returns the configuration for the rewriting client IP settings. +func getRewriteClientIPSettings(rewriteIPConfig dataplane.RewriteClientIPSettings) shared.RewriteClientIPSettings { + var proxyProtocol string + if rewriteIPConfig.Mode == dataplane.RewriteIPModeProxyProtocol { + proxyProtocol = shared.ProxyProtocolDirective + } + + return shared.RewriteClientIPSettings{ + RealIPHeader: string(rewriteIPConfig.Mode), + RealIPFrom: rewriteIPConfig.TrustedAddresses, + Recursive: rewriteIPConfig.IPRecursive, + ProxyProtocol: proxyProtocol, + } +} diff --git a/internal/mode/static/nginx/config/servers_template.go b/internal/mode/static/nginx/config/servers_template.go index 02b8fae97f..80b0847e2e 100644 --- a/internal/mode/static/nginx/config/servers_template.go +++ b/internal/mode/static/nginx/config/servers_template.go @@ -2,27 +2,44 @@ package config const serversTemplateText = ` js_preload_object matches from /etc/nginx/conf.d/matches.json; -{{ range $s := .Servers -}} + +{{- range $s := .Servers -}} {{ if $s.IsDefaultSSL -}} server { {{- if or ($.IPFamily.IPv4) ($s.IsSocket) }} - listen {{ $s.Listen }} ssl default_server; + listen {{ $s.Listen }} ssl default_server{{ $.RewriteClientIP.ProxyProtocol }}; {{- end }} {{- if and ($.IPFamily.IPv6) (not $s.IsSocket) }} - listen [::]:{{ $s.Listen }} ssl default_server; + listen [::]:{{ $s.Listen }} ssl default_server{{ $.RewriteClientIP.ProxyProtocol }}; {{- end }} - ssl_reject_handshake on; + {{- range $address := $.RewriteClientIP.RealIPFrom }} + set_real_ip_from {{ $address }}; + {{- end}} + {{- if $.RewriteClientIP.RealIPHeader}} + real_ip_header {{ $.RewriteClientIP.RealIPHeader }}; + {{- end}} + {{- if $.RewriteClientIP.Recursive}} + real_ip_recursive on; + {{- end }} } {{- else if $s.IsDefaultHTTP }} server { {{- if $.IPFamily.IPv4 }} - listen {{ $s.Listen }} default_server; + listen {{ $s.Listen }} default_server{{ $.RewriteClientIP.ProxyProtocol }}; {{- end }} {{- if $.IPFamily.IPv6 }} - listen [::]:{{ $s.Listen }} default_server; + listen [::]:{{ $s.Listen }} default_server{{ $.RewriteClientIP.ProxyProtocol }}; + {{- end }} + {{- range $address := $.RewriteClientIP.RealIPFrom }} + set_real_ip_from {{ $address }}; + {{- end}} + {{- if $.RewriteClientIP.RealIPHeader}} + real_ip_header {{ $.RewriteClientIP.RealIPHeader }}; + {{- end}} + {{- if $.RewriteClientIP.Recursive}} + real_ip_recursive on; {{- end }} - default_type text/html; return 404; } @@ -30,10 +47,10 @@ server { server { {{- if $s.SSL }} {{- if or ($.IPFamily.IPv4) ($s.IsSocket) }} - listen {{ $s.Listen }} ssl; + listen {{ $s.Listen }} ssl{{ $.RewriteClientIP.ProxyProtocol }}; {{- end }} {{- if and ($.IPFamily.IPv6) (not $s.IsSocket) }} - listen [::]:{{ $s.Listen }} ssl; + listen [::]:{{ $s.Listen }} ssl{{ $.RewriteClientIP.ProxyProtocol }}; {{- end }} ssl_certificate {{ $s.SSL.Certificate }}; ssl_certificate_key {{ $s.SSL.CertificateKey }}; @@ -43,10 +60,10 @@ server { } {{- else }} {{- if $.IPFamily.IPv4 }} - listen {{ $s.Listen }}; + listen {{ $s.Listen }}{{ $.RewriteClientIP.ProxyProtocol }}; {{- end }} {{- if $.IPFamily.IPv6 }} - listen [::]:{{ $s.Listen }}; + listen [::]:{{ $s.Listen }}{{ $.RewriteClientIP.ProxyProtocol }}; {{- end }} {{- end }} @@ -60,6 +77,16 @@ server { include {{ $i.Name }}; {{- end }} + {{- range $address := $.RewriteClientIP.RealIPFrom }} + set_real_ip_from {{ $address }}; + {{- end}} + {{- if $.RewriteClientIP.RealIPHeader}} + real_ip_header {{ $.RewriteClientIP.RealIPHeader }}; + {{- end}} + {{- if $.RewriteClientIP.Recursive}} + real_ip_recursive on; + {{- end }} + {{ range $l := $s.Locations }} location {{ $l.Path }} { {{ if eq $l.Type "internal" -}} diff --git a/internal/mode/static/nginx/config/servers_test.go b/internal/mode/static/nginx/config/servers_test.go index a912e6f2bc..e3a9122b23 100644 --- a/internal/mode/static/nginx/config/servers_test.go +++ b/internal/mode/static/nginx/config/servers_test.go @@ -259,6 +259,124 @@ func TestExecuteServers_IPFamily(t *testing.T) { "listen [::]:8443 ssl default_server;": 1, "listen [::]:8443 ssl;": 1, "status_zone": 0, + "real_ip_header proxy-protocol;": 0, + "real_ip_recursive on;": 0, + }, + }, + } + + for _, test := range tests { + t.Run(test.msg, func(t *testing.T) { + g := NewWithT(t) + + gen := GeneratorImpl{} + results := gen.executeServers(test.config, &policiesfakes.FakeGenerator{}) + g.Expect(results).To(HaveLen(2)) + serverConf := string(results[0].data) + httpMatchConf := string(results[1].data) + g.Expect(httpMatchConf).To(Equal("{}")) + + for expSubStr, expCount := range test.expectedHTTPConfig { + g.Expect(strings.Count(serverConf, expSubStr)).To(Equal(expCount)) + } + }) + } +} + +func TestExecuteServers_RewriteClientIP(t *testing.T) { + httpServers := []dataplane.VirtualServer{ + { + IsDefault: true, + Port: 8080, + }, + { + Hostname: "example.com", + Port: 8080, + }, + } + + sslServers := []dataplane.VirtualServer{ + { + IsDefault: true, + Port: 8443, + }, + { + Hostname: "example.com", + SSL: &dataplane.SSL{ + KeyPairID: "test-keypair", + }, + Port: 8443, + }, + } + tests := []struct { + msg string + expectedHTTPConfig map[string]int + config dataplane.Configuration + }{ + { + msg: "rewrite client IP settings configured with proxy protocol", + config: dataplane.Configuration{ + HTTPServers: httpServers, + SSLServers: sslServers, + BaseHTTPConfig: dataplane.BaseHTTPConfig{ + IPFamily: dataplane.Dual, + RewriteClientIPSettings: dataplane.RewriteClientIPSettings{ + Mode: dataplane.RewriteIPModeProxyProtocol, + TrustedAddresses: []string{"10.56.73.51/32"}, + IPRecursive: false, + }, + }, + }, + expectedHTTPConfig: map[string]int{ + "set_real_ip_from 10.56.73.51/32;": 4, + "real_ip_header proxy_protocol;": 4, + "listen 8080 default_server proxy_protocol;": 1, + "listen 8080 proxy_protocol;": 1, + "listen 8443 ssl default_server proxy_protocol;": 1, + "listen 8443 ssl proxy_protocol;": 1, + "server_name example.com;": 2, + "ssl_certificate /etc/nginx/secrets/test-keypair.pem;": 1, + "ssl_certificate_key /etc/nginx/secrets/test-keypair.pem;": 1, + "ssl_reject_handshake on;": 1, + "listen [::]:8080 default_server proxy_protocol;": 1, + "listen [::]:8080 proxy_protocol;": 1, + "listen [::]:8443 ssl default_server proxy_protocol;": 1, + "listen [::]:8443 ssl proxy_protocol;": 1, + "real_ip_recursive on;": 0, + }, + }, + { + msg: "rewrite client IP settings configured with x-forwarded-for", + config: dataplane.Configuration{ + HTTPServers: httpServers, + SSLServers: sslServers, + BaseHTTPConfig: dataplane.BaseHTTPConfig{ + IPFamily: dataplane.Dual, + RewriteClientIPSettings: dataplane.RewriteClientIPSettings{ + Mode: dataplane.RewriteIPModeXForwardedFor, + TrustedAddresses: []string{"10.1.1.3/32", "2.2.2.2", "2001:db8::/32"}, + IPRecursive: true, + }, + }, + }, + expectedHTTPConfig: map[string]int{ + "set_real_ip_from 10.1.1.3/32;": 4, + "set_real_ip_from 2.2.2.2;": 4, + "set_real_ip_from 2001:db8::/32;": 4, + "real_ip_header X-Forwarded-For;": 4, + "real_ip_recursive on;": 4, + "listen 8080 default_server;": 1, + "listen 8080;": 1, + "listen 8443 ssl default_server;": 1, + "listen 8443 ssl;": 1, + "server_name example.com;": 2, + "ssl_certificate /etc/nginx/secrets/test-keypair.pem;": 1, + "ssl_certificate_key /etc/nginx/secrets/test-keypair.pem;": 1, + "ssl_reject_handshake on;": 1, + "listen [::]:8080 default_server;": 1, + "listen [::]:8080;": 1, + "listen [::]:8443 ssl default_server;": 1, + "listen [::]:8443 ssl;": 1, }, }, } @@ -796,44 +914,44 @@ func TestCreateServers(t *testing.T) { }, } - httpServers := []dataplane.VirtualServer{ - { - IsDefault: true, - Port: 8080, - }, - { - Hostname: "cafe.example.com", - PathRules: cafePathRules, - Port: 8080, - Policies: []policies.Policy{ - &policiesfakes.FakePolicy{}, - &policiesfakes.FakePolicy{}, + conf := dataplane.Configuration{ + HTTPServers: []dataplane.VirtualServer{ + { + IsDefault: true, + Port: 8080, + }, + { + Hostname: "cafe.example.com", + PathRules: cafePathRules, + Port: 8080, + Policies: []policies.Policy{ + &policiesfakes.FakePolicy{}, + &policiesfakes.FakePolicy{}, + }, }, }, - } - - sslServers := []dataplane.VirtualServer{ - { - IsDefault: true, - Port: 8443, - }, - { - Hostname: "cafe.example.com", - SSL: &dataplane.SSL{KeyPairID: sslKeyPairID}, - PathRules: cafePathRules, - Port: 8443, - Policies: []policies.Policy{ - &policiesfakes.FakePolicy{}, - &policiesfakes.FakePolicy{}, + SSLServers: []dataplane.VirtualServer{ + { + IsDefault: true, + Port: 8443, + }, + { + Hostname: "cafe.example.com", + SSL: &dataplane.SSL{KeyPairID: sslKeyPairID}, + PathRules: cafePathRules, + Port: 8443, + Policies: []policies.Policy{ + &policiesfakes.FakePolicy{}, + &policiesfakes.FakePolicy{}, + }, }, }, - } - - tlsPassthroughServers := []dataplane.Layer4VirtualServer{ - { - Hostname: "app.example.com", - Port: 8443, - UpstreamName: "sup", + TLSPassthroughServers: []dataplane.Layer4VirtualServer{ + { + Hostname: "app.example.com", + Port: 8443, + UpstreamName: "sup", + }, }, } @@ -905,6 +1023,22 @@ func TestCreateServers(t *testing.T) { Name: "Connection", Value: "$connection_upgrade", }, + { + Name: "X-Real-IP", + Value: "$remote_addr", + }, + { + Name: "X-Forwarded-Proto", + Value: "$scheme", + }, + { + Name: "X-Forwarded-Host", + Value: "$host", + }, + { + Name: "X-Forwarded-Port", + Value: "$server_port", + }, } externalIncludes := []http.Include{ @@ -1172,6 +1306,22 @@ func TestCreateServers(t *testing.T) { Name: "Connection", Value: "$connection_upgrade", }, + { + Name: "X-Real-IP", + Value: "$remote_addr", + }, + { + Name: "X-Forwarded-Proto", + Value: "$scheme", + }, + { + Name: "X-Forwarded-Host", + Value: "$host", + }, + { + Name: "X-Forwarded-Port", + Value: "$server_port", + }, }, ResponseHeaders: http.ResponseHeaders{ Add: []http.Header{ @@ -1210,6 +1360,22 @@ func TestCreateServers(t *testing.T) { Name: "Connection", Value: "$connection_upgrade", }, + { + Name: "X-Real-IP", + Value: "$remote_addr", + }, + { + Name: "X-Forwarded-Proto", + Value: "$scheme", + }, + { + Name: "X-Forwarded-Host", + Value: "$host", + }, + { + Name: "X-Forwarded-Port", + Value: "$server_port", + }, }, ResponseHeaders: http.ResponseHeaders{ Add: []http.Header{ @@ -1315,7 +1481,7 @@ func TestCreateServers(t *testing.T) { }, }) - result, httpMatchPair := createServers(httpServers, sslServers, tlsPassthroughServers, fakeGenerator) + result, httpMatchPair := createServers(conf, fakeGenerator) g.Expect(httpMatchPair).To(Equal(allExpMatchPair)) g.Expect(helpers.Diff(expectedServers, result)).To(BeEmpty()) @@ -1530,12 +1696,7 @@ func TestCreateServersConflicts(t *testing.T) { g := NewWithT(t) - result, _ := createServers( - httpServers, - []dataplane.VirtualServer{}, - []dataplane.Layer4VirtualServer{}, - &policiesfakes.FakeGenerator{}, - ) + result, _ := createServers(dataplane.Configuration{HTTPServers: httpServers}, &policiesfakes.FakeGenerator{}) g.Expect(helpers.Diff(expectedServers, result)).To(BeEmpty()) }) } @@ -2385,6 +2546,22 @@ func TestGenerateProxySetHeaders(t *testing.T) { Name: "Connection", Value: "$connection_upgrade", }, + { + Name: "X-Real-IP", + Value: "$remote_addr", + }, + { + Name: "X-Forwarded-Proto", + Value: "$scheme", + }, + { + Name: "X-Forwarded-Host", + Value: "$host", + }, + { + Name: "X-Forwarded-Port", + Value: "$server_port", + }, }, }, { @@ -2423,6 +2600,22 @@ func TestGenerateProxySetHeaders(t *testing.T) { Name: "Connection", Value: "$connection_upgrade", }, + { + Name: "X-Real-IP", + Value: "$remote_addr", + }, + { + Name: "X-Forwarded-Proto", + Value: "$scheme", + }, + { + Name: "X-Forwarded-Host", + Value: "$host", + }, + { + Name: "X-Forwarded-Port", + Value: "$server_port", + }, }, }, { @@ -2470,6 +2663,22 @@ func TestGenerateProxySetHeaders(t *testing.T) { Name: "Authority", Value: "$gw_api_compliant_host", }, + { + Name: "X-Real-IP", + Value: "$remote_addr", + }, + { + Name: "X-Forwarded-Proto", + Value: "$scheme", + }, + { + Name: "X-Forwarded-Host", + Value: "$host", + }, + { + Name: "X-Forwarded-Port", + Value: "$server_port", + }, }, }, } diff --git a/internal/mode/static/nginx/config/shared/config.go b/internal/mode/static/nginx/config/shared/config.go index 65c0c873f5..62ea4bec82 100644 --- a/internal/mode/static/nginx/config/shared/config.go +++ b/internal/mode/static/nginx/config/shared/config.go @@ -19,3 +19,15 @@ type IPFamily struct { IPv4 bool IPv6 bool } + +// RewriteClientIP holds the configuration for the rewrite client IP settings. +type RewriteClientIPSettings struct { + RealIPHeader string + ProxyProtocol string + RealIPFrom []string + Recursive bool +} + +const ( + ProxyProtocolDirective = " proxy_protocol" +) diff --git a/internal/mode/static/nginx/config/stream/config.go b/internal/mode/static/nginx/config/stream/config.go index 6a3687306c..ddc215eea7 100644 --- a/internal/mode/static/nginx/config/stream/config.go +++ b/internal/mode/static/nginx/config/stream/config.go @@ -4,12 +4,13 @@ import "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/conf // Server holds all configuration for a stream server. type Server struct { - Listen string - StatusZone string - ProxyPass string - Pass string - SSLPreread bool - IsSocket bool + Listen string + StatusZone string + ProxyPass string + Pass string + RewriteClientIP shared.RewriteClientIPSettings + SSLPreread bool + IsSocket bool } // Upstream holds all configuration for a stream upstream. diff --git a/internal/mode/static/nginx/config/stream_servers.go b/internal/mode/static/nginx/config/stream_servers.go index b655818698..b6cb763cb5 100644 --- a/internal/mode/static/nginx/config/stream_servers.go +++ b/internal/mode/static/nginx/config/stream_servers.go @@ -5,6 +5,7 @@ import ( gotemplate "text/template" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/helpers" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/shared" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/stream" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/dataplane" ) @@ -46,12 +47,17 @@ func createStreamServers(conf dataplane.Configuration) []stream.Server { for _, server := range conf.TLSPassthroughServers { if u, ok := upstreams[server.UpstreamName]; ok && server.UpstreamName != "" { if server.Hostname != "" && len(u.Endpoints) > 0 { - streamServers = append(streamServers, stream.Server{ + streamServer := stream.Server{ Listen: getSocketNameTLS(server.Port, server.Hostname), StatusZone: server.Hostname, ProxyPass: server.UpstreamName, IsSocket: true, - }) + } + // set rewriteClientIP settings as this is a socket stream server + streamServer.RewriteClientIP = getRewriteClientIPSettingsForStream( + conf.BaseHTTPConfig.RewriteClientIPSettings, + ) + streamServers = append(streamServers, streamServer) } } @@ -60,13 +66,29 @@ func createStreamServers(conf dataplane.Configuration) []stream.Server { } portSet[server.Port] = struct{}{} - streamServers = append(streamServers, stream.Server{ + + // we do not evaluate rewriteClientIP settings for non-socket stream servers + streamServer := stream.Server{ Listen: fmt.Sprint(server.Port), StatusZone: server.Hostname, Pass: getTLSPassthroughVarName(server.Port), SSLPreread: true, - }) + } + streamServers = append(streamServers, streamServer) } - return streamServers } + +func getRewriteClientIPSettingsForStream( + rewriteConfig dataplane.RewriteClientIPSettings, +) shared.RewriteClientIPSettings { + proxyEnabled := rewriteConfig.Mode == dataplane.RewriteIPModeProxyProtocol + if proxyEnabled { + return shared.RewriteClientIPSettings{ + ProxyProtocol: shared.ProxyProtocolDirective, + RealIPFrom: rewriteConfig.TrustedAddresses, + } + } + + return shared.RewriteClientIPSettings{} +} diff --git a/internal/mode/static/nginx/config/stream_servers_template.go b/internal/mode/static/nginx/config/stream_servers_template.go index 4b619a9e0e..58a95a70b0 100644 --- a/internal/mode/static/nginx/config/stream_servers_template.go +++ b/internal/mode/static/nginx/config/stream_servers_template.go @@ -4,12 +4,15 @@ const streamServersTemplateText = ` {{- range $s := .Servers }} server { {{- if or ($.IPFamily.IPv4) ($s.IsSocket) }} - listen {{ $s.Listen }}; + listen {{ $s.Listen }}{{ $s.RewriteClientIP.ProxyProtocol }}; {{- end }} {{- if and ($.IPFamily.IPv6) (not $s.IsSocket) }} listen [::]:{{ $s.Listen }}; {{- end }} + {{- range $address := $s.RewriteClientIP.RealIPFrom }} + set_real_ip_from {{ $address }}; + {{- end}} {{- if $.Plus }} status_zone {{ $s.StatusZone }}; {{- end }} diff --git a/internal/mode/static/nginx/config/stream_servers_test.go b/internal/mode/static/nginx/config/stream_servers_test.go index 1e08874c8f..322e474e2f 100644 --- a/internal/mode/static/nginx/config/stream_servers_test.go +++ b/internal/mode/static/nginx/config/stream_servers_test.go @@ -297,6 +297,101 @@ func TestExecuteStreamServersForIPFamily(t *testing.T) { } } +func TestExecuteStreamServers_RewriteClientIP(t *testing.T) { + passThroughServers := []dataplane.Layer4VirtualServer{ + { + UpstreamName: "backend1", + Hostname: "cafe.example.com", + Port: 8443, + }, + } + streamUpstreams := []dataplane.Upstream{ + { + Name: "backend1", + Endpoints: []resolver.Endpoint{ + { + Address: "1.1.1.1", + }, + }, + }, + } + tests := []struct { + msg string + expectedStreamConfig map[string]int + config dataplane.Configuration + }{ + { + msg: "rewrite client IP not configured", + config: dataplane.Configuration{ + TLSPassthroughServers: passThroughServers, + StreamUpstreams: streamUpstreams, + }, + expectedStreamConfig: map[string]int{ + "listen 8443;": 1, + "listen [::]:8443;": 1, + "listen unix:/var/run/nginx/cafe.example.com-8443.sock;": 1, + }, + }, + { + msg: "rewrite client IP configured with proxy protocol", + config: dataplane.Configuration{ + BaseHTTPConfig: dataplane.BaseHTTPConfig{ + RewriteClientIPSettings: dataplane.RewriteClientIPSettings{ + Mode: dataplane.RewriteIPModeProxyProtocol, + TrustedAddresses: []string{"10.1.1.22/32", "::1/128", "3.4.5.6"}, + IPRecursive: false, + }, + }, + TLSPassthroughServers: passThroughServers, + StreamUpstreams: streamUpstreams, + }, + expectedStreamConfig: map[string]int{ + "listen 8443;": 1, + "listen [::]:8443;": 1, + "listen unix:/var/run/nginx/cafe.example.com-8443.sock proxy_protocol;": 1, + "set_real_ip_from 10.1.1.22/32;": 1, + "set_real_ip_from ::1/128;": 1, + "set_real_ip_from 3.4.5.6;": 1, + "real_ip_recursive on;": 0, + }, + }, + { + msg: "rewrite client IP configured with xforwardedfor", + config: dataplane.Configuration{ + BaseHTTPConfig: dataplane.BaseHTTPConfig{ + RewriteClientIPSettings: dataplane.RewriteClientIPSettings{ + Mode: dataplane.RewriteIPModeXForwardedFor, + TrustedAddresses: []string{"1.1.1.1/32"}, + IPRecursive: true, + }, + }, + TLSPassthroughServers: passThroughServers, + StreamUpstreams: streamUpstreams, + }, + expectedStreamConfig: map[string]int{ + "listen 8443;": 1, + "listen [::]:8443;": 1, + "listen unix:/var/run/nginx/cafe.example.com-8443.sock;": 1, + }, + }, + } + + for _, test := range tests { + t.Run(test.msg, func(t *testing.T) { + g := NewWithT(t) + + gen := GeneratorImpl{} + results := gen.executeStreamServers(test.config) + g.Expect(results).To(HaveLen(1)) + serverConf := string(results[0].data) + + for expSubStr, expCount := range test.expectedStreamConfig { + g.Expect(strings.Count(serverConf, expSubStr)).To(Equal(expCount)) + } + }) + } +} + func TestCreateStreamServersWithNone(t *testing.T) { conf := dataplane.Configuration{ TLSPassthroughServers: nil, diff --git a/internal/mode/static/state/dataplane/configuration.go b/internal/mode/static/state/dataplane/configuration.go index ebaf15bf0b..eefe5e4bb4 100644 --- a/internal/mode/static/state/dataplane/configuration.go +++ b/internal/mode/static/state/dataplane/configuration.go @@ -852,6 +852,27 @@ func buildBaseHTTPConfig(g *graph.Graph) BaseHTTPConfig { } } + if g.NginxProxy.Source.Spec.RewriteClientIP != nil { + if g.NginxProxy.Source.Spec.RewriteClientIP.Mode != nil { + switch *g.NginxProxy.Source.Spec.RewriteClientIP.Mode { + case ngfAPI.RewriteClientIPModeProxyProtocol: + baseConfig.RewriteClientIPSettings.Mode = RewriteIPModeProxyProtocol + case ngfAPI.RewriteClientIPModeXForwardedFor: + baseConfig.RewriteClientIPSettings.Mode = RewriteIPModeXForwardedFor + } + } + + if len(g.NginxProxy.Source.Spec.RewriteClientIP.TrustedAddresses) > 0 { + baseConfig.RewriteClientIPSettings.TrustedAddresses = convertAddresses( + g.NginxProxy.Source.Spec.RewriteClientIP.TrustedAddresses, + ) + } + + if g.NginxProxy.Source.Spec.RewriteClientIP.SetIPRecursively != nil { + baseConfig.RewriteClientIPSettings.IPRecursive = *g.NginxProxy.Source.Spec.RewriteClientIP.SetIPRecursively + } + } + return baseConfig } @@ -872,3 +893,11 @@ func buildPolicies(graphPolicies []*graph.Policy) []policies.Policy { return finalPolicies } + +func convertAddresses(addresses []ngfAPI.Address) []string { + trustedAddresses := make([]string, len(addresses)) + for i, addr := range addresses { + trustedAddresses[i] = addr.Value + } + return trustedAddresses +} diff --git a/internal/mode/static/state/dataplane/configuration_test.go b/internal/mode/static/state/dataplane/configuration_test.go index 0510a3719b..a2ae5dc910 100644 --- a/internal/mode/static/state/dataplane/configuration_test.go +++ b/internal/mode/static/state/dataplane/configuration_test.go @@ -2180,6 +2180,53 @@ func TestBuildConfiguration(t *testing.T) { }), msg: "NginxProxy with IPv6 IPFamily and no routes", }, + { + graph: getModifiedGraph(func(g *graph.Graph) *graph.Graph { + g.Gateway.Source.ObjectMeta = metav1.ObjectMeta{ + Name: "gw", + Namespace: "ns", + } + g.Gateway.Listeners = append(g.Gateway.Listeners, &graph.Listener{ + Name: "listener-80-1", + Source: listener80, + Valid: true, + Routes: map[graph.RouteKey]*graph.L7Route{}, + }) + g.NginxProxy = &graph.NginxProxy{ + Valid: true, + Source: &ngfAPI.NginxProxy{ + Spec: ngfAPI.NginxProxySpec{ + RewriteClientIP: &ngfAPI.RewriteClientIP{ + SetIPRecursively: helpers.GetPointer(true), + TrustedAddresses: []ngfAPI.Address{ + { + Type: ngfAPI.AddressTypeCIDR, + Value: "1.1.1.1/32", + }, + }, + Mode: helpers.GetPointer(ngfAPI.RewriteClientIPModeProxyProtocol), + }, + }, + }, + } + return g + }), + expConf: getModifiedExpectedConfiguration(func(conf Configuration) Configuration { + conf.SSLServers = []VirtualServer{} + conf.SSLKeyPairs = map[SSLKeyPairID]SSLKeyPair{} + conf.BaseHTTPConfig = BaseHTTPConfig{ + HTTP2: true, + IPFamily: Dual, + RewriteClientIPSettings: RewriteClientIPSettings{ + IPRecursive: true, + TrustedAddresses: []string{"1.1.1.1/32"}, + Mode: RewriteIPModeProxyProtocol, + }, + } + return conf + }), + msg: "NginxProxy with rewriteClientIP details set", + }, } for _, test := range tests { @@ -3552,3 +3599,123 @@ func TestBuildStreamUpstreams(t *testing.T) { g.Expect(streamUpstreams).To(ConsistOf(expectedStreamUpstreams)) } + +func TestBuildRewriteIPSettings(t *testing.T) { + tests := []struct { + msg string + g *graph.Graph + expRewriteIPSettings RewriteClientIPSettings + }{ + { + msg: "no rewrite IP settings configured", + g: &graph.Graph{ + NginxProxy: &graph.NginxProxy{ + Valid: true, + Source: &ngfAPI.NginxProxy{}, + }, + }, + expRewriteIPSettings: RewriteClientIPSettings{}, + }, + { + msg: "rewrite IP settings configured with proxyProtocol", + g: &graph.Graph{ + NginxProxy: &graph.NginxProxy{ + Valid: true, + Source: &ngfAPI.NginxProxy{ + Spec: ngfAPI.NginxProxySpec{ + RewriteClientIP: &ngfAPI.RewriteClientIP{ + Mode: helpers.GetPointer(ngfAPI.RewriteClientIPModeProxyProtocol), + TrustedAddresses: []ngfAPI.Address{ + { + Type: ngfAPI.AddressTypeCIDR, + Value: "10.9.9.4/32", + }, + }, + SetIPRecursively: helpers.GetPointer(true), + }, + }, + }, + }, + }, + expRewriteIPSettings: RewriteClientIPSettings{ + Mode: RewriteIPModeProxyProtocol, + TrustedAddresses: []string{"10.9.9.4/32"}, + IPRecursive: true, + }, + }, + { + msg: "rewrite IP settings configured with xForwardedFor", + g: &graph.Graph{ + NginxProxy: &graph.NginxProxy{ + Valid: true, + Source: &ngfAPI.NginxProxy{ + Spec: ngfAPI.NginxProxySpec{ + RewriteClientIP: &ngfAPI.RewriteClientIP{ + Mode: helpers.GetPointer(ngfAPI.RewriteClientIPModeXForwardedFor), + TrustedAddresses: []ngfAPI.Address{ + { + Type: ngfAPI.AddressTypeCIDR, + Value: "76.89.90.11/24", + }, + }, + SetIPRecursively: helpers.GetPointer(true), + }, + }, + }, + }, + }, + expRewriteIPSettings: RewriteClientIPSettings{ + Mode: RewriteIPModeXForwardedFor, + TrustedAddresses: []string{"76.89.90.11/24"}, + IPRecursive: true, + }, + }, + { + msg: "rewrite IP settings configured with recursive set to false and multiple trusted addresses", + g: &graph.Graph{ + NginxProxy: &graph.NginxProxy{ + Valid: true, + Source: &ngfAPI.NginxProxy{ + Spec: ngfAPI.NginxProxySpec{ + RewriteClientIP: &ngfAPI.RewriteClientIP{ + Mode: helpers.GetPointer(ngfAPI.RewriteClientIPModeXForwardedFor), + TrustedAddresses: []ngfAPI.Address{ + { + Type: ngfAPI.AddressTypeCIDR, + Value: "5.5.5.5/12", + }, + { + Type: ngfAPI.AddressTypeCIDR, + Value: "1.1.1.1/26", + }, + { + Type: ngfAPI.AddressTypeCIDR, + Value: "2.2.2.2/32", + }, + { + Type: ngfAPI.AddressTypeCIDR, + Value: "3.3.3.3/24", + }, + }, + SetIPRecursively: helpers.GetPointer(false), + }, + }, + }, + }, + }, + expRewriteIPSettings: RewriteClientIPSettings{ + Mode: RewriteIPModeXForwardedFor, + TrustedAddresses: []string{"5.5.5.5/12", "1.1.1.1/26", "2.2.2.2/32", "3.3.3.3/24"}, + IPRecursive: false, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.msg, func(t *testing.T) { + g := NewWithT(t) + baseConfig := buildBaseHTTPConfig(tc.g) + g.Expect(baseConfig.RewriteClientIPSettings).To(Equal(tc.expRewriteIPSettings)) + }) + } +} diff --git a/internal/mode/static/state/dataplane/types.go b/internal/mode/static/state/dataplane/types.go index 3741f131c8..59110f8cbb 100644 --- a/internal/mode/static/state/dataplane/types.go +++ b/internal/mode/static/state/dataplane/types.go @@ -38,10 +38,10 @@ type Configuration struct { StreamUpstreams []Upstream // BackendGroups holds all unique BackendGroups. BackendGroups []BackendGroup - // BaseHTTPConfig holds the configuration options at the http context. - BaseHTTPConfig BaseHTTPConfig // Telemetry holds the Otel configuration. Telemetry Telemetry + // BaseHTTPConfig holds the configuration options at the http context. + BaseHTTPConfig BaseHTTPConfig // Version represents the version of the generated configuration. Version int } @@ -309,10 +309,32 @@ type SpanAttribute struct { type BaseHTTPConfig struct { // IPFamily specifies the IP family for all servers. IPFamily IPFamilyType + // RewriteIPSettings defines configuration for rewriting the client IP to the original client's IP. + RewriteClientIPSettings RewriteClientIPSettings // HTTP2 specifies whether http2 should be enabled for all servers. HTTP2 bool } +// RewriteIPSettings defines configuration for rewriting the client IP to the original client's IP. +type RewriteClientIPSettings struct { + // Mode specifies the mode for rewriting the client IP. + Mode RewriteIPModeType + // TrustedAddresses specifies the addresses that are trusted to provide the client IP. + TrustedAddresses []string + // IPRecursive specifies whether a recursive search is used when selecting the client IP. + IPRecursive bool +} + +// RewriteIPModeType specifies the mode for rewriting the client IP. +type RewriteIPModeType string + +const ( + // RewriteIPModeProxyProtocol specifies that client IP will be rewrritten using the Proxy-Protocol header. + RewriteIPModeProxyProtocol RewriteIPModeType = "proxy_protocol" + // RewriteIPModeXForwardedFor specifies that client IP will be rewrritten using the X-Forwarded-For header. + RewriteIPModeXForwardedFor RewriteIPModeType = "X-Forwarded-For" +) + // IPFamilyType specifies the IP family to be used by NGINX. type IPFamilyType string diff --git a/internal/mode/static/state/graph/nginxproxy.go b/internal/mode/static/state/graph/nginxproxy.go index d36133308b..cf6dc70990 100644 --- a/internal/mode/static/state/graph/nginxproxy.go +++ b/internal/mode/static/state/graph/nginxproxy.go @@ -2,6 +2,7 @@ package graph import ( "k8s.io/apimachinery/pkg/types" + k8svalidation "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" v1 "sigs.k8s.io/gateway-api/apis/v1" @@ -126,5 +127,73 @@ func validateNginxProxy( npCfg.Spec.IPFamily = helpers.GetPointer[ngfAPI.IPFamilyType](ngfAPI.Dual) } + allErrs = append(allErrs, validateRewriteClientIP(npCfg)...) + + return allErrs +} + +func validateRewriteClientIP(npCfg *ngfAPI.NginxProxy) field.ErrorList { + var allErrs field.ErrorList + spec := field.NewPath("spec") + + if npCfg.Spec.RewriteClientIP != nil { + rewriteClientIP := npCfg.Spec.RewriteClientIP + rewriteClientIPPath := spec.Child("rewriteClientIP") + trustedAddressesPath := rewriteClientIPPath.Child("trustedAddresses") + + if rewriteClientIP.Mode != nil { + mode := *rewriteClientIP.Mode + if len(rewriteClientIP.TrustedAddresses) == 0 { + allErrs = append( + allErrs, + field.Required(rewriteClientIPPath, "trustedAddresses field required when mode is set"), + ) + } + + switch mode { + case ngfAPI.RewriteClientIPModeProxyProtocol, ngfAPI.RewriteClientIPModeXForwardedFor: + default: + allErrs = append( + allErrs, + field.NotSupported( + rewriteClientIPPath.Child("mode"), + mode, + []string{string(ngfAPI.RewriteClientIPModeProxyProtocol), string(ngfAPI.RewriteClientIPModeXForwardedFor)}, + ), + ) + } + } + + if len(rewriteClientIP.TrustedAddresses) > 16 { + allErrs = append( + allErrs, + field.TooLongMaxLength(trustedAddressesPath, rewriteClientIP.TrustedAddresses, 16), + ) + } + + for _, addr := range rewriteClientIP.TrustedAddresses { + switch addr.Type { + case ngfAPI.AddressTypeCIDR: + if err := k8svalidation.IsValidCIDR(trustedAddressesPath, addr.Value); err != nil { + allErrs = append( + allErrs, + field.Invalid(trustedAddressesPath.Child(addr.Value), + addr, + err.ToAggregate().Error(), + ), + ) + } + default: + allErrs = append( + allErrs, + field.NotSupported(trustedAddressesPath.Child("type"), + addr.Type, + []string{string(ngfAPI.AddressTypeCIDR)}, + ), + ) + } + } + } + return allErrs } diff --git a/internal/mode/static/state/graph/nginxproxy_test.go b/internal/mode/static/state/graph/nginxproxy_test.go index efec06e865..8c9f236237 100644 --- a/internal/mode/static/state/graph/nginxproxy_test.go +++ b/internal/mode/static/state/graph/nginxproxy_test.go @@ -222,27 +222,27 @@ func TestGCReferencesAnyNginxProxy(t *testing.T) { } } -func TestValidateNginxProxy(t *testing.T) { - createValidValidator := func() *validationfakes.FakeGenericValidator { - v := &validationfakes.FakeGenericValidator{} - v.ValidateEscapedStringNoVarExpansionReturns(nil) - v.ValidateEndpointReturns(nil) - v.ValidateServiceNameReturns(nil) - v.ValidateNginxDurationReturns(nil) +func createValidValidator() *validationfakes.FakeGenericValidator { + v := &validationfakes.FakeGenericValidator{} + v.ValidateEscapedStringNoVarExpansionReturns(nil) + v.ValidateEndpointReturns(nil) + v.ValidateServiceNameReturns(nil) + v.ValidateNginxDurationReturns(nil) - return v - } + return v +} - createInvalidValidator := func() *validationfakes.FakeGenericValidator { - v := &validationfakes.FakeGenericValidator{} - v.ValidateEscapedStringNoVarExpansionReturns(errors.New("error")) - v.ValidateEndpointReturns(errors.New("error")) - v.ValidateServiceNameReturns(errors.New("error")) - v.ValidateNginxDurationReturns(errors.New("error")) +func createInvalidValidator() *validationfakes.FakeGenericValidator { + v := &validationfakes.FakeGenericValidator{} + v.ValidateEscapedStringNoVarExpansionReturns(errors.New("error")) + v.ValidateEndpointReturns(errors.New("error")) + v.ValidateServiceNameReturns(errors.New("error")) + v.ValidateNginxDurationReturns(errors.New("error")) - return v - } + return v +} +func TestValidateNginxProxy(t *testing.T) { tests := []struct { np *ngfAPI.NginxProxy validator *validationfakes.FakeGenericValidator @@ -266,6 +266,20 @@ func TestValidateNginxProxy(t *testing.T) { }, }, IPFamily: helpers.GetPointer[ngfAPI.IPFamilyType](ngfAPI.Dual), + RewriteClientIP: &ngfAPI.RewriteClientIP{ + SetIPRecursively: helpers.GetPointer(true), + TrustedAddresses: []ngfAPI.Address{ + { + Type: ngfAPI.AddressTypeCIDR, + Value: "2001:db8:a0b:12f0::1/32", + }, + { + Type: ngfAPI.AddressTypeCIDR, + Value: "1.1.1.1/24", + }, + }, + Mode: helpers.GetPointer(ngfAPI.RewriteClientIPModeProxyProtocol), + }, }, }, expectErrCount: 0, @@ -356,3 +370,185 @@ func TestValidateNginxProxy(t *testing.T) { }) } } + +func TestValidateRewriteClientIP(t *testing.T) { + tests := []struct { + np *ngfAPI.NginxProxy + validator *validationfakes.FakeGenericValidator + name string + errorString string + expectErrCount int + }{ + { + name: "valid rewriteClientIP", + validator: createValidValidator(), + np: &ngfAPI.NginxProxy{ + Spec: ngfAPI.NginxProxySpec{ + RewriteClientIP: &ngfAPI.RewriteClientIP{ + SetIPRecursively: helpers.GetPointer(true), + TrustedAddresses: []ngfAPI.Address{ + { + Type: ngfAPI.AddressTypeCIDR, + Value: "2001:db8:a0b:12f0::1/32", + }, + { + Type: ngfAPI.AddressTypeCIDR, + Value: "10.56.32.11/32", + }, + }, + Mode: helpers.GetPointer(ngfAPI.RewriteClientIPModeProxyProtocol), + }, + }, + }, + expectErrCount: 0, + }, + { + name: "invalid CIDR in trustedAddresses", + validator: createInvalidValidator(), + np: &ngfAPI.NginxProxy{ + Spec: ngfAPI.NginxProxySpec{ + RewriteClientIP: &ngfAPI.RewriteClientIP{ + SetIPRecursively: helpers.GetPointer(true), + TrustedAddresses: []ngfAPI.Address{ + { + Type: ngfAPI.AddressTypeCIDR, + Value: "2001:db8::/129", + }, + { + Type: ngfAPI.AddressTypeCIDR, + Value: "10.0.0.1/32", + }, + }, + Mode: helpers.GetPointer(ngfAPI.RewriteClientIPModeProxyProtocol), + }, + }, + }, + expectErrCount: 1, + errorString: "spec.rewriteClientIP.trustedAddresses.2001:db8::/129: " + + "Invalid value: v1alpha1.Address{Type:\"cidr\", Value:\"2001:db8::/129\"}: " + + "spec.rewriteClientIP.trustedAddresses: Invalid value: " + + "\"2001:db8::/129\": must be a valid CIDR value, (e.g. 10.9.8.0/24 or 2001:db8::/64)", + }, + { + name: "invalid when mode is set and trustedAddresses is empty", + validator: createInvalidValidator(), + np: &ngfAPI.NginxProxy{ + Spec: ngfAPI.NginxProxySpec{ + RewriteClientIP: &ngfAPI.RewriteClientIP{ + Mode: helpers.GetPointer(ngfAPI.RewriteClientIPModeProxyProtocol), + }, + }, + }, + expectErrCount: 1, + errorString: "spec.rewriteClientIP: Required value: trustedAddresses field required when mode is set", + }, + { + name: "invalid when trustedAddresses is greater in length than 16", + validator: createInvalidValidator(), + np: &ngfAPI.NginxProxy{ + Spec: ngfAPI.NginxProxySpec{ + RewriteClientIP: &ngfAPI.RewriteClientIP{ + Mode: helpers.GetPointer(ngfAPI.RewriteClientIPModeProxyProtocol), + TrustedAddresses: []ngfAPI.Address{ + {Type: ngfAPI.AddressTypeCIDR, Value: "2001:db8:a0b:12f0::1/32"}, + {Type: ngfAPI.AddressTypeCIDR, Value: "2001:db8:a0b:12f0::1/32"}, + {Type: ngfAPI.AddressTypeCIDR, Value: "2001:db8:a0b:12f0::1/32"}, + {Type: ngfAPI.AddressTypeCIDR, Value: "2001:db8:a0b:12f0::1/32"}, + {Type: ngfAPI.AddressTypeCIDR, Value: "2001:db8:a0b:12f0::1/32"}, + {Type: ngfAPI.AddressTypeCIDR, Value: "2001:db8:a0b:12f0::1/32"}, + {Type: ngfAPI.AddressTypeCIDR, Value: "2001:db8:a0b:12f0::1/32"}, + {Type: ngfAPI.AddressTypeCIDR, Value: "2001:db8:a0b:12f0::1/32"}, + {Type: ngfAPI.AddressTypeCIDR, Value: "2001:db8:a0b:12f0::1/32"}, + {Type: ngfAPI.AddressTypeCIDR, Value: "2001:db8:a0b:12f0::1/32"}, + {Type: ngfAPI.AddressTypeCIDR, Value: "2001:db8:a0b:12f0::1/32"}, + {Type: ngfAPI.AddressTypeCIDR, Value: "2001:db8:a0b:12f0::1/32"}, + {Type: ngfAPI.AddressTypeCIDR, Value: "2001:db8:a0b:12f0::1/32"}, + {Type: ngfAPI.AddressTypeCIDR, Value: "2001:db8:a0b:12f0::1/32"}, + {Type: ngfAPI.AddressTypeCIDR, Value: "2001:db8:a0b:12f0::1/32"}, + {Type: ngfAPI.AddressTypeCIDR, Value: "2001:db8:a0b:12f0::1/32"}, + {Type: ngfAPI.AddressTypeCIDR, Value: "2001:db8:a0b:12f0::1/32"}, + {Type: ngfAPI.AddressTypeCIDR, Value: "2001:db8:a0b:12f0::1/32"}, + {Type: ngfAPI.AddressTypeCIDR, Value: "2001:db8:a0b:12f0::1/32"}, + {Type: ngfAPI.AddressTypeCIDR, Value: "2001:db8:a0b:12f0::1/32"}, + {Type: ngfAPI.AddressTypeCIDR, Value: "2001:db8:a0b:12f0::1/32"}, + }, + }, + }, + }, + expectErrCount: 1, + errorString: "spec.rewriteClientIP.trustedAddresses: Too long: may not be longer than 16", + }, + { + name: "invalid when mode is not proxyProtocol or XForwardedFor", + validator: createInvalidValidator(), + np: &ngfAPI.NginxProxy{ + Spec: ngfAPI.NginxProxySpec{ + RewriteClientIP: &ngfAPI.RewriteClientIP{ + Mode: helpers.GetPointer(ngfAPI.RewriteClientIPModeType("invalid")), + TrustedAddresses: []ngfAPI.Address{ + { + Type: ngfAPI.AddressTypeCIDR, + Value: "2001:db8:a0b:12f0::1/32", + }, + { + Type: ngfAPI.AddressTypeCIDR, + Value: "10.0.0.1/32", + }, + }, + }, + }, + }, + expectErrCount: 1, + errorString: "spec.rewriteClientIP.mode: Unsupported value: \"invalid\": " + + "supported values: \"ProxyProtocol\", \"XForwardedFor\"", + }, + { + name: "invalid when mode is not proxyProtocol or XForwardedFor and trustedAddresses is empty", + validator: createInvalidValidator(), + np: &ngfAPI.NginxProxy{ + Spec: ngfAPI.NginxProxySpec{ + RewriteClientIP: &ngfAPI.RewriteClientIP{ + Mode: helpers.GetPointer(ngfAPI.RewriteClientIPModeType("invalid")), + }, + }, + }, + expectErrCount: 2, + errorString: "[spec.rewriteClientIP: Required value: trustedAddresses field " + + "required when mode is set, spec.rewriteClientIP.mode: " + + "Unsupported value: \"invalid\": supported values: \"ProxyProtocol\", \"XForwardedFor\"]", + }, + { + name: "invalid address type in trustedAddresses", + validator: createInvalidValidator(), + np: &ngfAPI.NginxProxy{ + Spec: ngfAPI.NginxProxySpec{ + RewriteClientIP: &ngfAPI.RewriteClientIP{ + SetIPRecursively: helpers.GetPointer(true), + TrustedAddresses: []ngfAPI.Address{ + { + Type: ngfAPI.AddressType("invalid"), + Value: "2001:db8::/129", + }, + }, + Mode: helpers.GetPointer(ngfAPI.RewriteClientIPModeProxyProtocol), + }, + }, + }, + expectErrCount: 1, + errorString: "spec.rewriteClientIP.trustedAddresses.type: " + + "Unsupported value: \"invalid\": supported values: \"cidr\"", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + g := NewWithT(t) + + allErrs := validateRewriteClientIP(test.np) + g.Expect(allErrs).To(HaveLen(test.expectErrCount)) + if len(allErrs) > 0 { + g.Expect(allErrs.ToAggregate().Error()).To(Equal(test.errorString)) + } + }) + } +} diff --git a/site/content/how-to/monitoring/troubleshooting.md b/site/content/how-to/monitoring/troubleshooting.md index 4d4b266ad8..b1fc476890 100644 --- a/site/content/how-to/monitoring/troubleshooting.md +++ b/site/content/how-to/monitoring/troubleshooting.md @@ -83,56 +83,56 @@ You can see logs for a crashed or killed container by adding the `-p` flag to th 1. Container Logs - To see logs for the control plane container: + To see logs for the control plane container: - ```shell - kubectl -n nginx-gateway logs -c nginx-gateway - ``` + ```shell + kubectl -n nginx-gateway logs -c nginx-gateway + ``` - To see logs for the data plane container: + To see logs for the data plane container: - ```shell - kubectl -n nginx-gateway logs -c nginx - ``` + ```shell + kubectl -n nginx-gateway logs -c nginx + ``` 1. Error Logs - For the _nginx-gateway_ container, you can `grep` the logs for the word `error`: + For the _nginx-gateway_ container, you can `grep` the logs for the word `error`: - ```shell - kubectl -n nginx-gateway logs -c nginx-gateway | grep error - ``` + ```shell + kubectl -n nginx-gateway logs -c nginx-gateway | grep error + ``` - For example, an error message when telemetry is not enabled for NGINX Plus installations: + For example, an error message when telemetry is not enabled for NGINX Plus installations: - ```text - kubectl logs -n nginx-gateway nginx-gateway-nginx-gateway-fabric-77f8746996-j6z6v | grep error - Defaulted container "nginx-gateway" out of: nginx-gateway, nginx - {"level":"error","ts":"2024-06-13T18:22:16Z","logger":"usageReporter","msg":"Usage reporting must be enabled when using NGINX Plus; redeploy with usage reporting enabled","error":"usage reporting not enabled","stacktrace":"github.com/nginxinc/nginx-gateway-fabric/internal/mode/static.createUsageWarningJob.func1\n\tgithub.com/nginxinc/nginx-gateway-fabric/internal/mode/static/manager.go:616\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext.func1\n\tk8s.io/apimachinery@v0.30.1/pkg/util/wait/backoff.go:259\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\tk8s.io/apimachinery@v0.30.1/pkg/util/wait/backoff.go:226\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\tk8s.io/apimachinery@v0.30.1/pkg/util/wait/backoff.go:227\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\tk8s.io/apimachinery@v0.30.1/pkg/util/wait/backoff.go:204\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\tk8s.io/apimachinery@v0.30.1/pkg/util/wait/backoff.go:259\ngithub.com/nginxinc/nginx-gateway-fabric/internal/framework/runnables.(*CronJob).Start\n\tgithub.com/nginxinc/nginx-gateway-fabric/internal/framework/runnables/cronjob.go:53\nsigs.k8s.io/controller-runtime/pkg/manager.(*runnableGroup).reconcile.func1\n\tsigs.k8s.io/controller-runtime@v0.18.4/pkg/manager/runnable_group.go:226"} - ``` + ```text + kubectl logs -n nginx-gateway nginx-gateway-nginx-gateway-fabric-77f8746996-j6z6v | grep error + Defaulted container "nginx-gateway" out of: nginx-gateway, nginx + {"level":"error","ts":"2024-06-13T18:22:16Z","logger":"usageReporter","msg":"Usage reporting must be enabled when using NGINX Plus; redeploy with usage reporting enabled","error":"usage reporting not enabled","stacktrace":"github.com/nginxinc/nginx-gateway-fabric/internal/mode/static.createUsageWarningJob.func1\n\tgithub.com/nginxinc/nginx-gateway-fabric/internal/mode/static/manager.go:616\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext.func1\n\tk8s.io/apimachinery@v0.30.1/pkg/util/wait/backoff.go:259\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\tk8s.io/apimachinery@v0.30.1/pkg/util/wait/backoff.go:226\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\tk8s.io/apimachinery@v0.30.1/pkg/util/wait/backoff.go:227\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\tk8s.io/apimachinery@v0.30.1/pkg/util/wait/backoff.go:204\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\tk8s.io/apimachinery@v0.30.1/pkg/util/wait/backoff.go:259\ngithub.com/nginxinc/nginx-gateway-fabric/internal/framework/runnables.(*CronJob).Start\n\tgithub.com/nginxinc/nginx-gateway-fabric/internal/framework/runnables/cronjob.go:53\nsigs.k8s.io/controller-runtime/pkg/manager.(*runnableGroup).reconcile.func1\n\tsigs.k8s.io/controller-runtime@v0.18.4/pkg/manager/runnable_group.go:226"} + ``` - For the _nginx_ container you can `grep` for various [error](https://nginx.org/en/docs/ngx_core_module.html#error_log) logs. For example, to search for all logs logged at the `emerg` level: + For the _nginx_ container you can `grep` for various [error](https://nginx.org/en/docs/ngx_core_module.html#error_log) logs. For example, to search for all logs logged at the `emerg` level: - ```shell - kubectl -n nginx-gateway logs -c nginx | grep emerg - ``` + ```shell + kubectl -n nginx-gateway logs -c nginx | grep emerg + ``` - For example, if a variable is too long, NGINX may display such an error message: + For example, if a variable is too long, NGINX may display such an error message: - ```text - kubectl logs -n nginx-gateway ngf-nginx-gateway-fabric-bb8598998-jwk2m -c nginx | grep emerg - 2024/06/13 20:04:17 [emerg] 27#27: too long parameter, probably missing terminating """ character in /etc/nginx/conf.d/http.conf:78 - ``` + ```text + kubectl logs -n nginx-gateway ngf-nginx-gateway-fabric-bb8598998-jwk2m -c nginx | grep emerg + 2024/06/13 20:04:17 [emerg] 27#27: too long parameter, probably missing terminating """ character in /etc/nginx/conf.d/http.conf:78 + ``` 1. Access Logs - NGINX access logs record all requests processed by the NGINX server. These logs provide detailed information about each request, which can be useful for troubleshooting and analyzing web traffic. - Access logs can be viewed with the above method of using `kubectl logs`, or by viewing the access log file directly. To do that, get shell access to your NGINX container using these [steps](#get-shell-access-to-nginx-container). The access logs are located in the file `/var/log/nginx/access.log` in the NGINX container. + NGINX access logs record all requests processed by the NGINX server. These logs provide detailed information about each request, which can be useful for troubleshooting and analyzing web traffic. + Access logs can be viewed with the above method of using `kubectl logs`, or by viewing the access log file directly. To do that, get shell access to your NGINX container using these [steps](#get-shell-access-to-nginx-container). The access logs are located in the file `/var/log/nginx/access.log` in the NGINX container. 1. Modify Log Levels - To modify log levels for the control plane in NGINX Gateway Fabric, edit the `NginxGateway` configuration. This can be done either before or after deploying NGINX Gateway Fabric. Refer to this [guide](https://docs.nginx.com/nginx-gateway-fabric/how-to/configuration/control-plane-configuration) to do so. - To check error logs, modify the log level to `error` to view error logs. Similarly, change the log level to `debug` and `grep` for the word `debug` to view debug logs. + To modify log levels for the control plane in NGINX Gateway Fabric, edit the `NginxGateway` configuration. This can be done either before or after deploying NGINX Gateway Fabric. Refer to this [guide](https://docs.nginx.com/nginx-gateway-fabric/how-to/configuration/control-plane-configuration) to do so. + To check error logs, modify the log level to `error` to view error logs. Similarly, change the log level to `debug` and `grep` for the word `debug` to view debug logs. #### Understanding the generated NGINX configuration @@ -167,18 +167,18 @@ metadata: name: coffee spec: parentRefs: - - name: gateway - sectionName: http + - name: gateway + sectionName: http hostnames: - - "cafe.example.com" + - "cafe.example.com" rules: - - matches: - - path: - type: PathPrefix - value: /coffee - backendRefs: - - name: coffee - port: 80 + - matches: + - path: + type: PathPrefix + value: /coffee + backendRefs: + - name: coffee + port: 80 ``` The modified `nginx.conf`: @@ -231,7 +231,7 @@ upstream default_coffee_80 { Key information to note is: 1. A new `server` block is created with the hostname of the HTTPRoute. When a request is sent to this hostname, it will be handled by this `server` block. -2. Within the `server` block, three new `location` blocks are added for *coffee*, each with distinct prefix and exact paths. Requests directed to the *coffee* application with a path prefix `/coffee/hello` will be managed by the first location block, while those with an exact path `/coffee` will be handled by the second location block. Any other requests not recognized by the server block for this hostname will default to the third location block, returning a 404 Not Found status. +2. Within the `server` block, three new `location` blocks are added for _coffee_, each with distinct prefix and exact paths. Requests directed to the _coffee_ application with a path prefix `/coffee/hello` will be managed by the first location block, while those with an exact path `/coffee` will be handled by the second location block. Any other requests not recognized by the server block for this hostname will default to the third location block, returning a 404 Not Found status. 3. Each `location` block has headers and directives that configure the NGINX proxy to forward requests to the `/coffee` path correctly, preserving important client information and ensuring compatibility with the upstream server. 4. The `upstream` block in the given NGINX configuration defines a group of backend servers and configures how NGINX should load balance requests among them. @@ -294,19 +294,19 @@ Verify that the port number (for example, `8080`) matches the port number you ha ### Common errors {{< bootstrap-table "table table-striped table-bordered" >}} -| Problem Area | Symptom | Troubleshooting Method | Common Cause | +| Problem Area | Symptom | Troubleshooting Method | Common Cause | |------------------------------|----------------------------------------|---------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------| -| Startup | NGINX Gateway Fabric fails to start. | Check logs for _nginx_ and _nginx-gateway_ containers. | Readiness probe failed. | -| Resources not configured | Status missing on resources. | Check referenced resources. | Referenced resources do not belong to NGINX Gateway Fabric. | -| NGINX errors | Reload failures on NGINX | Fix permissions for control plane. | Security context not configured. | -| Usage reporting | Errors logs related to usage reporting | Enable usage reporting. Refer to [Usage Reporting]({{< relref "installation/usage-reporting.md" >}}) | Usage reporting disabled. | -| Client Settings | Request entity too large error | Adjust client settings. Refer to [Client Settings Policy]({{< relref "../traffic-management/client-settings.md" >}}) | Payload is greater than the [`client_max_body_size`](https://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size) value.| +| Startup | NGINX Gateway Fabric fails to start. | Check logs for _nginx_ and _nginx-gateway_ containers. | Readiness probe failed. | +| Resources not configured | Status missing on resources. | Check referenced resources. | Referenced resources do not belong to NGINX Gateway Fabric. | +| NGINX errors | Reload failures on NGINX | Fix permissions for control plane. | Security context not configured. | +| Usage reporting | Errors logs related to usage reporting | Enable usage reporting. Refer to [Usage Reporting]({{< relref "installation/usage-reporting.md" >}}) | Usage reporting disabled. | +| Client Settings | Request entity too large error | Adjust client settings. Refer to [Client Settings Policy]({{< relref "../traffic-management/client-settings.md" >}}) | Payload is greater than the [`client_max_body_size`](https://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size) value.| {{< /bootstrap-table >}} ##### NGINX fails to reload NGINX reload errors can occur for various reasons, including syntax errors in configuration files, permission issues, and more. To determine if NGINX has failed to reload, check logs for your _nginx-gateway_ and _nginx_ containers. -You will see the following error in the _nginx-gateway_ logs: `failed to reload NGINX:`, followed by the reason for the failure. Similarly, error logs in _nginx_ container start with `emerg`. For example, `2024/06/12 14:25:11 [emerg] 12345#0: open() "/var/run/nginx.pid" failed (13: Permission denied)` shows a critical error, such as a permission problem preventing NGINX from accessing necessary files. +You will see the following error in the _nginx-gateway_ logs: `failed to reload NGINX:`, followed by the reason for the failure. Similarly, error logs in _nginx_ container start with `emerg`. For example, `2024/06/12 14:25:11 [emerg] 12345#0: open() "/var/run/nginx.pid" failed (13: Permission denied)` shows a critical error, such as a permission problem preventing NGINX from accessing necessary files. To debug why your reload has failed, start with verifying the syntax of your configuration files by opening a shell in the NGINX container following these [steps](#get-shell-access-to-nginx-container) and running `nginx -T`. If there are errors in your configuration file, the reload will fail and specify the reason for it. @@ -455,6 +455,20 @@ This means you are attempting to attach a Policy to a Route that has an overlapp - Combine the Route rules for the overlapping path into a single Route. - If the Policy allows it, specify both Routes in the `targetRefs` list. +##### Broken Header error + +If you check your _nginx_ container logs and see the following error: + +```text + 2024/07/25 00:50:45 [error] 211#211: *22 broken header: "GET /coffee HTTP/1.1" while reading PROXY protocol, client: 127.0.0.1, server: 0.0.0.0:80 +``` + +It indicates that `proxy_protocol` is enabled for the gateway listeners, but the request sent to the application endpoint does not contain proxy information. To **resolve** this, you can do one of the following: + +- Unassign the field [`rewriteClientIP.mode`](({{< relref "reference/api.md" >}})) in the NginxProxy configuration. + +- Send valid proxy information with requests being handled by your application. + ### Further reading You can view the [Kubernetes Troubleshooting Guide](https://kubernetes.io/docs/tasks/debug/debug-application/) for more debugging guidance. diff --git a/site/content/reference/api.md b/site/content/reference/api.md index d9dd3d2de2..d5b191193b 100644 --- a/site/content/reference/api.md +++ b/site/content/reference/api.md @@ -329,6 +329,20 @@ Telemetry +rewriteClientIP
+ + +RewriteClientIP + + + + +(Optional) +

RewriteClientIP defines configuration for rewriting the client IP to the original client’s IP.

+ + + + disableHTTP2
bool @@ -453,6 +467,76 @@ sigs.k8s.io/gateway-api/apis/v1alpha2.PolicyStatus +

Address + +

+

+(Appears on: +RewriteClientIP) +

+

+

Address is a struct that specifies address type and value.

+

+ + + + + + + + + + + + + + + + + +
FieldDescription
+type
+ + +AddressType + + +		 +(Optional) +

Type specifies the type of address. +Default is “cidr” which specifies that the address is a CIDR block.

+
+value
+ +string + +		 +(Optional) +

Value specifies the address value.

+
+

AddressType +(string alias)

+

+

+(Appears on: +Address) +

+

+

AddressType specifies the type of address.

+

+ + + + + + + + + + +
ValueDescription

"cidr"

AddressTypeCIDR specifies that the address is a CIDR block. +kubebuilder:validation:Pattern=^[\.a-zA-Z0-9:]*(\/([0-9]?[0-9]?[0-9]))$

+

ClientBody

@@ -951,6 +1035,20 @@ Telemetry +rewriteClientIP
+ + +RewriteClientIP + + + + +(Optional) +

RewriteClientIP defines configuration for rewriting the client IP to the original client’s IP.

+ + + + disableHTTP2
bool @@ -1013,6 +1111,116 @@ Support: HTTPRoute, GRPCRoute.

+

RewriteClientIP + +

+

+(Appears on: +NginxProxySpec) +

+

+

RewriteClientIP specifies the configuration for rewriting the client’s IP address.

+

+ + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+mode
+ + +RewriteClientIPModeType + + +		 +(Optional) +

Mode defines how NGINX will rewrite the client’s IP address. +There are two possible modes: +- ProxyProtocol: NGINX will rewrite the client’s IP using the PROXY protocol header. +- XForwardedFor: NGINX will rewrite the client’s IP using the X-Forwarded-For header. +Sets NGINX directive real_ip_header: https://nginx.org/en/docs/http/ngx_http_realip_module.html#real_ip_header

+
+setIPRecursively
+ +bool + +		 +(Optional) +

SetIPRecursively configures whether recursive search is used when selecting the client’s address from +the X-Forwarded-For header. It is used in conjunction with TrustedAddresses. +If enabled, NGINX will recurse on the values in X-Forwarded-Header from the end of array +to start of array and select the first untrusted IP. +For example, if X-Forwarded-For is [11.11.11.11, 22.22.22.22, 55.55.55.1], +and TrustedAddresses is set to 55.55.55.1⁄32, NGINX will rewrite the client IP to 22.22.22.22. +If disabled, NGINX will select the IP at the end of the array. +In the previous example, 55.55.55.1 would be selected. +Sets NGINX directive real_ip_recursive: https://nginx.org/en/docs/http/ngx_http_realip_module.html#real_ip_recursive

+
+trustedAddresses
+ + +[]Address + + +		 +(Optional) +

TrustedAddresses specifies the addresses that are trusted to send correct client IP information. +If a request comes from a trusted address, NGINX will rewrite the client IP information, +and forward it to the backend in the X-Forwarded-For* and X-Real-IP headers. +If the request does not come from a trusted address, NGINX will not rewrite the client IP information. +TrustedAddresses only supports CIDR blocks: 192.33.21.1⁄24, fe80::1⁄64. +To trust all addresses (not recommended for production), set to 0.0.0.0/0. +If no addresses are provided, NGINX will not rewrite the client IP information. +Sets NGINX directive set_real_ip_from: https://nginx.org/en/docs/http/ngx_http_realip_module.html#set_real_ip_from +This field is required if mode is set.

+
+

RewriteClientIPModeType +(string alias)

+

+

+(Appears on: +RewriteClientIP) +

+

+

RewriteClientIPModeType defines how NGINX Gateway Fabric will determine the client’s original IP address.

+

+ + + + + + + + + + + + +
ValueDescription

"ProxyProtocol"

RewriteClientIPModeProxyProtocol configures NGINX to accept PROXY protocol and +set the client’s IP address to the IP address in the PROXY protocol header. +Sets the proxy_protocol parameter on the listen directive of all servers and sets real_ip_header +to proxy_protocol: https://nginx.org/en/docs/http/ngx_http_realip_module.html#real_ip_header.

+

"XForwardedFor"

RewriteClientIPModeXForwardedFor configures NGINX to set the client’s IP address to the +IP address in the X-Forwarded-For HTTP header. +https://nginx.org/en/docs/http/ngx_http_realip_module.html#real_ip_header.

+

Size (string alias)