Add support for NGINX Service Mesh internal routes

cmd/nginx-ingress/main.go

@@ -152,9 +152,12 @@ var (
 		"Enable TLS Passthrough on port 443. Requires -enable-custom-resources")
 
 	spireAgentAddress = flag.String("spire-agent-address", "",
-		`Specifies the address of the running Spire agent. For use with NGINX Service Mesh only. If the flag is set,
+		`Specifies the address of the running Spire agent. Requires -nginx-plus and is for use with NGINX Service Mesh only. If the flag is set,
 			but the Ingress Controller is not able to connect with the Spire Agent, the Ingress Controller will fail to start.`)
 
+	enableInternalRoutes = flag.Bool("enable-internal-routes", false,
+		`Enable support for internal routes with NGINX Service Mesh. Requires -spire-agent-address and -nginx-plus. Is for use with NGINX Service Mesh only.`)
+
 	readyStatus = flag.Bool("ready-status", true, "Enables the readiness endpoint '/nginx-ready'. The endpoint returns a success code when NGINX has loaded all the config after the startup")
 
 	readyStatusPort = flag.Int("ready-status-port", 8081, "Set the port where the readiness endpoint is exposed. [1024 - 65535]")
@@ -211,6 +214,14 @@ func main() {
 		glog.Fatal("NGINX App Protect support is for NGINX Plus only")
 	}
 
+	if *spireAgentAddress != "" && !*nginxPlus {
+		glog.Fatal("spire-agent-address support is for NGINX Plus only")
+	}
+
+	if *enableInternalRoutes && *spireAgentAddress == "" {
+		glog.Fatal("enable-internal-routes flag requires spire-agent-address")
+	}
+
 	glog.Infof("Starting NGINX Ingress controller Version=%v GitCommit=%v\n", version, gitCommit)
 
 	var config *rest.Config
@@ -519,6 +530,7 @@ func main() {
 		GlobalConfigurationValidator: globalConfigurationValidator,
 		TransportServerValidator:     transportServerValidator,
 		SpireAgentAddress:            *spireAgentAddress,
+		InternalRoutesEnabled:        *enableInternalRoutes,
 	}
 
 	lbc := k8s.NewLoadBalancerController(lbcInput)

docs-web/configuration/global-configuration/command-line-arguments.md

@@ -178,9 +178,17 @@ Below we describe the available command-line arguments:
 .. option:: -spire-agent-address <string>
 
 	Specifies the address of a running Spire agent. **For use with NGINX Service Mesh only**.
+    Requires :option:`-nginx-plus`.
 
 	- If the argument is set, but the Ingress Controller is unable to connect to the Spire Agent, the Ingress Controller will fail to start.
 
+.. option:: -enable-internal-routes
+
+	Enable support for internal routes with NGINX Service Mesh. **For use with NGINX Service Mesh only**.
+    Requires :option:`-nginx-plus` and :option:`-spire-agent-address`.
+
+    - If the argument is set, but `nginx-plus` is set to false, or the `spire-agent-address` is not provided, the Ingress Controller will fail to start.
+
 .. option:: -enable-app-protect
 
 	 Enables support for App Protect.

internal/configs/annotations.go

@@ -20,6 +20,9 @@ const AppProtectLogConfAnnotation = "appprotect.f5.com/app-protect-security-log"
 // AppProtectLogConfDstAnnotation is where the NGINX AppProtect Log Configuration is specified
 const AppProtectLogConfDstAnnotation = "appprotect.f5.com/app-protect-security-log-destination"
 
+// nginxMeshInternalRoute specifies if the ingress resource is an internal route.
+const nginxMeshInternalRouteAnnotation = "nsm.nginx.com/internal-route"
+
 var masterBlacklist = map[string]bool{
 	"nginx.org/rewrites":                      true,
 	"nginx.org/ssl-services":                  true,
@@ -67,7 +70,7 @@ var minionInheritanceList = map[string]bool{
 	"nginx.org/fail-timeout":             true,
 }
 
-func parseAnnotations(ingEx *IngressEx, baseCfgParams *ConfigParams, isPlus bool, hasAppProtect bool) ConfigParams {
+func parseAnnotations(ingEx *IngressEx, baseCfgParams *ConfigParams, isPlus bool, hasAppProtect bool, enableInternalRoutes bool) ConfigParams {
 	cfgParams := *baseCfgParams
 
 	if lbMethod, exists := ingEx.Ingress.Annotations["nginx.org/lb-method"]; exists {
@@ -347,6 +350,15 @@ func parseAnnotations(ingEx *IngressEx, baseCfgParams *ConfigParams, isPlus bool
 		}
 
 	}
+	if enableInternalRoutes {
+		if spiffeServerCerts, exists, err := GetMapKeyAsBool(ingEx.Ingress.Annotations, nginxMeshInternalRouteAnnotation, ingEx.Ingress); exists {
+			if err != nil {
+				glog.Error(err)
+			} else {
+				cfgParams.SpiffeServerCerts = spiffeServerCerts
+			}
+		}
+	}
 	return cfgParams
 }
 

internal/configs/config_params.go

@@ -96,7 +96,7 @@ type ConfigParams struct {
 	Ports    []int
 	SSLPorts []int
 
-	SpiffeCerts bool
+	SpiffeServerCerts bool
 }
 
 // StaticConfigParams holds immutable NGINX configuration parameters that affect the main NGINX config.
@@ -110,7 +110,9 @@ type StaticConfigParams struct {
 	TLSPassthrough                 bool
 	EnableSnippets                 bool
 	SpiffeCerts                    bool
+	EnableInternalRoutes           bool
 	MainAppProtectLoadModule       bool
+	PodName                        string
 }
 
 // GlobalConfigParams holds global configuration parameters. For now, it only holds listeners.

internal/configs/configmaps.go

@@ -4,8 +4,9 @@ import (
 	"strings"
 
 	"github.com/golang/glog"
-	"github.com/nginxinc/kubernetes-ingress/internal/configs/version1"
 	v1 "k8s.io/api/core/v1"
+
+	"github.com/nginxinc/kubernetes-ingress/internal/configs/version1"
 )
 
 // ParseConfigMap parses ConfigMap into ConfigParams.
@@ -542,6 +543,8 @@ func GenerateNginxMainConfig(staticCfgParams *StaticConfigParams, config *Config
 		AppProtectCookieSeed:               config.MainAppProtectCookieSeed,
 		AppProtectCPUThresholds:            config.MainAppProtectCPUThresholds,
 		AppProtectPhysicalMemoryThresholds: config.MainAppProtectPhysicalMemoryThresholds,
+		InternalRouteServer:                staticCfgParams.EnableInternalRoutes,
+		InternalRouteServerName:            staticCfgParams.PodName,
 	}
 	return nginxCfg
 }

internal/configs/configurator.go

@@ -693,7 +693,7 @@ func (cnf *Configurator) updatePlusEndpointsForTransportServer(transportServerEx
 }
 
 func (cnf *Configurator) updatePlusEndpoints(ingEx *IngressEx) error {
-	ingCfg := parseAnnotations(ingEx, cnf.cfgParams, cnf.isPlus, cnf.staticCfgParams.MainAppProtectLoadModule)
+	ingCfg := parseAnnotations(ingEx, cnf.cfgParams, cnf.isPlus, cnf.staticCfgParams.MainAppProtectLoadModule, cnf.staticCfgParams.EnableInternalRoutes)
 
 	cfg := nginx.ServerConfig{
 		MaxFails:    ingCfg.MaxFails,
@@ -990,7 +990,7 @@ func (cnf *Configurator) updateApResources(ingEx *IngressEx) map[string]string {
 		policyContent := generateApResourceFileContent(ingEx.AppProtectPolicy)
 		cnf.nginxManager.CreateAppProtectResourceFile(policyFileName, policyContent)
 		apRes[appProtectPolicyKey] = policyFileName
-	
+
 	}
 
 	if ingEx.AppProtectLogConf != nil {
@@ -1094,3 +1094,20 @@ func (cnf *Configurator) DeleteAppProtectLogConf(logConfNamespaceame string, ing
 
 	return nil
 }
+
+// AddInternalRouteConfig adds internal route server to NGINX Configuration and
+// reloads NGINX
+func (cnf *Configurator) AddInternalRouteConfig() error {
+	cnf.staticCfgParams.EnableInternalRoutes = true
+	cnf.staticCfgParams.PodName = os.Getenv("POD_NAME")
+	mainCfg := GenerateNginxMainConfig(cnf.staticCfgParams, cnf.cfgParams)
+	mainCfgContent, err := cnf.templateExecutor.ExecuteMainConfigTemplate(mainCfg)
+	if err != nil {
+		return fmt.Errorf("Error when writing main Config: %v", err)
+	}
+	cnf.nginxManager.CreateMainConfig(mainCfgContent)
+	if err := cnf.nginxManager.Reload(); err != nil {
+		return fmt.Errorf("Error when reloading nginx: %v", err)
+	}
+	return nil
+}

internal/configs/configurator_test.go

@@ -1,6 +1,7 @@
 package configs
 
 import (
+	"os"
 	"reflect"
 	"testing"
 
@@ -428,3 +429,26 @@ func TestGenerateTLSPassthroughHostsConfig(t *testing.T) {
 		t.Errorf("generateTLSPassthroughHostsConfig() returned %v but expected %v", resultDuplicatedHosts, expectedDuplicatedHosts)
 	}
 }
+
+func TestAddInternalRouteConfig(t *testing.T) {
+	cnf, err := createTestConfigurator()
+	if err != nil {
+		t.Errorf("Failed to create a test configurator: %v", err)
+	}
+	// set pod name in env
+	err = os.Setenv("POD_NAME", "nginx-ingress")
+	if err != nil {
+		t.Errorf("Failed to set pod name in environment: %v", err)
+	}
+	err = cnf.AddInternalRouteConfig()
+	if err != nil {
+		t.Errorf("AddInternalRouteConfig returned:  \n%v, but expected: \n%v", err, nil)
+	}
+
+	if !cnf.staticCfgParams.EnableInternalRoutes {
+		t.Errorf("AddInternalRouteConfig failed to set EnableInteralRoutes field of staticCfgParams to true")
+	}
+	if cnf.staticCfgParams.PodName != "nginx-ingress" {
+		t.Errorf("AddInternalRouteConfig failed to set PodName field of staticCfgParams")
+	}
+}

internal/configs/ingress.go

@@ -9,8 +9,9 @@ import (
 	api_v1 "k8s.io/api/core/v1"
 	extensions "k8s.io/api/extensions/v1beta1"
 
-	"github.com/nginxinc/kubernetes-ingress/internal/configs/version1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+
+	"github.com/nginxinc/kubernetes-ingress/internal/configs/version1"
 )
 
 const emptyHost = ""
@@ -52,7 +53,7 @@ type MergeableIngresses struct {
 
 func generateNginxCfg(ingEx *IngressEx, pems map[string]string, apResources map[string]string, isMinion bool, baseCfgParams *ConfigParams, isPlus bool, isResolverConfigured bool, jwtKeyFileName string, staticParams *StaticConfigParams) version1.IngressNginxConfig {
 	hasAppProtect := staticParams.MainAppProtectLoadModule
-	cfgParams := parseAnnotations(ingEx, baseCfgParams, isPlus, hasAppProtect)
+	cfgParams := parseAnnotations(ingEx, baseCfgParams, isPlus, hasAppProtect, staticParams.EnableInternalRoutes)
 
 	wsServices := getWebsocketServices(ingEx)
 	spServices := getSessionPersistenceServices(ingEx)
@@ -116,6 +117,7 @@ func generateNginxCfg(ingEx *IngressEx, pems map[string]string, apResources map[
 			TLSPassthrough:        staticParams.TLSPassthrough,
 			AppProtectEnable:      cfgParams.AppProtectEnable,
 			AppProtectLogEnable:   cfgParams.AppProtectLogEnable,
+			SpiffeCerts:           cfgParams.SpiffeServerCerts,
 		}
 
 		if pemFile, ok := pems[serverName]; ok {
@@ -179,7 +181,7 @@ func generateNginxCfg(ingEx *IngressEx, pems map[string]string, apResources map[
 				upstreams[upsName] = upstream
 			}
 
-			ssl := sslServices[path.Backend.ServiceName] || staticParams.SpiffeCerts
+			ssl := isSSLEnabled(sslServices[path.Backend.ServiceName], cfgParams, staticParams)
 			proxySSLName := generateProxySSLName(path.Backend.ServiceName, ingEx.Ingress.Namespace)
 			loc := createLocation(pathOrDefault(path.Path), upstreams[upsName], &cfgParams, wsServices[path.Backend.ServiceName], rewrites[path.Backend.ServiceName],
 				ssl, grpcServices[path.Backend.ServiceName], proxySSLName)
@@ -207,7 +209,7 @@ func generateNginxCfg(ingEx *IngressEx, pems map[string]string, apResources map[
 
 		if !rootLocation && ingEx.Ingress.Spec.Backend != nil {
 			upsName := getNameForUpstream(ingEx.Ingress, emptyHost, ingEx.Ingress.Spec.Backend)
-			ssl := sslServices[ingEx.Ingress.Spec.Backend.ServiceName] || staticParams.SpiffeCerts
+			ssl := isSSLEnabled(sslServices[ingEx.Ingress.Spec.Backend.ServiceName], cfgParams, staticParams)
 			proxySSLName := generateProxySSLName(ingEx.Ingress.Spec.Backend.ServiceName, ingEx.Ingress.Namespace)
 
 			loc := createLocation(pathOrDefault("/"), upstreams[upsName], &cfgParams, wsServices[ingEx.Ingress.Spec.Backend.ServiceName], rewrites[ingEx.Ingress.Spec.Backend.ServiceName],
@@ -246,7 +248,7 @@ func generateNginxCfg(ingEx *IngressEx, pems map[string]string, apResources map[
 			Namespace:   ingEx.Ingress.Namespace,
 			Annotations: ingEx.Ingress.Annotations,
 		},
-		SpiffeCerts: staticParams.SpiffeCerts,
+		SpiffeClientCerts: staticParams.SpiffeCerts && !cfgParams.SpiffeServerCerts,
 	}
 }
 
@@ -451,10 +453,14 @@ func generateNginxCfgForMergeableIngresses(mergeableIngs *MergeableIngresses, ma
 	masterServer.Locations = locations
 
 	return version1.IngressNginxConfig{
-		Servers:     []version1.Server{masterServer},
-		Upstreams:   upstreams,
-		Keepalive:   keepalive,
-		Ingress:     masterNginxCfg.Ingress,
-		SpiffeCerts: staticParams.SpiffeCerts,
+		Servers:           []version1.Server{masterServer},
+		Upstreams:         upstreams,
+		Keepalive:         keepalive,
+		Ingress:           masterNginxCfg.Ingress,
+		SpiffeClientCerts: staticParams.SpiffeCerts && !baseCfgParams.SpiffeServerCerts,
 	}
 }
+
+func isSSLEnabled(isSSLService bool, cfgParams ConfigParams, staticCfgParams *StaticConfigParams) bool {
+	return isSSLService || staticCfgParams.SpiffeCerts && !cfgParams.SpiffeServerCerts
+}