fix: Fix reload errors due long matching conditions (nginx#1829)

Add `js_preload_directive` to read http match conditions

Problem: A large number of match conditions for a single hostname/path cause reload errors with NGINX about being parameter being too long

Solution: The solution is to use js_preload_objects to store the http_matches and then parse them through NGINX using NJS to map the correct match condition.

Author: miledxz

internal/mode/static/nginx/config/generator.go

@@ -23,6 +23,9 @@ const (
 
 	// configVersionFile is the path to the config version configuration file.
 	configVersionFile = httpFolder + "/config-version.conf"
+
+	// httpMatchVarsFile is the path to the http_match pairs configuration file.
+	httpMatchVarsFile = httpFolder + "/matches.json"
 )
 
 // ConfigFolders is a list of folders where NGINX configuration files are stored.
@@ -52,8 +55,13 @@ func NewGeneratorImpl(plus bool) GeneratorImpl {
 	return GeneratorImpl{plus: plus}
 }
 
+type executeResult struct {
+	dest string
+	data []byte
+}
+
 // executeFunc is a function that generates NGINX configuration from internal representation.
-type executeFunc func(configuration dataplane.Configuration) []byte
+type executeFunc func(configuration dataplane.Configuration) []executeResult
 
 // Generate generates NGINX configuration files from internal representation.
 // It is the responsibility of the caller to validate the configuration before calling this function.
@@ -66,7 +74,7 @@ func (g GeneratorImpl) Generate(conf dataplane.Configuration) []file.File {
 		files = append(files, generatePEM(id, pair.Cert, pair.Key))
 	}
 
-	files = append(files, g.generateHTTPConfig(conf))
+	files = append(files, g.generateHTTPConfig(conf)...)
 
 	files = append(files, generateConfigVersion(conf.Version))
 
@@ -106,24 +114,33 @@ func generateCertBundleFileName(id dataplane.CertBundleID) string {
 	return filepath.Join(secretsFolder, string(id)+".crt")
 }
 
-func (g GeneratorImpl) generateHTTPConfig(conf dataplane.Configuration) file.File {
-	var c []byte
+func (g GeneratorImpl) generateHTTPConfig(conf dataplane.Configuration) []file.File {
+	fileBytes := make(map[string][]byte)
+
 	for _, execute := range g.getExecuteFuncs() {
-		c = append(c, execute(conf)...)
+		results := execute(conf)
+		for _, res := range results {
+			fileBytes[res.dest] = append(fileBytes[res.dest], res.data...)
+		}
 	}
 
-	return file.File{
-		Content: c,
-		Path:    httpConfigFile,
-		Type:    file.TypeRegular,
+	files := make([]file.File, 0, len(fileBytes))
+	for filepath, bytes := range fileBytes {
+		files = append(files, file.File{
+			Path:    filepath,
+			Content: bytes,
+			Type:    file.TypeRegular,
+		})
 	}
+
+	return files
 }
 
 func (g GeneratorImpl) getExecuteFuncs() []executeFunc {
 	return []executeFunc{
+		executeServers,
 		g.executeUpstreams,
 		executeSplitClients,
-		executeServers,
 		executeMaps,
 	}
 }

internal/mode/static/nginx/config/generator_test.go

@@ -2,6 +2,7 @@ package config_test
 
 import (
 	"fmt"
+	"sort"
 	"testing"
 
 	. "github.com/onsi/gomega"
@@ -70,13 +71,16 @@ func TestGenerate(t *testing.T) {
 
 	files := generator.Generate(conf)
 
-	g.Expect(files).To(HaveLen(4))
+	g.Expect(files).To(HaveLen(5))
+	arrange := func(i, j int) bool {
+		return files[i].Path < files[j].Path
+	}
+	sort.Slice(files, arrange)
 
-	g.Expect(files[0]).To(Equal(file.File{
-		Type:    file.TypeSecret,
-		Path:    "/etc/nginx/secrets/test-keypair.pem",
-		Content: []byte("test-cert\ntest-key"),
-	}))
+	g.Expect(files[0].Type).To(Equal(file.TypeRegular))
+	g.Expect(files[0].Path).To(Equal("/etc/nginx/conf.d/config-version.conf"))
+	configVersion := string(files[0].Content)
+	g.Expect(configVersion).To(ContainSubstring(fmt.Sprintf("return 200 %d", conf.Version)))
 
 	g.Expect(files[1].Type).To(Equal(file.TypeRegular))
 	g.Expect(files[1].Path).To(Equal("/etc/nginx/conf.d/http.conf"))
@@ -88,12 +92,18 @@ func TestGenerate(t *testing.T) {
 	g.Expect(httpCfg).To(ContainSubstring("upstream"))
 	g.Expect(httpCfg).To(ContainSubstring("split_clients"))
 
+	g.Expect(files[2].Path).To(Equal("/etc/nginx/conf.d/matches.json"))
 	g.Expect(files[2].Type).To(Equal(file.TypeRegular))
-	g.Expect(files[2].Path).To(Equal("/etc/nginx/conf.d/config-version.conf"))
-	configVersion := string(files[2].Content)
-	g.Expect(configVersion).To(ContainSubstring(fmt.Sprintf("return 200 %d", conf.Version)))
+	expString := "{}"
+	g.Expect(string(files[2].Content)).To(Equal(expString))
 
 	g.Expect(files[3].Path).To(Equal("/etc/nginx/secrets/test-certbundle.crt"))
 	certBundle := string(files[3].Content)
 	g.Expect(certBundle).To(Equal("test-cert"))
+
+	g.Expect(files[4]).To(Equal(file.File{
+		Type:    file.TypeSecret,
+		Path:    "/etc/nginx/secrets/test-keypair.pem",
+		Content: []byte("test-cert\ntest-key"),
+	}))
 }

internal/mode/static/nginx/config/http/config.go

@@ -12,13 +12,13 @@ type Server struct {
 
 // Location holds all configuration for an HTTP location.
 type Location struct {
-	Return          *Return
-	ProxySSLVerify  *ProxySSLVerify
 	Path            string
 	ProxyPass       string
-	HTTPMatchVar    string
-	Rewrites        []string
+	HTTPMatchKey    string
 	ProxySetHeaders []Header
+	ProxySSLVerify  *ProxySSLVerify
+	Return          *Return
+	Rewrites        []string
 }
 
 // Header defines a HTTP header to be passed to the proxied server.

internal/mode/static/nginx/config/maps.go

@@ -10,9 +10,13 @@ import (
 
 var mapsTemplate = gotemplate.Must(gotemplate.New("maps").Parse(mapsTemplateText))
 
-func executeMaps(conf dataplane.Configuration) []byte {
+func executeMaps(conf dataplane.Configuration) []executeResult {
 	maps := buildAddHeaderMaps(append(conf.HTTPServers, conf.SSLServers...))
-	return execute(mapsTemplate, maps)
+	result := executeResult{
+		dest: httpConfigFile,
+		data: execute(mapsTemplate, maps),
+	}
+	return []executeResult{result}
 }
 
 func buildAddHeaderMaps(servers []dataplane.VirtualServer) []http.Map {

internal/mode/static/nginx/config/maps_test.go

@@ -88,7 +88,10 @@ func TestExecuteMaps(t *testing.T) {
 		"map $http_upgrade $connection_upgrade {":                             1,
 	}
 
-	maps := string(executeMaps(conf))
+	mapResult := executeMaps(conf)
+	g.Expect(mapResult).To(HaveLen(1))
+	maps := string(mapResult[0].data)
+	g.Expect(mapResult[0].dest).To(Equal(httpConfigFile))
 	for expSubStr, expCount := range expSubStrings {
 		g.Expect(expCount).To(Equal(strings.Count(maps, expSubStr)))
 	}