diff --git a/.changelog/14749.txt b/.changelog/14749.txt new file mode 100644 index 00000000000..6cee0e98819 --- /dev/null +++ b/.changelog/14749.txt @@ -0,0 +1,3 @@ +```release-note:feature +config-entry(ingress-gateway): Added support for `max_connections` for upstream clusters +``` diff --git a/agent/proxycfg/ingress_gateway.go b/agent/proxycfg/ingress_gateway.go index c19a298caa4..652f8c85e71 100644 --- a/agent/proxycfg/ingress_gateway.go +++ b/agent/proxycfg/ingress_gateway.go @@ -94,6 +94,9 @@ func (s *handlerIngressGateway) handleUpdate(ctx context.Context, u cache.Update snap.IngressGateway.GatewayConfigLoaded = true snap.IngressGateway.TLSConfig = gatewayConf.TLS + if gatewayConf.Defaults != nil { + snap.IngressGateway.Defaults = *gatewayConf.Defaults + } // Load each listener's config from the config entry so we don't have to // pass listener config through "upstreams" types as that grows. diff --git a/agent/proxycfg/snapshot.go b/agent/proxycfg/snapshot.go index 7cf669a89d2..51084ac3bfe 100644 --- a/agent/proxycfg/snapshot.go +++ b/agent/proxycfg/snapshot.go @@ -387,6 +387,9 @@ type configSnapshotIngressGateway struct { // Listeners is the original listener config from the ingress-gateway config // entry to save us trying to pass fields through Upstreams Listeners map[IngressListenerKey]structs.IngressListener + + // Defaults is the default configuration for upstream service instances + Defaults structs.IngressServiceConfig } // isEmpty is a test helper diff --git a/agent/structs/config_entry_gateways.go b/agent/structs/config_entry_gateways.go index 0f891709899..26c4cb763ec 100644 --- a/agent/structs/config_entry_gateways.go +++ b/agent/structs/config_entry_gateways.go @@ -31,11 +31,20 @@ type IngressGatewayConfigEntry struct { // what services to associated to those ports. Listeners []IngressListener + // Defaults contains default configuration for all upstream service instances + Defaults *IngressServiceConfig `json:",omitempty"` + Meta map[string]string `json:",omitempty"` acl.EnterpriseMeta `hcl:",squash" mapstructure:",squash"` RaftIndex } +type IngressServiceConfig struct { + MaxConnections uint32 + MaxPendingRequests uint32 + MaxConcurrentRequests uint32 +} + type IngressListener struct { // Port declares the port on which the ingress gateway should listen for traffic. Port int @@ -90,6 +99,10 @@ type IngressService struct { RequestHeaders *HTTPHeaderModifiers `json:",omitempty" alias:"request_headers"` ResponseHeaders *HTTPHeaderModifiers `json:",omitempty" alias:"response_headers"` + MaxConnections uint32 `json:",omitempty" alias:"max_connections"` + MaxPendingRequests uint32 `json:",omitempty" alias:"max_pending_requests"` + MaxConcurrentRequests uint32 `json:",omitempty" alias:"max_concurrent_requests"` + Meta map[string]string `json:",omitempty"` acl.EnterpriseMeta `hcl:",squash" mapstructure:",squash"` } diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index c1e1abfa3d2..478f2c2caba 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -444,7 +444,7 @@ func (s *ResourceGenerator) injectGatewayServiceAddons(cfgSnap *proxycfg.ConfigS func (s *ResourceGenerator) clustersFromSnapshotIngressGateway(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) { var clusters []proto.Message createdClusters := make(map[proxycfg.UpstreamID]bool) - for _, upstreams := range cfgSnap.IngressGateway.Upstreams { + for listenerKey, upstreams := range cfgSnap.IngressGateway.Upstreams { for _, u := range upstreams { uid := proxycfg.NewUpstreamID(&u) @@ -472,6 +472,7 @@ func (s *ResourceGenerator) clustersFromSnapshotIngressGateway(cfgSnap *proxycfg } for _, c := range upstreamClusters { + s.configIngressUpstreamCluster(c, cfgSnap, listenerKey, &u) clusters = append(clusters, c) } createdClusters[uid] = true @@ -480,6 +481,50 @@ func (s *ResourceGenerator) clustersFromSnapshotIngressGateway(cfgSnap *proxycfg return clusters, nil } +func (s *ResourceGenerator) configIngressUpstreamCluster(c *envoy_cluster_v3.Cluster, cfgSnap *proxycfg.ConfigSnapshot, listenerKey proxycfg.IngressListenerKey, u *structs.Upstream) { + var threshold *envoy_cluster_v3.CircuitBreakers_Thresholds + setThresholdLimit := func(limitType string, limit int) { + if limit <= 0 { + return + } + + if threshold == nil { + threshold = &envoy_cluster_v3.CircuitBreakers_Thresholds{} + } + + switch limitType { + case "max_connections": + threshold.MaxConnections = makeUint32Value(limit) + case "max_pending_requests": + threshold.MaxPendingRequests = makeUint32Value(limit) + case "max_requests": + threshold.MaxRequests = makeUint32Value(limit) + } + } + + setThresholdLimit("max_connections", int(cfgSnap.IngressGateway.Defaults.MaxConnections)) + setThresholdLimit("max_pending_requests", int(cfgSnap.IngressGateway.Defaults.MaxPendingRequests)) + setThresholdLimit("max_requests", int(cfgSnap.IngressGateway.Defaults.MaxConcurrentRequests)) + + // Adjust the limit for upstream service + // Lookup listener and service config details from ingress gateway + // definition. + var svc *structs.IngressService + if lCfg, ok := cfgSnap.IngressGateway.Listeners[listenerKey]; ok { + svc = findIngressServiceMatchingUpstream(lCfg, *u) + } + + if svc != nil { + setThresholdLimit("max_connections", int(svc.MaxConnections)) + setThresholdLimit("max_pending_requests", int(svc.MaxPendingRequests)) + setThresholdLimit("max_requests", int(svc.MaxConcurrentRequests)) + } + + if threshold != nil { + c.CircuitBreakers.Thresholds = []*envoy_cluster_v3.CircuitBreakers_Thresholds{threshold} + } +} + func (s *ResourceGenerator) makeAppCluster(cfgSnap *proxycfg.ConfigSnapshot, name, pathProtocol string, port int) (*envoy_cluster_v3.Cluster, error) { var c *envoy_cluster_v3.Cluster var err error diff --git a/agent/xds/clusters_test.go b/agent/xds/clusters_test.go index 655d152512d..0bfc80e5819 100644 --- a/agent/xds/clusters_test.go +++ b/agent/xds/clusters_test.go @@ -489,6 +489,45 @@ func TestClustersFromSnapshot(t *testing.T) { "simple", nil, nil, nil) }, }, + { + name: "ingress-with-service-max-connections", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", + "simple", nil, + func(entry *structs.IngressGatewayConfigEntry) { + entry.Listeners[0].Services[0].MaxConnections = 4096 + }, nil) + }, + }, + { + name: "ingress-with-defaults-service-max-connections", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", + "simple", nil, + func(entry *structs.IngressGatewayConfigEntry) { + entry.Defaults = &structs.IngressServiceConfig{ + MaxConnections: 2048, + MaxPendingRequests: 512, + MaxConcurrentRequests: 4096, + } + }, nil) + }, + }, + { + name: "ingress-with-overwrite-defaults-service-max-connections", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", + "simple", nil, + func(entry *structs.IngressGatewayConfigEntry) { + entry.Defaults = &structs.IngressServiceConfig{ + MaxConnections: 2048, + MaxPendingRequests: 512, + } + entry.Listeners[0].Services[0].MaxConnections = 4096 + entry.Listeners[0].Services[0].MaxPendingRequests = 2048 + }, nil) + }, + }, { name: "ingress-with-chain-external-sni", create: func(t testinf.T) *proxycfg.ConfigSnapshot { diff --git a/agent/xds/testdata/clusters/ingress-with-defaults-service-max-connections.latest.golden b/agent/xds/testdata/clusters/ingress-with-defaults-service-max-connections.latest.golden new file mode 100644 index 00000000000..08d2c471e53 --- /dev/null +++ b/agent/xds/testdata/clusters/ingress-with-defaults-service-max-connections.latest.golden @@ -0,0 +1,71 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "altStatName": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": { + + }, + "resourceApiVersion": "V3" + } + }, + "connectTimeout": "33s", + "circuitBreakers": { + "thresholds":[ + { + "maxConnections": 2048, + "maxPendingRequests": 512, + "maxRequests": 4096 + } + ] + }, + "outlierDetection": { + + }, + "commonLbConfig": { + "healthyPanicThreshold": { + + } + }, + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": { + + }, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + }, + "matchSubjectAltNames": [ + { + "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/db" + } + ] + } + }, + "sni": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + } + ], + "typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/clusters/ingress-with-overwrite-defaults-service-max-connections.latest.golden b/agent/xds/testdata/clusters/ingress-with-overwrite-defaults-service-max-connections.latest.golden new file mode 100644 index 00000000000..61101551cd5 --- /dev/null +++ b/agent/xds/testdata/clusters/ingress-with-overwrite-defaults-service-max-connections.latest.golden @@ -0,0 +1,70 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "altStatName": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": { + + }, + "resourceApiVersion": "V3" + } + }, + "connectTimeout": "33s", + "circuitBreakers": { + "thresholds":[ + { + "maxConnections": 4096, + "maxPendingRequests": 2048 + } + ] + }, + "outlierDetection": { + + }, + "commonLbConfig": { + "healthyPanicThreshold": { + + } + }, + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": { + + }, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + }, + "matchSubjectAltNames": [ + { + "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/db" + } + ] + } + }, + "sni": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + } + ], + "typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/clusters/ingress-with-service-max-connections.latest.golden b/agent/xds/testdata/clusters/ingress-with-service-max-connections.latest.golden new file mode 100644 index 00000000000..6ca8d60c6ab --- /dev/null +++ b/agent/xds/testdata/clusters/ingress-with-service-max-connections.latest.golden @@ -0,0 +1,69 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "altStatName": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": { + + }, + "resourceApiVersion": "V3" + } + }, + "connectTimeout": "33s", + "circuitBreakers": { + "thresholds":[ + { + "maxConnections": 4096 + } + ] + }, + "outlierDetection": { + + }, + "commonLbConfig": { + "healthyPanicThreshold": { + + } + }, + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": { + + }, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + }, + "matchSubjectAltNames": [ + { + "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/db" + } + ] + } + }, + "sni": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + } + ], + "typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "nonce": "00000001" +} \ No newline at end of file diff --git a/api/config_entry_gateways.go b/api/config_entry_gateways.go index 56d949ea573..63b323e6ba8 100644 --- a/api/config_entry_gateways.go +++ b/api/config_entry_gateways.go @@ -27,6 +27,9 @@ type IngressGatewayConfigEntry struct { Meta map[string]string `json:",omitempty"` + // Defaults is default configuration for all upstream services + Defaults *IngressServiceConfig `json:",omitempty"` + // CreateIndex is the Raft index this entry was created at. This is a // read-only field. CreateIndex uint64 @@ -37,6 +40,12 @@ type IngressGatewayConfigEntry struct { ModifyIndex uint64 } +type IngressServiceConfig struct { + MaxConnections *uint32 + MaxPendingRequests *uint32 + MaxConcurrentRequests *uint32 +} + type GatewayTLSConfig struct { // Indicates that TLS should be enabled for this gateway service. Enabled bool @@ -124,6 +133,10 @@ type IngressService struct { // Allow HTTP header manipulation to be configured. RequestHeaders *HTTPHeaderModifiers `json:",omitempty" alias:"request_headers"` ResponseHeaders *HTTPHeaderModifiers `json:",omitempty" alias:"response_headers"` + + MaxConnections *uint32 `json:",omitempty" alias:"max_connections"` + MaxPendingRequests *uint32 `json:",omitempty" alias:"max_pending_requests"` + MaxConcurrentRequests *uint32 `json:",omitempty" alias:"max_concurrent_requests"` } func (i *IngressGatewayConfigEntry) GetKind() string { return i.Kind } diff --git a/api/config_entry_gateways_test.go b/api/config_entry_gateways_test.go index 0e2acd728dd..0602be24fe5 100644 --- a/api/config_entry_gateways_test.go +++ b/api/config_entry_gateways_test.go @@ -29,6 +29,10 @@ func TestAPI_ConfigEntries_IngressGateway(t *testing.T) { Enabled: true, TLSMinVersion: "TLSv1_2", }, + Defaults: &IngressServiceConfig{ + MaxConnections: uint32Pointer(2048), + MaxPendingRequests: uint32Pointer(4096), + }, } global := &ProxyConfigEntry{ @@ -93,6 +97,9 @@ func TestAPI_ConfigEntries_IngressGateway(t *testing.T) { CertResource: "bar", }, }, + MaxConnections: uint32Pointer(5120), + MaxPendingRequests: uint32Pointer(512), + MaxConcurrentRequests: uint32Pointer(2048), }, }, TLS: &GatewayTLSConfig{ @@ -168,6 +175,9 @@ func TestAPI_ConfigEntries_IngressGateway(t *testing.T) { require.True(t, ok) require.Equal(t, ingress2.Kind, readIngress.Kind) require.Equal(t, ingress2.Name, readIngress.Name) + require.Equal(t, *ingress2.Defaults.MaxConnections, *readIngress.Defaults.MaxConnections) + require.Equal(t, uint32(4096), *readIngress.Defaults.MaxPendingRequests) + require.Equal(t, uint32(0), *readIngress.Defaults.MaxConcurrentRequests) require.Len(t, readIngress.Listeners, 1) require.Len(t, readIngress.Listeners[0].Services, 1) // Set namespace and partition to blank so that OSS and ent can utilize the same tests diff --git a/test/integration/connect/envoy/case-ingress-gateway-multiple-services/config_entries.hcl b/test/integration/connect/envoy/case-ingress-gateway-multiple-services/config_entries.hcl index b5fbaf22419..c1c7c58320f 100644 --- a/test/integration/connect/envoy/case-ingress-gateway-multiple-services/config_entries.hcl +++ b/test/integration/connect/envoy/case-ingress-gateway-multiple-services/config_entries.hcl @@ -11,6 +11,11 @@ config_entries { kind = "ingress-gateway" name = "ingress-gateway" + Defaults { + MaxConnections = 10 + MaxPendingRequests = 20 + MaxConcurrentRequests = 30 + } listeners = [ { port = 9999 @@ -28,6 +33,9 @@ config_entries { { name = "s1" hosts = ["test.example.com"] + MaxConnections = 100 + MaxPendingRequests = 200 + MaxConcurrentRequests = 300 } ] } diff --git a/test/integration/connect/envoy/case-ingress-gateway-multiple-services/verify.bats b/test/integration/connect/envoy/case-ingress-gateway-multiple-services/verify.bats index 17cd62f2e90..97c712a7f91 100644 --- a/test/integration/connect/envoy/case-ingress-gateway-multiple-services/verify.bats +++ b/test/integration/connect/envoy/case-ingress-gateway-multiple-services/verify.bats @@ -23,11 +23,28 @@ load helpers } @test "ingress-gateway should have healthy endpoints for s1" { - assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s1 HEALTHY 1 + assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s1 HEALTHY 1 } @test "ingress-gateway should have healthy endpoints for s2" { - assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s2 HEALTHY 1 + assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s2 HEALTHY 1 +} + +@test "s2 proxy should have been configured with connection threshold from defaults" { + CLUSTER_THRESHOLD=$(get_envoy_cluster_config 127.0.0.1:20000 s2.default.primary | jq '.circuit_breakers.thresholds[0]') + echo $CLUSTER_THRESHOLD + + MAX_CONNS=$(echo $CLUSTER_THRESHOLD | jq --raw-output '.max_connections') + MAX_PENDING_REQS=$(echo $CLUSTER_THRESHOLD | jq --raw-output '.max_pending_requests') + MAX_REQS=$(echo $CLUSTER_THRESHOLD | jq --raw-output '.max_requests') + + echo "MAX_CONNS = $MAX_CONNS" + echo "MAX_PENDING_REQS = $MAX_PENDING_REQS" + echo "MAX_REQS = $MAX_REQS" + + [ "$MAX_CONNS" = "10" ] + [ "$MAX_PENDING_REQS" = "20" ] + [ "$MAX_REQS" = "30" ] } @test "ingress should be able to connect to s1 using Host header" { diff --git a/test/integration/connect/envoy/case-ingress-gateway-simple/config_entries.hcl b/test/integration/connect/envoy/case-ingress-gateway-simple/config_entries.hcl index dfc7bc7b930..88a76594a80 100644 --- a/test/integration/connect/envoy/case-ingress-gateway-simple/config_entries.hcl +++ b/test/integration/connect/envoy/case-ingress-gateway-simple/config_entries.hcl @@ -2,7 +2,11 @@ config_entries { bootstrap { kind = "ingress-gateway" name = "ingress-gateway" - + Defaults { + MaxConnections = 10 + MaxPendingRequests = 20 + MaxConcurrentRequests = 30 + } listeners = [ { port = 9999 @@ -10,6 +14,8 @@ config_entries { services = [ { name = "s1" + MaxConnections = 100 + MaxPendingRequests = 200 } ] } diff --git a/test/integration/connect/envoy/case-ingress-gateway-simple/verify.bats b/test/integration/connect/envoy/case-ingress-gateway-simple/verify.bats index 73c09773d57..9d0735b42ce 100644 --- a/test/integration/connect/envoy/case-ingress-gateway-simple/verify.bats +++ b/test/integration/connect/envoy/case-ingress-gateway-simple/verify.bats @@ -19,7 +19,24 @@ load helpers } @test "ingress-gateway should have healthy endpoints for s1" { - assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s1 HEALTHY 1 + assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s1 HEALTHY 1 +} + +@test "s1 proxy should have been configured with connection threshold from defaults and service" { + CLUSTER_THRESHOLD=$(get_envoy_cluster_config 127.0.0.1:20000 s1.default.primary | jq '.circuit_breakers.thresholds[0]') + echo $CLUSTER_THRESHOLD + + MAX_CONNS=$(echo $CLUSTER_THRESHOLD | jq --raw-output '.max_connections') + MAX_PENDING_REQS=$(echo $CLUSTER_THRESHOLD | jq --raw-output '.max_pending_requests') + MAX_REQS=$(echo $CLUSTER_THRESHOLD | jq --raw-output '.max_requests') + + echo "MAX_CONNS = $MAX_CONNS" + echo "MAX_PENDING_REQS = $MAX_PENDING_REQS" + echo "MAX_REQS = $MAX_REQS" + + [ "$MAX_CONNS" = "100" ] + [ "$MAX_PENDING_REQS" = "200" ] + [ "$MAX_REQS" = "30" ] } @test "ingress should be able to connect to s1 via configured port" { diff --git a/website/content/docs/connect/config-entries/ingress-gateway.mdx b/website/content/docs/connect/config-entries/ingress-gateway.mdx index b1f92ff9f1c..3cc27b534c3 100644 --- a/website/content/docs/connect/config-entries/ingress-gateway.mdx +++ b/website/content/docs/connect/config-entries/ingress-gateway.mdx @@ -328,6 +328,7 @@ In the following example, two listeners are configured on an ingress gateway nam - The first listener is configured to listen on port `8080` and uses a wildcard (`*`) to proxy traffic to all services in the datacenter. - The second listener exposes the `api` and `web` services on port `4567` at user-provided hosts. - TLS is enabled on every listener. +- The `max_connections` of the ingress gateway proxy to each upstream cluster is set to 4096. The Consul Enterprise version implements the following additional configurations: @@ -346,6 +347,10 @@ TLS { Enabled = true } +Defaults { + MaxConnections = 4096 +} + Listeners = [ { Port = 8080 @@ -1044,6 +1049,38 @@ You can specify the following parameters to configure ingress gateway configurat }, ], }, + { + name: 'Defaults', + type: 'IngressServiceConfig: ', + description: `Default configuration that applies to all upstreams.`, + children: [ + { + name: 'MaxConnections', + type: 'int: 0', + description: `The maximum number of connections a service instance + will be allowed to establish against the given upstream. Use this to limit + HTTP/1.1 traffic, since HTTP/1.1 has a request per connection. + If not specified, it uses the default value. For example, 1024 for Envoy proxy.`, + }, + { + name: 'MaxPendingRequests', + type: 'int: 0', + description: `The maximum number of requests that will be queued + while waiting for a connection to be established. For this configuration to + be respected, a L7 protocol must be defined in the \`protocol\` field. + If not specified, it uses the default value. For example, 1024 for Envoy proxy.`, + }, + { + name: 'MaxConcurrentRequests', + type: 'int: 0', + description: `The maximum number of concurrent requests that + will be allowed at a single point in time. Use this to limit HTTP/2 traffic, + since HTTP/2 has many requests per connection. For this configuration to be + respected, a L7 protocol must be defined in the \`protocol\` field. + If not specified, it uses the default value. For example, 1024 for Envoy proxy.`, + }, + ], + }, { name: 'Listeners', type: 'array: )', @@ -1159,6 +1196,21 @@ You can specify the following parameters to configure ingress gateway configurat }, ], }, + { + name: 'MaxConnections', + type: 'int: 0', + description: 'overrides for the [`Defaults` field](#available-fields)', + }, + { + name: 'MaxPendingRequests', + type: 'int: 0', + description: 'overrides for the [`Defaults` field](#available-fields)', + }, + { + name: 'MaxConcurrentRequests', + type: 'int: 0', + description: 'overrides for the [`Defaults` field](#available-fields)', + }, ], }, {