support many Gateway Listeners

apis/projectcontour/v1/detailedconditions.go

@@ -193,4 +193,8 @@ const (
 	// ConditionTypeVirtualHostError describes an error condition relating
 	// to the VirtualHost configuration section of an HTTPProxy resource.
 	ConditionTypeVirtualHostError = "VirtualHostError"
+
+	// ConditionTypeListenerError describes an error condition relating
+	// to the configuration of Listeners.
+	ConditionTypeListenerError = "ListenerError"
 )

changelogs/unreleased/5160-skriss-major.md

@@ -0,0 +1,76 @@
+## Support for Gateway Listeners on more than two ports
+
+Contour now supports Gateway Listeners with many different ports.
+Previously, Contour only allowed a single port for HTTP, and a single port for HTTPS/TLS.
+
+As an example, the following Gateway, with two HTTP ports and two HTTPS ports, is now fully supported:
+
+```yaml
+kind: Gateway
+apiVersion: gateway.networking.k8s.io/v1beta1
+metadata:
+  name: contour
+spec:
+  gatewayClassName: contour
+  listeners:
+    - name: http-1
+      protocol: HTTP
+      port: 80
+      allowedRoutes:
+        namespaces:
+          from: Same
+    - name: http-2
+      protocol: HTTP
+      port: 81
+      allowedRoutes:
+        namespaces:
+          from: Same
+    - name: https-1
+      protocol: HTTPS
+      port: 443
+      allowedRoutes:
+        namespaces:
+          from: Same
+      tls:
+        mode: Terminate
+        certificateRefs:
+        - name: tls-cert-1
+    - name: https-2
+      protocol: HTTPS
+      port: 444
+      allowedRoutes:
+        namespaces:
+          from: Same
+      tls:
+        mode: Terminate
+        certificateRefs:
+        - name: tls-cert-2
+```
+
+If you are using the Contour Gateway Provisioner, ports for all valid Listeners will automatically be exposed via the Envoy service, and will update when any Listener changes are made.
+If you are using static provisioning, you must keep the Service definition in sync with the Gateway spec manually.
+
+Note that if you are using the Contour Gateway Provisioner along with HTTPProxy or Ingress for routing, then your Gateway must have exactly one HTTP Listener and one HTTPS Listener.
+For this case, Contour supports a custom HTTPS Listener protocol value, to avoid having to specify TLS details in the Listener (since they're specified in the HTTPProxy or Ingress instead):
+
+```yaml
+kind: Gateway
+apiVersion: gateway.networking.k8s.io/v1beta1
+metadata:
+  name: contour-with-httpproxy
+spec:
+  gatewayClassName: contour
+  listeners:
+    - name: http
+      protocol: HTTP
+      port: 80
+      allowedRoutes:
+        namespaces:
+          from: All
+    - name: https
+      protocol: projectcontour.io/https
+      port: 443
+      allowedRoutes:
+        namespaces:
+          from: All
+```

cmd/contour/shutdownmanager.go

@@ -24,7 +24,6 @@ import (
 	"time"
 
 	"github.com/alecthomas/kingpin/v2"
-	xdscache_v3 "github.com/projectcontour/contour/internal/xdscache/v3"
 	"github.com/prometheus/common/expfmt"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -43,10 +42,6 @@ const shutdownReadyFile = "/admin/ok"
 // shutdownReadyCheckInterval is the default polling interval for the file used in the /shutdown endpoint.
 const shutdownReadyCheckInterval = time.Second * 1
 
-func prometheusLabels() []string {
-	return []string{xdscache_v3.ENVOY_HTTP_LISTENER, xdscache_v3.ENVOY_HTTPS_LISTENER}
-}
-
 type shutdownmanagerContext struct {
 	// httpServePort defines what port the shutdown-manager listens on
 	httpServePort int
@@ -273,10 +268,11 @@ func parseOpenConnections(stats io.Reader) (int, error) {
 	// Look up open connections value
 	for _, metrics := range metricFamilies[prometheusStat].Metric {
 		for _, labels := range metrics.Label {
-			for _, item := range prometheusLabels() {
-				if item == labels.GetValue() {
-					openConnections += int(metrics.Gauge.GetValue())
-				}
+			switch labels.GetValue() {
+			// don't count connections to these listeners.
+			case "admin", "envoy-admin", "stats", "health", "stats-health":
+			default:
+				openConnections += int(metrics.Gauge.GetValue())
 			}
 		}
 	}

cmd/contour/shutdownmanager_test.go

@@ -177,6 +177,12 @@ func TestParseOpenConnections(t *testing.T) {
 		wantError:       nil,
 	})
 
+	run(t, "many listeners", testcase{
+		stats:           strings.NewReader(VALIDMANYLISTENERS),
+		wantConnections: 16,
+		wantError:       nil,
+	})
+
 	run(t, "missing values", testcase{
 		stats:           strings.NewReader(MISSING_STATS),
 		wantConnections: -1,
@@ -218,6 +224,10 @@ envoy_server_days_until_first_cert_expiring{} 82
 envoy_server_hot_restart_epoch{} 0
 # TYPE envoy_http_downstream_cx_active gauge
 envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="ingress_http"} 4
+envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="envoy-admin"} 7
+envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="stats"} 77
+envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="health"} 777
+envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="stats-health"} 7777
 `
 	VALIDHTTPS = `envoy_cluster_circuit_breakers_default_cx_pool_open{envoy_cluster_name="projectcontour_envoy-admin_9001"} 0
 envoy_cluster_max_host_weight{envoy_cluster_name="projectcontour_envoy-admin_9001"} 0
@@ -244,7 +254,11 @@ envoy_server_days_until_first_cert_expiring{} 82
 # TYPE envoy_server_hot_restart_epoch gauge
 envoy_server_hot_restart_epoch{} 0
 # TYPE envoy_http_downstream_cx_active gauge
-envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="ingress_http"} 4
+envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="ingress_https"} 4
+envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="envoy-admin"} 7
+envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="stats"} 77
+envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="health"} 777
+envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="stats-health"} 7777
 `
 	VALIDBOTH = `envoy_cluster_circuit_breakers_default_cx_pool_open{envoy_cluster_name="projectcontour_envoy-admin_9001"} 0
 envoy_cluster_max_host_weight{envoy_cluster_name="projectcontour_envoy-admin_9001"} 0
@@ -273,6 +287,45 @@ envoy_server_hot_restart_epoch{} 0
 # TYPE envoy_http_downstream_cx_active gauge
 envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="ingress_http"} 4
 envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="ingress_https"} 4
+envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="envoy-admin"} 7
+envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="stats"} 77
+envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="health"} 777
+envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="stats-health"} 7777
+`
+
+	VALIDMANYLISTENERS = `envoy_cluster_circuit_breakers_default_cx_pool_open{envoy_cluster_name="projectcontour_envoy-admin_9001"} 0
+envoy_cluster_max_host_weight{envoy_cluster_name="projectcontour_envoy-admin_9001"} 0
+envoy_cluster_upstream_rq_pending_active{envoy_cluster_name="projectcontour_envoy-admin_9001"} 0
+envoy_cluster_circuit_breakers_high_rq_retry_open{envoy_cluster_name="projectcontour_envoy-admin_9001"} 0
+envoy_cluster_circuit_breakers_high_cx_pool_open{envoy_cluster_name="projectcontour_envoy-admin_9001"} 0
+envoy_cluster_upstream_cx_tx_bytes_buffered{envoy_cluster_name="projectcontour_envoy-admin_9001"} 0
+envoy_cluster_version{envoy_cluster_name="projectcontour_envoy-admin_9001"} 0
+envoy_cluster_circuit_breakers_default_cx_open{envoy_cluster_name="projectcontour_envoy-admin_9001"} 0
+# TYPE envoy_http_downstream_cx_ssl_active gauge
+envoy_http_downstream_cx_ssl_active{envoy_http_conn_manager_prefix="admin"} 0
+# TYPE envoy_server_total_connections gauge
+envoy_server_total_connections{} 1
+# TYPE envoy_runtime_num_layers gauge
+envoy_runtime_num_layers{} 2
+# TYPE envoy_server_parent_connections gauge
+envoy_server_parent_connections{} 0
+# TYPE envoy_server_stats_recent_lookups gauge
+envoy_server_stats_recent_lookups{} 0
+# TYPE envoy_cluster_manager_warming_clusters gauge
+envoy_cluster_manager_warming_clusters{} 0
+# TYPE envoy_server_days_until_first_cert_expiring gauge
+envoy_server_days_until_first_cert_expiring{} 82
+# TYPE envoy_server_hot_restart_epoch gauge
+envoy_server_hot_restart_epoch{} 0
+# TYPE envoy_http_downstream_cx_active gauge
+envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="http-80"} 4
+envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="http-81"} 4
+envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="https-443"} 4
+envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="https-444"} 4
+envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="envoy-admin"} 7
+envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="stats"} 77
+envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="health"} 777
+envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="stats-health"} 7777
 `
 
 	MISSING_STATS = `envoy_cluster_circuit_breakers_default_cx_pool_open{envoy_cluster_name="projectcontour_envoy-admin_9001"} 0

internal/dag/accessors.go

@@ -16,6 +16,7 @@ package dag
 import (
 	"fmt"
 	"strconv"
+	"strings"
 
 	"github.com/projectcontour/contour/internal/annotation"
 	"github.com/projectcontour/contour/internal/xds"
@@ -118,6 +119,30 @@ func externalName(svc *v1.Service) string {
 	return svc.Spec.ExternalName
 }
 
+// GetSingleListener returns the sole listener with the specified protocol,
+// or an error if there is not exactly one listener with that protocol.
+func (d *DAG) GetSingleListener(protocol string) (*Listener, error) {
+	var res *Listener
+
+	for _, listener := range d.Listeners {
+		if listener.Protocol != protocol {
+			continue
+		}
+
+		if res != nil {
+			return nil, fmt.Errorf("more than one %s listener configured", strings.ToUpper(protocol))
+		}
+
+		res = listener
+	}
+
+	if res == nil {
+		return nil, fmt.Errorf("no %s listener configured", strings.ToUpper(protocol))
+	}
+
+	return res, nil
+}
+
 // GetSecureVirtualHost returns the secure virtual host in the DAG that
 // matches the provided name, or nil if no matching secure virtual host
 // is found.

internal/dag/accessors_test.go

@@ -196,3 +196,78 @@ func TestBuilderLookupService(t *testing.T) {
 		})
 	}
 }
+
+func TestGetSingleListener(t *testing.T) {
+	t.Run("happy path", func(t *testing.T) {
+		d := &DAG{
+			Listeners: map[string]*Listener{
+				"http": {
+					Protocol: "http",
+					Port:     80,
+				},
+				"https": {
+					Protocol: "https",
+					Port:     443,
+				},
+			},
+		}
+
+		got, gotErr := d.GetSingleListener("http")
+		assert.Equal(t, d.Listeners["http"], got)
+		assert.NoError(t, gotErr)
+
+		got, gotErr = d.GetSingleListener("https")
+		assert.Equal(t, d.Listeners["https"], got)
+		assert.NoError(t, gotErr)
+	})
+
+	t.Run("one HTTP listener, no HTTPS listener", func(t *testing.T) {
+		d := &DAG{
+			Listeners: map[string]*Listener{
+				"http": {
+					Protocol: "http",
+					Port:     80,
+				},
+			},
+		}
+
+		got, gotErr := d.GetSingleListener("http")
+		assert.Equal(t, d.Listeners["http"], got)
+		assert.NoError(t, gotErr)
+
+		got, gotErr = d.GetSingleListener("https")
+		assert.Nil(t, got)
+		assert.EqualError(t, gotErr, "no HTTPS listener configured")
+	})
+
+	t.Run("many HTTP listeners, one HTTPS listener", func(t *testing.T) {
+		d := &DAG{
+			Listeners: map[string]*Listener{
+				"http-1": {
+					Protocol: "http",
+					Port:     80,
+				},
+				"http-2": {
+					Protocol: "http",
+					Port:     81,
+				},
+				"http-3": {
+					Protocol: "http",
+					Port:     82,
+				},
+				"https-1": {
+					Protocol: "https",
+					Port:     443,
+				},
+			},
+		}
+
+		got, gotErr := d.GetSingleListener("http")
+		assert.Nil(t, got)
+		assert.EqualError(t, gotErr, "more than one HTTP listener configured")
+
+		got, gotErr = d.GetSingleListener("https")
+		assert.Equal(t, d.Listeners["https-1"], got)
+		assert.NoError(t, gotErr)
+	})
+}