Backport of Add stricter validation and some normalization code for API Gateway ConfigEntries into release/1.15.x

agent/proxycfg/snapshot.go

@@ -734,6 +734,8 @@ type configSnapshotAPIGateway struct {
 	// Listeners is the original listener config from the api-gateway config
 	// entry to save us trying to pass fields through Upstreams
 	Listeners map[string]structs.APIGatewayListener
+	// this acts as an intermediary for inlining certificates
+	ListenerCertificates map[IngressListenerKey][]structs.InlineCertificateConfigEntry
 
 	BoundListeners map[string]structs.BoundAPIGatewayListener
 }
@@ -751,6 +753,9 @@ func (c *configSnapshotAPIGateway) ToIngress(datacenter string) (configSnapshotI
 	watchedUpstreamEndpoints := make(map[UpstreamID]map[string]structs.CheckServiceNodes)
 	watchedGatewayEndpoints := make(map[UpstreamID]map[string]structs.CheckServiceNodes)
 
+	// reset the cached certificates
+	c.ListenerCertificates = make(map[IngressListenerKey][]structs.InlineCertificateConfigEntry)
+
 	for name, listener := range c.Listeners {
 		boundListener, ok := c.BoundListeners[name]
 		if !ok {
@@ -802,17 +807,18 @@ func (c *configSnapshotAPIGateway) ToIngress(datacenter string) (configSnapshotI
 			watchedGatewayEndpoints[id] = gatewayEndpoints
 		}
 
+		key := IngressListenerKey{
+			Port:     listener.Port,
+			Protocol: string(listener.Protocol),
+		}
+
 		// Configure TLS for the ingress listener
-		tls, err := c.toIngressTLS()
+		tls, err := c.toIngressTLS(key, listener, boundListener)
 		if err != nil {
 			return configSnapshotIngressGateway{}, err
 		}
-		ingressListener.TLS = tls
 
-		key := IngressListenerKey{
-			Port:     listener.Port,
-			Protocol: string(listener.Protocol),
-		}
+		ingressListener.TLS = tls
 		ingressListeners[key] = ingressListener
 		ingressUpstreams[key] = upstreams
 	}
@@ -905,9 +911,25 @@ DOMAIN_LOOP:
 	return services, upstreams, compiled, err
 }
 
-func (c *configSnapshotAPIGateway) toIngressTLS() (*structs.GatewayTLSConfig, error) {
-	// TODO (t-eckert) this is dependent on future SDS work.
-	return &structs.GatewayTLSConfig{}, nil
+func (c *configSnapshotAPIGateway) toIngressTLS(key IngressListenerKey, listener structs.APIGatewayListener, bound structs.BoundAPIGatewayListener) (*structs.GatewayTLSConfig, error) {
+	if len(listener.TLS.Certificates) == 0 {
+		return nil, nil
+	}
+
+	for _, certRef := range bound.Certificates {
+		cert, ok := c.Certificates.Get(certRef)
+		if !ok {
+			continue
+		}
+		c.ListenerCertificates[key] = append(c.ListenerCertificates[key], *cert)
+	}
+
+	return &structs.GatewayTLSConfig{
+		Enabled:       true,
+		TLSMinVersion: listener.TLS.MinVersion,
+		TLSMaxVersion: listener.TLS.MaxVersion,
+		CipherSuites:  listener.TLS.CipherSuites,
+	}, nil
 }
 
 type configSnapshotIngressGateway struct {

agent/proxycfg/testing_api_gateway.go

@@ -0,0 +1,135 @@
+package proxycfg
+
+import (
+	"fmt"
+
+	"github.com/mitchellh/go-testing-interface"
+
+	"github.com/hashicorp/consul/agent/connect"
+	"github.com/hashicorp/consul/agent/consul/discoverychain"
+	"github.com/hashicorp/consul/agent/structs"
+)
+
+func TestConfigSnapshotAPIGateway(t testing.T) *ConfigSnapshot {
+	roots, placeholderLeaf := TestCerts(t)
+
+	entries := []structs.ConfigEntry{
+		&structs.ProxyConfigEntry{
+			Kind: structs.ProxyDefaults,
+			Name: structs.ProxyConfigGlobal,
+			Config: map[string]interface{}{
+				"protocol": "tcp",
+			},
+		},
+		&structs.ServiceResolverConfigEntry{
+			Kind:           structs.ServiceResolver,
+			Name:           "api-gateway",
+		},
+	}
+
+	baseEvents := []UpdateEvent{
+		{
+			CorrelationID: rootsWatchID,
+			Result:        roots,
+		},
+		{
+			CorrelationID: leafWatchID,
+			Result:        placeholderLeaf,
+		},
+		{
+			CorrelationID: gatewayConfigWatchID,
+			Result: &structs.ConfigEntryResponse{
+				Entry: &structs.APIGatewayConfigEntry{
+					Kind: structs.APIGateway,
+					Name: "api-gateway",
+					Listeners: []structs.APIGatewayListener{
+						{
+							Name:     "",
+							Hostname: "",
+							Port:     8080,
+							Protocol: structs.ListenerProtocolTCP,
+							TLS: structs.APIGatewayTLSConfiguration{
+								Certificates: []structs.ResourceReference{
+									{
+										Kind: structs.InlineCertificate,
+										Name: "my-inline-certificate",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			CorrelationID: gatewayConfigWatchID,
+			Result: &structs.ConfigEntryResponse{
+				Entry: &structs.BoundAPIGatewayConfigEntry{
+					Kind: structs.BoundAPIGateway,
+					Name: "api-gateway",
+					Listeners: []structs.BoundAPIGatewayListener{
+						{
+							Name: "",
+							Certificates: []structs.ResourceReference{
+								{
+									Kind: structs.InlineCertificate,
+									Name: "my-inline-certificate",
+								},
+							},
+							Routes: []structs.ResourceReference{
+								{
+									Kind: structs.TCPRoute,
+									Name: "my-tcp-route",
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			CorrelationID: routeConfigWatchID,
+			Result: &structs.ConfigEntryResponse{
+				Entry: &structs.TCPRouteConfigEntry{
+					Kind: structs.TCPRoute,
+					Name: "my-tcp-route",
+					Parents: []structs.ResourceReference{
+						{
+							Kind: structs.APIGateway,
+							Name: "api-gateway",
+						},
+					},
+					Services: []structs.TCPService{
+						{Name: "my-tcp-service"},
+					},
+				},
+			},
+		},
+		{
+			CorrelationID: inlineCertificateConfigWatchID,
+			Result: &structs.ConfigEntryResponse{
+				Entry: &structs.InlineCertificateConfigEntry{
+					Kind:        structs.InlineCertificate,
+					Name:        "my-inline-certificate",
+					Certificate: "certificate",
+					PrivateKey:  "private key",
+				},
+			},
+		},
+		{
+			CorrelationID: fmt.Sprintf("discovery-chain:%s", UpstreamIDString("","","my-tcp-service",nil, "")),
+			Result: &structs.DiscoveryChainResponse{
+				Chain: discoverychain.TestCompileConfigEntries(t,"my-tcp-service","default","default","dc1", connect.TestClusterID+".consul",nil,entries...),
+			},
+		},
+	}
+
+	return testConfigSnapshotFixture(t, &structs.NodeService{
+		Kind:            structs.ServiceKindAPIGateway,
+		Service:         "api-gateway",
+		Port:            9999,
+		Address:         "1.2.3.4",
+		Meta:            nil,
+		TaggedAddresses: nil,
+	}, nil, nil, testSpliceEvents(baseEvents, nil))
+}

agent/structs/config_entry_status.go

@@ -1,6 +1,7 @@
 package structs
 
 import (
+	"fmt"
 	"sort"
 	"time"
 
@@ -24,6 +25,10 @@ type ResourceReference struct {
 	acl.EnterpriseMeta
 }
 
+func (r *ResourceReference) String() string {
+	return fmt.Sprintf("%s:%s/%s/%s/%s", r.Kind, r.PartitionOrDefault(), r.NamespaceOrDefault(), r.Name, r.SectionName)
+}
+
 func (r *ResourceReference) IsSame(other *ResourceReference) bool {
 	if r == nil && other == nil {
 		return true

agent/xds/delta.go

@@ -466,20 +466,21 @@ func (s *Server) applyEnvoyExtensions(resources *xdscommon.IndexedResources, cfg
 	return nil
 }
 
+// https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol#eventual-consistency-considerations
 var xDSUpdateOrder = []xDSUpdateOperation{
-	// TODO Update comments
+	// 1. SDS updates (if any) can be pushed here with no harm.
 	{TypeUrl: xdscommon.SecretType, Upsert: true},
-	// 1. CDS updates (if any) must always be pushed first.
+	// 2. CDS updates (if any) must always be pushed before the following types.
 	{TypeUrl: xdscommon.ClusterType, Upsert: true},
-	// 2. EDS updates (if any) must arrive after CDS updates for the respective clusters.
+	// 3. EDS updates (if any) must arrive after CDS updates for the respective clusters.
 	{TypeUrl: xdscommon.EndpointType, Upsert: true},
-	// 3. LDS updates must arrive after corresponding CDS/EDS updates.
+	// 4. LDS updates must arrive after corresponding CDS/EDS updates.
 	{TypeUrl: xdscommon.ListenerType, Upsert: true, Remove: true},
-	// 4. RDS updates related to the newly added listeners must arrive after CDS/EDS/LDS updates.
+	// 5. RDS updates related to the newly added listeners must arrive after CDS/EDS/LDS updates.
 	{TypeUrl: xdscommon.RouteType, Upsert: true, Remove: true},
-	// 5. (NOT IMPLEMENTED YET IN CONSUL) VHDS updates (if any) related to the newly added RouteConfigurations must arrive after RDS updates.
+	// 6. (NOT IMPLEMENTED YET IN CONSUL) VHDS updates (if any) related to the newly added RouteConfigurations must arrive after RDS updates.
 	// {},
-	// 6. Stale CDS clusters, related EDS endpoints (ones no longer being referenced) and SDS secrets can then be removed.
+	// 7. Stale CDS clusters, related EDS endpoints (ones no longer being referenced) and SDS secrets can then be removed.
 	{TypeUrl: xdscommon.ClusterType, Remove: true},
 	{TypeUrl: xdscommon.EndpointType, Remove: true},
 	{TypeUrl: xdscommon.SecretType, Remove: true},

agent/xds/listeners_ingress.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/hashicorp/consul/agent/proxycfg"
 	"github.com/hashicorp/consul/agent/structs"
+	"github.com/hashicorp/consul/lib"
 	"github.com/hashicorp/consul/types"
 )
 
@@ -25,7 +26,12 @@ func (s *ResourceGenerator) makeIngressGatewayListeners(address string, cfgSnap
 			return nil, fmt.Errorf("no listener config found for listener on proto/port %s/%d", listenerKey.Protocol, listenerKey.Port)
 		}
 
-		tlsContext, err := makeDownstreamTLSContextFromSnapshotListenerConfig(cfgSnap, listenerCfg)
+		var certs []structs.InlineCertificateConfigEntry
+		if cfgSnap.APIGateway.ListenerCertificates != nil {
+			certs = cfgSnap.APIGateway.ListenerCertificates[listenerKey]
+		}
+
+		tlsContext, err := makeDownstreamTLSContextFromSnapshotListenerConfig(cfgSnap, listenerCfg, certs)
 		if err != nil {
 			return nil, err
 		}
@@ -160,10 +166,10 @@ func (s *ResourceGenerator) makeIngressGatewayListeners(address string, cfgSnap
 	return resources, nil
 }
 
-func makeDownstreamTLSContextFromSnapshotListenerConfig(cfgSnap *proxycfg.ConfigSnapshot, listenerCfg structs.IngressListener) (*envoy_tls_v3.DownstreamTlsContext, error) {
+func makeDownstreamTLSContextFromSnapshotListenerConfig(cfgSnap *proxycfg.ConfigSnapshot, listenerCfg structs.IngressListener, certs []structs.InlineCertificateConfigEntry) (*envoy_tls_v3.DownstreamTlsContext, error) {
 	var downstreamContext *envoy_tls_v3.DownstreamTlsContext
 
-	tlsContext, err := makeCommonTLSContextFromSnapshotListenerConfig(cfgSnap, listenerCfg)
+	tlsContext, err := makeCommonTLSContextFromSnapshotListenerConfig(cfgSnap, listenerCfg, certs)
 	if err != nil {
 		return nil, err
 	}
@@ -181,7 +187,7 @@ func makeDownstreamTLSContextFromSnapshotListenerConfig(cfgSnap *proxycfg.Config
 	return downstreamContext, nil
 }
 
-func makeCommonTLSContextFromSnapshotListenerConfig(cfgSnap *proxycfg.ConfigSnapshot, listenerCfg structs.IngressListener) (*envoy_tls_v3.CommonTlsContext, error) {
+func makeCommonTLSContextFromSnapshotListenerConfig(cfgSnap *proxycfg.ConfigSnapshot, listenerCfg structs.IngressListener, certs []structs.InlineCertificateConfigEntry) (*envoy_tls_v3.CommonTlsContext, error) {
 	var tlsContext *envoy_tls_v3.CommonTlsContext
 
 	// Enable connect TLS if it is enabled at the Gateway or specific listener
@@ -199,6 +205,10 @@ func makeCommonTLSContextFromSnapshotListenerConfig(cfgSnap *proxycfg.ConfigSnap
 		return nil, err
 	}
 
+	if len(certs) != 0 {
+		return makeInlineTLSContextFromGatewayTLSConfig(*tlsCfg, certs), nil
+	}
+
 	if tlsCfg.SDS != nil {
 		// Set up listener TLS from SDS
 		tlsContext = makeCommonTLSContextFromGatewayTLSConfig(*tlsCfg)
@@ -383,6 +393,29 @@ func makeTLSParametersFromGatewayTLSConfig(tlsCfg structs.GatewayTLSConfig) *env
 	return makeTLSParametersFromTLSConfig(tlsCfg.TLSMinVersion, tlsCfg.TLSMaxVersion, tlsCfg.CipherSuites)
 }
 
+func makeInlineTLSContextFromGatewayTLSConfig(tlsCfg structs.GatewayTLSConfig, certs []structs.InlineCertificateConfigEntry) *envoy_tls_v3.CommonTlsContext {
+	tlsParams := makeTLSParametersFromGatewayTLSConfig(tlsCfg)
+	tlsCerts := []*envoy_tls_v3.TlsCertificate{}
+	for _, cert := range certs {
+		tlsCerts = append(tlsCerts, &envoy_tls_v3.TlsCertificate{
+			CertificateChain: &envoy_core_v3.DataSource{
+				Specifier: &envoy_core_v3.DataSource_InlineString{
+					InlineString: lib.EnsureTrailingNewline(cert.Certificate),
+				},
+			},
+			PrivateKey: &envoy_core_v3.DataSource{
+				Specifier: &envoy_core_v3.DataSource_InlineString{
+					InlineString: lib.EnsureTrailingNewline(cert.PrivateKey),
+				},
+			},
+		})
+	}
+	return &envoy_tls_v3.CommonTlsContext{
+		TlsParams:       tlsParams,
+		TlsCertificates: tlsCerts,
+	}
+}
+
 func makeCommonTLSContextFromGatewayTLSConfig(tlsCfg structs.GatewayTLSConfig) *envoy_tls_v3.CommonTlsContext {
 	return &envoy_tls_v3.CommonTlsContext{
 		TlsParams:                      makeTLSParametersFromGatewayTLSConfig(tlsCfg),
@@ -396,6 +429,21 @@ func makeCommonTLSContextFromGatewayServiceTLSConfig(tlsCfg structs.GatewayServi
 		TlsCertificateSdsSecretConfigs: makeTLSCertificateSdsSecretConfigsFromSDS(*tlsCfg.SDS),
 	}
 }
+
+func makeTLSCertificateSdsSecretConfigsFromSDSUsingADS(sdsCfg structs.GatewayTLSSDSConfig) []*envoy_tls_v3.SdsSecretConfig {
+	return []*envoy_tls_v3.SdsSecretConfig{
+		{
+			Name: sdsCfg.CertResource,
+			SdsConfig: &envoy_core_v3.ConfigSource{
+				ConfigSourceSpecifier: &envoy_core_v3.ConfigSource_Ads{
+					Ads: &envoy_core_v3.AggregatedConfigSource{},
+				},
+				ResourceApiVersion: envoy_core_v3.ApiVersion_V3,
+			},
+		},
+	}
+}
+
 func makeTLSCertificateSdsSecretConfigsFromSDS(sdsCfg structs.GatewayTLSSDSConfig) []*envoy_tls_v3.SdsSecretConfig {
 	return []*envoy_tls_v3.SdsSecretConfig{
 		{

agent/xds/resources.go

@@ -35,8 +35,7 @@ func NewResourceGenerator(
 
 func (g *ResourceGenerator) AllResourcesFromSnapshot(cfgSnap *proxycfg.ConfigSnapshot) (map[string][]proto.Message, error) {
 	all := make(map[string][]proto.Message)
-	// TODO Add xdscommon.SecretType
-	for _, typeUrl := range []string{xdscommon.ListenerType, xdscommon.RouteType, xdscommon.ClusterType, xdscommon.EndpointType} {
+	for _, typeUrl := range []string{xdscommon.ListenerType, xdscommon.RouteType, xdscommon.ClusterType, xdscommon.EndpointType, xdscommon.SecretType} {
 		res, err := g.resourcesFromSnapshot(typeUrl, cfgSnap)
 		if err != nil {
 			return nil, fmt.Errorf("failed to generate xDS resources for %q: %v", typeUrl, err)