From 10edc5d0fc72e002b9b392659b6689858fcc3182 Mon Sep 17 00:00:00 2001 From: zirain Date: Mon, 4 Aug 2025 13:59:10 +0800 Subject: [PATCH 1/3] fix: don't block deployment creating when missing secret in EnvoyProxy Signed-off-by: zirain --- .../backend-tls-settings-invalid.in.yaml | 130 ++++++++++ .../backend-tls-settings-invalid.out.yaml | 242 ++++++++++++++++++ internal/provider/kubernetes/controller.go | 32 +-- 3 files changed, 380 insertions(+), 24 deletions(-) create mode 100644 internal/gatewayapi/testdata/backend-tls-settings-invalid.in.yaml create mode 100644 internal/gatewayapi/testdata/backend-tls-settings-invalid.out.yaml diff --git a/internal/gatewayapi/testdata/backend-tls-settings-invalid.in.yaml b/internal/gatewayapi/testdata/backend-tls-settings-invalid.in.yaml new file mode 100644 index 0000000000..5b8c5ff0e0 --- /dev/null +++ b/internal/gatewayapi/testdata/backend-tls-settings-invalid.in.yaml @@ -0,0 +1,130 @@ +envoyProxyForGatewayClass: + apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + namespace: envoy-gateway-system + name: test + spec: + backendTLS: + clientCertificateRef: + group: "" + kind: Secret + namespace: envoy-gateway-system + name: client-auth + ciphers: + - ECDHE-RSA-AES128-GCM-SHA256 + - ECDHE-ECDSA-AES256-GCM-SHA384 + ecdhCurves: + - ECDHE-RSA-AES128-GCM-SHA256 + - ECDHE-ECDSA-AES256-GCM-SHA384 + maxVersion: tls1.3 + minVersion: tls1.2 + SignatureAlgorithms: + - RSA-PSS-RSAE-SHA256 + - ECDSA-SECP256R1-SHA256 + alpnProtocols: + - HTTP/1.1 + - HTTP/2 + +gateways: + - apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All +httpRoutes: + - apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - matches: + - path: + value: "/" + backendRefs: + - group: gateway.envoyproxy.io + kind: Backend + name: backend-1 + +configMaps: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: ca-cmap + namespace: default + data: + ca.crt: | + -----BEGIN CERTIFICATE----- + MIIDJzCCAg+gAwIBAgIUAl6UKIuKmzte81cllz5PfdN2IlIwDQYJKoZIhvcNAQEL + BQAwIzEQMA4GA1UEAwwHbXljaWVudDEPMA0GA1UECgwGa3ViZWRiMB4XDTIzMTAw + MjA1NDE1N1oXDTI0MTAwMTA1NDE1N1owIzEQMA4GA1UEAwwHbXljaWVudDEPMA0G + A1UECgwGa3ViZWRiMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwSTc + 1yj8HW62nynkFbXo4VXKv2jC0PM7dPVky87FweZcTKLoWQVPQE2p2kLDK6OEszmM + yyr+xxWtyiveremrWqnKkNTYhLfYPhgQkczib7eUalmFjUbhWdLvHakbEgCodn3b + kz57mInX2VpiDOKg4kyHfiuXWpiBqrCx0KNLpxo3DEQcFcsQTeTHzh4752GV04RU + Ti/GEWyzIsl4Rg7tGtAwmcIPgUNUfY2Q390FGqdH4ahn+mw/6aFbW31W63d9YJVq + ioyOVcaMIpM5B/c7Qc8SuhCI1YGhUyg4cRHLEw5VtikioyE3X04kna3jQAj54YbR + bpEhc35apKLB21HOUQIDAQABo1MwUTAdBgNVHQ4EFgQUyvl0VI5vJVSuYFXu7B48 + 6PbMEAowHwYDVR0jBBgwFoAUyvl0VI5vJVSuYFXu7B486PbMEAowDwYDVR0TAQH/ + BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAMLxrgFVMuNRq2wAwcBt7SnNR5Cfz + 2MvXq5EUmuawIUi9kaYjwdViDREGSjk7JW17vl576HjDkdfRwi4E28SydRInZf6J + i8HZcZ7caH6DxR335fgHVzLi5NiTce/OjNBQzQ2MJXVDd8DBmG5fyatJiOJQ4bWE + A7FlP0RdP3CO3GWE0M5iXOB2m1qWkE2eyO4UHvwTqNQLdrdAXgDQlbam9e4BG3Gg + d/6thAkWDbt/QNT+EJHDCvhDRKh1RuGHyg+Y+/nebTWWrFWsktRrbOoHCZiCpXI1 + 3eXE6nt0YkgtDxG22KqnhpAg9gUSs2hlhoxyvkzyF0mu6NhPlwAgnq7+/Q== + -----END CERTIFICATE----- +backendTLSPolicies: + - apiVersion: gateway.networking.k8s.io/v1alpha3 + kind: BackendTLSPolicy + metadata: + name: policy-btls-for-backend-1 + namespace: default + spec: + targetRefs: + - group: gateway.envoyproxy.io + kind: Backend + name: backend-1 + validation: + caCertificateRefs: + - kind: ConfigMap + group: "" + name: istio-ca-root-cert + hostname: example.com + subjectAltNames: + - type: URI + uri: spiffe://cluster.local/ns/istio-demo/sa/echo-v1 + - type: Hostname + hostname: subdomain.secondexample.com + +backends: + - apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: Backend + metadata: + name: backend-1 + namespace: default + spec: + # the BackendTLSPolicy should override the one from Backend + # the generated ir tls settings should contain the tls settings from Backend, BackendTLSPolicy and EnvoyProxy + tls: + caCertificateRefs: + - name: ca-cmap + group: "" + kind: ConfigMap + endpoints: + - ip: + address: 1.1.1.1 + port: 3001 diff --git a/internal/gatewayapi/testdata/backend-tls-settings-invalid.out.yaml b/internal/gatewayapi/testdata/backend-tls-settings-invalid.out.yaml new file mode 100644 index 0000000000..4056a8cfe2 --- /dev/null +++ b/internal/gatewayapi/testdata/backend-tls-settings-invalid.out.yaml @@ -0,0 +1,242 @@ +backendTLSPolicies: +- apiVersion: gateway.networking.k8s.io/v1alpha3 + kind: BackendTLSPolicy + metadata: + creationTimestamp: null + name: policy-btls-for-backend-1 + namespace: default + spec: + targetRefs: + - group: gateway.envoyproxy.io + kind: Backend + name: backend-1 + validation: + caCertificateRefs: + - group: "" + kind: ConfigMap + name: istio-ca-root-cert + hostname: example.com + subjectAltNames: + - type: URI + uri: spiffe://cluster.local/ns/istio-demo/sa/echo-v1 + - hostname: subdomain.secondexample.com + type: Hostname + status: + ancestors: + - ancestorRef: + name: gateway-1 + namespace: envoy-gateway + conditions: + - lastTransitionTime: null + message: Configmap istio-ca-root-cert not found in namespace default. + reason: Invalid + status: "False" + type: Accepted + controllerName: gateway.envoyproxy.io/gatewayclass-controller +backends: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: Backend + metadata: + creationTimestamp: null + name: backend-1 + namespace: default + spec: + endpoints: + - ip: + address: 1.1.1.1 + port: 3001 + tls: + caCertificateRefs: + - group: "" + kind: ConfigMap + name: ca-cmap + status: + conditions: + - lastTransitionTime: null + message: The Backend was accepted + reason: Accepted + status: "True" + type: Accepted +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + creationTimestamp: null + name: gateway-1 + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: All + name: http + port: 80 + protocol: HTTP + status: + listeners: + - attachedRoutes: 1 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: http + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + creationTimestamp: null + name: httproute-1 + namespace: default + spec: + parentRefs: + - name: gateway-1 + namespace: envoy-gateway + rules: + - backendRefs: + - group: gateway.envoyproxy.io + kind: Backend + name: backend-1 + matches: + - path: + value: / + status: + parents: + - conditions: + - lastTransitionTime: null + message: Route is accepted + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: 'Failed to process route rule 0 backendRef 0: configmap istio-ca-root-cert + not found in namespace default.' + reason: InvalidBackendTLS + status: "False" + type: ResolvedRefs + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parentRef: + name: gateway-1 + namespace: envoy-gateway +infraIR: + envoy-gateway/gateway-1: + proxy: + config: + apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + creationTimestamp: null + name: test + namespace: envoy-gateway-system + spec: + backendTLS: + alpnProtocols: + - HTTP/1.1 + - HTTP/2 + ciphers: + - ECDHE-RSA-AES128-GCM-SHA256 + - ECDHE-ECDSA-AES256-GCM-SHA384 + clientCertificateRef: + group: "" + kind: Secret + name: client-auth + namespace: envoy-gateway-system + ecdhCurves: + - ECDHE-RSA-AES128-GCM-SHA256 + - ECDHE-ECDSA-AES256-GCM-SHA384 + maxVersion: tls1.3 + minVersion: tls1.2 + signatureAlgorithms: + - RSA-PSS-RSAE-SHA256 + - ECDSA-SECP256R1-SHA256 + logging: {} + status: {} + listeners: + - address: null + name: envoy-gateway/gateway-1/http + ports: + - containerPort: 10080 + name: http-80 + protocol: HTTP + servicePort: 80 + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + ownerReference: + kind: GatewayClass + name: envoy-gateway-class + name: envoy-gateway/gateway-1 + namespace: envoy-gateway-system +xdsIR: + envoy-gateway/gateway-1: + accessLog: + json: + - path: /dev/stdout + globalResources: + proxyServiceCluster: + name: envoy-gateway/gateway-1 + settings: + - addressType: IP + endpoints: + - host: 7.6.5.4 + port: 8080 + zone: zone1 + metadata: + name: envoy-envoy-gateway-gateway-1-196ae069 + namespace: envoy-gateway-system + sectionName: "8080" + name: envoy-gateway/gateway-1 + protocol: TCP + http: + - address: 0.0.0.0 + externalPort: 80 + hostnames: + - '*' + isHTTP2: false + metadata: + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + sectionName: http + name: envoy-gateway/gateway-1/http + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 10080 + routes: + - directResponse: + statusCode: 500 + hostname: '*' + isHTTP2: false + metadata: + kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/0/match/0/* + pathMatch: + distinct: false + name: "" + prefix: / + readyListener: + address: 0.0.0.0 + ipFamily: IPv4 + path: /ready + port: 19003 diff --git a/internal/provider/kubernetes/controller.go b/internal/provider/kubernetes/controller.go index 7db2b157c6..304d4549f8 100644 --- a/internal/provider/kubernetes/controller.go +++ b/internal/provider/kubernetes/controller.go @@ -1363,7 +1363,6 @@ func (r *gatewayAPIReconciler) processGateways(ctx context.Context, managedGC *g } for _, gtw := range gatewayList.Items { - gtw := gtw //nolint:copyloopvar if r.namespaceLabel != nil { if ok, err := r.checkObjectNamespaceLabels(>w); err != nil { // If the error is transient, we return it to allow Reconcile to retry. @@ -1385,19 +1384,14 @@ func (r *gatewayAPIReconciler) processGateways(ctx context.Context, managedGC *g if terminatesTLS(&listener) { for _, certRef := range listener.TLS.CertificateRefs { if refsSecret(&certRef) { - if err := r.processSecretRef( - ctx, - resourceMap, - resourceTree, - resource.KindGateway, - gtw.Namespace, - gtw.Name, + if err := r.processSecretRef(ctx, + resourceMap, resourceTree, + resource.KindGateway, gtw.Namespace, gtw.Name, certRef); err != nil { if isTransientError(err) { return err } - r.log.Error(err, - "failed to process TLS SecretRef for gateway", + r.log.Error(err, "failed to process TLS SecretRef for gateway", "gateway", gtw, "secretRef", certRef) } } @@ -1455,11 +1449,6 @@ func (r *gatewayAPIReconciler) processGateways(ctx context.Context, managedGC *g if isTransientError(err) { return err } - // Update the Gateway status to not accepted if there is an error processing the parametersRef. - // These not-accepted gateways will not be processed by the gateway-api layer, but their status will be - // updated in the gateway-api layer along with other gateways. This is to avoid the potential race condition - // of updating the status in both the controller and the gateway-api layer. - status.UpdateGatewayStatusNotAccepted(>w, gwapiv1.GatewayReasonInvalidParameters, err.Error()) r.log.Error(err, "failed to process infrastructure.parametersRef for gateway", "namespace", gtw.Namespace, "name", gtw.Name) } @@ -2294,15 +2283,10 @@ func (r *gatewayAPIReconciler) processGatewayParamsRef(ctx context.Context, gtw if ep.Spec.BackendTLS != nil && ep.Spec.BackendTLS.ClientCertificateRef != nil { certRef := ep.Spec.BackendTLS.ClientCertificateRef if refsSecret(certRef) { - if err := r.processSecretRef( - ctx, - resourceMap, - resourceTree, - resource.KindGateway, - gtw.Namespace, - gtw.Name, - *certRef); err != nil { - return fmt.Errorf("failed to process TLS SecretRef for gateway %s/%s: %w", gtw.Namespace, gtw.Name, err) + if err := r.processSecretRef(ctx, + resourceMap, resourceTree, resource.KindGateway, + gtw.Namespace, gtw.Name, *certRef); err != nil { + r.log.Error(err, "failed to process ClientCertificateRef for EnvoyProxy", "namespace", gtw.Namespace, "name", gtw.Name) } } } From 2ea90b1d6d74a4abe1adb636dad6a6d93dc0258d Mon Sep 17 00:00:00 2001 From: zirain Date: Mon, 4 Aug 2025 17:33:23 +0800 Subject: [PATCH 2/3] revert Signed-off-by: zirain --- internal/provider/kubernetes/controller.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/provider/kubernetes/controller.go b/internal/provider/kubernetes/controller.go index 304d4549f8..c806496f0d 100644 --- a/internal/provider/kubernetes/controller.go +++ b/internal/provider/kubernetes/controller.go @@ -1449,6 +1449,11 @@ func (r *gatewayAPIReconciler) processGateways(ctx context.Context, managedGC *g if isTransientError(err) { return err } + // Update the Gateway status to not accepted if there is an error processing the parametersRef. + // These not-accepted gateways will not be processed by the gateway-api layer, but their status will be + // updated in the gateway-api layer along with other gateways. This is to avoid the potential race condition + // of updating the status in both the controller and the gateway-api layer. + status.UpdateGatewayStatusNotAccepted(>w, gwapiv1.GatewayReasonInvalidParameters, err.Error()) r.log.Error(err, "failed to process infrastructure.parametersRef for gateway", "namespace", gtw.Namespace, "name", gtw.Name) } @@ -2280,6 +2285,7 @@ func (r *gatewayAPIReconciler) processGatewayParamsRef(ctx context.Context, gtw return err } + // Missing secret shouldn't stop the Gateway infrastructure from coming up if ep.Spec.BackendTLS != nil && ep.Spec.BackendTLS.ClientCertificateRef != nil { certRef := ep.Spec.BackendTLS.ClientCertificateRef if refsSecret(certRef) { From 9bb4823b468d4ffc5f227be4e2194cb4a09d397f Mon Sep 17 00:00:00 2001 From: zirain Date: Tue, 5 Aug 2025 07:30:01 +0800 Subject: [PATCH 3/3] update test Signed-off-by: zirain --- .../backend-tls-settings-invalid.in.yaml | 29 +++++- .../backend-tls-settings-invalid.out.yaml | 97 +++++++++++++++++-- 2 files changed, 118 insertions(+), 8 deletions(-) diff --git a/internal/gatewayapi/testdata/backend-tls-settings-invalid.in.yaml b/internal/gatewayapi/testdata/backend-tls-settings-invalid.in.yaml index 5b8c5ff0e0..c9fdaa9eca 100644 --- a/internal/gatewayapi/testdata/backend-tls-settings-invalid.in.yaml +++ b/internal/gatewayapi/testdata/backend-tls-settings-invalid.in.yaml @@ -59,6 +59,23 @@ httpRoutes: - group: gateway.envoyproxy.io kind: Backend name: backend-1 + - apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-2 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - matches: + - path: + value: "/" + backendRefs: + - group: gateway.envoyproxy.io + kind: Backend + name: backend-2 configMaps: - apiVersion: v1 @@ -102,7 +119,7 @@ backendTLSPolicies: caCertificateRefs: - kind: ConfigMap group: "" - name: istio-ca-root-cert + name: ca-cmap hostname: example.com subjectAltNames: - type: URI @@ -128,3 +145,13 @@ backends: - ip: address: 1.1.1.1 port: 3001 + - apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: Backend + metadata: + name: backend-2 + namespace: default + spec: + endpoints: + - ip: + address: 2.2.2.2 + port: 3001 diff --git a/internal/gatewayapi/testdata/backend-tls-settings-invalid.out.yaml b/internal/gatewayapi/testdata/backend-tls-settings-invalid.out.yaml index 4056a8cfe2..7c6f33d270 100644 --- a/internal/gatewayapi/testdata/backend-tls-settings-invalid.out.yaml +++ b/internal/gatewayapi/testdata/backend-tls-settings-invalid.out.yaml @@ -14,7 +14,7 @@ backendTLSPolicies: caCertificateRefs: - group: "" kind: ConfigMap - name: istio-ca-root-cert + name: ca-cmap hostname: example.com subjectAltNames: - type: URI @@ -28,9 +28,9 @@ backendTLSPolicies: namespace: envoy-gateway conditions: - lastTransitionTime: null - message: Configmap istio-ca-root-cert not found in namespace default. - reason: Invalid - status: "False" + message: Policy has been accepted. + reason: Accepted + status: "True" type: Accepted controllerName: gateway.envoyproxy.io/gatewayclass-controller backends: @@ -57,6 +57,24 @@ backends: reason: Accepted status: "True" type: Accepted +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: Backend + metadata: + creationTimestamp: null + name: backend-2 + namespace: default + spec: + endpoints: + - ip: + address: 2.2.2.2 + port: 3001 + status: + conditions: + - lastTransitionTime: null + message: The Backend was accepted + reason: Accepted + status: "True" + type: Accepted gateways: - apiVersion: gateway.networking.k8s.io/v1 kind: Gateway @@ -75,7 +93,7 @@ gateways: protocol: HTTP status: listeners: - - attachedRoutes: 1 + - attachedRoutes: 2 conditions: - lastTransitionTime: null message: Sending translated listener configuration to the data plane @@ -126,8 +144,9 @@ httpRoutes: status: "True" type: Accepted - lastTransitionTime: null - message: 'Failed to process route rule 0 backendRef 0: configmap istio-ca-root-cert - not found in namespace default.' + message: 'Failed to process route rule 0 backendRef 0: failed to locate TLS + secret for client auth: envoy-gateway-system/client-auth specified in EnvoyProxy + envoy-gateway-system/test.' reason: InvalidBackendTLS status: "False" type: ResolvedRefs @@ -135,6 +154,41 @@ httpRoutes: parentRef: name: gateway-1 namespace: envoy-gateway +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + creationTimestamp: null + name: httproute-2 + namespace: default + spec: + parentRefs: + - name: gateway-1 + namespace: envoy-gateway + rules: + - backendRefs: + - group: gateway.envoyproxy.io + kind: Backend + name: backend-2 + matches: + - path: + value: / + status: + parents: + - conditions: + - lastTransitionTime: null + message: Route is accepted + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Resolved all the Object references for the Route + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parentRef: + name: gateway-1 + namespace: envoy-gateway infraIR: envoy-gateway/gateway-1: proxy: @@ -235,6 +289,35 @@ xdsIR: distinct: false name: "" prefix: / + - destination: + metadata: + kind: HTTPRoute + name: httproute-2 + namespace: default + name: httproute/default/httproute-2/rule/0 + settings: + - addressType: IP + endpoints: + - host: 2.2.2.2 + port: 3001 + metadata: + kind: Backend + name: backend-2 + namespace: default + name: httproute/default/httproute-2/rule/0/backend/0 + protocol: HTTP + weight: 1 + hostname: '*' + isHTTP2: false + metadata: + kind: HTTPRoute + name: httproute-2 + namespace: default + name: httproute/default/httproute-2/rule/0/match/0/* + pathMatch: + distinct: false + name: "" + prefix: / readyListener: address: 0.0.0.0 ipFamily: IPv4